From 6acf137cd257e43cf4ddf74cca79e481c3f6f20f Mon Sep 17 00:00:00 2001 From: Luwei Date: Tue, 17 Jun 2025 14:19:49 +0800 Subject: [PATCH 001/572] [enhancement](time_series) increase the version limit for the time series table (#51371) (#51656) pick #51371 --- be/src/agent/task_worker_pool.cpp | 3 ++- be/src/cloud/cloud_delete_task.cpp | 11 +++++++---- be/src/cloud/cloud_rowset_builder.cpp | 8 +++++--- be/src/cloud/cloud_tablet_mgr.cpp | 4 +++- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/olap/base_tablet.cpp | 8 ++++++++ be/src/olap/base_tablet.h | 2 ++ be/src/olap/olap_server.cpp | 3 ++- be/src/olap/push_handler.cpp | 9 ++++++--- be/src/olap/rowset_builder.cpp | 10 ++++++---- 11 files changed, 45 insertions(+), 17 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 5fe8da388b17a6..7eb7716e4130b5 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1893,8 +1893,9 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest& if (!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) { tablet->published_count.fetch_add(1); int64_t published_count = tablet->published_count.load(); + int32_t max_version_config = tablet->max_version_config(); if (tablet->exceed_version_limit( - config::max_tablet_version_num * + max_version_config * config::load_trigger_compaction_version_percent / 100) && published_count % 20 == 0) { auto st = _engine.submit_compaction_task( diff --git a/be/src/cloud/cloud_delete_task.cpp b/be/src/cloud/cloud_delete_task.cpp index 35c48841d38c53..5698fb632cd1ea 100644 --- a/be/src/cloud/cloud_delete_task.cpp +++ b/be/src/cloud/cloud_delete_task.cpp @@ -48,14 +48,17 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine, const TPushReq& requ tablet->last_load_time_ms = duration_cast(system_clock::now().time_since_epoch()).count(); // check if version number exceed limit - if (tablet->fetch_add_approximate_num_rowsets(0) > config::max_tablet_version_num) { + + int32_t max_version_config = tablet->max_version_config(); + if (tablet->fetch_add_approximate_num_rowsets(0) > max_version_config) { LOG_WARNING("tablet exceeds max version num limit") - .tag("limit", config::max_tablet_version_num) + .tag("limit", max_version_config) .tag("tablet_id", tablet->tablet_id()); return Status::Error( "too many versions, versions={} tablet={}. Please reduce the frequency of loading " - "data or adjust the max_tablet_version_num in be.conf to a larger value.", - config::max_tablet_version_num, tablet->tablet_id()); + "data or adjust the max_tablet_version_num or time_series_max_tablet_version_num " + "in be.conf to a larger value.", + max_version_config, tablet->tablet_id()); } // check delete condition if push for delete diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index 9466dd1062803e..8715d90340c50a 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -89,12 +89,14 @@ Status CloudRowsetBuilder::init() { Status CloudRowsetBuilder::check_tablet_version_count() { int version_count = cloud_tablet()->fetch_add_approximate_num_rowsets(0); // TODO(plat1ko): load backoff algorithm - if (version_count > config::max_tablet_version_num) { + int32_t max_version_config = cloud_tablet()->max_version_config(); + if (version_count > max_version_config) { return Status::Error( "failed to init rowset builder. version count: {}, exceed limit: {}, " "tablet: {}. Please reduce the frequency of loading data or adjust the " - "max_tablet_version_num in be.conf to a larger value.", - version_count, config::max_tablet_version_num, _tablet->tablet_id()); + "max_tablet_version_num or time_series_max_tablet_version_numin be.conf to a " + "larger value.", + version_count, max_version_config, _tablet->tablet_id()); } return Status::OK(); } diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 17d1b98a95fc77..e8d06f72577d87 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -343,11 +343,13 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( } // If tablet has too many rowsets but not be compacted for a long time, compaction should be performed // regardless of whether there is a load job recently. + + int32_t max_version_config = t->max_version_config(); return now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms || now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms || (now - t->last_load_time_ms > config::cu_compaction_freeze_interval_s * 1000 && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_interval_s * 1000 - && t->fetch_add_approximate_num_rowsets(0) < config::max_tablet_version_num / 2); + && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); }; // We don't schedule tablets that are disabled for compaction auto disable = [](CloudTablet* t) { return t->tablet_meta()->tablet_schema()->disable_auto_compaction(); }; diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 04e04aa4b4b165..02cd267cc2456d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -824,6 +824,8 @@ DEFINE_Int32(query_cache_max_partition_count, "1024"); // This is to avoid too many version num. DEFINE_mInt32(max_tablet_version_num, "2000"); +DEFINE_mInt32(time_series_max_tablet_version_num, "20000"); + // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server, // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 51a275a55b8f17..b60e10fe49d609 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -868,6 +868,8 @@ DECLARE_Int32(query_cache_max_partition_count); // This is to avoid too many version num. DECLARE_mInt32(max_tablet_version_num); +DECLARE_mInt32(time_series_max_tablet_version_num); + // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server, // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport DECLARE_String(thrift_server_type_of_fe); diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 8c8e52be30f90e..1aa74582d84d9c 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -29,6 +29,7 @@ #include "common/logging.h" #include "common/status.h" #include "olap/calc_delete_bitmap_executor.h" +#include "olap/cumulative_compaction_time_series_policy.h" #include "olap/delete_bitmap_calculator.h" #include "olap/iterators.h" #include "olap/memtable.h" @@ -1736,4 +1737,11 @@ void BaseTablet::get_base_rowset_delete_bitmap_count( } } +int32_t BaseTablet::max_version_config() { + int32_t max_version = tablet_meta()->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY + ? config::time_series_max_tablet_version_num + : config::max_tablet_version_num; + return max_version; +} + } // namespace doris diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 4e9ddaac1b5e0d..4df16de7eb35ef 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -79,6 +79,8 @@ class BaseTablet { // Property encapsulated in TabletMeta const TabletMetaSharedPtr& tablet_meta() { return _tablet_meta; } + int32 max_version_config(); + // FIXME(plat1ko): It is not appropriate to expose this lock std::shared_mutex& get_header_lock() { return _meta_lock; } diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 8469503b8edf28..df201c7d699112 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -1700,8 +1700,9 @@ void StorageEngine::_process_async_publish() { continue; } if (version != max_version + 1) { + int32_t max_version_config = tablet->max_version_config(); // Keep only the most recent versions - while (tablet_iter->second.size() > config::max_tablet_version_num) { + while (tablet_iter->second.size() > max_version_config) { need_removed_tasks.emplace_back(tablet, version); task_iter = tablet_iter->second.erase(task_iter); version = task_iter->first; diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 575b002b2f6086..8cfe6a1a7f9223 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -39,6 +39,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "olap/cumulative_compaction_time_series_policy.h" #include "olap/delete_handler.h" #include "olap/olap_define.h" #include "olap/rowset/pending_rowset_helper.h" @@ -160,13 +161,15 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR } } + int32_t max_version_config = tablet->max_version_config(); // check if version number exceed limit - if (tablet->exceed_version_limit(config::max_tablet_version_num)) { + if (tablet->exceed_version_limit(max_version_config)) { return Status::Status::Error( "failed to push data. version count: {}, exceed limit: {}, tablet: {}. Please " - "reduce the frequency of loading data or adjust the max_tablet_version_num in " + "reduce the frequency of loading data or adjust the max_tablet_version_num or " + "time_series_max_tablet_version_num in " "be.conf to a larger value.", - tablet->version_count(), config::max_tablet_version_num, tablet->tablet_id()); + tablet->version_count(), max_version_config, tablet->tablet_id()); } int version_count = tablet->version_count() + tablet->stale_version_count(); diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 97e140748cf21e..e9b518eaae02e3 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -151,9 +151,10 @@ Status RowsetBuilder::check_tablet_version_count() { bool injection = false; DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version", { injection = true; }); + int32_t max_version_config = _tablet->max_version_config(); if (injection) { // do not return if injection - } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num - 100) || + } else if (!_tablet->exceed_version_limit(max_version_config - 100) || GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { return Status::OK(); } @@ -167,12 +168,13 @@ Status RowsetBuilder::check_tablet_version_count() { int version_count = tablet()->version_count(); DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version", { version_count = INT_MAX; }); - if (version_count > config::max_tablet_version_num) { + if (version_count > max_version_config) { return Status::Error( "failed to init rowset builder. version count: {}, exceed limit: {}, " "tablet: {}. Please reduce the frequency of loading data or adjust the " - "max_tablet_version_num in be.conf to a larger value.", - version_count, config::max_tablet_version_num, _tablet->tablet_id()); + "max_tablet_version_num or time_series_max_tablet_version_num in be.conf to a " + "larger value.", + version_count, max_version_config, _tablet->tablet_id()); } return Status::OK(); } From 1b67a32737476297a524fb8373e78bdcb80d51a8 Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 17 Jun 2025 14:32:56 +0800 Subject: [PATCH 002/572] branch-3.0: Pick [fix](group commit) replay wal set cloud cluster name (#51517) (#51666) pick https://github.com/apache/doris/pull/51517 --- .../apache/doris/load/StreamLoadHandler.java | 2 +- .../ExternalFileTableValuedFunction.java | 2 +- .../test_group_commit_replay_wal.csv | 5 + .../test_group_commit_replay_wal.groovy | 95 +++++++++++++++++++ .../insert_p0/insert_group_commit_into.groovy | 2 +- 5 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 regression-test/data/cloud_p0/multi_cluster/test_group_commit_replay_wal.csv create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_group_commit_replay_wal.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java index 61e084386d00d3..0e19ac55e11c7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/StreamLoadHandler.java @@ -139,7 +139,7 @@ public void setCloudCluster() throws UserException { Preconditions.checkState(currentUser.size() == 1); ctx.setCurrentUserIdentity(currentUser.get(0)); } - if (request.isSetAuthCode() && request.isSetBackendId()) { + if ((request.isSetToken() || request.isSetAuthCode()) && request.isSetBackendId()) { long backendId = request.getBackendId(); Backend backend = Env.getCurrentSystemInfo().getBackend(backendId); Preconditions.checkNotNull(backend); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index bda5aa7134213f..5a15ccd21f50d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -91,7 +91,7 @@ import java.util.stream.Collectors; /** - * ExternalFileTableValuedFunction is used for S3/HDFS/LOCAL table-valued-function + * ExternalFileTableValuedFunction is used for S3/HDFS/LOCAL/HTTP_STREAM/GROUP_COMMIT table-valued-function */ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctionIf { public static final Logger LOG = LogManager.getLogger(ExternalFileTableValuedFunction.class); diff --git a/regression-test/data/cloud_p0/multi_cluster/test_group_commit_replay_wal.csv b/regression-test/data/cloud_p0/multi_cluster/test_group_commit_replay_wal.csv new file mode 100644 index 00000000000000..6ab7bd6bcdfde4 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/test_group_commit_replay_wal.csv @@ -0,0 +1,5 @@ +1,1 +2,2 +3,3 +4,4 +5,5 \ No newline at end of file diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_group_commit_replay_wal.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_group_commit_replay_wal.groovy new file mode 100644 index 00000000000000..7188d8c49b2a83 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/test_group_commit_replay_wal.groovy @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.awaitility.Awaitility +import static java.util.concurrent.TimeUnit.SECONDS + +suite('test_group_commit_replay_wal', 'multi_cluster,docker') { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'enable_workload_group=false', + 'sys_log_verbose_modules=org.apache.doris.qe.ConnectContext' + ] + options.beConfigs += [ + 'enable_debug_points=true' + ] + + docker(options) { + // add cluster1 + cluster.addBackend(1, "cpuster1") + def ret = sql_return_maparray """show clusters""" + logger.info("clusters: " + ret) + def cluster0 = ret.stream().filter(cluster -> cluster.is_current == "TRUE").findFirst().orElse(null) + def cluster0Name = cluster0['cluster'] as String + logger.info("current cluster: " + cluster0Name) + def cluster1 = ret.stream().filter(cluster -> cluster.cluster == "cpuster1").findFirst().orElse(null) + assertTrue(cluster1 != null) + sql """set property 'DEFAULT_CLOUD_CLUSTER' = '$cluster0Name'""" + + def backends = sql_return_maparray """show backends""" + logger.info("backends: " + backends) + long cluster1BeId = 0 + def cluster1BeHost = "" + int cluster1BePort = 0 + for (final def backend in backends) { + if (backend['Tag'].toString().contains("cpuster1")) { + cluster1BeId = backend['BackendId'] as long + cluster1BeHost = backend['Host'] + cluster1BePort = backend['HttpPort'] as int + } + } + logger.info("cluster1BeId: " + cluster1BeId + ", cluster1BeHost: " + cluster1BeHost + ", cluster1BePort: " + cluster1BePort) + assertTrue(cluster1BeId > 0) + + def testTable = "test_group_commit_replay_wal" + sql """ + create table ${testTable} (`k` int, `v` int) + UNIQUE KEY(`k`) + DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES ( + "group_commit_interval_ms"="100" + ); + """ + + GetDebugPoint().enableDebugPointForAllBEs("LoadBlockQueue._finish_group_commit_load.load_error") + streamLoad { + table "${testTable}" + set 'column_separator', ',' + set 'group_commit', 'async_mode' + unset 'label' + file 'test_group_commit_replay_wal.csv' + time 10000 + directToBe cluster1BeHost, cluster1BePort + } + def rowCount = 0 + for (int i = 0; i < 30; i++) { + def result = sql "select count(*) from ${testTable}" + logger.info("rowCount: ${result}") + rowCount = result[0][0] + if (rowCount == 5) { + break + } + Thread.sleep(500) + } + assertEquals(5, rowCount) + } +} diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy b/regression-test/suites/insert_p0/insert_group_commit_into.groovy index a64d66f7f87f2d..bade0c8f279da0 100644 --- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy +++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy @@ -66,7 +66,7 @@ suite("insert_group_commit_into") { } catch (Exception e) { logger.warn("group_commit_insert failed, retry: " + retry + ", error: " + e.getMessage()) retry++ - if ((e.getMessage().contains("is blocked on schema change") || e.getMessage().contains("can not get a block queue")) && retry < 20) { + if ((e.getMessage().contains("is blocked on schema change") || e.getMessage().contains("can not get a block queue") || e.getMessage().contains("schema version not match")) && retry < 20) { sleep(1500) continue } else { From 5a7618fe4c6ac21578ff7f90b4f3ac407d88cbf9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 17 Jun 2025 19:31:47 +0800 Subject: [PATCH 003/572] branch-3.0: [fix](nereids)allow in partition contains MAX_VALUE #46076 (#51806) Cherry-picked from #46076 Co-authored-by: starocean999 --- .../doris/nereids/trees/plans/commands/info/InPartition.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java index 611da907e15eed..079a648b0d8d55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java @@ -49,9 +49,6 @@ public void validate(Map properties) { } catch (Exception e) { throw new AnalysisException(e.getMessage(), e.getCause()); } - if (values.stream().anyMatch(l -> l.stream().anyMatch(MaxValue.class::isInstance))) { - throw new AnalysisException("MAXVALUE cannot be used in 'in partition'"); - } } @Override From 5ae9cf7ee6fe2ef53f8af34efe4b34de98c7c41f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:14:32 +0800 Subject: [PATCH 004/572] branch-3.0: [Fix](case) Fix case `test_key_bounds_truncation_write_scenarios` #51761 (#51788) Cherry-picked from #51761 Co-authored-by: bobhan1 --- ...y_bounds_truncation_write_scenarios.groovy | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy index bbb479a7bd3328..27dc9c32aa7011 100644 --- a/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy +++ b/regression-test/suites/unique_with_mow_p0/key_bounds/test_key_bounds_truncation_write_scenarios.groovy @@ -242,41 +242,7 @@ suite("test_key_bounds_truncation_write_scenarios", "nonConcurrent") { logger.info("============= compaction ==============") set_be_param("segments_key_bounds_truncation_threshold", 8) Thread.sleep(2000) - def triggerFullCompaction = { - def beNodes = sql_return_maparray("show backends;") - def tabletStat = sql_return_maparray("show tablets from ${tableName};").get(0) - def tabletBackendId = tabletStat.BackendId - def tabletId = tabletStat.TabletId - def tabletBackend; - for (def be : beNodes) { - if (be.BackendId == tabletBackendId) { - tabletBackend = be - break; - } - } - - logger.info("trigger compaction on another BE ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}") - def (code, out, err) = be_run_full_compaction(tabletBackend.Host, tabletBackend.HttpPort, tabletId) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactJson = parseJson(out.trim()) - assertEquals("success", compactJson.status.toLowerCase()) - - // wait for full compaction to complete - Awaitility.await().atMost(3, TimeUnit.SECONDS).pollDelay(200, TimeUnit.MILLISECONDS).pollInterval(100, TimeUnit.MILLISECONDS).until( - { - (code, out, err) = be_get_compaction_status(tabletBackend.Host, tabletBackend.HttpPort, tabletId) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - return !compactionStatus.run_status - } - ) - } - - // trigger full compaction on tablet - triggerFullCompaction() + trigger_and_wait_compaction(tableName, "full") checkKeyBounds(8) qt_sql "select count(*) from ${tableName};" From b8a9344e2263985e8cc58fa1404e5adf6457e360 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:16:29 +0800 Subject: [PATCH 005/572] branch-3.0: [opt](insert) return root cause of exception when executing insert #51721 (#51758) Cherry-picked from #51721 Co-authored-by: Mingyu Chen (Rayner) --- .../plans/commands/insert/BaseExternalTableInsertExecutor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index 082f1bab7d66f8..ab7b6d3e36170a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -22,6 +22,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -113,7 +114,7 @@ protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink p @Override protected void onFail(Throwable t) { - errMsg = t.getMessage() == null ? "unknown reason" : t.getMessage(); + errMsg = Util.getRootCauseMessage(t); String queryId = DebugUtil.printId(ctx.queryId()); // if any throwable being thrown during insert operation, first we should abort this txn LOG.warn("insert [{}] with query id {} failed", labelName, queryId, t); From e6f690ee6e8d04d5ce8315e501ac03180485f84e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:17:29 +0800 Subject: [PATCH 006/572] branch-3.0: [fix](doris-compose) fix docker file create directory #51644 (#51737) Cherry-picked from #51644 Co-authored-by: yujun --- docker/runtime/doris-compose/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/runtime/doris-compose/Dockerfile b/docker/runtime/doris-compose/Dockerfile index 7dc7221d6e3707..d3f6f1a22ce498 100644 --- a/docker/runtime/doris-compose/Dockerfile +++ b/docker/runtime/doris-compose/Dockerfile @@ -74,7 +74,8 @@ RUN sed -i -e s@/deb.debian.org/@/mirrors.aliyun.com/@g -e s@/security.debian.or && dpkg-reconfigure -f noninteractive tzdata \ && apt-get clean -RUN mkdir -p /opt/apache-doris/{fdb,coverage} +RUN mkdir -p /opt/apache-doris/fdb \ + && mkdir -p /opt/apache-doris/coverage RUN curl -f https://repo1.maven.org/maven2/org/jacoco/jacoco/${JACOCO_VERSION}/jacoco-${JACOCO_VERSION}.zip -o jacoco.zip \ && mkdir /jacoco \ From 987b5b502ea2b3db540bf083f46f11fdc5aabb1f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:18:01 +0800 Subject: [PATCH 007/572] branch-3.0: [bugfix](query) Fix for incorrect query results of variant type #51651 (#51705) Cherry-picked from #51651 Co-authored-by: lw112 <131352377+felixwluo@users.noreply.github.com> --- ...calResultSinkToShortCircuitPointQuery.java | 3 ++ .../data/point_query_p0/test_point_query.out | 4 ++- .../data/variant_p0/variant_with_rowstore.out | 2 +- .../point_query_p0/test_point_query.groovy | 31 ++++++++++++++++--- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalResultSinkToShortCircuitPointQuery.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalResultSinkToShortCircuitPointQuery.java index c087dcbb37b2e6..dfcd2ea289cc6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalResultSinkToShortCircuitPointQuery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/LogicalResultSinkToShortCircuitPointQuery.java @@ -65,6 +65,9 @@ private boolean scanMatchShortCircuitCondition(LogicalOlapScan olapScan) { return false; } OlapTable olapTable = olapScan.getTable(); + if (olapTable.hasVariantColumns()) { + return false; + } return olapTable.getEnableLightSchemaChange() && olapTable.getEnableUniqueKeyMergeOnWrite() && olapTable.storeRowColumn(); } diff --git a/regression-test/data/point_query_p0/test_point_query.out b/regression-test/data/point_query_p0/test_point_query.out index 8572ed52a68bf5..60aff86a589c0a 100644 --- a/regression-test/data/point_query_p0/test_point_query.out +++ b/regression-test/data/point_query_p0/test_point_query.out @@ -158,7 +158,7 @@ -- !sql -- -- !sql -- -10 20 aabc +10 20 aabc -- !sql -- -10 20 aabc update val @@ -202,3 +202,5 @@ user_guid feature sk feature_value 2021-01-01T00:00 -- !sql -- 4 +-- !sql -- +v1 \ No newline at end of file diff --git a/regression-test/data/variant_p0/variant_with_rowstore.out b/regression-test/data/variant_p0/variant_with_rowstore.out index a2aa68f2270ce9..347df4dcd41d61 100644 --- a/regression-test/data/variant_p0/variant_with_rowstore.out +++ b/regression-test/data/variant_p0/variant_with_rowstore.out @@ -33,5 +33,5 @@ -1 {"a":1123} {"a":1123} -- !point_select -- -1 1|[""] +1 "1|[\\"\\"]" diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy index fbf96d9f4783d9..04afed2ed74fee 100644 --- a/regression-test/suites/point_query_p0/test_point_query.groovy +++ b/regression-test/suites/point_query_p0/test_point_query.groovy @@ -280,8 +280,8 @@ suite("test_point_query", "nonConcurrent") { PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "store_row_column" = "true" - ); - """ + ); + """ sql "insert into test_ODS_EBA_LLREPORT(RPTNO) values('567890')" sql "select /*+ SET_VAR(enable_nereids_planner=true) */ substr(RPTNO,2,5) from test_ODS_EBA_LLREPORT where RPTNO = '567890'" @@ -298,14 +298,14 @@ suite("test_point_query", "nonConcurrent") { "enable_unique_key_merge_on_write" = "true", "store_row_column" = "true" ); - """ + """ sql """insert into `test_cc_aaaid2` values('1111111')""" qt_sql """SELECT `__DORIS_DELETE_SIGN__`, aaaid FROM - `test_cc_aaaid2` + `test_cc_aaaid2` WHERE aaaid = '1111111'""" } finally { @@ -333,7 +333,7 @@ suite("test_point_query", "nonConcurrent") { explain { sql("select * from table_3821461 where col1 = -10 and col2 = 20 and loc3 = 'aabc'") contains "SHORT-CIRCUIT" - } + } qt_sql "select * from table_3821461 where col1 = 10 and col2 = 20 and loc3 = 'aabc';" sql "delete from table_3821461 where col1 = 10 and col2 = 20 and loc3 = 'aabc';" // read delete sign @@ -444,4 +444,25 @@ suite("test_point_query", "nonConcurrent") { sql "insert into table_with_chars values (20, 30, 'aabc', 'value');" sql "set enable_short_circuit_query = true" qt_sql "select length(loc3) from table_with_chars where col1 = 10" + + // test variant type + sql "DROP TABLE IF EXISTS test_with_variant" + sql """ + CREATE TABLE `test_with_variant` ( + `col1` bigint NULL, + `col2` variant NULL + ) ENGINE=OLAP + UNIQUE KEY(`col1`) + DISTRIBUTED BY HASH(`col1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "true", + "store_row_column" = "true" + ); + """ + sql """ + INSERT INTO test_with_variant VALUES(1, '{"k1":"v1", "k2": 200}'); + """ + qt_sql "select col2['k1'] from test_with_variant where col1=1" + } From f6d7f7023bec128d59d6d22521a6ca900a7bd511 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:19:01 +0800 Subject: [PATCH 008/572] branch-3.0: [fix](regression) fix test_alter_column_comment case failed in 3 FE environment #51655 (#51691) Cherry-picked from #51655 Co-authored-by: camby --- .../suites/alter_p0/test_alter_column_comment.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/alter_p0/test_alter_column_comment.groovy b/regression-test/suites/alter_p0/test_alter_column_comment.groovy index b86f57182728cf..2e9d648288bbfa 100644 --- a/regression-test/suites/alter_p0/test_alter_column_comment.groovy +++ b/regression-test/suites/alter_p0/test_alter_column_comment.groovy @@ -19,7 +19,7 @@ suite('test_alter_column_comment', 'nonConcurrent') { def tbl = 'test_alter_column_comment_tbl' def cmt = '0123456789012345678901234567890123456789012345678901234567890123456789' - sql "ADMIN SET FRONTEND CONFIG ('column_comment_length_limit' = '64')" + sql "ADMIN SET ALL FRONTENDS CONFIG ('column_comment_length_limit' = '64')" sql "DROP TABLE IF EXISTS ${tbl} FORCE" @@ -101,5 +101,5 @@ suite('test_alter_column_comment', 'nonConcurrent') { sql "DROP TABLE IF EXISTS ${tbl} FORCE" // restore column_comment_length_limit - sql """ ADMIN SET FRONTEND CONFIG ("column_comment_length_limit" = "-1"); """ + sql """ ADMIN SET ALL FRONTENDS CONFIG ("column_comment_length_limit" = "-1"); """ } From d199e24a8a72655c517021948ef864f3bcb86824 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 18 Jun 2025 12:21:03 +0800 Subject: [PATCH 009/572] branch-3.0: [Fix](cloud-mow) Remove potential existing split delete bitmap KVs before update them in schema change (#51353) (#51531) pick https://github.com/apache/doris/pull/51353 --- cloud/src/meta-service/meta_service.cpp | 4 +- cloud/src/meta-service/meta_service_job.cpp | 6 ++ cloud/test/meta_service_job_test.cpp | 59 +++++++++++ cloud/test/meta_service_test.cpp | 111 ++++++++++++++++++++ 4 files changed, 178 insertions(+), 2 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index d26e2fde1ec988..3041d20f428e14 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -2097,13 +2097,13 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont meta_delete_bitmap_key(key_info, &key); delete_bitmap_keys.add_delete_bitmap_keys(key); } - // no need to record pending key for compaction or schema change, + // no need to record pending key for compaction, // because delete bitmap will attach to new rowset, just delete new rowset if failed // lock_id > 0 : load // lock_id = -1 : compaction // lock_id = -2 : schema change // lock_id = -3 : compaction update delete bitmap without lock - if (request->lock_id() > 0) { + if (request->lock_id() > 0 || request->lock_id() == -2) { std::string pending_val; if (!delete_bitmap_keys.SerializeToString(&pending_val)) { code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 3dd89afbb6123d..8d8da279b733e0 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -29,6 +29,7 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" +#include "keys.h" #include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_tablet_stats.h" @@ -1324,6 +1325,11 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str if (!success) { return; } + + std::string pending_key = meta_pending_delete_bitmap_key({instance_id, new_tablet_id}); + txn->remove(pending_key); + LOG(INFO) << "xxx sc remove delete bitmap pending key, pending_key=" << hex(pending_key) + << " tablet_id=" << new_tablet_id << "job_id=" << schema_change.id(); } for (size_t i = 0; i < schema_change.txn_ids().size(); ++i) { diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp index 998a24256f43c0..cf1b8cd7d581d8 100644 --- a/cloud/test/meta_service_job_test.cpp +++ b/cloud/test/meta_service_job_test.cpp @@ -166,6 +166,31 @@ MetaServiceCode get_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_t t return res.status().code(); } +MetaServiceCode update_delete_bitmap(MetaServiceProxy* meta_service, int64_t table_id, + int64_t partition_id, int64_t tablet_id, int64_t lock_id, + int64_t initor, + std::string cloud_unique_id = "test_cloud_unique_id") { + brpc::Controller cntl; + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + update_delete_bitmap_req.set_cloud_unique_id(cloud_unique_id); + update_delete_bitmap_req.set_table_id(table_id); + update_delete_bitmap_req.set_partition_id(partition_id); + update_delete_bitmap_req.set_lock_id(lock_id); + update_delete_bitmap_req.set_initiator(initor); + update_delete_bitmap_req.set_tablet_id(tablet_id); + for (int i = 0; i < 3; i++) { + update_delete_bitmap_req.add_rowset_ids("0200000003ea308a3647dbea83220ed4b8897f2288244a91"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(i); + update_delete_bitmap_req.add_segment_delete_bitmaps("1"); + } + meta_service->update_delete_bitmap(reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, + nullptr); + return update_delete_bitmap_res.status().code(); +} + void remove_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_t table_id) { std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id, table_id, -1}); std::unique_ptr txn; @@ -1789,11 +1814,45 @@ TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) { res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 12345); ASSERT_EQ(res_code, MetaServiceCode::OK); + + std::string pending_key = meta_pending_delete_bitmap_key({instance_id, new_tablet_id}); + std::string pending_val; + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn->get(pending_key, &pending_val), TxnErrorCode::TXN_KEY_NOT_FOUND); + + res_code = update_delete_bitmap(meta_service.get(), table_id, partition_id, new_tablet_id, + -2, 12345); + ASSERT_EQ(res_code, MetaServiceCode::OK); + + // schema change job should write pending delete bitmap key + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn->get(pending_key, &pending_val), TxnErrorCode::TXN_OK); + PendingDeleteBitmapPB pending_info; + ASSERT_TRUE(pending_info.ParseFromString(pending_val)); + ASSERT_EQ(pending_info.delete_bitmap_keys_size(), 3); + for (int i = 0; i < 3; ++i) { + std::string_view k1 = pending_info.delete_bitmap_keys(i); + k1.remove_prefix(1); + std::vector, int, int>> out; + decode_key(&k1, &out); + // 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id} ${version} ${segment_id} -> roaringbitmap + ASSERT_EQ(std::get(std::get<0>(out[3])), new_tablet_id); + ASSERT_EQ(std::get(std::get<0>(out[4])), + "0200000003ea308a3647dbea83220ed4b8897f2288244a91"); + ASSERT_EQ(std::get(std::get<0>(out[5])), i); + ASSERT_EQ(std::get(std::get<0>(out[6])), 0); + } + finish_schema_change_job(meta_service.get(), tablet_id, new_tablet_id, "job1", "be1", output_rowsets, res); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); remove_delete_bitmap_lock(meta_service.get(), table_id); res.Clear(); + + // pending delete bitmap key on new tablet should be removed after schema change job finishes + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn->get(pending_key, &pending_val), TxnErrorCode::TXN_KEY_NOT_FOUND); } { diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index bd593ce783c86d..669d84025a9920 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -5305,6 +5305,117 @@ TEST(MetaServiceTest, UpdateDeleteBitmapFailCase) { ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(0), data1); } +TEST(MetaServiceTest, UpdateDeleteBitmapScOverrideExistingKey) { + auto meta_service = get_meta_service(); + brpc::Controller cntl; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + + extern std::string get_instance_id(const std::shared_ptr& rc_mgr, + const std::string& cloud_unique_id); + auto instance_id = get_instance_id(meta_service->resource_mgr(), "test_cloud_unique_id"); + + { + // schema change should use pending delete bitmap to clear previous failed trials + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, + index_id, t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + int64_t version = 100; + + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + + { + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, + update_delete_bitmap_res, table_id, t1p1, lock_id, initiator, + tablet_id, txn_id, version, data1); + ASSERT_EQ(update_delete_bitmap_res.status().code(), MetaServiceCode::OK); + + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(tablet_id); + get_delete_bitmap_req.add_rowset_ids("123"); + get_delete_bitmap_req.add_begin_versions(0); + get_delete_bitmap_req.add_end_versions(version); + meta_service->get_delete_bitmap( + reinterpret_cast(&cntl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + ASSERT_EQ(get_delete_bitmap_res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(get_delete_bitmap_res.rowset_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.versions_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(0), data1); + } + + { + std::string pending_key = meta_pending_delete_bitmap_key({instance_id, tablet_id}); + std::string pending_val; + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn->get(pending_key, &pending_val), TxnErrorCode::TXN_OK); + PendingDeleteBitmapPB pending_info; + ASSERT_TRUE(pending_info.ParseFromString(pending_val)); + ASSERT_EQ(pending_info.delete_bitmap_keys_size(), 1); + + std::string_view k1 = pending_info.delete_bitmap_keys(0); + k1.remove_prefix(1); + std::vector, int, int>> out; + decode_key(&k1, &out); + // 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id} ${version} ${segment_id} -> roaringbitmap + auto encoded_tablet_id = std::get(std::get<0>(out[3])); + ASSERT_EQ(encoded_tablet_id, tablet_id); + auto encoded_rowset_id = std::get(std::get<0>(out[4])); + ASSERT_EQ(encoded_rowset_id, "123"); + auto encoded_version = std::get(std::get<0>(out[5])); + ASSERT_EQ(encoded_version, version); + auto encoded_segment_id = std::get(std::get<0>(out[6])); + ASSERT_EQ(encoded_segment_id, 0); + } + + { + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 3 KVs + // if we don't remove previous splited KVs, will crash when reading + std::string data2(split_size * 3, 'a'); + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, + update_delete_bitmap_res, table_id, t1p1, lock_id, initiator, + tablet_id, txn_id, version, data2); + ASSERT_EQ(update_delete_bitmap_res.status().code(), MetaServiceCode::OK); + + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(tablet_id); + get_delete_bitmap_req.add_rowset_ids("123"); + get_delete_bitmap_req.add_begin_versions(0); + get_delete_bitmap_req.add_end_versions(version); + meta_service->get_delete_bitmap( + reinterpret_cast(&cntl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + ASSERT_EQ(get_delete_bitmap_res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(get_delete_bitmap_res.rowset_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.versions_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_ids_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps_size(), 1); + ASSERT_EQ(get_delete_bitmap_res.segment_delete_bitmaps(0), data2); + } + } +} + TEST(MetaServiceTest, UpdateDeleteBitmap) { auto meta_service = get_meta_service(); From 029197d9372c2b24befc1e8a6c93c082c6dfc976 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Wed, 18 Jun 2025 12:22:31 +0800 Subject: [PATCH 010/572] [feat](cloud) Add unused rowset state for CloudTablet (#51674) the pr is the manual partitial pick https://github.com/apache/doris/pull/51573 --- be/src/cloud/cloud_tablet.cpp | 57 +++++++++++++++++++++++++++++++ be/src/cloud/cloud_tablet.h | 9 +++++ be/src/cloud/cloud_tablet_mgr.cpp | 16 +++++++++ 3 files changed, 82 insertions(+) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 039cdb85f22e0e..c044b8361b7af3 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -56,6 +56,8 @@ namespace doris { #include "common/compile_check_begin.h" using namespace ErrorCode; +bvar::Adder g_unused_rowsets_count("unused_rowsets_count"); + static constexpr int LOAD_INITIATOR_ID = -1; CloudTablet::CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta) @@ -337,6 +339,19 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ // replace existed rowset with `to_add` rowset. This may occur when: // 1. schema change converts rowsets which have been double written to new tablet // 2. cumu compaction picks single overlapping input rowset to perform compaction + + // add existed rowset to unused_rowsets to remove delete bitmap and recycle cached data + std::vector unused_rowsets; + if (auto find_it = _rs_version_map.find(rs->version()); + find_it != _rs_version_map.end()) { + DCHECK(find_it->second->rowset_id() != rs->rowset_id()) + << "tablet_id=" << tablet_id() + << ", rowset_id=" << rs->rowset_id().to_string() + << ", existed rowset_id=" << find_it->second->rowset_id().to_string(); + unused_rowsets.push_back(find_it->second); + } + add_unused_rowsets(unused_rowsets); + _tablet_meta->delete_rs_meta_by_version(rs->version(), nullptr); _rs_version_map[rs->version()] = rs; _tablet_meta->add_rowsets_unchecked({rs}); @@ -447,9 +462,51 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() { if (config::enable_mow_verbose_log) { LOG_INFO("finish delete_expired_stale_rowset for tablet={}", tablet_id()); } + add_unused_rowsets(expired_rowsets); return expired_rowsets.size(); } +bool CloudTablet::need_remove_unused_rowsets() { + std::lock_guard lock(_gc_mutex); + return !_unused_rowsets.empty(); +} + +void CloudTablet::add_unused_rowsets(const std::vector& rowsets) { + std::lock_guard lock(_gc_mutex); + for (const auto& rowset : rowsets) { + _unused_rowsets[rowset->rowset_id()] = rowset; + } + g_unused_rowsets_count << rowsets.size(); +} + +void CloudTablet::remove_unused_rowsets() { + int64_t removed_rowsets_num = 0; + OlapStopWatch watch; + std::lock_guard lock(_gc_mutex); + // 1. remove unused rowsets's cache data and delete bitmap + for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) { + // it->second is std::shared_ptr + auto&& rs = it->second; + if (rs.use_count() > 1) { + LOG(WARNING) << "tablet_id:" << tablet_id() << " rowset: " << rs->rowset_id() << " has " + << rs.use_count() << " references, it cannot be removed"; + ++it; + continue; + } + rs->clear_cache(); + it = _unused_rowsets.erase(it); + g_unused_rowsets_count << -1; + removed_rowsets_num++; + } + + if (removed_rowsets_num > 0) { + LOG(INFO) << "tablet_id=" << tablet_id() + << ", unused_rowset size=" << _unused_rowsets.size() + << ", removed_rowsets_num=" << removed_rowsets_num + << ", cost(us)=" << watch.get_elapse_time_us(); + } +} + void CloudTablet::update_base_size(const Rowset& rs) { // Define base rowset as the rowset of version [2-x] if (rs.start_version() == 2) { diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 6c41af26d77cce..ee3fdc35c6a2c5 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -261,6 +261,11 @@ class CloudTablet final : public BaseTablet { // check that if the delete bitmap in delete bitmap cache has the same cardinality with the expected_delete_bitmap's Status check_delete_bitmap_cache(int64_t txn_id, DeleteBitmap* expected_delete_bitmap) override; + bool need_remove_unused_rowsets(); + + void add_unused_rowsets(const std::vector& rowsets); + void remove_unused_rowsets(); + private: // FIXME(plat1ko): No need to record base size if rowsets are ordered by version void update_base_size(const Rowset& rs); @@ -320,6 +325,10 @@ class CloudTablet final : public BaseTablet { // Schema will be merged from all rowsets when sync_rowsets TabletSchemaSPtr _merged_tablet_schema; + + // unused_rowsets, [start_version, end_version] + std::mutex _gc_mutex; + std::unordered_map _unused_rowsets; }; using CloudTabletSPtr = std::shared_ptr; diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index e8d06f72577d87..efcfdde73616f8 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -262,6 +262,22 @@ void CloudTabletMgr::vacuum_stale_rowsets(const CountDownLatch& stop_latch) { LOG_INFO("finish vacuum stale rowsets") .tag("num_vacuumed", num_vacuumed) .tag("num_tablets", tablets_to_vacuum.size()); + + { + LOG_INFO("begin to remove unused rowsets"); + std::vector> tablets_to_remove_unused_rowsets; + tablets_to_remove_unused_rowsets.reserve(_tablet_map->size()); + _tablet_map->traverse([&tablets_to_remove_unused_rowsets](auto&& t) { + if (t->need_remove_unused_rowsets()) { + tablets_to_remove_unused_rowsets.push_back(t); + } + }); + for (auto& t : tablets_to_remove_unused_rowsets) { + t->remove_unused_rowsets(); + } + LOG_INFO("finish remove unused rowsets") + .tag("num_tablets", tablets_to_remove_unused_rowsets.size()); + } } std::vector> CloudTabletMgr::get_weak_tablets() { From 2404371f3c8e050a3d0ce5da44183daba2724aed Mon Sep 17 00:00:00 2001 From: yujun Date: Wed, 18 Jun 2025 12:23:11 +0800 Subject: [PATCH 011/572] branch-3.0: [fix](nereids) fix cast ipv4 to string #51546 (#51739) cherry pick from #51546 --- .../apache/doris/common/util/NetUtils.java | 28 ++++++++ .../expressions/literal/IPv4Literal.java | 68 +++++++++++++++++-- .../doris/common/util/NetUtilsTest.java | 37 ++++++++++ .../pull_up_predicate_literal.out | 2 +- .../fold_constant/fold_constant_ip.groovy | 30 ++++++++ 5 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/common/util/NetUtilsTest.java create mode 100644 regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_ip.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/NetUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/NetUtils.java index 9b787f52bf47a2..13fa4fbaf9c60f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/NetUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/NetUtils.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.net.DatagramSocket; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.NetworkInterface; @@ -161,4 +162,31 @@ public static SystemInfoService.HostInfo resolveHostInfoFromHostPort(String host return new SystemInfoService.HostInfo(pair[0], Integer.valueOf(pair[1])); } + /** + * Convert IPv4 address to long + * @param inet4Address IPv4 address + * @return The corresponding long value + */ + public static long inet4AddressToLong(Inet4Address inet4Address) { + byte[] bytes = inet4Address.getAddress(); + long result = 0; + for (byte b : bytes) { + result = result << 8 | (b & 0xFF); + } + return result; + } + + /** + * Convert long value back to IPv4 address + * @param value IP address as a long value + * @return The corresponding IPv4 address + */ + public static Inet4Address longToInet4Address(long value) throws Exception { + byte[] bytes = new byte[4]; + bytes[0] = (byte) ((value >> 24) & 0xFF); + bytes[1] = (byte) ((value >> 16) & 0xFF); + bytes[2] = (byte) ((value >> 8) & 0xFF); + bytes[3] = (byte) (value & 0xFF); + return (Inet4Address) Inet4Address.getByAddress(bytes); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java index e4e931738b02a2..fd012561260869 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv4Literal.java @@ -18,10 +18,13 @@ package org.apache.doris.nereids.trees.expressions.literal; import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.common.util.NetUtils; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IPv4Type; +import java.net.Inet4Address; +import java.util.Objects; import java.util.regex.Pattern; /** @@ -32,20 +35,67 @@ public class IPv4Literal extends Literal { private static final Pattern IPV4_STD_REGEX = Pattern.compile("^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"); - private long value; + /** + * Add a class Inet4Addr wrap in Inet4Address, + * When cast ipv4 literal to string, it will call `new StringLiteral(ipv4Literal.getValue().toString())`, + * but Inet4Address.toString() contains a prefix "/", like "/192.168.1.10". + * Use Inet4Addr can solve this problem. + */ + public static class Inet4Addr { + final Inet4Address address; + + public Inet4Addr(Inet4Address addr) { + this.address = addr; + } + + public Inet4Address getAddress() { + return this.address; + } + + public long toLong() { + return NetUtils.inet4AddressToLong(address); + } + + @Override + public String toString() { + return address.getHostAddress(); + } + + @Override + public int hashCode() { + return Objects.hash(address); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof Inet4Addr)) { + return false; + } + Inet4Addr otherAddr = (Inet4Addr) other; + return address.equals(otherAddr.address); + } + } + + private Inet4Addr value; public IPv4Literal(String ipv4) throws AnalysisException { super(IPv4Type.INSTANCE); init(ipv4); } - protected IPv4Literal(long value) { + protected IPv4Literal(long value) throws AnalysisException { super(IPv4Type.INSTANCE); - this.value = value; + Inet4Address address; + try { + address = NetUtils.longToInet4Address(value); + } catch (Exception e) { + throw new AnalysisException(e.getMessage()); + } + this.value = new Inet4Addr(address); } @Override - public Long getValue() { + public Inet4Addr getValue() { return value; } @@ -56,7 +106,7 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public LiteralExpr toLegacyLiteral() { - return new org.apache.doris.analysis.IPv4Literal(value); + return new org.apache.doris.analysis.IPv4Literal(value.toLong()); } void init(String ipv4) throws AnalysisException { @@ -80,7 +130,13 @@ void init(String ipv4) throws AnalysisException { } value = (value << 8) | octet; } - this.value = value; + Inet4Address address; + try { + address = NetUtils.longToInet4Address(value); + } catch (Exception e) { + throw new AnalysisException(e.getMessage()); + } + this.value = new Inet4Addr(address); } private void checkValueValid(String ipv4) throws AnalysisException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/NetUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/NetUtilsTest.java new file mode 100644 index 00000000000000..bbebfdfd33f7b8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/NetUtilsTest.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import org.junit.Assert; +import org.junit.Test; + +import java.net.Inet4Address; +import java.net.InetAddress; + +public class NetUtilsTest { + + @Test + public void testConvertIp() throws Exception { + long ipValue = 3232235786L; + InetAddress ip = InetAddress.getByName("192.168.1.10"); + Assert.assertTrue(ip instanceof Inet4Address); + Assert.assertEquals(ipValue, NetUtils.inet4AddressToLong((Inet4Address) ip)); + Inet4Address convertIp = NetUtils.longToInet4Address(ipValue); + Assert.assertEquals(ip, convertIp); + } +} diff --git a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out index 88d7fa790d9d61..1b4c185e280013 100644 --- a/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out +++ b/regression-test/data/nereids_rules_p0/infer_predicate/pull_up_predicate_literal.out @@ -2471,7 +2471,7 @@ PhysicalResultSink ----------PhysicalProject ------------PhysicalStorageLayerAggregate[test_pull_up_predicate_literal] ------PhysicalProject ---------filter((t2.d_ipv4 = 2130706433)) +--------filter((t2.d_ipv4 = 127.0.0.1)) ----------PhysicalOlapScan[test_types] -- !const_value_and_join_column_type239 -- diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_ip.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_ip.groovy new file mode 100644 index 00000000000000..375057a01a88a7 --- /dev/null +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_ip.groovy @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("fold_constant_ip") { + // cast function + for (def ipv4 : ["1", "256.256.256.256", "192.168.1.10"]) { + testFoldConst("SELECT cast('${ipv4}' as ipv4)") + testFoldConst("SELECT cast(cast('${ipv4}' as ipv4) as string)") + testFoldConst("SELECT cast(cast(cast('${ipv4}' as ipv4) as string) as ipv4)") + } + for (def ipv6 : ["1", "ef8d:3d6a:869b:2582:7200:aa46:4dcd:2bd4"]) { + testFoldConst("SELECT cast('${ipv6}' as ipv6)") + testFoldConst("SELECT cast(cast('${ipv6}' as ipv6) as string)") + testFoldConst("SELECT cast(cast(cast('${ipv6}' as ipv6) as string) as ipv6)") + } +} From d499904f4073db892a57213861eb5a5f17582a0b Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 18 Jun 2025 12:24:03 +0800 Subject: [PATCH 012/572] [cherry-pick](branch-30) support split_by_regexp (#38259) (#47676) (#50272) bp (#38259) (#47676) also need pick https://github.com/apache/doris/pull/51293 --------- Co-authored-by: James --- .../functions/function_split_by_regexp.cpp | 378 ++++++++++++++++++ .../vec/functions/simple_function_factory.h | 2 + .../doris/catalog/BuiltinScalarFunctions.java | 2 + .../functions/scalar/SplitByRegexp.java | 97 +++++ .../visitor/ScalarFunctionVisitor.java | 5 + gensrc/script/doris_builtins_functions.py | 2 + .../string_functions/test_split_by_regexp.out | 70 ++++ .../test_split_by_regexp.groovy | 74 ++++ 8 files changed, 630 insertions(+) create mode 100644 be/src/vec/functions/function_split_by_regexp.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitByRegexp.java create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy diff --git a/be/src/vec/functions/function_split_by_regexp.cpp b/be/src/vec/functions/function_split_by_regexp.cpp new file mode 100644 index 00000000000000..40628ee2017fc7 --- /dev/null +++ b/be/src/vec/functions/function_split_by_regexp.cpp @@ -0,0 +1,378 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "common/status.h" +#include "vec/columns/column_array.h" +#include "vec/columns/column_const.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_string.h" +#include "vec/functions/function.h" +#include "vec/functions/function_string.h" +#include "vec/functions/simple_function_factory.h" + +namespace doris::vectorized { + +struct Match { + std::string::size_type offset; + std::string::size_type length; +}; + +class RegexpSplit { +public: + void init(re2::RE2* re2, int32_t max_splits); + void set(const char* pos, const char* end); + bool get(const char*& token_begin, const char*& token_end); + +private: + const char* _pos; + const char* _end; + + std::int32_t _max_splits = 0; + std::vector _matches; + int32_t _splits; + re2::RE2* _re2 = nullptr; + unsigned _number_of_subpatterns = 0; + + unsigned match(const char* subject, size_t subject_size, std::vector& matches, + unsigned limit) const; +}; + +unsigned RegexpSplit::match(const char* subject, size_t subject_size, std::vector& matches, + unsigned limit) const { + matches.clear(); + + if (limit == 0) { + return 0; + } + + limit = std::min(limit, _number_of_subpatterns + 1); + std::vector pieces(limit); + + if (!_re2->Match({subject, subject_size}, 0, subject_size, re2::RE2::UNANCHORED, pieces.data(), + limit)) { + return 0; + } else { + matches.resize(limit); + for (size_t i = 0; i < limit; ++i) { + if (pieces[i].empty()) { + matches[i].offset = std::string::npos; + matches[i].length = 0; + } else { + matches[i].offset = pieces[i].data() - subject; + matches[i].length = pieces[i].length(); + } + } + return limit; + } +} + +void RegexpSplit::init(re2::RE2* re2, int32_t max_splits) { + _max_splits = max_splits; + _re2 = re2; + if (_re2) { + _number_of_subpatterns = _re2->NumberOfCapturingGroups(); + } +} + +// Called for each next string. +void RegexpSplit::set(const char* pos, const char* end) { + _pos = pos; + _end = end; + _splits = 0; +} + +// Get the next token, if any, or return false. +bool RegexpSplit::get(const char*& token_begin, const char*& token_end) { + if (!_re2) { + if (_pos == _end) { + return false; + } + + token_begin = _pos; + if (_max_splits != -1) { + if (_splits == _max_splits - 1) { + token_end = _end; + _pos = _end; + return true; + } + } + + _pos += 1; + token_end = _pos; + ++_splits; + } else { + if (!_pos || _pos > _end) { + return false; + } + + token_begin = _pos; + if (_max_splits != -1) { + if (_splits == _max_splits - 1) { + token_end = _end; + _pos = nullptr; + return true; + } + } + + if (!match(_pos, _end - _pos, _matches, _number_of_subpatterns + 1) || + !_matches[0].length) { + token_end = _end; + _pos = _end + 1; + } else { + token_end = _pos + _matches[0].offset; + _pos = token_end + _matches[0].length; + ++_splits; + } + } + + return true; +} + +template +class SplitByRegexp : public IFunction { +public: + static constexpr auto name = "split_by_regexp"; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { + return get_variadic_argument_types_impl().size(); + } + + bool is_variadic() const override { return true; } + + DataTypes get_variadic_argument_types_impl() const override { + return Impl::get_variadic_argument_types(); + } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + DCHECK(is_string(arguments[0])) + << "first argument for function: " << name << " should be string" + << " and arguments[0] is " << arguments[0]->get_name(); + DCHECK(is_string(arguments[1])) + << "second argument for function: " << name << " should be string" + << " and arguments[1] is " << arguments[1]->get_name(); + auto nullable_string_type = make_nullable(std::make_shared()); + return std::make_shared(nullable_string_type); + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) const override { + return Impl::execute_impl(context, block, arguments, result, input_rows_count); + } +}; + +struct ExecuteImpl { + using NullMapType = PaddedPODArray; + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, size_t result, + size_t input_rows_count) { + const auto& [first_column, left_const] = + unpack_if_const(block.get_by_position(arguments[0]).column); + const auto& [second_column, right_const] = + unpack_if_const(block.get_by_position(arguments[1]).column); + const auto& [three_column, three_is_const] = + unpack_if_const(block.get_by_position(arguments[2]).column); + auto limit_value = assert_cast(*three_column).get_int(0); + const auto& src_column = assert_cast(*first_column); + const auto& pattern_column = assert_cast(*second_column); + + auto nullable_string_type = make_nullable(std::make_shared()); + auto dest_column_ptr = ColumnArray::create(nullable_string_type->create_column(), + ColumnArray::ColumnOffsets::create()); + IColumn* dest_nested_column = &dest_column_ptr->get_data(); + auto& dest_offsets = dest_column_ptr->get_offsets(); + DCHECK(dest_nested_column != nullptr); + + NullMapType* dest_nested_null_map = nullptr; + auto* dest_nullable_col = assert_cast(dest_nested_column); + auto& dest_column_string = + assert_cast(*(dest_nullable_col->get_nested_column_ptr())); + dest_nested_null_map = &dest_nullable_col->get_null_map_column().get_data(); + RE2::Options opts; + opts.set_never_nl(false); + opts.set_dot_nl(true); + // split_by_regexp(ColumnString, "xxx") + if (right_const) { + RETURN_IF_ERROR(_execute_constant_pattern( + src_column, pattern_column.get_data_at(0), dest_column_string, dest_offsets, + dest_nested_null_map, limit_value, input_rows_count, &opts)); + } else if (left_const) { + // split_by_regexp("xxx", ColumnString) + _execute_constant_src_string(src_column.get_data_at(0), pattern_column, + dest_column_string, dest_offsets, dest_nested_null_map, + limit_value, input_rows_count, &opts); + } else { + // split_by_regexp(ColumnString, ColumnString) + _execute_vector_vector(src_column, pattern_column, dest_column_string, dest_offsets, + dest_nested_null_map, limit_value, input_rows_count, &opts); + } + + block.replace_by_position(result, std::move(dest_column_ptr)); + return Status::OK(); + } + +private: + static Status _execute_constant_pattern(const ColumnString& src_column_string, + const StringRef& pattern_ref, + ColumnString& dest_column_string, + ColumnArray::Offsets64& dest_offsets, + NullMapType* dest_nested_null_map, Int64 limit_value, + size_t input_rows_count, RE2::Options* opts) { + const char* token_begin = nullptr; + const char* token_end = nullptr; + UInt64 index = 0; + std::unique_ptr re2_ptr = nullptr; + if (pattern_ref.size) { + re2_ptr = std::make_unique(pattern_ref.to_string_view(), *opts); + } + if (!re2_ptr->ok()) { + return Status::RuntimeError("Invalid pattern: {}", pattern_ref.debug_string()); + } + RegexpSplit RegexpSplit; + RegexpSplit.init(re2_ptr.get(), limit_value); + for (int row = 0; row < input_rows_count; ++row) { + auto str_data = src_column_string.get_data_at(row); + RegexpSplit.set(str_data.begin(), str_data.end()); + while (RegexpSplit.get(token_begin, token_end)) { + size_t token_size = token_end - token_begin; + dest_column_string.insert_data(token_begin, token_size); + dest_nested_null_map->push_back(false); + index += 1; + } + dest_offsets.push_back(index); + } + return Status::OK(); + } + + static void _execute_constant_src_string(const StringRef& str_ref, + const ColumnString& pattern_column, + ColumnString& dest_column_string, + ColumnArray::Offsets64& dest_offsets, + NullMapType* dest_nested_null_map, Int64 limit_value, + size_t input_rows_count, RE2::Options* opts) { + const char* token_begin = nullptr; + const char* token_end = nullptr; + UInt64 index = 0; + RegexpSplit RegexpSplit; + + for (int row = 0; row < input_rows_count; ++row) { + std::unique_ptr re2_ptr = nullptr; + auto pattern = pattern_column.get_data_at(row); + if (pattern.size) { + re2_ptr = std::make_unique(pattern.to_string_view(), *opts); + if (!re2_ptr->ok()) { + dest_column_string.insert_default(); + dest_nested_null_map->push_back(true); + index += 1; + dest_offsets.push_back(index); + continue; + } + } + + RegexpSplit.init(re2_ptr.get(), limit_value); + RegexpSplit.set(str_ref.begin(), str_ref.end()); + while (RegexpSplit.get(token_begin, token_end)) { + size_t token_size = token_end - token_begin; + dest_column_string.insert_data(token_begin, token_size); + dest_nested_null_map->push_back(false); + index += 1; + } + dest_offsets.push_back(index); + } + } + + static void _execute_vector_vector(const ColumnString& src_column_string, + const ColumnString& pattern_column, + ColumnString& dest_column_string, + ColumnArray::Offsets64& dest_offsets, + NullMapType* dest_nested_null_map, Int64 limit_value, + size_t input_rows_count, RE2::Options* opts) { + const char* token_begin = nullptr; + const char* token_end = nullptr; + UInt64 index = 0; + RegexpSplit RegexpSplit; + + for (int row = 0; row < input_rows_count; ++row) { + std::unique_ptr re2_ptr = nullptr; + auto str_data = src_column_string.get_data_at(row); + auto pattern = pattern_column.get_data_at(row); + if (pattern.size) { + re2_ptr = std::make_unique(pattern.to_string_view(), *opts); + if (!re2_ptr->ok()) { + dest_column_string.insert_default(); + dest_nested_null_map->push_back(true); + index += 1; + dest_offsets.push_back(index); + continue; + } + } + RegexpSplit.init(re2_ptr.get(), limit_value); + RegexpSplit.set(str_data.begin(), str_data.end()); + while (RegexpSplit.get(token_begin, token_end)) { + size_t token_size = token_end - token_begin; + dest_column_string.insert_data(token_begin, token_size); + dest_nested_null_map->push_back(false); + index += 1; + } + dest_offsets.push_back(index); + } + } +}; + +struct TwoArgumentImpl { + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared()}; + } + + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, size_t result, + size_t input_rows_count) { + DCHECK_EQ(arguments.size(), 2); + auto max_limit = ColumnConst::create(ColumnInt32::create(1, -1), input_rows_count); + block.insert({std::move(max_limit), std::make_shared(), "max_limit"}); + ColumnNumbers temp_arguments = {arguments[0], arguments[1], block.columns() - 1}; + return ExecuteImpl::execute_impl(context, block, temp_arguments, result, input_rows_count); + } +}; + +struct ThreeArgumentImpl { + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared(), + std::make_shared()}; + } + static Status execute_impl(FunctionContext* context, Block& block, + const ColumnNumbers& arguments, size_t result, + size_t input_rows_count) { + DCHECK_EQ(arguments.size(), 3); + return ExecuteImpl::execute_impl(context, block, arguments, result, input_rows_count); + } +}; + +void register_function_split_by_regexp(SimpleFunctionFactory& factory) { + factory.register_function>(); + factory.register_function>(); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/functions/simple_function_factory.h b/be/src/vec/functions/simple_function_factory.h index 7619858153c203..727cf98cda38fd 100644 --- a/be/src/vec/functions/simple_function_factory.h +++ b/be/src/vec/functions/simple_function_factory.h @@ -109,6 +109,7 @@ void register_function_url(SimpleFunctionFactory& factory); void register_function_ip(SimpleFunctionFactory& factory); void register_function_multi_match(SimpleFunctionFactory& factory); void register_function_assert_true(SimpleFunctionFactory& factory); +void register_function_split_by_regexp(SimpleFunctionFactory& factory); class SimpleFunctionFactory { using Creator = std::function; @@ -313,6 +314,7 @@ class SimpleFunctionFactory { register_function_variant_element(instance); register_function_multi_match(instance); register_function_assert_true(instance); + register_function_split_by_regexp(instance); }); return instance; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 1c1a820952f263..7d9314cb74fd75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -385,6 +385,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4Encrypt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Space; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByChar; +import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByRegexp; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByString; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitPart; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sqrt; @@ -881,6 +882,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Sm4Encrypt.class, "sm4_encrypt"), scalar(Space.class, "space"), scalar(SplitByChar.class, "split_by_char"), + scalar(SplitByRegexp.class, "split_by_regexp"), scalar(SplitByString.class, "split_by_string"), scalar(SplitPart.class, "split_part"), scalar(Sqrt.class, "sqrt"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitByRegexp.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitByRegexp.java new file mode 100644 index 00000000000000..a72ed434cc35f7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/SplitByRegexp.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.types.StringType; +import org.apache.doris.nereids.types.VarcharType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'split_by_regexp'. This class is generated by + * GenerateFunction. + */ +public class SplitByRegexp extends ScalarFunction + implements ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(ArrayType.of(VarcharType.SYSTEM_DEFAULT)) + .args(StringType.INSTANCE, StringType.INSTANCE), + FunctionSignature.ret(ArrayType.of(VarcharType.SYSTEM_DEFAULT)) + .args(StringType.INSTANCE, StringType.INSTANCE, IntegerType.INSTANCE)); + + /** + * constructor with 2 arguments. + */ + public SplitByRegexp(Expression arg0, Expression arg1) { + super("split_by_regexp", arg0, arg1); + } + + /** + * constructor with 3 arguments. + */ + public SplitByRegexp(Expression arg0, Expression arg1, Expression arg2) { + super("split_by_regexp", arg0, arg1, arg2); + } + + /** + * withChildren. + */ + @Override + public SplitByRegexp withChildren(List children) { + Preconditions.checkArgument(children.size() == 2 || children.size() == 3); + if (children.size() == 2) { + return new SplitByRegexp(children.get(0), children.get(1)); + } else { + return new SplitByRegexp(children.get(0), children.get(1), children.get(2)); + } + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (children().size() == 3) { + if (!child(2).isConstant() || !(child(2) instanceof IntegerLikeLiteral) + || (((IntegerLikeLiteral) child(2)).getIntValue() < 0)) { + throw new AnalysisException("the third parameter of " + + getName() + " function must be a positive constant: " + toSql()); + } + } + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitSplitByRegexp(this, context); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index b061e2f8d6a6f3..f85ce2fbed3b80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -383,6 +383,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Sm4Encrypt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Space; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByChar; +import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByRegexp; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitByString; import org.apache.doris.nereids.trees.expressions.functions.scalar.SplitPart; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sqrt; @@ -1912,6 +1913,10 @@ default R visitSplitByChar(SplitByChar splitByChar, C context) { return visitScalarFunction(splitByChar, context); } + default R visitSplitByRegexp(SplitByRegexp splitByRegexp, C context) { + return visitScalarFunction(splitByRegexp, context); + } + default R visitSplitByString(SplitByString splitByString, C context) { return visitScalarFunction(splitByString, context); } diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py index 99c17f7dfc835a..3f4614dc337e51 100644 --- a/gensrc/script/doris_builtins_functions.py +++ b/gensrc/script/doris_builtins_functions.py @@ -1659,6 +1659,8 @@ [['money_format'], 'VARCHAR', ['DECIMAL64'], ''], [['money_format'], 'VARCHAR', ['DECIMAL128'], ''], [['split_by_string'],'ARRAY_VARCHAR',['STRING','STRING'], ''], + [['split_by_regexp'],'ARRAY_VARCHAR',['STRING','STRING'], ''], + [['split_by_regexp'],'ARRAY_VARCHAR',['STRING','STRING', 'INT'], ''], [['split_part'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'INT'], 'ALWAYS_NULLABLE'], [['substring_index'], 'VARCHAR', ['VARCHAR', 'VARCHAR', 'INT'], 'DEPEND_ON_ARGUMENT'], [['extract_url_parameter'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], ''], diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out new file mode 100644 index 00000000000000..1fb99f58ab1c7c --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out @@ -0,0 +1,70 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +["a", "b", "c", "d", "e"] + +-- !select2 -- +["a", "bc", "de", "f"] + +-- !select3 -- +\N + +-- !select4 -- +\N + +-- !select5 -- +["abcde"] +["12553"] +[""] +[""] +[""] +["a1b1c1d"] +["", "", "", ""] +["a", "b", "c"] +["a", "b", "c", ""] +\N +["a", "b", "c", "12345", ""] + +-- !select6 -- +["d", "o", ",", "r", "i", "s"] +["d", "o", ",", "r", "i", "s"] +["d", "o", ",", "r", "i", "s"] +["do", "ris"] +["do,ris"] +["do,ris"] +["do", "ris"] +["do", "ris"] +["do", "ris"] +["do", "ris"] +["do", "ris"] + +-- !select7 -- +["a", "b", "c", "d", "e"] +["1", "2", "5", "5", "3"] +[] +[""] +[""] +["a", "b", "c", "d"] +["", "", "", ""] +["a", "b", "c"] +["a", "b", "c", ""] +\N +["a", "b", "c", "12345", ""] + +-- !select8 -- +["aa,bbb,cccc"] + +-- !select9 -- +["aa", "bbb,cccc"] + +-- !select10 -- +["aa", "bbb", "cccc"] + +-- !select11 -- +["aa", "bbb", "cccc"] + +-- !select12 -- +["aa", "bbb", "cccc"] + +-- !select13 -- +["aa", "bbb", "cccc"] + diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy new file mode 100644 index 00000000000000..4b9719068e6606 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_split_by_regexp") { + qt_select1 "select split_by_regexp('abcde','');" + qt_select2 "select split_by_regexp('a12bc23de345f','\\\\d+');" + qt_select3 "select split_by_regexp('a12bc23de345f',NULL);" + qt_select4 "select split_by_regexp(NULL, 'a12bc23de345f');" + + def tableName1 = "test_split_by_regexp" + + sql """DROP TABLE IF EXISTS ${tableName1}""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName1} ( + `k1` int(11) NULL COMMENT "", + `v1` varchar(20) NULL COMMENT "", + `v2` varchar(1) NOT NULL COMMENT "" + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2" + ) + """ + sql """ INSERT INTO ${tableName1} VALUES(1, 'abcde', '') """ + sql """ INSERT INTO ${tableName1} VALUES(2, '12553', '') """ + sql """ INSERT INTO ${tableName1} VALUES(3, '', '') """ + sql """ INSERT INTO ${tableName1} VALUES(4, '', ',') """ + sql """ INSERT INTO ${tableName1} VALUES(5, '', 'a') """ + sql """ INSERT INTO ${tableName1} VALUES(6, 'a1b1c1d', '1') """ + sql """ INSERT INTO ${tableName1} VALUES(7, ',,,', ',') """ + sql """ INSERT INTO ${tableName1} VALUES(8, 'a,b,c', ',') """ + sql """ INSERT INTO ${tableName1} VALUES(9, 'a,b,c,', ',') """ + sql """ INSERT INTO ${tableName1} VALUES(10, null, ',') """ + sql """ INSERT INTO ${tableName1} VALUES(11, 'a,b,c,12345,', ',') """ + + test { + sql " select split_by_regexp(NULL, 'a12bc23de345f', k1) from test_split_by_regexp" + exception "function must be a positive constant" + } + test { + sql " select split_by_regexp(NULL, 'a12bc23de345f', -10) from test_split_by_regexp" + exception "function must be a positive constant" + } + test { + sql " select split_by_regexp(NULL, 'a12bc23de345f', 1 + 2) from test_split_by_regexp" + exception "function must be a positive constant" + } + qt_select5 "select split_by_regexp(v1, ',') from test_split_by_regexp order by k1;" + qt_select6 "select split_by_regexp('do,ris', v2) from test_split_by_regexp order by k1;" + qt_select7 "select split_by_regexp(v1, v2) from test_split_by_regexp order by k1;" + qt_select8 "select split_by_regexp('aa,bbb,cccc', ',', 1);" + qt_select9 "select split_by_regexp('aa,bbb,cccc', ',', 2);" + qt_select10 "select split_by_regexp('aa,bbb,cccc', ',', 3);" + qt_select11 "select split_by_regexp('aa,bbb,cccc', ',', 4);" + qt_select12 "select split_by_regexp('aa,bbb,cccc', ',', 100000000);" + qt_select13 "select split_by_regexp('aa,bbb,cccc', ',', 10000000000000);" +} + From 9d727d6c61f25ac35cc0a7e60295dc5f1a9e585a Mon Sep 17 00:00:00 2001 From: Luwei Date: Wed, 18 Jun 2025 15:24:24 +0800 Subject: [PATCH 013/572] [fix](cloud) compaction and schema change potential data race when retrying prepare rowset (#51048) (#51852) related PR #51129 pick master #51048 --- be/src/cloud/cloud_delete_task.cpp | 2 +- be/src/cloud/cloud_delta_writer.cpp | 2 +- be/src/cloud/cloud_meta_mgr.cpp | 6 +- be/src/cloud/cloud_meta_mgr.h | 4 +- be/src/cloud/cloud_rowset_builder.cpp | 2 +- be/src/cloud/cloud_schema_change_job.cpp | 4 +- be/src/olap/compaction.cpp | 5 +- cloud/src/common/config.h | 2 + cloud/src/meta-service/meta_service.cpp | 75 ++++- cloud/test/meta_service_test.cpp | 337 +++++++++++++++++++++++ gensrc/proto/cloud.proto | 2 + 11 files changed, 429 insertions(+), 12 deletions(-) diff --git a/be/src/cloud/cloud_delete_task.cpp b/be/src/cloud/cloud_delete_task.cpp index 5698fb632cd1ea..cf7a6a371bc940 100644 --- a/be/src/cloud/cloud_delete_task.cpp +++ b/be/src/cloud/cloud_delete_task.cpp @@ -94,7 +94,7 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine, const TPushReq& requ RETURN_IF_ERROR(rowset_writer->build(rowset)); rowset->rowset_meta()->set_delete_predicate(std::move(del_pred)); - auto st = engine.meta_mgr().commit_rowset(*rowset->rowset_meta()); + auto st = engine.meta_mgr().commit_rowset(*rowset->rowset_meta(), ""); // Update tablet stats tablet->fetch_add_approximate_num_rowsets(1); diff --git a/be/src/cloud/cloud_delta_writer.cpp b/be/src/cloud/cloud_delta_writer.cpp index 7beaeb3e086703..2533028314f4d9 100644 --- a/be/src/cloud/cloud_delta_writer.cpp +++ b/be/src/cloud/cloud_delta_writer.cpp @@ -113,7 +113,7 @@ Status CloudDeltaWriter::commit_rowset() { RETURN_IF_ERROR(_rowset_builder->init()); RETURN_IF_ERROR(_rowset_builder->build_rowset()); } - return _engine.meta_mgr().commit_rowset(*rowset_meta()); + return _engine.meta_mgr().commit_rowset(*rowset_meta(), ""); } Status CloudDeltaWriter::set_txn_related_delete_bitmap() { diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 4054c3b20fdaaa..2971ff33f8e75e 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -860,7 +860,7 @@ Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t old_ return Status::OK(); } -Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, +Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, const std::string& job_id, RowsetMetaSharedPtr* existed_rs_meta) { VLOG_DEBUG << "prepare rowset, tablet_id: " << rs_meta.tablet_id() << ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id(); @@ -872,6 +872,7 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, CreateRowsetResponse resp; req.set_cloud_unique_id(config::cloud_unique_id); req.set_txn_id(rs_meta.txn_id()); + req.set_tablet_job_id(job_id); RowsetMetaPB doris_rs_meta = rs_meta.get_rowset_pb(/*skip_schema=*/true); doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(doris_rs_meta)); @@ -889,7 +890,7 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, return st; } -Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, +Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id, RowsetMetaSharedPtr* existed_rs_meta) { VLOG_DEBUG << "commit rowset, tablet_id: " << rs_meta.tablet_id() << ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id(); @@ -902,6 +903,7 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, CreateRowsetResponse resp; req.set_cloud_unique_id(config::cloud_unique_id); req.set_txn_id(rs_meta.txn_id()); + req.set_tablet_job_id(job_id); RowsetMetaPB rs_meta_pb = rs_meta.get_rowset_pb(); doris_rowset_meta_to_cloud(req.mutable_rowset_meta(), std::move(rs_meta_pb)); diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index cf86f09929dfce..0cc58e48166963 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -70,10 +70,10 @@ class CloudMetaMgr { bool warmup_delta_data = false, bool sync_delete_bitmap = true, bool full_sync = false, SyncRowsetStats* sync_stats = nullptr); - Status prepare_rowset(const RowsetMeta& rs_meta, + Status prepare_rowset(const RowsetMeta& rs_meta, const std::string& job_id, std::shared_ptr* existed_rs_meta = nullptr); - Status commit_rowset(const RowsetMeta& rs_meta, + Status commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id, std::shared_ptr* existed_rs_meta = nullptr); Status update_tmp_rowset(const RowsetMeta& rs_meta); diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index 8715d90340c50a..b495ae953caf7e 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -80,7 +80,7 @@ Status CloudRowsetBuilder::init() { _calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor()->create_token(); - RETURN_IF_ERROR(_engine.meta_mgr().prepare_rowset(*_rowset_writer->rowset_meta())); + RETURN_IF_ERROR(_engine.meta_mgr().prepare_rowset(*_rowset_writer->rowset_meta(), "")); _is_init = true; return Status::OK(); diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index ce88be5264983e..05d29383af4213 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -290,7 +290,7 @@ Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam RowsetMetaSharedPtr existed_rs_meta; auto st = _cloud_storage_engine.meta_mgr().prepare_rowset(*rowset_writer->rowset_meta(), - &existed_rs_meta); + _job_id, &existed_rs_meta); if (!st.ok()) { if (st.is()) { LOG(INFO) << "Rowset " << rs_reader->version() << " has already existed in tablet " @@ -325,7 +325,7 @@ Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam st.to_string()); } - st = _cloud_storage_engine.meta_mgr().commit_rowset(*rowset_writer->rowset_meta(), + st = _cloud_storage_engine.meta_mgr().commit_rowset(*rowset_writer->rowset_meta(), _job_id, &existed_rs_meta); if (!st.ok()) { if (st.is()) { diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 9b173db26fb1e1..1f21795e1c9bd5 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1352,7 +1352,7 @@ Status CloudCompactionMixin::execute_compact_impl(int64_t permits) { << _output_rowset->rowset_meta()->rowset_id().to_string(); }) - RETURN_IF_ERROR(_engine.meta_mgr().commit_rowset(*_output_rowset->rowset_meta().get())); + RETURN_IF_ERROR(_engine.meta_mgr().commit_rowset(*_output_rowset->rowset_meta().get(), _uuid)); // 4. modify rowsets in memory RETURN_IF_ERROR(modify_rowsets()); @@ -1433,7 +1433,8 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext& compaction_type() == ReaderType::READER_BASE_COMPACTION); ctx.file_cache_ttl_sec = _tablet->ttl_seconds(); _output_rs_writer = DORIS_TRY(_tablet->create_rowset_writer(ctx, _is_vertical)); - RETURN_IF_ERROR(_engine.meta_mgr().prepare_rowset(*_output_rs_writer->rowset_meta().get())); + RETURN_IF_ERROR( + _engine.meta_mgr().prepare_rowset(*_output_rs_writer->rowset_meta().get(), _uuid)); return Status::OK(); } diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 3bb791f71e9080..14047123b4cbb3 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -222,6 +222,8 @@ CONF_mBool(enable_distinguish_hdfs_path, "true"); // If enabled, the txn status will be checked when preapre/commit rowset CONF_mBool(enable_load_txn_status_check, "true"); +CONF_mBool(enable_tablet_job_check, "true"); + // Declare a selection strategy for those servers have many ips. // Note that there should at most one ip match this list. // this is a list in semicolon-delimited format, in CIDR notation, diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 3041d20f428e14..9ecb08363cdc65 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -991,6 +992,60 @@ static void fill_schema_from_dict(MetaServiceCode& code, std::string& msg, existed_rowset_meta->CopyFrom(metas.Get(0)); } +bool check_job_existed(Transaction* txn, MetaServiceCode& code, std::string& msg, + const std::string& instance_id, int64_t tablet_id, + const std::string& rowset_id, const std::string& job_id) { + TabletIndexPB tablet_idx; + get_tablet_idx(code, msg, txn, instance_id, tablet_id, tablet_idx); + if (code != MetaServiceCode::OK) { + return false; + } + + std::string job_key = job_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_id}); + std::string job_val; + auto err = txn->get(job_key, &job_val); + if (err != TxnErrorCode::TXN_OK) { + std::stringstream ss; + ss << (err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "job not found," : "internal error,") + << " instance_id=" << instance_id << " tablet_id=" << tablet_id + << " rowset_id=" << rowset_id << " err=" << err; + msg = ss.str(); + code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::STALE_PREPARE_ROWSET + : cast_as(err); + return false; + } + + TabletJobInfoPB job_pb; + job_pb.ParseFromString(job_val); + bool match = false; + if (!job_pb.compaction().empty()) { + for (auto c : job_pb.compaction()) { + if (c.id() == job_id) { + match = true; + } + } + } + + if (job_pb.has_schema_change()) { + if (job_pb.schema_change().id() == job_id) { + match = true; + } + } + + if (!match) { + std::stringstream ss; + ss << " stale perpare rowset request," + << " instance_id=" << instance_id << " tablet_id=" << tablet_id << " job id=" << job_id + << " rowset_id=" << rowset_id; + msg = ss.str(); + code = MetaServiceCode::STALE_PREPARE_ROWSET; + return false; + } + + return true; +} + /** * Check if the transaction status is as expected. * If the transaction is not in the expected state, return false and set the error code and message. @@ -1100,6 +1155,15 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll return; } + // Check if the compaction/sc tablet job has finished + if (config::enable_tablet_job_check && request->has_tablet_job_id() && + !request->tablet_job_id().empty()) { + if (!check_job_existed(txn.get(), code, msg, instance_id, tablet_id, rowset_id, + request->tablet_job_id())) { + return; + } + } + // Check if the prepare rowset request is invalid. // If the transaction has been finished, it means this prepare rowset is a timeout retry request. // In this case, do not write the recycle key again, otherwise it may cause data loss. @@ -1237,6 +1301,15 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle return; } + // Check if the compaction/sc tablet job has finished + if (config::enable_tablet_job_check && request->has_tablet_job_id() && + !request->tablet_job_id().empty()) { + if (!check_job_existed(txn.get(), code, msg, instance_id, tablet_id, rowset_id, + request->tablet_job_id())) { + return; + } + } + // Check if the commit rowset request is invalid. // If the transaction has been finished, it means this commit rowset is a timeout retry request. // In this case, do not write the recycle key again, otherwise it may cause data loss. @@ -2882,4 +2955,4 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control response->mutable_schema_dict()->Swap(&schema_dict); } -} // namespace doris::cloud \ No newline at end of file +} // namespace doris::cloud diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 669d84025a9920..b5e609be3a8005 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -320,6 +320,32 @@ static void add_tablet_metas(MetaServiceProxy* meta_service, std::string instanc ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); } +static void start_compaction_job(MetaService* meta_service, int64_t tablet_id, + const std::string& job_id, const std::string& initiator, + int base_compaction_cnt, int cumu_compaction_cnt, + TabletCompactionJobPB::CompactionType type, + StartTabletJobResponse& res, + std::pair input_version = {0, 0}) { + brpc::Controller cntl; + StartTabletJobRequest req; + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto compaction = req.mutable_job()->add_compaction(); + compaction->set_id(job_id); + compaction->set_initiator(initiator); + compaction->set_base_compaction_cnt(base_compaction_cnt); + compaction->set_cumulative_compaction_cnt(cumu_compaction_cnt); + compaction->set_type(type); + long now = time(nullptr); + compaction->set_expiration(now + 12); + compaction->set_lease(now + 3); + if (input_version.second > 0) { + compaction->add_input_versions(input_version.first); + compaction->add_input_versions(input_version.second); + compaction->set_check_input_versions_range(true); + } + meta_service->start_tablet_job(&cntl, &req, &res, nullptr); +}; + TEST(MetaServiceTest, GetInstanceIdTest) { extern std::string get_instance_id(const std::shared_ptr& rc_mgr, const std::string& cloud_unique_id); @@ -9042,6 +9068,317 @@ TEST(MetaServiceTest, AddObjInfoWithRole) { SyncPoint::get_instance()->clear_all_call_backs(); } +TEST(MetaServiceTest, CheckJobExisted) { + auto meta_service = get_meta_service(); + + std::string instance_id = "check_job_existed_instance_id"; + auto sp = SyncPoint::get_instance(); + std::unique_ptr> defer( + (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + + // OK + { + constexpr auto table_id = 952701, index_id = 952702, partition_id = 952703, + tablet_id = 952704; + int64_t txn_id = 952705; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id("compaction1"); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + res.Clear(); + } + + // job does not exist, + { + constexpr auto table_id = 952801, index_id = 952802, partition_id = 952803, + tablet_id = 952804; + int64_t txn_id = 952805; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id("compaction1"); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::STALE_PREPARE_ROWSET) << res.status().msg(); + res.Clear(); + } + + // compaction job exists, job id not match + { + constexpr auto table_id = 952901, index_id = 952902, partition_id = 952903, + tablet_id = 952904; + int64_t txn_id = 952905; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id("compaction2"); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::STALE_PREPARE_ROWSET) << res.status().msg(); + res.Clear(); + } + + // do not set job id + { + constexpr auto table_id = 953501, index_id = 953502, partition_id = 953503, + tablet_id = 953504; + int64_t txn_id = 953505; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + res.Clear(); + } + + // job id is empty string + { + constexpr auto table_id = 953601, index_id = 953602, partition_id = 953603, + tablet_id = 953604; + int64_t txn_id = 953605; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id(""); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + res.Clear(); + } + + // commit rowset OK + { + constexpr auto table_id = 953001, index_id = 953002, partition_id = 953003, + tablet_id = 953004; + int64_t txn_id = 953005; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id("compaction1"); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + res.Clear(); + } + + // commit rowset, job does not exist, + { + constexpr auto table_id = 953101, index_id = 953102, partition_id = 953103, + tablet_id = 953104; + int64_t txn_id = 952805; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id("compaction1"); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::STALE_PREPARE_ROWSET) << res.status().msg(); + res.Clear(); + } + + // commit rowset, compaction job exists, job id not match + { + constexpr auto table_id = 953201, index_id = 953202, partition_id = 953203, + tablet_id = 953204; + int64_t txn_id = 952905; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id("compaction2"); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::STALE_PREPARE_ROWSET) << res.status().msg(); + res.Clear(); + } + + // do not set job id when commit rowset + { + constexpr auto table_id = 953301, index_id = 953302, partition_id = 953303, + tablet_id = 953304; + int64_t txn_id = 953305; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + res.Clear(); + } + + // job id is empty string when commit rowset + { + constexpr auto table_id = 953401, index_id = 953402, partition_id = 953403, + tablet_id = 953404; + int64_t txn_id = 953405; + std::string label = "update_rowset_meta_test_label1"; + CreateRowsetResponse res; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id)); + + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + + { + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + } + + brpc::Controller cntl; + auto arena = res.GetArena(); + auto req = google::protobuf::Arena::CreateMessage(arena); + req->set_tablet_job_id(""); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) delete req; + + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + res.Clear(); + } +} + TEST(MetaServiceTest, StalePrepareRowset) { auto meta_service = get_meta_service(); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 87c3ef5e542ce6..7b50747856c265 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -976,6 +976,7 @@ message CreateRowsetRequest { optional doris.RowsetMetaCloudPB rowset_meta = 2; optional bool temporary = 3; optional int64 txn_id = 4; + optional string tablet_job_id = 5; } message CreateRowsetResponse { @@ -1376,6 +1377,7 @@ enum MetaServiceCode { VERSION_NOT_FOUND = 2010; TABLET_NOT_FOUND = 2011; STALE_TABLET_CACHE = 2012; + STALE_PREPARE_ROWSET = 2013; CLUSTER_NOT_FOUND = 3001; ALREADY_EXISTED = 3002; From 3d28b86615b204924a395cbbee4c90b3e99edf43 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 17:44:33 +0800 Subject: [PATCH 014/572] branch-3.0: [Bug](function) fix wrong result of hll_to_base64 #51831 (#51849) Cherry-picked from #51831 Co-authored-by: Pxl --- be/src/vec/functions/function_hll.cpp | 4 ++-- .../data/load_p0/http_stream/test_http_stream_compress.out | 3 +++ .../load_p0/http_stream/test_http_stream_compress.groovy | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/be/src/vec/functions/function_hll.cpp b/be/src/vec/functions/function_hll.cpp index a6b91e27c2dd1f..e2f4c15064a983 100644 --- a/be/src/vec/functions/function_hll.cpp +++ b/be/src/vec/functions/function_hll.cpp @@ -296,8 +296,8 @@ struct HllToBase64 { last_ser_size = cur_ser_size; ser_buff.resize(cur_ser_size); } - hll_val.serialize(reinterpret_cast(ser_buff.data())); - auto outlen = base64_encode((const unsigned char*)ser_buff.data(), cur_ser_size, + size_t real_size = hll_val.serialize(reinterpret_cast(ser_buff.data())); + auto outlen = base64_encode((const unsigned char*)ser_buff.data(), real_size, chars_data + encoded_offset); DCHECK(outlen > 0); diff --git a/regression-test/data/load_p0/http_stream/test_http_stream_compress.out b/regression-test/data/load_p0/http_stream/test_http_stream_compress.out index 80a422c7e72de7..2d3ba016dbf21c 100644 --- a/regression-test/data/load_p0/http_stream/test_http_stream_compress.out +++ b/regression-test/data/load_p0/http_stream/test_http_stream_compress.out @@ -2504,3 +2504,6 @@ -- !sql2 -- 1 2 1025 1028 +-- !test -- +1028 + diff --git a/regression-test/suites/load_p0/http_stream/test_http_stream_compress.groovy b/regression-test/suites/load_p0/http_stream/test_http_stream_compress.groovy index 5ce35278928681..41e001c8a2bcec 100644 --- a/regression-test/suites/load_p0/http_stream/test_http_stream_compress.groovy +++ b/regression-test/suites/load_p0/http_stream/test_http_stream_compress.groovy @@ -103,6 +103,7 @@ suite("test_http_stream_compress", "p0") { } sql "sync" qt_sql2 "select k1, k2, bitmap_union_count(v1), HLL_UNION_AGG(v2) from ${tableName2} group by k1, k2 order by k1" + qt_test "select HLL_CARDINALITY(hll_from_base64(hll_to_base64(v2))) from test_http_stream_bz4;" } finally { try_sq2 "DROP TABLE IF EXISTS ${tableName2}" } From 0922e52f685e70d16eff4381a72be81bca98a0b9 Mon Sep 17 00:00:00 2001 From: seawinde Date: Wed, 18 Jun 2025 17:45:48 +0800 Subject: [PATCH 015/572] [opt](mtmv) optimize mtmv rewrite performance (#49514) (#51312) ### What problem does this PR solve? commitId: 4da1c8ab pr: https://github.com/apache/doris/pull/49514 Co-authored-by: zhangdong --- .../doris/common/profile/SummaryProfile.java | 18 +- .../doris/mtmv/MTMVRelationManager.java | 24 +- .../apache/doris/mtmv/MTMVRewriteUtil.java | 45 +- .../apache/doris/nereids/CascadesContext.java | 5 + .../apache/doris/nereids/NereidsPlanner.java | 21 +- .../org/apache/doris/nereids/PlannerHook.java | 12 + .../doris/nereids/StatementContext.java | 43 +- .../executor/TablePartitionCollector.java | 46 ++ .../doris/nereids/memo/StructInfoMap.java | 4 + .../apache/doris/nereids/rules/RuleType.java | 1 + .../mv/AbstractMaterializedViewRule.java | 227 +++----- .../exploration/mv/HyperGraphComparator.java | 41 +- .../mv/InitMaterializationContextHook.java | 17 +- .../mv/MaterializationContext.java | 37 +- ...lizedViewAggregateOnNoneAggregateRule.java | 12 +- .../exploration/mv/PartitionCompensator.java | 199 +++++++ .../rules/exploration/mv/StructInfo.java | 113 +--- .../rules/rewrite/PruneFileScanPartition.java | 1 - .../rewrite/QueryPartitionCollector.java | 95 +++ .../visitor/ExpressionLineageReplacer.java | 27 +- .../org/apache/doris/qe/SessionVariable.java | 9 + .../doris/mtmv/MTMVRewriteUtilTest.java | 41 +- .../doris/nereids/memo/StructInfoMapTest.java | 10 +- .../doris/nereids/mv/IdStatisticsMapTest.java | 3 +- .../mv/MtmvCacheNewConnectContextTest.java | 6 +- .../doris/nereids/mv/MvTableIdIsLongTest.java | 4 +- .../mv/OptimizeGetAvailableMvsTest.java | 269 +++++++++ .../mv/PartitionCompensatorTest.java | 186 ++++++ .../doris/nereids/util/PlanChecker.java | 12 +- .../partition_mv_rewrite.out | 143 +++++ .../unioin_rewrite_grace_big.out} | 46 +- .../mv/availability/grace_period.groovy | 4 +- .../partition_mv_rewrite.groovy | 293 +++++----- .../rewrite_duration_exceeded.groovy | 156 +++++ .../unioin_rewrite_grace_big.groovy | 549 ++++++++++++++++++ 35 files changed, 2251 insertions(+), 468 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java create mode 100644 regression-test/data/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.out rename regression-test/data/nereids_rules_p0/mv/{partition_mv_rewrite.out => union_rewrite_grace_big/unioin_rewrite_grace_big.out} (69%) rename regression-test/suites/nereids_rules_p0/mv/{ => partition_union_rewrite}/partition_mv_rewrite.groovy (68%) create mode 100644 regression-test/suites/nereids_rules_p0/mv/rewrite_duration_exceeded/rewrite_duration_exceeded.groovy create mode 100644 regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index f875de5745c72a..70253613bfcd54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -99,6 +99,8 @@ public class SummaryProfile { public static final String NEREIDS_LOCK_TABLE_TIME = "Nereids Lock Table Time"; public static final String NEREIDS_ANALYSIS_TIME = "Nereids Analysis Time"; public static final String NEREIDS_REWRITE_TIME = "Nereids Rewrite Time"; + + public static final String NEREIDS_COLLECT_TABLE_PARTITION_TIME = "Nereids Collect Table Partition Time"; public static final String NEREIDS_OPTIMIZE_TIME = "Nereids Optimize Time"; public static final String NEREIDS_TRANSLATE_TIME = "Nereids Translate Time"; public static final String NEREIDS_DISTRIBUTE_TIME = "Nereids Distribute Time"; @@ -226,6 +228,9 @@ public class SummaryProfile { private long parseSqlFinishTime = -1; @SerializedName(value = "nereidsLockTableFinishTime") private long nereidsLockTableFinishTime = -1; + + @SerializedName(value = "nereidsCollectTablePartitionFinishTime") + private long nereidsCollectTablePartitionFinishTime = -1; @SerializedName(value = "nereidsAnalysisFinishTime") private long nereidsAnalysisFinishTime = -1; @SerializedName(value = "nereidsRewriteFinishTime") @@ -415,6 +420,8 @@ private void updateExecutionSummaryProfile() { executionSummaryProfile.addInfoString(NEREIDS_LOCK_TABLE_TIME, getPrettyNereidsLockTableTime()); executionSummaryProfile.addInfoString(NEREIDS_ANALYSIS_TIME, getPrettyNereidsAnalysisTime()); executionSummaryProfile.addInfoString(NEREIDS_REWRITE_TIME, getPrettyNereidsRewriteTime()); + executionSummaryProfile.addInfoString(NEREIDS_COLLECT_TABLE_PARTITION_TIME, + getPrettyNereidsCollectTablePartitionTime()); executionSummaryProfile.addInfoString(NEREIDS_OPTIMIZE_TIME, getPrettyNereidsOptimizeTime()); executionSummaryProfile.addInfoString(NEREIDS_TRANSLATE_TIME, getPrettyNereidsTranslateTime()); executionSummaryProfile.addInfoString(NEREIDS_DISTRIBUTE_TIME, getPrettyNereidsDistributeTime()); @@ -513,6 +520,10 @@ public void setNereidsLockTableFinishTime() { this.nereidsLockTableFinishTime = TimeUtils.getStartTimeMs(); } + public void setNereidsCollectTablePartitionFinishTime() { + this.nereidsCollectTablePartitionFinishTime = TimeUtils.getStartTimeMs(); + } + public void setNereidsAnalysisTime() { this.nereidsAnalysisFinishTime = TimeUtils.getStartTimeMs(); } @@ -787,8 +798,13 @@ public String getPrettyNereidsRewriteTime() { return getPrettyTime(nereidsRewriteFinishTime, nereidsAnalysisFinishTime, TUnit.TIME_MS); } + + public String getPrettyNereidsCollectTablePartitionTime() { + return getPrettyTime(nereidsCollectTablePartitionFinishTime, nereidsRewriteFinishTime, TUnit.TIME_MS); + } + public String getPrettyNereidsOptimizeTime() { - return getPrettyTime(nereidsOptimizeFinishTime, nereidsRewriteFinishTime, TUnit.TIME_MS); + return getPrettyTime(nereidsOptimizeFinishTime, nereidsCollectTablePartitionFinishTime, TUnit.TIME_MS); } public String getPrettyNereidsTranslateTime() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index 7858aa952777c9..bd9244af61e71f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -27,6 +28,7 @@ import org.apache.doris.job.exception.JobException; import org.apache.doris.job.extensions.mtmv.MTMVTask; import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; +import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator; import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo; import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo; @@ -43,6 +45,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -82,13 +85,21 @@ public Set getAvailableMTMVs(List tableInfos, ConnectContex boolean forceConsistent, BiPredicate predicate) { Set res = Sets.newLinkedHashSet(); Set mvInfos = getMTMVInfos(tableInfos); + Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( + ctx.getStatementContext()); + for (BaseTableInfo tableInfo : mvInfos) { try { MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo); if (predicate.test(ctx, mtmv)) { continue; } - if (isMVPartitionValid(mtmv, ctx, forceConsistent)) { + if (!mtmv.isUseForRewrite()) { + continue; + } + BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); + if (isMVPartitionValid(mtmv, ctx, forceConsistent, + relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) { res.add(mtmv); } } catch (Exception e) { @@ -117,10 +128,15 @@ public Set getAllMTMVs(List tableInfos) { } @VisibleForTesting - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent, + Set relatedPartitions) { long currentTimeMillis = System.currentTimeMillis(); - return !CollectionUtils - .isEmpty(MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMillis, forceConsistent)); + Collection mtmvCanRewritePartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions( + mtmv, ctx, currentTimeMillis, forceConsistent, relatedPartitions); + // MTMVRewriteUtil.getMTMVCanRewritePartitions is time-consuming behavior, So record for used later + ctx.getStatementContext().getMvCanRewritePartitionsMap().putIfAbsent( + new BaseTableInfo(mtmv), mtmvCanRewritePartitions); + return !CollectionUtils.isEmpty(mtmvCanRewritePartitions); } private Set getMTMVInfos(List tableInfos) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index ff1b3263d3409e..afaad55a34bbc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -25,12 +25,16 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.Collection; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; public class MTMVRewriteUtil { private static final Logger LOG = LogManager.getLogger(MTMVRewriteUtil.class); @@ -43,7 +47,8 @@ public class MTMVRewriteUtil { * @return */ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, ConnectContext ctx, - long currentTimeMills, boolean forceConsistent) { + long currentTimeMills, boolean forceConsistent, + Set relatedPartitions) { List res = Lists.newArrayList(); Collection allPartitions = mtmv.getPartitions(); MTMVRelation mtmvRelation = mtmv.getRelation(); @@ -55,6 +60,11 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne if (mtmvStatus.getState() != MTMVState.NORMAL || mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) { return res; } + // if relatedPartitions is empty but not null, which means query no partitions + if (relatedPartitions != null && relatedPartitions.size() == 0) { + return res; + } + Set mtmvNeedComparePartitions = null; MTMVRefreshContext refreshContext = null; // check gracePeriod long gracePeriodMills = mtmv.getGracePeriod(); @@ -73,6 +83,14 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne return res; } } + if (mtmvNeedComparePartitions == null) { + mtmvNeedComparePartitions = getMtmvPartitionsByRelatedPartitions(mtmv, refreshContext, + relatedPartitions); + } + // if the partition which query not used, should not compare partition version + if (!mtmvNeedComparePartitions.contains(partition.getName())) { + continue; + } try { if (MTMVPartitionUtil.isMTMVPartitionSync(refreshContext, partition.getName(), mtmvRelation.getBaseTablesOneLevel(), @@ -86,4 +104,29 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne } return res; } + + private static Set getMtmvPartitionsByRelatedPartitions(MTMV mtmv, MTMVRefreshContext refreshContext, + Set relatedPartitions) { + // if relatedPartitions is null, which means QueryPartitionCollector visitLogicalCatalogRelation can not + // get query used partitions, should get all mtmv partitions + if (relatedPartitions == null) { + return mtmv.getPartitionNames(); + } + Set res = Sets.newHashSet(); + Map relatedToMv = getRelatedToMv(refreshContext.getPartitionMappings()); + for (String relatedPartition : relatedPartitions) { + res.add(relatedToMv.get(relatedPartition)); + } + return res; + } + + private static Map getRelatedToMv(Map> mvToRelated) { + Map res = Maps.newHashMap(); + for (Entry> entry : mvToRelated.entrySet()) { + for (String relatedPartition : entry.getValue()) { + res.put(relatedPartition, entry.getKey()); + } + } + return res; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 1486f03e269b13..d431fc545396b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.executor.Analyzer; import org.apache.doris.nereids.jobs.executor.TableCollectAndHookInitializer; +import org.apache.doris.nereids.jobs.executor.TablePartitionCollector; import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.RootRewriteJobContext; @@ -228,6 +229,10 @@ public TableCollectAndHookInitializer newTableCollector() { return new TableCollectAndHookInitializer(this); } + public TablePartitionCollector newTablePartitionCollector() { + return new TablePartitionCollector(this); + } + public Analyzer newAnalyzer() { return new Analyzer(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 85e40c6c7b37fb..4e0fabc05c0b46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -358,7 +358,22 @@ protected void collectAndLockTable(boolean showPlanProcess) { } } - private void analyze(boolean showPlanProcess) { + protected void collectTableUsedPartitions(boolean showPlanProcess) { + if (LOG.isDebugEnabled()) { + LOG.debug("Start to collect table used partition"); + } + keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newTablePartitionCollector().execute()); + NereidsTracer.logImportantTime("EndCollectTablePartitions"); + if (LOG.isDebugEnabled()) { + LOG.debug("Start to collect table used partition"); + } + if (statementContext.getConnectContext().getExecutor() != null) { + statementContext.getConnectContext().getExecutor().getSummaryProfile() + .setNereidsCollectTablePartitionFinishTime(); + } + } + + protected void analyze(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start analyze plan"); } @@ -390,6 +405,10 @@ private void rewrite(boolean showPlanProcess) { if (statementContext.getConnectContext().getExecutor() != null) { statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsRewriteTime(); } + // collect partitions table used, this is for query rewrite by materialized view + // this is needed before init hook + collectTableUsedPartitions(showPlanProcess); + cascadesContext.getStatementContext().getPlannerHooks().forEach(hook -> hook.afterRewrite(this)); } // DependsRules: EnsureProjectOnTopJoin.class diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java index 18d71b539a94b3..76a68f0b22c763 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerHook.java @@ -35,4 +35,16 @@ default void beforeAnalyze(NereidsPlanner planner) { */ default void afterAnalyze(NereidsPlanner planner) { } + + /** + * the hook before rewrite + */ + default void beforeRewrite(NereidsPlanner planner) { + } + + /** + * the hook after rewrite + */ + default void afterRewrite(NereidsPlanner planner) { + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index c9e1e23d0b1212..6c2e0d1ab12201 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; import org.apache.doris.catalog.constraint.TableIdentifier; @@ -29,6 +30,7 @@ import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.datasource.mvcc.MvccTableInfo; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.hint.Hint; import org.apache.doris.nereids.memo.Group; @@ -59,6 +61,7 @@ import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.base.Throwables; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import com.google.common.collect.Multimap; @@ -105,6 +108,7 @@ public enum TableFrom { private ConnectContext connectContext; private final Stopwatch stopwatch = Stopwatch.createUnstarted(); + private final Stopwatch materializedViewStopwatch = Stopwatch.createUnstarted(); @GuardedBy("this") private final Map> contextCacheMap = Maps.newLinkedHashMap(); @@ -173,7 +177,14 @@ public enum TableFrom { // tables in this query directly private final Map, TableIf> tables = Maps.newHashMap(); - // tables maybe used by mtmv rewritten in this query + // tables maybe used by mtmv rewritten in this query, + // this contains mvs which use table in tables and the tables in mvs + // such as + // mv1 use t1, t2. + // mv2 use mv1, t3, t4. + // mv3 use t3, t4, t5 + // if query is: select * from t2 join t5 + // mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5 private final Map, TableIf> mtmvRelatedTables = Maps.newHashMap(); // insert into target tables private final Map, TableIf> insertTargetTables = Maps.newHashMap(); @@ -209,6 +220,16 @@ public enum TableFrom { private boolean privChecked; + // if greater than 0 means the duration has used + private long materializedViewRewriteDuration = 0L; + + // Record used table and it's used partitions + private final Multimap, Pair>> tableUsedPartitionNameMap = + HashMultimap.create(); + + // Record mtmv and valid partitions map because this is time-consuming behavior + private final Map> mvCanRewritePartitionsMap = new HashMap<>(); + private final Map snapshots = Maps.newHashMap(); public StatementContext() { @@ -349,6 +370,18 @@ public Stopwatch getStopwatch() { return stopwatch; } + public Stopwatch getMaterializedViewStopwatch() { + return materializedViewStopwatch; + } + + public long getMaterializedViewRewriteDuration() { + return materializedViewRewriteDuration; + } + + public void addMaterializedViewRewriteDuration(long millisecond) { + materializedViewRewriteDuration += millisecond; + } + public void setMaxNAryInnerJoin(int maxNAryInnerJoin) { if (maxNAryInnerJoin > this.maxNAryInnerJoin) { this.maxNAryInnerJoin = maxNAryInnerJoin; @@ -753,4 +786,12 @@ public boolean isPrivChecked() { public void setPrivChecked(boolean privChecked) { this.privChecked = privChecked; } + + public Multimap, Pair>> getTableUsedPartitionNameMap() { + return tableUsedPartitionNameMap; + } + + public Map> getMvCanRewritePartitionsMap() { + return mvCanRewritePartitionsMap; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java new file mode 100644 index 00000000000000..e67b94d1314e89 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.jobs.executor; + +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.jobs.rewrite.RewriteJob; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.QueryPartitionCollector; + +import java.util.List; + +/** + * Collect partitions which query used, this is useful for optimizing get available mvs, + * should collect after RBO + */ +public class TablePartitionCollector extends AbstractBatchJobExecutor { + public TablePartitionCollector(CascadesContext cascadesContext) { + super(cascadesContext); + } + + @Override + public List getJobs() { + return buildCollectorJobs(); + } + + private static List buildCollectorJobs() { + return jobs( + custom(RuleType.COLLECT_PARTITIONS, QueryPartitionCollector::new) + ); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index c39be5b569089f..36079fd3763475 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -25,6 +25,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.BitSet; @@ -42,6 +44,8 @@ * Representation for group in cascades optimizer. */ public class StructInfoMap { + + public static final Logger LOG = LogManager.getLogger(StructInfoMap.class); private final Map>> groupExpressionMap = new HashMap<>(); private final Map infoMap = new HashMap<>(); private long refreshVersion = 0; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index b7cbe5343d1129..ab82f08217993d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -357,6 +357,7 @@ public enum RuleType { LEADING_JOIN(RuleTypeClass.REWRITE), REWRITE_SENTINEL(RuleTypeClass.REWRITE), COLLECT_COLUMNS(RuleTypeClass.REWRITE), + COLLECT_PARTITIONS(RuleTypeClass.REWRITE), // topn opts DEFER_MATERIALIZE_TOP_N_RESULT(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 80200af0a51e00..49f657911d2681 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -18,17 +18,13 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.catalog.MTMV; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.constraint.TableIdentifier; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseTableInfo; -import org.apache.doris.mtmv.MTMVPartitionInfo; -import org.apache.doris.mtmv.MTMVRewriteUtil; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.exploration.ExplorationRuleFactory; @@ -66,7 +62,6 @@ import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -75,15 +70,13 @@ import java.util.ArrayList; import java.util.BitSet; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; /** @@ -109,11 +102,22 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac */ public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { List rewrittenPlans = new ArrayList<>(); + SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); // if available materialization list is empty, bail out + StatementContext statementContext = cascadesContext.getStatementContext(); if (cascadesContext.getMaterializationContexts().isEmpty()) { return rewrittenPlans; } + if (statementContext.getMaterializedViewRewriteDuration() + > sessionVariable.materializedViewRewriteDurationThresholdMs) { + LOG.warn("materialized view rewrite duration is exceeded, the query queryId is {}", + cascadesContext.getConnectContext().getQueryIdentifier()); + MaterializationContext.makeFailWithDurationExceeded(queryPlan, cascadesContext.getMaterializationContexts(), + statementContext.getMaterializedViewRewriteDuration()); + return rewrittenPlans; + } for (MaterializationContext context : cascadesContext.getMaterializationContexts()) { + statementContext.getMaterializedViewStopwatch().reset().start(); if (checkIfRewritten(queryPlan, context)) { continue; } @@ -127,15 +131,31 @@ public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { if (queryStructInfos.isEmpty()) { continue; } + statementContext.addMaterializedViewRewriteDuration( + statementContext.getMaterializedViewStopwatch().elapsed(TimeUnit.MILLISECONDS)); for (StructInfo queryStructInfo : queryStructInfos) { + statementContext.getMaterializedViewStopwatch().reset().start(); + if (statementContext.getMaterializedViewRewriteDuration() + > sessionVariable.materializedViewRewriteDurationThresholdMs) { + statementContext.getMaterializedViewStopwatch().stop(); + LOG.warn("materialized view rewrite duration is exceeded, the queryId is {}", + cascadesContext.getConnectContext().getQueryIdentifier()); + MaterializationContext.makeFailWithDurationExceeded(queryStructInfo.getOriginalPlan(), + cascadesContext.getMaterializationContexts(), + statementContext.getMaterializedViewRewriteDuration()); + return rewrittenPlans; + } try { - if (rewrittenPlans.size() < cascadesContext.getConnectContext() - .getSessionVariable().getMaterializedViewRewriteSuccessCandidateNum()) { + if (rewrittenPlans.size() < sessionVariable.getMaterializedViewRewriteSuccessCandidateNum()) { rewrittenPlans.addAll(doRewrite(queryStructInfo, cascadesContext, context)); } } catch (Exception exception) { + LOG.warn("Materialized view rule exec fail", exception); context.recordFailReason(queryStructInfo, "Materialized view rule exec fail", exception::toString); + } finally { + statementContext.addMaterializedViewRewriteDuration( + statementContext.getMaterializedViewStopwatch().elapsed(TimeUnit.MILLISECONDS)); } } } @@ -266,43 +286,70 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca continue; } Pair>, Map>> invalidPartitions; - if (materializationContext instanceof AsyncMaterializationContext) { + if (PartitionCompensator.needUnionRewrite(materializationContext) + && sessionVariable.isEnableMaterializedViewUnionRewrite()) { + MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); + BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); + Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( + cascadesContext.getConnectContext().getStatementContext()); + Set relateTableUsedPartitions = queryUsedPartitions.get(relatedTableInfo.toList()); + if (relateTableUsedPartitions == null) { + materializationContext.recordFailReason(queryStructInfo, + String.format("queryUsedPartition is null but needUnionRewrite, table is %s, queryId is %s", + relatedTableInfo.toList(), + cascadesContext.getConnectContext().getQueryIdentifier()), + () -> String.format( + "queryUsedPartition is null but needUnionRewrite, table is %s, queryId is %s", + relatedTableInfo.toList(), + cascadesContext.getConnectContext().getQueryIdentifier())); + LOG.warn(String.format( + "queryUsedPartition is null but needUnionRewrite, table is %s, queryId is %s", + relatedTableInfo.toList(), cascadesContext.getConnectContext().getQueryIdentifier())); + return rewriteResults; + } + if (relateTableUsedPartitions.isEmpty()) { + materializationContext.recordFailReason(queryStructInfo, + String.format("queryUsedPartition is empty, table is %s, queryId is %s", + relatedTableInfo.toList(), + cascadesContext.getConnectContext().getQueryIdentifier()), + () -> String.format("queryUsedPartition is empty, table is %s, queryId is %s", + relatedTableInfo.toList(), + cascadesContext.getConnectContext().getQueryIdentifier())); + LOG.debug(String.format("queryUsedPartition is empty, table is %s, queryId is %s", + relatedTableInfo.toList(), cascadesContext.getConnectContext().getQueryIdentifier())); + // no need to rewrite by current mv, becaus + return rewriteResults; + } try { - invalidPartitions = calcInvalidPartitions(queryPlan, rewrittenPlan, - (AsyncMaterializationContext) materializationContext, cascadesContext); + invalidPartitions = calcInvalidPartitions(relateTableUsedPartitions, rewrittenPlan, + cascadesContext, (AsyncMaterializationContext) materializationContext); } catch (AnalysisException e) { materializationContext.recordFailReason(queryStructInfo, "Calc invalid partitions fail", () -> String.format("Calc invalid partitions fail, mv partition names are %s", - ((AsyncMaterializationContext) materializationContext).getMtmv().getPartitions())); + mtmv.getPartitions())); LOG.warn("Calc invalid partitions fail", e); continue; } if (invalidPartitions == null) { - // if mv can not offer any partition for query, query rewrite bail out to avoid cycle run materializationContext.recordFailReason(queryStructInfo, "mv can not offer any partition for query", - () -> String.format("mv partition info %s", - ((AsyncMaterializationContext) materializationContext).getMtmv() - .getMvPartitionInfo())); + () -> String.format("mv partition info %s", mtmv.getMvPartitionInfo())); + // if mv can not offer any partition for query, query rewrite bail out to avoid cycle run return rewriteResults; } - boolean partitionNeedUnion = needUnionRewrite(invalidPartitions, cascadesContext); - boolean canUnionRewrite = canUnionRewrite(queryPlan, - ((AsyncMaterializationContext) materializationContext).getMtmv(), - cascadesContext); + boolean partitionNeedUnion = PartitionCompensator.needUnionRewrite(invalidPartitions, cascadesContext); + boolean canUnionRewrite = canUnionRewrite(queryPlan, mtmv, cascadesContext); if (partitionNeedUnion && !canUnionRewrite) { materializationContext.recordFailReason(queryStructInfo, "need compensate union all, but can not, because the query structInfo", () -> String.format("mv partition info is %s, and the query plan is %s", - ((AsyncMaterializationContext) materializationContext).getMtmv() - .getMvPartitionInfo(), queryPlan.treeString())); + mtmv.getMvPartitionInfo(), queryPlan.treeString())); return rewriteResults; } final Pair>, Map>> finalInvalidPartitions = invalidPartitions; if (partitionNeedUnion) { - MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); Pair planAndNeedAddFilterPair = StructInfo.addFilterOnTableScan(queryPlan, invalidPartitions.value(), mtmv.getMvPartitionInfo().getRelatedCol(), cascadesContext); @@ -384,13 +431,6 @@ private static void trySetStatistics(MaterializationContext context, CascadesCon } } - protected boolean needUnionRewrite( - Pair>, Map>> invalidPartitions, - CascadesContext cascadesContext) { - return invalidPartitions != null - && (!invalidPartitions.key().isEmpty() || !invalidPartitions.value().isEmpty()); - } - /** * Not all query after rewritten successfully can compensate union all * Such as: @@ -438,98 +478,13 @@ protected boolean isOutputValid(Plan sourcePlan, Plan rewrittenPlan) { * @return the key in pair is mvNeedRemovePartitionNameSet, the value in pair is baseTableNeedUnionPartitionNameSet */ protected Pair>, Map>> calcInvalidPartitions( - Plan queryPlan, Plan rewrittenPlan, - AsyncMaterializationContext materializationContext, CascadesContext cascadesContext) + Set queryUsedPartition, + Plan rewrittenPlan, + CascadesContext cascadesContext, + AsyncMaterializationContext materializationContext) throws AnalysisException { - Set mvNeedRemovePartitionNameSet = new HashSet<>(); - Set baseTableNeedUnionPartitionNameSet = new HashSet<>(); - // check partition is valid or not - MTMV mtmv = materializationContext.getMtmv(); - PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); - if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) { - // if not partition, if rewrite success, it means mv is available - return Pair.of(ImmutableMap.of(), ImmutableMap.of()); - } - MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo(); - BaseTableInfo relatedPartitionTable = mvCustomPartitionInfo.getRelatedTableInfo(); - if (relatedPartitionTable == null) { - return Pair.of(ImmutableMap.of(), ImmutableMap.of()); - } - // Collect the mv related base table partitions which query used - Map> queryUsedBaseTablePartitions = new LinkedHashMap<>(); - queryUsedBaseTablePartitions.put(relatedPartitionTable, new HashSet<>()); - queryPlan.accept(new StructInfo.QueryScanPartitionsCollector(), queryUsedBaseTablePartitions); - // Bail out, not check invalid partition if not olap scan, support later - if (queryUsedBaseTablePartitions.isEmpty()) { - return Pair.of(ImmutableMap.of(), ImmutableMap.of()); - } - Set queryUsedBaseTablePartitionNameSet = queryUsedBaseTablePartitions.get(relatedPartitionTable) - .stream() - .map(Partition::getName) - .collect(Collectors.toSet()); - - Collection mvValidPartitions = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, - cascadesContext.getConnectContext(), System.currentTimeMillis(), false); - Set mvValidPartitionNameSet = new HashSet<>(); - Set mvValidBaseTablePartitionNameSet = new HashSet<>(); - Set mvValidHasDataRelatedBaseTableNameSet = new HashSet<>(); - Pair>, Map> partitionMapping = mtmv.calculateDoublyPartitionMappings(); - for (Partition mvValidPartition : mvValidPartitions) { - mvValidPartitionNameSet.add(mvValidPartition.getName()); - Set relatedBaseTablePartitions = partitionMapping.key().get(mvValidPartition.getName()); - if (relatedBaseTablePartitions != null) { - mvValidBaseTablePartitionNameSet.addAll(relatedBaseTablePartitions); - } - if (!mtmv.selectNonEmptyPartitionIds(ImmutableList.of(mvValidPartition.getId())).isEmpty()) { - if (relatedBaseTablePartitions != null) { - mvValidHasDataRelatedBaseTableNameSet.addAll(relatedBaseTablePartitions); - } - } - } - if (Sets.intersection(mvValidHasDataRelatedBaseTableNameSet, queryUsedBaseTablePartitionNameSet).isEmpty()) { - // if mv can not offer any partition for query, query rewrite bail out - return null; - } - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("calcInvalidPartitions mv name is %s,\n mvValidBaseTablePartitionNameSet is %s,\n " - + "mvValidHasDataRelatedBaseTableNameSet is %s,\n" - + "queryUsedBaseTablePartitionNameSet is %s,\n " - + "partitionMapping is %s \n, sql hash is %s", - materializationContext.generateMaterializationIdentifier(), - mvValidBaseTablePartitionNameSet, - mvValidHasDataRelatedBaseTableNameSet, - queryUsedBaseTablePartitionNameSet, - partitionMapping, cascadesContext.getConnectContext().getSqlHash())); - } - // Check when mv partition relates base table partition data change or delete partition - Set rewrittenPlanUsePartitionNameSet = new HashSet<>(); - List mvOlapScanList = rewrittenPlan.collectToList(node -> - node instanceof LogicalOlapScan - && Objects.equals(((CatalogRelation) node).getTable().getName(), mtmv.getName())); - for (Object olapScanObj : mvOlapScanList) { - LogicalOlapScan olapScan = (LogicalOlapScan) olapScanObj; - olapScan.getSelectedPartitionIds().forEach(id -> - rewrittenPlanUsePartitionNameSet.add(olapScan.getTable().getPartition(id).getName())); - } - // If rewritten plan use but not in mv valid partition name set, need remove in mv and base table union - Sets.difference(rewrittenPlanUsePartitionNameSet, mvValidPartitionNameSet) - .copyInto(mvNeedRemovePartitionNameSet); - for (String partitionName : mvNeedRemovePartitionNameSet) { - baseTableNeedUnionPartitionNameSet.addAll(partitionMapping.key().get(partitionName)); - } - // If related base table create partitions or mv is created with ttl, need base table union - Sets.difference(queryUsedBaseTablePartitionNameSet, mvValidBaseTablePartitionNameSet) - .copyInto(baseTableNeedUnionPartitionNameSet); - // Construct result map - Map> mvPartitionNeedRemoveNameMap = new HashMap<>(); - if (!mvNeedRemovePartitionNameSet.isEmpty()) { - mvPartitionNeedRemoveNameMap.put(new BaseTableInfo(mtmv), mvNeedRemovePartitionNameSet); - } - Map> baseTablePartitionNeedUnionNameMap = new HashMap<>(); - if (!baseTableNeedUnionPartitionNameSet.isEmpty()) { - baseTablePartitionNeedUnionNameMap.put(relatedPartitionTable, baseTableNeedUnionPartitionNameSet); - } - return Pair.of(mvPartitionNeedRemoveNameMap, baseTablePartitionNeedUnionNameMap); + return PartitionCompensator.calcInvalidPartitions(queryUsedPartition, rewrittenPlan, + materializationContext, cascadesContext); } /** @@ -907,6 +862,18 @@ protected boolean checkIfRewritten(Plan plan, MaterializationContext context) { // check mv plan is valid or not, this can use cache for performance private boolean isMaterializationValid(Plan queryPlan, CascadesContext cascadesContext, MaterializationContext context) { + if (!context.getStructInfo().isValid()) { + context.recordFailReason(context.getStructInfo(), + "View original struct info is invalid", () -> String.format("view plan is %s", + context.getStructInfo().getOriginalPlan().treeString())); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format("View struct info is invalid, mv identifier is %s, query plan is %s," + + "view plan is %s", + context.generateMaterializationIdentifier(), queryPlan.treeString(), + context.getStructInfo().getTopPlan().treeString())); + } + return false; + } long materializationId = context.generateMaterializationIdentifier().hashCode(); Boolean cachedCheckResult = cascadesContext.getMemo().materializationHasChecked(this.getClass(), materializationId); @@ -943,18 +910,6 @@ private boolean isMaterializationValid(Plan queryPlan, CascadesContext cascadesC } return false; } - if (!context.getStructInfo().isValid()) { - context.recordFailReason(context.getStructInfo(), - "View original struct info is invalid", () -> String.format("view plan is %s", - context.getStructInfo().getOriginalPlan().treeString())); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("View struct info is invalid, mv identifier is %s, query plan is %s," - + "view plan is %s", - context.generateMaterializationIdentifier(), queryPlan.treeString(), - context.getStructInfo().getTopPlan().treeString())); - } - return false; - } return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 22282a2351627b..fce8e2f520f0d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -406,13 +406,20 @@ private Map getQueryToViewNodeIdMap() { } private Map constructQueryToViewJoinMapWithExpr() { - Map viewExprToEdge = getViewJoinEdges().stream() - .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) - .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - Map queryExprToEdge = getQueryJoinEdges().stream() - .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) - .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - + Map viewExprToEdge = new HashMap<>(); + List viewJoinEdges = getViewJoinEdges(); + for (JoinEdge viewJoin : viewJoinEdges) { + for (Expression expression : viewJoin.getExpressions()) { + viewExprToEdge.put(expression, viewJoin); + } + } + Map queryExprToEdge = new HashMap<>(); + List queryJoinEdges = getQueryJoinEdges(); + for (JoinEdge queryJoin : queryJoinEdges) { + for (Expression expression : queryJoin.getExpressions()) { + queryExprToEdge.put(expression, queryJoin); + } + } HashMap edgeMap = new HashMap<>(); for (Entry entry : queryExprToEdge.entrySet()) { if (edgeMap.containsKey(entry.getValue())) { @@ -444,15 +451,19 @@ private Map constructQueryToViewJoinMapWithExpr() { // +--LogicalOlapScan private Map constructQueryToViewFilterMapWithExpr() { Multimap viewExprToEdge = HashMultimap.create(); - getViewFilterEdges().stream() - .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) - .forEach(pair -> viewExprToEdge.put(pair.key(), pair.value())); - + List viewFilterEdges = getViewFilterEdges(); + for (FilterEdge viewEdge : viewFilterEdges) { + for (Expression expression : viewEdge.getExpressions()) { + viewExprToEdge.put(expression, viewEdge); + } + } Multimap queryExprToEdge = HashMultimap.create(); - getQueryFilterEdges().stream() - .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) - .forEach(pair -> queryExprToEdge.put(pair.key(), pair.value())); - + List queryFilterEdges = getQueryFilterEdges(); + for (FilterEdge queryEdge : queryFilterEdges) { + for (Expression expression : queryEdge.getExpressions()) { + queryExprToEdge.put(expression, queryEdge); + } + } HashMap queryToViewEdgeMap = new HashMap<>(); for (Entry> entry : queryExprToEdge.asMap().entrySet()) { Expression queryExprViewBased = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 5ffb4b5f332295..5baeff7c5852ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -61,7 +61,7 @@ public class InitMaterializationContextHook implements PlannerHook { public static final InitMaterializationContextHook INSTANCE = new InitMaterializationContextHook(); @Override - public void afterAnalyze(NereidsPlanner planner) { + public void afterRewrite(NereidsPlanner planner) { initMaterializationContext(planner.getCascadesContext()); } @@ -123,13 +123,13 @@ private List createAsyncMaterializationContext(CascadesC try { availableMTMVs = getAvailableMTMVs(usedTables, cascadesContext); } catch (Exception e) { - LOG.warn(String.format("MaterializationContext getAvailableMTMVs generate fail, current queryId is %s", - cascadesContext.getConnectContext().getQueryIdentifier()), e); + LOG.warn(String.format("MaterializationContext getAvailableMTMVs generate fail, current sqlHash is %s", + cascadesContext.getConnectContext().getSqlHash()), e); return ImmutableList.of(); } if (CollectionUtils.isEmpty(availableMTMVs)) { - LOG.debug("Enable materialized view rewrite but availableMTMVs is empty, current queryId " - + "is {}", cascadesContext.getConnectContext().getQueryIdentifier()); + LOG.debug("Enable materialized view rewrite but availableMTMVs is empty, current sqlHash " + + "is {}", cascadesContext.getConnectContext().getSqlHash()); return ImmutableList.of(); } List asyncMaterializationContext = new ArrayList<>(); @@ -137,13 +137,6 @@ private List createAsyncMaterializationContext(CascadesC MTMVCache mtmvCache = null; try { mtmvCache = materializedView.getOrGenerateCache(cascadesContext.getConnectContext()); - // If mv property use_for_rewrite is set false, should not partition in - // query rewrite by materialized view - if (!materializedView.isUseForRewrite()) { - LOG.debug("mv doesn't part in query rewrite process because " - + "use_for_rewrite is false, mv is {}", materializedView.getName()); - continue; - } if (mtmvCache == null) { continue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 38eba2ac3406ff..46acd9861a4d96 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Id; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; @@ -326,7 +327,7 @@ public boolean isSuccess() { } /** - * Record fail reason when in rewriting + * Record fail reason when in rewriting by struct info */ public void recordFailReason(StructInfo structInfo, String summary, Supplier failureReasonSupplier) { // record it's rewritten @@ -342,6 +343,24 @@ public void recordFailReason(StructInfo structInfo, String summary, Supplier failureReasonSupplier) { + // record it's rewritten + if (queryGroupPlan.getGroupExpression().isPresent()) { + this.addMatchedGroup(queryGroupPlan.getGroupExpression().get().getOwnerGroup().getGroupId(), + false); + } + // once success, do not record the fail reason + if (this.success) { + return; + } + this.failReason.put(queryGroupPlan.getGroupExpression() + .map(GroupExpression::getId).orElseGet(() -> new ObjectId(-1)), + Pair.of(summary, this.isEnableRecordFailureDetail() ? failureReasonSupplier.get() : "")); + } + @Override public String toString() { return getStringInfo(); @@ -410,6 +429,22 @@ public Void visitPhysicalRelation(PhysicalRelation physicalRelation, Void contex return builder.toString(); } + /** + * If materialized view rewrite duration is exceeded, make all materializationContexts with reason + * materialized view rewrite duration is exceeded + * */ + public static void makeFailWithDurationExceeded(Plan queryPlan, + List materializationContexts, long duration) { + for (MaterializationContext context : materializationContexts) { + if (context.isSuccess()) { + continue; + } + context.recordFailReason(queryPlan, + "materialized view rewrite duration is exceeded, the duration is " + duration, + () -> "materialized view rewrite duration is exceeded, the duration is " + duration); + } + } + private static String generateIdentifierName(List qualifiers) { return String.join(".", qualifiers); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java index 45514f6fb15e6e..06e1e77e56bc33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateOnNoneAggregateRule.java @@ -92,11 +92,15 @@ protected boolean checkMaterializationPattern(StructInfo structInfo, CascadesCon @Override protected Pair>, Map>> calcInvalidPartitions( - Plan queryPlan, Plan rewrittenPlan, AsyncMaterializationContext materializationContext, - CascadesContext cascadesContext) throws AnalysisException { + Set queryUsedPartition, + Plan rewrittenPlan, + CascadesContext cascadesContext, + AsyncMaterializationContext materializationContext) + throws AnalysisException { Pair>, Map>> invalidPartitions - = super.calcInvalidPartitions(queryPlan, rewrittenPlan, materializationContext, cascadesContext); - if (needUnionRewrite(invalidPartitions, cascadesContext)) { + = super.calcInvalidPartitions(queryUsedPartition, rewrittenPlan, cascadesContext, + materializationContext); + if (PartitionCompensator.needUnionRewrite(invalidPartitions, cascadesContext)) { // if query use some invalid partition in mv, bail out return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java new file mode 100644 index 00000000000000..98629f86028b81 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -0,0 +1,199 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Pair; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Handle materialized view partition union compensate handler + * */ +public class PartitionCompensator { + + public static final Logger LOG = LogManager.getLogger(PartitionCompensator.class); + // if partition pair is null which means can not get partitions from table in QueryPartitionCollector, + // we think this table scan query all partitions default + public static final Pair> ALL_PARTITIONS = Pair.of(null, null); + public static final Collection>> ALL_PARTITIONS_LIST = + ImmutableList.of(ALL_PARTITIONS); + + /** + * Maybe only some partitions is invalid in materialized view, or base table maybe add, modify, delete partition + * So we should calc the invalid partition used in query + * @param queryUsedBaseTablePartitionNameSet partitions used by query related partition table + * @param rewrittenPlan tmp rewrittenPlan when mv rewrite + * @param materializationContext the context of materialization,which hold materialized view meta and other info + * @param cascadesContext the context of cascades + * @return the key in pair is mvNeedRemovePartitionNameSet, the value in pair is baseTableNeedUnionPartitionNameSet + */ + public static Pair>, Map>> calcInvalidPartitions( + Set queryUsedBaseTablePartitionNameSet, Plan rewrittenPlan, + AsyncMaterializationContext materializationContext, CascadesContext cascadesContext) + throws AnalysisException { + Set mvNeedRemovePartitionNameSet = new HashSet<>(); + Set baseTableNeedUnionPartitionNameSet = new HashSet<>(); + // check partition is valid or not + MTMV mtmv = materializationContext.getMtmv(); + PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); + if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) { + // if not partition, if rewrite success, it means mv is available + return Pair.of(ImmutableMap.of(), ImmutableMap.of()); + } + MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo(); + BaseTableInfo relatedPartitionTable = mvCustomPartitionInfo.getRelatedTableInfo(); + if (relatedPartitionTable == null || queryUsedBaseTablePartitionNameSet.isEmpty()) { + // if mv is not partitioned or query not query any partition, doesn't compensate + return Pair.of(ImmutableMap.of(), ImmutableMap.of()); + } + Collection mvValidPartitions = cascadesContext.getStatementContext() + .getMvCanRewritePartitionsMap().get(new BaseTableInfo(mtmv)); + Set mvValidPartitionNameSet = new HashSet<>(); + Set mvValidBaseTablePartitionNameSet = new HashSet<>(); + Set mvValidHasDataRelatedBaseTableNameSet = new HashSet<>(); + Pair>, Map> partitionMapping = mtmv.calculateDoublyPartitionMappings(); + for (Partition mvValidPartition : mvValidPartitions) { + mvValidPartitionNameSet.add(mvValidPartition.getName()); + Set relatedBaseTablePartitions = partitionMapping.key().get(mvValidPartition.getName()); + if (relatedBaseTablePartitions != null) { + mvValidBaseTablePartitionNameSet.addAll(relatedBaseTablePartitions); + } + if (!mtmv.selectNonEmptyPartitionIds(ImmutableList.of(mvValidPartition.getId())).isEmpty()) { + if (relatedBaseTablePartitions != null) { + mvValidHasDataRelatedBaseTableNameSet.addAll(relatedBaseTablePartitions); + } + } + } + if (Sets.intersection(mvValidHasDataRelatedBaseTableNameSet, queryUsedBaseTablePartitionNameSet).isEmpty()) { + // if mv can not offer any partition for query, query rewrite bail out + return null; + } + // Check when mv partition relates base table partition data change or delete partition + Set rewrittenPlanUsePartitionNameSet = new HashSet<>(); + List mvOlapScanList = rewrittenPlan.collectToList(node -> + node instanceof LogicalOlapScan + && Objects.equals(((CatalogRelation) node).getTable().getName(), mtmv.getName())); + for (Object olapScanObj : mvOlapScanList) { + LogicalOlapScan olapScan = (LogicalOlapScan) olapScanObj; + olapScan.getSelectedPartitionIds().forEach(id -> + rewrittenPlanUsePartitionNameSet.add(olapScan.getTable().getPartition(id).getName())); + } + // If rewritten plan use but not in mv valid partition name set, need remove in mv and base table union + Sets.difference(rewrittenPlanUsePartitionNameSet, mvValidPartitionNameSet) + .copyInto(mvNeedRemovePartitionNameSet); + for (String partitionName : mvNeedRemovePartitionNameSet) { + baseTableNeedUnionPartitionNameSet.addAll(partitionMapping.key().get(partitionName)); + } + // If related base table create partitions or mv is created with ttl, need base table union + Sets.difference(queryUsedBaseTablePartitionNameSet, mvValidBaseTablePartitionNameSet) + .copyInto(baseTableNeedUnionPartitionNameSet); + // Construct result map + Map> mvPartitionNeedRemoveNameMap = new HashMap<>(); + if (!mvNeedRemovePartitionNameSet.isEmpty()) { + mvPartitionNeedRemoveNameMap.put(new BaseTableInfo(mtmv), mvNeedRemovePartitionNameSet); + } + Map> baseTablePartitionNeedUnionNameMap = new HashMap<>(); + if (!baseTableNeedUnionPartitionNameSet.isEmpty()) { + baseTablePartitionNeedUnionNameMap.put(relatedPartitionTable, baseTableNeedUnionPartitionNameSet); + } + return Pair.of(mvPartitionNeedRemoveNameMap, baseTablePartitionNeedUnionNameMap); + } + + public static boolean needUnionRewrite( + Pair>, Map>> invalidPartitions, + CascadesContext cascadesContext) { + return invalidPartitions != null + && (!invalidPartitions.key().isEmpty() || !invalidPartitions.value().isEmpty()); + } + + /** + * Check if need union compensate or not + */ + public static boolean needUnionRewrite(MaterializationContext materializationContext) { + if (!(materializationContext instanceof AsyncMaterializationContext)) { + return false; + } + MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); + PartitionType type = mtmv.getPartitionInfo().getType(); + BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); + return !PartitionType.UNPARTITIONED.equals(type) && relatedTableInfo != null; + } + + /** + * Get query used partitions + * this is calculated from tableUsedPartitionNameMap and tables in statementContext + * */ + public static Map, Set> getQueryUsedPartitions(StatementContext statementContext) { + // get table used partitions + // if table is not in statementContext().getTables() which means the table is partition prune as empty relation + Multimap, Pair>> tableUsedPartitionNameMap = statementContext + .getTableUsedPartitionNameMap(); + // if value is empty, means query no partitions + // if value is null, means query all partitions + // if value is not empty, means query some partitions + Map, Set> queryUsedRelatedTablePartitionsMap = new HashMap<>(); + outer: + for (Map.Entry, TableIf> queryUsedTableEntry : statementContext.getTables().entrySet()) { + Set usedPartitionSet = new HashSet<>(); + Collection>> tableUsedPartitions = + tableUsedPartitionNameMap.get(queryUsedTableEntry.getKey()); + if (!tableUsedPartitions.isEmpty()) { + if (ALL_PARTITIONS_LIST.equals(tableUsedPartitions)) { + queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), null); + continue; + } + for (Pair> partitionPair : tableUsedPartitions) { + if (ALL_PARTITIONS.equals(partitionPair)) { + queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), null); + continue outer; + } + usedPartitionSet.addAll(partitionPair.value()); + } + } + queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), usedPartitionSet); + } + return queryUsedRelatedTablePartitionsMap; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 2e2119efe7176c..2003e7c12ae113 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -17,8 +17,6 @@ package org.apache.doris.nereids.rules.exploration.mv; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.CascadesContext; @@ -33,7 +31,6 @@ import org.apache.doris.nereids.trees.copier.DeepCopierContext; import org.apache.doris.nereids.trees.copier.LogicalPlanDeepCopier; import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -50,7 +47,6 @@ import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand.PredicateAddContext; import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand.PredicateAdder; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; -import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -60,12 +56,9 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; -import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.HashMultimap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import com.google.common.collect.Multimap; import com.google.common.collect.Sets; @@ -131,8 +124,6 @@ public class StructInfo { // this is for building LogicalCompatibilityContext later. private final Map> expressionToShuttledExpressionToMap; - // Record the exprId and the corresponding expr map, this is used by expression shuttled - private final Map namedExprIdAndExprMapping; private final List planOutputShuttledExpressions; /** @@ -145,7 +136,6 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG Map>> shuttledExpressionsToExpressionsMap, Map> expressionToShuttledExpressionToMap, - Map namedExprIdAndExprMapping, BitSet tableIdSet, SplitPredicate splitPredicate, EquivalenceClass equivalenceClass, @@ -164,7 +154,6 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG this.equivalenceClass = equivalenceClass; this.shuttledExpressionsToExpressionsMap = shuttledExpressionsToExpressionsMap; this.expressionToShuttledExpressionToMap = expressionToShuttledExpressionToMap; - this.namedExprIdAndExprMapping = namedExprIdAndExprMapping; this.planOutputShuttledExpressions = planOutputShuttledExpressions; } @@ -175,8 +164,7 @@ public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, - this.namedExprIdAndExprMapping, this.tableBitSet, - null, null, this.planOutputShuttledExpressions); + this.tableBitSet, null, null, this.planOutputShuttledExpressions); } /** @@ -186,8 +174,7 @@ public StructInfo withTableBitSet(BitSet tableBitSet) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, this.predicates, this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, - this.namedExprIdAndExprMapping, tableBitSet, - this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions); + tableBitSet, this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, @@ -195,7 +182,6 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, Map>> shuttledExpressionsToExpressionsMap, Map> expressionToShuttledExpressionToMap, - Map namedExprIdAndExprMapping, List relations, Map relationIdStructInfoNodeMap, BitSet hyperTableBitSet, @@ -213,22 +199,16 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, // plan relation collector and set to map StructInfoNode structInfoNode = (StructInfoNode) node; // record expressions in node - if (structInfoNode.getExpressions() != null) { - structInfoNode.getExpressions().forEach(expression -> { - ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = - new ExpressionLineageReplacer.ExpressionReplaceContext( - Lists.newArrayList(expression), ImmutableSet.of(), - ImmutableSet.of(), new BitSet()); - structInfoNode.getPlan().accept(ExpressionLineageReplacer.INSTANCE, replaceContext); - // Replace expressions by expression map - List replacedExpressions = replaceContext.getReplacedExpressions(); + List nodeExpressions = structInfoNode.getExpressions(); + if (nodeExpressions != null) { + List shuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + nodeExpressions, structInfoNode.getPlan(), + new BitSet()); + for (int index = 0; index < nodeExpressions.size(); index++) { putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, expressionToShuttledExpressionToMap, - ExpressionPosition.NODE, replacedExpressions.get(0), expression, node); - // Record this, will be used in top level expression shuttle later, see the method - // ExpressionLineageReplacer#visitGroupPlan - namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); - }); + ExpressionPosition.NODE, shuttledExpressions.get(index), nodeExpressions.get(index), node); + } } // every node should only have one relation, this is for LogicalCompatibilityContext if (!nodeRelations.isEmpty()) { @@ -240,37 +220,27 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, List joinConjunctExpressions = edge.getExpressions(); // shuttle expression in edge for the build of LogicalCompatibilityContext later. // Record the exprId to expr map in the processing to strut info - // TODO get exprId to expr map when complex project is ready in join dege - ExpressionLineageReplacer.ExpressionReplaceContext replaceContext = - new ExpressionLineageReplacer.ExpressionReplaceContext( - joinConjunctExpressions.stream().map(expr -> (Expression) expr) - .collect(Collectors.toList()), - ImmutableSet.of(), ImmutableSet.of(), new BitSet()); - topPlan.accept(ExpressionLineageReplacer.INSTANCE, replaceContext); // Replace expressions by expression map - List replacedExpressions = replaceContext.getReplacedExpressions(); - for (int i = 0; i < replacedExpressions.size(); i++) { + List shuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + joinConjunctExpressions, topPlan, new BitSet()); + for (int i = 0; i < shuttledExpressions.size(); i++) { putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, expressionToShuttledExpressionToMap, - ExpressionPosition.JOIN_EDGE, replacedExpressions.get(i), joinConjunctExpressions.get(i), - edge); + ExpressionPosition.JOIN_EDGE, shuttledExpressions.get(i), + joinConjunctExpressions.get(i), edge); } - // Record this, will be used in top level expression shuttle later, see the method - // ExpressionLineageReplacer#visitGroupPlan - namedExprIdAndExprMapping.putAll(replaceContext.getExprIdExpressionMap()); } // Collect expression from where in hyper graph hyperGraph.getFilterEdges().forEach(filterEdge -> { List filterExpressions = filterEdge.getExpressions(); - filterExpressions.forEach(predicate -> { - // this is used for LogicalCompatibilityContext - ExpressionUtils.extractConjunction(predicate).forEach(expr -> - putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, - expressionToShuttledExpressionToMap, - ExpressionPosition.FILTER_EDGE, - ExpressionUtils.shuttleExpressionWithLineage(predicate, topPlan, new BitSet()), - predicate, filterEdge)); - }); + List shuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( + filterExpressions, topPlan, new BitSet()); + for (int i = 0; i < shuttledExpressions.size(); i++) { + putShuttledExpressionToExpressionsMap(shuttledExpressionsToExpressionsMap, + expressionToShuttledExpressionToMap, + ExpressionPosition.FILTER_EDGE, shuttledExpressions.get(i), + filterExpressions.get(i), filterEdge); + } }); return true; } @@ -343,12 +313,10 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable Map relationIdStructInfoNodeMap = new LinkedHashMap<>(); Map>> shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); - Map namedExprIdAndExprMapping = new LinkedHashMap<>(); BitSet tableBitSet = new BitSet(); Map> expressionToShuttledExpressionToMap = new HashMap<>(); boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, expressionToShuttledExpressionToMap, - namedExprIdAndExprMapping, relationList, relationIdStructInfoNodeMap, tableBitSet, @@ -370,7 +338,7 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, predicates, shuttledHashConjunctsToConjunctsMap, expressionToShuttledExpressionToMap, - namedExprIdAndExprMapping, tableBitSet, null, null, + tableBitSet, null, null, planOutputShuttledExpressions); } @@ -471,10 +439,6 @@ public ObjectId getOriginalPlanId() { return originalPlanId; } - public Map getNamedExprIdAndExprMapping() { - return namedExprIdAndExprMapping; - } - public BitSet getTableBitSet() { return tableBitSet; } @@ -764,35 +728,6 @@ public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, } } - /** - * Collect partitions on base table - */ - public static class QueryScanPartitionsCollector extends DefaultPlanVisitor>> { - @Override - public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, - Map> targetTablePartitionMap) { - TableIf table = catalogRelation.getTable(); - BaseTableInfo relatedPartitionTable = new BaseTableInfo(table); - if (!targetTablePartitionMap.containsKey(relatedPartitionTable)) { - return catalogRelation; - } - if (catalogRelation instanceof LogicalOlapScan) { - // Handle olap table - LogicalOlapScan logicalOlapScan = (LogicalOlapScan) catalogRelation; - Set tablePartitions = targetTablePartitionMap.get(relatedPartitionTable); - for (Long partitionId : logicalOlapScan.getSelectedPartitionIds()) { - tablePartitions.add(logicalOlapScan.getTable().getPartition(partitionId)); - } - } else { - // todo Support other type partition table - // Not support to partition check now when query external catalog table, support later. - targetTablePartitionMap.clear(); - } - return catalogRelation; - } - } - /** * Add filter on table scan according to table filter map * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java index e99906f5e13dc4..89dfb75f2ebbad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java @@ -64,7 +64,6 @@ public Rule build() { // set isPruned so that it won't go pass the partition prune again selectedPartitions = new SelectedPartitions(0, ImmutableMap.of(), true); } - LogicalFileScan rewrittenScan = scan.withSelectedPartitions(selectedPartitions); return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); }).toRule(RuleType.FILE_SCAN_PARTITION_PRUNE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java new file mode 100644 index 00000000000000..2ad993b361d43a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Pair; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Multimap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Used to collect query partitions, only collect once + * */ +public class QueryPartitionCollector extends DefaultPlanRewriter implements CustomRewriter { + + public static final Logger LOG = LogManager.getLogger(QueryPartitionCollector.class); + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getSessionVariable().internalSession) { + return plan; + } + plan.accept(this, connectContext); + return plan; + } + + @Override + public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, ConnectContext context) { + + TableIf table = catalogRelation.getTable(); + if (table.getDatabase() == null) { + LOG.error("QueryPartitionCollector visitLogicalCatalogRelation database is null, table is " + + table.getName()); + return catalogRelation; + } + Multimap, Pair>> tableUsedPartitionNameMap = context.getStatementContext() + .getTableUsedPartitionNameMap(); + Set tablePartitions = new HashSet<>(); + if (catalogRelation instanceof LogicalOlapScan) { + // Handle olap table + LogicalOlapScan logicalOlapScan = (LogicalOlapScan) catalogRelation; + for (Long partitionId : logicalOlapScan.getSelectedPartitionIds()) { + tablePartitions.add(logicalOlapScan.getTable().getPartition(partitionId).getName()); + } + tableUsedPartitionNameMap.put(table.getFullQualifiers(), + Pair.of(catalogRelation.getRelationId(), tablePartitions)); + } else if (catalogRelation instanceof LogicalFileScan + && catalogRelation.getTable() != null + && ((ExternalTable) catalogRelation.getTable()).supportInternalPartitionPruned()) { + LogicalFileScan logicalFileScan = (LogicalFileScan) catalogRelation; + SelectedPartitions selectedPartitions = logicalFileScan.getSelectedPartitions(); + tablePartitions.addAll(selectedPartitions.selectedPartitions.keySet()); + tableUsedPartitionNameMap.put(table.getFullQualifiers(), + Pair.of(catalogRelation.getRelationId(), tablePartitions)); + } else { + // not support get partition scene, we consider query all partitions from table + tableUsedPartitionNameMap.put(table.getFullQualifiers(), PartitionCompensator.ALL_PARTITIONS); + } + return catalogRelation; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java index 1252f3b4bbf899..b9f1cc907c538b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/ExpressionLineageReplacer.java @@ -18,8 +18,6 @@ package org.apache.doris.nereids.trees.plans.visitor; import org.apache.doris.catalog.TableIf.TableType; -import org.apache.doris.nereids.memo.Group; -import org.apache.doris.nereids.rules.exploration.mv.StructInfo; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -31,12 +29,14 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer.ExpressionReplaceContext; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.BitSet; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -47,6 +47,7 @@ */ public class ExpressionLineageReplacer extends DefaultPlanVisitor { + public static final Logger LOG = LogManager.getLogger(ExpressionLineageReplacer.class); public static final ExpressionLineageReplacer INSTANCE = new ExpressionLineageReplacer(); @Override @@ -63,25 +64,7 @@ public Expression visit(Plan plan, ExpressionReplaceContext context) { @Override public Expression visitGroupPlan(GroupPlan groupPlan, ExpressionReplaceContext context) { - Group group = groupPlan.getGroup(); - if (group == null) { - return visit(groupPlan, context); - } - Collection structInfos = group.getstructInfoMap().getStructInfos(); - if (structInfos.isEmpty()) { - return visit(groupPlan, context); - } - // Find first info which the context's bitmap contains all to make sure that - // the expression lineage is correct - Optional structInfoOptional = structInfos.stream() - .filter(info -> (context.getTableBitSet().isEmpty() - || StructInfo.containsAll(context.getTableBitSet(), info.getTableBitSet())) - && !info.getNamedExprIdAndExprMapping().isEmpty()) - .findFirst(); - if (!structInfoOptional.isPresent()) { - return visit(groupPlan, context); - } - context.getExprIdExpressionMap().putAll(structInfoOptional.get().getNamedExprIdAndExprMapping()); + LOG.error("ExpressionLineageReplacer should not meet groupPlan, plan is {}", groupPlan.toString()); return null; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 15affdff26b0c3..78ea69113e6f09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -616,6 +616,9 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_SYNC_MV_COST_BASED_REWRITE = "enable_sync_mv_cost_based_rewrite"; + public static final String MATERIALIZED_VIEW_REWRITE_DURATION_THRESHOLD_MS + = "materialized_view_rewrite_duration_threshold_ms"; + public static final String MATERIALIZED_VIEW_RELATION_MAPPING_MAX_COUNT = "materialized_view_relation_mapping_max_count"; @@ -2109,6 +2112,12 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { "Whether enable cost based rewrite for sync mv"}) public boolean enableSyncMvCostBasedRewrite = true; + @VariableMgr.VarAttr(name = MATERIALIZED_VIEW_REWRITE_DURATION_THRESHOLD_MS, needForward = true, + description = {"物化视图透明改写允许的最长耗时，超过此时长不再进行透明改写", + "The maximum duration allowed for transparent rewriting of materialized views; " + + "if this duration is exceeded, transparent rewriting will no longer be performed."}) + public long materializedViewRewriteDurationThresholdMs = 1000L; + @VariableMgr.VarAttr(name = CREATE_TABLE_PARTITION_MAX_NUM, needForward = true, description = {"建表时创建分区的最大数量", "The maximum number of partitions created during table creation"}) diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 25ad6b05dd8454..82c7eaac631e81 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -27,6 +27,7 @@ import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import mockit.Expectations; import mockit.Mocked; import org.junit.Assert; @@ -64,6 +65,14 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = Lists.newArrayList(p1); + mtmv.getPartitionNames(); + minTimes = 0; + result = Sets.newHashSet("p1"); + + p1.getName(); + minTimes = 0; + result = "p1"; + p1.getVisibleVersionTime(); minTimes = 0; result = 1L; @@ -136,14 +145,15 @@ public void testGetMTMVCanRewritePartitionsForceConsistent() throws AnalysisExce // currentTimeMills is 3, grace period is 2, and partition getVisibleVersionTime is 1 // if forceConsistent this should get 0 partitions which mtmv can use. Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, true); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, true, null); Assert.assertEquals(0, mtmvCanRewritePartitions.size()); } @Test public void testGetMTMVCanRewritePartitionsNormal() { Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(1, mtmvCanRewritePartitions.size()); } @@ -164,7 +174,8 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(1, mtmvCanRewritePartitions.size()); } @@ -185,7 +196,8 @@ public void testGetMTMVCanRewritePartitionsNotInGracePeriod() throws AnalysisExc }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(0, mtmvCanRewritePartitions.size()); } @@ -199,7 +211,8 @@ public void testGetMTMVCanRewritePartitionsDisableMaterializedViewRewrite() { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); // getMTMVCanRewritePartitions only check the partition is valid or not, it doesn't care the // isEnableMaterializedViewRewriteWhenBaseTableUnawareness Assert.assertEquals(1, mtmvCanRewritePartitions.size()); @@ -217,7 +230,8 @@ public void testGetMTMVCanRewritePartitionsNotSync() throws AnalysisException { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(0, mtmvCanRewritePartitions.size()); } @@ -235,7 +249,8 @@ public void testGetMTMVCanRewritePartitionsEnableContainExternalTable() { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(1, mtmvCanRewritePartitions.size()); } @@ -253,7 +268,8 @@ public void testGetMTMVCanRewritePartitionsDisableContainExternalTable() { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); // getMTMVCanRewritePartitions only check the partition is valid or not, it doesn't care the // isEnableMaterializedViewRewriteWhenBaseTableUnawareness Assert.assertEquals(1, mtmvCanRewritePartitions.size()); @@ -269,7 +285,8 @@ public void testGetMTMVCanRewritePartitionsStateAbnormal() { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(0, mtmvCanRewritePartitions.size()); } @@ -283,7 +300,8 @@ public void testGetMTMVCanRewritePartitionsRefreshStateAbnormal() { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(1, mtmvCanRewritePartitions.size()); } @@ -297,7 +315,8 @@ public void testGetMTMVCanRewritePartitionsRefreshStateInit() { } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil - .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false); + .getMTMVCanRewritePartitions(mtmv, ctx, currentTimeMills, false, + null); Assert.assertEquals(0, mtmvCanRewritePartitions.size()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 6b10176e22aa9c..19d1efdbbd82d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -65,7 +65,8 @@ public BitSet getDisableNereidsRules() { Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent, + Set relatedPartitions) { return true; } }; @@ -123,7 +124,8 @@ public BitSet getDisableNereidsRules() { Assertions.assertEquals(1, tableMaps.size()); new MockUp() { @Mock - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent, + Set relatedPartitions) { return true; } }; @@ -147,7 +149,6 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPar .printlnBestPlanTree(); root = c1.getMemo().getRoot(); root.getstructInfoMap().refresh(root, c1, new HashSet<>()); - root.getstructInfoMap().refresh(root, c1, new HashSet<>()); tableMaps = root.getstructInfoMap().getTableMaps(); Assertions.assertEquals(2, tableMaps.size()); dropMvByNereids("drop materialized view mv1"); @@ -171,7 +172,8 @@ public BitSet getDisableNereidsRules() { ); new MockUp() { @Mock - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent, + Set relatedPartitions) { return true; } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java index 0090982db00898..1403a9fee5e254 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java @@ -54,7 +54,8 @@ public BitSet getDisableNereidsRules() { }; new MockUp() { @Mock - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid, + Set queryUsedRelatedTablePartitionsMap) { return true; } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MtmvCacheNewConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MtmvCacheNewConnectContextTest.java index 0134d5df4e7166..a15aad19de4c77 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MtmvCacheNewConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MtmvCacheNewConnectContextTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.mv; import org.apache.doris.catalog.MTMV; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVRelationManager; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.sqltest.SqlTestBase; @@ -31,6 +32,8 @@ import org.junit.jupiter.api.Test; import java.util.BitSet; +import java.util.Map; +import java.util.Set; /** * The connectContext would new instance when generate MTMVCache, after generate, the connectContext should @@ -52,7 +55,8 @@ public BitSet getDisableNereidsRules() { }; new MockUp() { @Mock - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceConsistent) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid, + Map> queryUsedRelatedTablePartitionsMap) { return true; } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java index dd15b5e06c7899..4fa0a68e77c6ab 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java @@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test; import java.util.BitSet; +import java.util.Set; /** * Test mv rewrite when base table id is lager then integer @@ -49,7 +50,8 @@ public BitSet getDisableNereidsRules() { }; new MockUp() { @Mock - public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid) { + public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPartitionValid, + Set queryUsedRelatedTablePartitionsMap) { return true; } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java new file mode 100644 index 00000000000000..da327b2ba6ca60 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/OptimizeGetAvailableMvsTest.java @@ -0,0 +1,269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.mv; + +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.common.Pair; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.expression.rules.PartitionPruner; +import org.apache.doris.nereids.rules.expression.rules.PartitionPruner.PartitionTableType; +import org.apache.doris.nereids.sqltest.SqlTestBase; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.Lists; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.BitSet; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Test get available mvs after rewrite by rules + */ +public class OptimizeGetAvailableMvsTest extends SqlTestBase { + + @Test + void testWhenNotPartitionPrune() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); + new MockUp() { + @Mock + public BitSet getDisableNereidsRules() { + return disableNereidsRules; + } + }; + + new MockUp() { + @Mock + public Partition getPartition(long partitionId) { + return new Partition() { + @Override + public long getId() { + return 1L; + } + + @Override + public String getName() { + return "mock_partition"; + } + + @Override + public PartitionState getState() { + return PartitionState.NORMAL; + } + + @Override + public MaterializedIndex getIndex(long indexId) { + return new MaterializedIndex(1L, IndexState.NORMAL); + } + + @Override + public DistributionInfo getDistributionInfo() { + return new DistributionInfo() { + @Override + public DistributionInfoType getType() { + return DistributionInfoType.RANDOM; + } + }; + } + }; + } + }; + + new MockUp() { + @Mock + public List getSelectedPartitionIds() { + return Lists.newArrayList(1L); + } + }; + + connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; + createMvByNereids("create materialized view mv1 " + + " BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + + " PARTITION BY (id)\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as " + + " select T4.id from T4 inner join T2 " + + " on T4.id = T2.id;"); + CascadesContext c1 = createCascadesContext( + "select T4.id " + + "from T4 " + + "inner join T2 on T4.id = T2.id " + + "inner join T3 on T4.id = T3.id", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .optimize() + .printlnBestPlanTree(); + Multimap, Pair>> tableUsedPartitionNameMap = c1.getStatementContext() + .getTableUsedPartitionNameMap(); + Map> mvCanRewritePartitionsMap = c1.getStatementContext() + .getMvCanRewritePartitionsMap(); + Assertions.assertFalse(tableUsedPartitionNameMap.isEmpty()); + + for (Map.Entry, Pair>> tableInfoEntry + : tableUsedPartitionNameMap.entries()) { + if (tableInfoEntry.getKey().contains("T2")) { + Assertions.assertEquals(tableInfoEntry.getValue().value(), Sets.newHashSet("mock_partition")); + } else if (tableInfoEntry.getKey().contains("T3")) { + Assertions.assertEquals(tableInfoEntry.getValue().value(), Sets.newHashSet("mock_partition")); + } else if (tableInfoEntry.getKey().contains("T4")) { + Assertions.assertEquals(tableInfoEntry.getValue().value(), Sets.newHashSet("mock_partition")); + } + } + + Assertions.assertEquals(1, mvCanRewritePartitionsMap.size()); + Assertions.assertTrue(mvCanRewritePartitionsMap.keySet().iterator().next().getTableName() + .equalsIgnoreCase("mv1")); + + dropMvByNereids("drop materialized view mv1"); + } + + @Test + void testWhenPartitionPrune() throws Exception { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + BitSet disableNereidsRules = connectContext.getSessionVariable().getDisableNereidsRules(); + new MockUp() { + @Mock + public BitSet getDisableNereidsRules() { + return disableNereidsRules; + } + }; + + new MockUp() { + @Mock + public > List prune(List partitionSlots, Expression partitionPredicate, + Map idToPartitions, CascadesContext cascadesContext, + PartitionTableType partitionTableType) { + return Lists.newArrayList(1L); + } + }; + + new MockUp() { + @Mock + public Partition getPartition(long partitionId) { + return new Partition() { + @Override + public long getId() { + return 1L; + } + + @Override + public String getName() { + return "mock_partition"; + } + + @Override + public PartitionState getState() { + return PartitionState.NORMAL; + } + + @Override + public MaterializedIndex getIndex(long indexId) { + return new MaterializedIndex(1L, IndexState.NORMAL); + } + + @Override + public DistributionInfo getDistributionInfo() { + return new DistributionInfo() { + @Override + public DistributionInfoType getType() { + return DistributionInfoType.RANDOM; + } + }; + } + }; + } + }; + + new MockUp() { + @Mock + public List getSelectedPartitionIds() { + return Lists.newArrayList(1L); + } + }; + + connectContext.getSessionVariable().enableMaterializedViewRewrite = true; + connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; + createMvByNereids("create materialized view mv2 " + + " BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" + + " PARTITION BY (id)\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as " + + " select T4.id from T4 inner join T2 " + + " on T4.id = T2.id;"); + CascadesContext c1 = createCascadesContext( + "select T4.id " + + "from T4 " + + "inner join T2 on T4.id = T2.id " + + "inner join T3 on T4.id = T3.id " + + "where T4.id > 0", + connectContext + ); + PlanChecker.from(c1) + .analyze() + .rewrite() + .optimize() + .printlnBestPlanTree(); + Multimap, Pair>> tableUsedPartitionNameMap = c1.getStatementContext() + .getTableUsedPartitionNameMap(); + Map> mvCanRewritePartitionsMap = c1.getStatementContext() + .getMvCanRewritePartitionsMap(); + Assertions.assertFalse(tableUsedPartitionNameMap.isEmpty()); + + for (Map.Entry, Pair>> tableInfoEntry + : tableUsedPartitionNameMap.entries()) { + if (tableInfoEntry.getKey().contains("T2")) { + Assertions.assertEquals(tableInfoEntry.getValue().value(), Sets.newHashSet("mock_partition")); + } else if (tableInfoEntry.getKey().contains("T3")) { + Assertions.assertEquals(tableInfoEntry.getValue().value(), Sets.newHashSet("mock_partition")); + } else if (tableInfoEntry.getKey().contains("T4")) { + Assertions.assertEquals(tableInfoEntry.getValue().value(), Sets.newHashSet("mock_partition")); + } + } + + Assertions.assertEquals(1, mvCanRewritePartitionsMap.size()); + Assertions.assertTrue(mvCanRewritePartitionsMap.keySet().iterator().next().getTableName() + .equalsIgnoreCase("mv2")); + + dropMvByNereids("drop materialized view mv2"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java new file mode 100644 index 00000000000000..76246e52f9d542 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class PartitionCompensatorTest extends TestWithFeService { + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("partition_compensate_test"); + useDatabase("partition_compensate_test"); + + createTable("CREATE TABLE `lineitem_list_partition` (\n" + + " `l_orderkey` BIGINT not NULL,\n" + + " `l_linenumber` INT NULL,\n" + + " `l_partkey` INT NULL,\n" + + " `l_suppkey` INT NULL,\n" + + " `l_quantity` DECIMAL(15, 2) NULL,\n" + + " `l_extendedprice` DECIMAL(15, 2) NULL,\n" + + " `l_discount` DECIMAL(15, 2) NULL,\n" + + " `l_tax` DECIMAL(15, 2) NULL,\n" + + " `l_returnflag` VARCHAR(1) NULL,\n" + + " `l_linestatus` VARCHAR(1) NULL,\n" + + " `l_commitdate` DATE NULL,\n" + + " `l_receiptdate` DATE NULL,\n" + + " `l_shipinstruct` VARCHAR(25) NULL,\n" + + " `l_shipmode` VARCHAR(10) NULL,\n" + + " `l_comment` VARCHAR(44) NULL,\n" + + " `l_shipdate` DATE NULL\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey )\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY list(l_orderkey) (\n" + + " PARTITION p1 VALUES in ('1'),\n" + + " PARTITION p2 VALUES in ('2'),\n" + + " PARTITION p3 VALUES in ('3')\n" + + " )\n" + + " DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 3\n" + + " PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + " )"); + + createTable("CREATE TABLE `orders_list_partition` (\n" + + " `o_orderkey` BIGINT not NULL,\n" + + " `o_custkey` INT NULL,\n" + + " `o_orderstatus` VARCHAR(1) NULL,\n" + + " `o_totalprice` DECIMAL(15, 2) NULL,\n" + + " `o_orderpriority` VARCHAR(15) NULL,\n" + + " `o_clerk` VARCHAR(15) NULL,\n" + + " `o_shippriority` INT NULL,\n" + + " `o_comment` VARCHAR(79) NULL,\n" + + " `o_orderdate` DATE NULL\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(`o_orderkey`, `o_custkey`)\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY list(o_orderkey) (\n" + + " PARTITION p1 VALUES in ('1'),\n" + + " PARTITION p2 VALUES in ('2'),\n" + + " PARTITION p3 VALUES in ('3'),\n" + + " PARTITION p4 VALUES in ('4')\n" + + " )\n" + + " DISTRIBUTED BY HASH(`o_orderkey`) BUCKETS 3\n" + + " PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + " )"); + + // Should not make scan to empty relation when the table used by materialized view has no data + connectContext.getSessionVariable().setDisableNereidsRules( + "OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION,ELIMINATE_GROUP_BY_KEY_BY_UNIFORM"); + } + + // Test when join both side are all partition table and partition column name is same + @Test + public void testGetQueryTableUsedPartition() { + PlanChecker.from(connectContext) + .checkExplain("select l1.*, O_CUSTKEY \n" + + "from lineitem_list_partition l1\n" + + "left outer join orders_list_partition\n" + + "on l1.l_shipdate = o_orderdate\n", + nereidsPlanner -> { + Map, Set> queryUsedPartitions + = PartitionCompensator.getQueryUsedPartitions( + nereidsPlanner.getCascadesContext().getStatementContext()); + + List itmeQualifier = ImmutableList.of( + "internal", "partition_compensate_test", "lineitem_list_partition"); + Set queryTableUsedPartition = queryUsedPartitions.get(itmeQualifier); + Assertions.assertEquals(queryTableUsedPartition, ImmutableSet.of("p1", "p2", "p3")); + + List orderQualifier = ImmutableList.of( + "internal", "partition_compensate_test", "orders_list_partition"); + Set orderTableUsedPartition = queryUsedPartitions.get(orderQualifier); + Assertions.assertEquals(orderTableUsedPartition, ImmutableSet.of("p1", "p2", "p3", "p4")); + }); + } + + @Test + public void testGetAllTableUsedPartition() { + PlanChecker.from(connectContext) + .checkExplain("select l1.*, O_CUSTKEY \n" + + "from lineitem_list_partition l1\n" + + "left outer join orders_list_partition\n" + + "on l1.l_shipdate = o_orderdate\n", + nereidsPlanner -> { + List qualifier = ImmutableList.of( + "internal", "partition_compensate_test", "lineitem_list_partition"); + + Multimap, Pair>> tableUsedPartitionNameMap + = connectContext.getStatementContext().getTableUsedPartitionNameMap(); + tableUsedPartitionNameMap.put(qualifier, PartitionCompensator.ALL_PARTITIONS); + + Map, Set> queryUsedPartitions + = PartitionCompensator.getQueryUsedPartitions( + nereidsPlanner.getCascadesContext().getStatementContext()); + Set queryTableUsedPartition = queryUsedPartitions.get(qualifier); + // if tableUsedPartitionNameMap contain any PartitionCompensator.ALL_PARTITIONS + // consider query all partitions from table + Assertions.assertNull(queryTableUsedPartition); + + List orderQualifier = ImmutableList.of( + "internal", "partition_compensate_test", "orders_list_partition"); + Set orderTableUsedPartition = queryUsedPartitions.get(orderQualifier); + Assertions.assertEquals(orderTableUsedPartition, ImmutableSet.of("p1", "p2", "p3", "p4")); + }); + } + + @Test + public void testGetAllTableUsedPartitionList() { + PlanChecker.from(connectContext) + .checkExplain("select l1.*, O_CUSTKEY \n" + + "from lineitem_list_partition l1\n" + + "left outer join orders_list_partition\n" + + "on l1.l_shipdate = o_orderdate\n", + nereidsPlanner -> { + List qualifier = ImmutableList.of( + "internal", "partition_compensate_test", "lineitem_list_partition"); + + Multimap, Pair>> tableUsedPartitionNameMap + = connectContext.getStatementContext().getTableUsedPartitionNameMap(); + tableUsedPartitionNameMap.removeAll(qualifier); + tableUsedPartitionNameMap.put(qualifier, PartitionCompensator.ALL_PARTITIONS); + + Map, Set> queryUsedPartitions + = PartitionCompensator.getQueryUsedPartitions( + nereidsPlanner.getCascadesContext().getStatementContext()); + Set queryTableUsedPartition = queryUsedPartitions.get(qualifier); + // if tableUsedPartitionNameMap contain only PartitionCompensator.ALL_PARTITIONS + // consider query all partitions from table + Assertions.assertNull(queryTableUsedPartition); + + List orderQualifier = ImmutableList.of( + "internal", "partition_compensate_test", "orders_list_partition"); + Set orderTableUsedPartition = queryUsedPartitions.get(orderQualifier); + Assertions.assertEquals(orderTableUsedPartition, ImmutableSet.of("p1", "p2", "p3", "p4")); + }); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index e0cf7f66a02fe7..e900937d2baa68 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -121,7 +121,6 @@ public PlanChecker parse(String sql) { public PlanChecker analyze() { this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.toMemo(); - InitMaterializationContextHook.INSTANCE.initMaterializationContext(this.cascadesContext); return this; } @@ -245,6 +244,8 @@ public Rule build() { public PlanChecker rewrite() { Rewriter.getWholeTreeRewriter(cascadesContext).execute(); + cascadesContext.newTablePartitionCollector().execute(); + InitMaterializationContextHook.INSTANCE.initMaterializationContext(this.cascadesContext); cascadesContext.toMemo(); return this; } @@ -553,8 +554,10 @@ public void onInvokeRule(RuleType ruleType) { public PlanChecker checkExplain(String sql, Consumer consumer) { LogicalPlan parsed = new NereidsParser().parseSingle(sql); + StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(sql, 0)); NereidsPlanner nereidsPlanner = new NereidsPlanner( - new StatementContext(connectContext, new OriginStatement(sql, 0))); + statementContext); + connectContext.setStatementContext(statementContext); LogicalPlanAdapter adapter = LogicalPlanAdapter.of(parsed); adapter.setIsExplain(new ExplainOptions(ExplainLevel.ALL_PLAN, false)); nereidsPlanner.plan(adapter); @@ -564,8 +567,9 @@ public PlanChecker checkExplain(String sql, Consumer consumer) { public PlanChecker checkPlannerResult(String sql, Consumer consumer) { LogicalPlan parsed = new NereidsParser().parseSingle(sql); - NereidsPlanner nereidsPlanner = new NereidsPlanner( - new StatementContext(connectContext, new OriginStatement(sql, 0))); + StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(sql, 0)); + NereidsPlanner nereidsPlanner = new NereidsPlanner(statementContext); + connectContext.setStatementContext(statementContext); nereidsPlanner.plan(LogicalPlanAdapter.of(parsed)); consumer.accept(nereidsPlanner); return this; diff --git a/regression-test/data/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.out b/regression-test/data/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.out new file mode 100644 index 00000000000000..9c3b0f5506395d --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.out @@ -0,0 +1,143 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query_3_0_before -- +2023-10-17 2023-10-17 2 3 1990.00 +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 +2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_3_0_after -- +2023-10-17 2023-10-17 2 3 1990.00 +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 +2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_4_0_before -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_4_0_after -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_7_0_before -- +2023-10-17 2023-10-17 2 3 1990.00 +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_7_0_after -- +2023-10-17 2023-10-17 2 3 1990.00 +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_8_0_before -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_8_0_after -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_11_0_before -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_11_0_after -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_12_0_before -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_12_0_after -- +2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_16_0_before -- +2023-10-18 2023-10-18 2 3 436.80 +2023-10-19 2023-10-19 2 3 398.00 + +-- !query_16_0_after -- +2023-10-18 2023-10-18 2 3 436.80 +2023-10-19 2023-10-19 2 3 398.00 + +-- !query_17_0_before -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_17_0_after -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_18_0_before -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_18_0_after -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_19_0_before -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-11-21 2023-11-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_19_0_after -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-11-21 2023-11-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_20_0_before -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_20_0_after -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_21_0_before -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_21_0_after -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_22_0_before -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_22_0_after -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + diff --git a/regression-test/data/nereids_rules_p0/mv/partition_mv_rewrite.out b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out similarity index 69% rename from regression-test/data/nereids_rules_p0/mv/partition_mv_rewrite.out rename to regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out index c6b66a59749cae..1e718b4e6127d3 100644 --- a/regression-test/data/nereids_rules_p0/mv/partition_mv_rewrite.out +++ b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out @@ -3,11 +3,13 @@ 2023-10-17 2023-10-17 2 3 1990.00 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 +2023-12-19 2023-12-19 2 3 1592.00 -- !query_3_0_after -- -2023-10-17 2023-10-17 2 3 1990.00 +2023-10-17 2023-10-17 2 3 1592.00 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 +2023-12-19 2023-12-19 2 3 1592.00 -- !query_4_0_before -- 2023-10-18 2023-10-18 2 3 1747.20 @@ -22,12 +24,13 @@ 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 -- !query_7_0_after -- 2023-10-17 2023-10-17 2 3 1990.00 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 -2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 -- !query_8_0_before -- 2023-10-18 2023-10-18 2 3 1747.20 @@ -41,11 +44,14 @@ 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 -- !query_11_0_after -- +2023-10-17 2023-10-17 2 3 1990.00 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 \N 2 3 \N +2023-12-19 2023-12-19 2 3 1592.00 -- !query_12_0_before -- 2023-10-18 2023-10-18 2 3 1747.20 @@ -64,16 +70,17 @@ 2023-10-19 2023-10-19 2 3 398.00 -- !query_17_0_before -- +2023-09-17 2023-09-17 \N 2 3 \N 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N -2023-11-21 2023-11-21 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_17_0_after -- 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N -2023-11-21 2023-11-21 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_18_0_before -- 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 @@ -84,18 +91,20 @@ 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 -- !query_19_0_before -- +2023-09-17 2023-09-17 \N 2 3 \N 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N 2023-11-21 2023-11-21 \N 2 3 \N 2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_19_0_after -- +2023-09-17 2023-09-17 \N 2 3 \N 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 2023-10-21 2023-10-21 \N 2 3 \N -2023-11-21 2023-11-21 \N 2 3 \N -2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 -- !query_20_0_before -- 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 @@ -105,3 +114,28 @@ 2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +-- !query_21_0_before -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_21_0_after -- +2023-09-17 2023-09-17 \N 2 3 \N +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 +2023-10-21 2023-10-21 \N 2 3 \N +2023-11-21 2023-11-21 \N 2 3 \N +2023-11-22 2023-11-22 \N 2 3 \N +2023-12-19 2023-12-19 2023-12-19 2 3 1592.00 + +-- !query_22_0_before -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + +-- !query_22_0_after -- +2023-10-18 2023-10-18 2023-10-18 2 3 1747.20 +2023-10-19 2023-10-19 2023-10-19 2 3 1592.00 + diff --git a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy index b9e671f2636b82..d4e3a80dbeb481 100644 --- a/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/availability/grace_period.groovy @@ -149,7 +149,7 @@ suite("grace_period") { """ // force consistency when partition table, and query use the partition changed, should fail - mv_rewrite_fail(""" + mv_not_part_in(""" select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem_partition @@ -310,7 +310,7 @@ suite("grace_period") { sql "SET enable_materialized_view_rewrite=true" Thread.sleep(15000); // after 10s when partition table, and query use the partition changed, should fail - mv_rewrite_fail( + mv_not_part_in( """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy similarity index 68% rename from regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy rename to regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy index eba1edd94b6c3e..0126d5993bbb2d 100644 --- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/partition_mv_rewrite.groovy @@ -1,3 +1,5 @@ +package mv.partition_union_rewrite + import java.text.SimpleDateFormat // Licensed to the Apache Software Foundation (ASF) under one @@ -40,7 +42,7 @@ suite("partition_mv_rewrite") { ) DUPLICATE KEY(o_orderkey, o_custkey) PARTITION BY RANGE(o_orderdate)( - FROM ('2023-10-16') TO ('2023-11-30') INTERVAL 1 DAY + FROM ('2023-09-16') TO ('2023-12-30') INTERVAL 1 DAY ) DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 PROPERTIES ( @@ -74,7 +76,7 @@ suite("partition_mv_rewrite") { ) DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) PARTITION BY RANGE(l_shipdate) - (FROM ('2023-10-16') TO ('2023-11-30') INTERVAL 1 DAY) + (FROM ('2023-09-16') TO ('2023-12-30') INTERVAL 1 DAY) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 PROPERTIES ( "replication_num" = "1" @@ -94,7 +96,11 @@ suite("partition_mv_rewrite") { (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), - (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'); + (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'); """ sql """ @@ -110,7 +116,11 @@ suite("partition_mv_rewrite") { (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), - (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'); + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'); """ @@ -126,7 +136,7 @@ suite("partition_mv_rewrite") { l_suppkey; """ - def all_partition_sql = """ + def query_all_partition_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate @@ -138,7 +148,7 @@ suite("partition_mv_rewrite") { """ - def partition_sql = """ + def query_partition_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate @@ -151,10 +161,21 @@ suite("partition_mv_rewrite") { """ - sql """DROP MATERIALIZED VIEW IF EXISTS mv_10086""" - sql """DROP TABLE IF EXISTS mv_10086""" + multi_sql """ + analyze table lineitem with sync; + analyze table orders with sync; + """ + + sql """alter table orders modify column o_comment set stats ('row_count'='20');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='20');""" + + + def mv_1_partition_name = "mv_10086" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_1_partition_name}""" + sql """DROP TABLE IF EXISTS ${mv_1_partition_name}""" sql""" - CREATE MATERIALIZED VIEW mv_10086 + CREATE MATERIALIZED VIEW ${mv_1_partition_name} BUILD IMMEDIATE REFRESH AUTO ON MANUAL partition by(l_shipdate) DISTRIBUTED BY RANDOM BUCKETS 2 @@ -163,98 +184,96 @@ suite("partition_mv_rewrite") { ${mv_def_sql} """ - waitingMTMVTaskFinished(getJobName(db, "mv_10086")) + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) multi_sql """ - analyze table lineitem with sync; - analyze table orders with sync; - analyze table mv_10086 with sync; + analyze table ${mv_1_partition_name} with sync; """ - sleep(10000) - mv_rewrite_success(all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - mv_rewrite_success(partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - // base table partition data change + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + // test base table partition data change sql """ insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'); """ - waitingPartitionIsExpected("mv_10086", "p_20231017_20231018", false) + waitingPartitionIsExpected(mv_1_partition_name, "p_20231017_20231018", false) + - // enable union rewrite sql "SET enable_materialized_view_rewrite=false" - order_qt_query_3_0_before "${all_partition_sql}" + order_qt_query_3_0_before "${query_all_partition_sql}" sql "SET enable_materialized_view_rewrite=true" - sql "analyze table mv_10086 with sync" - def memo = sql "explain memo plan ${all_partition_sql}" - print(memo) + + // should rewrite successful when union rewrite enalbe if sub partition is invalid - mv_rewrite_success(all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_3_0_after "${all_partition_sql}" + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_3_0_after "${query_all_partition_sql}" sql "SET enable_materialized_view_rewrite=false" - order_qt_query_4_0_before "${partition_sql}" + order_qt_query_4_0_before "${query_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successfully when union rewrite enable if doesn't query invalid partition - mv_rewrite_success(partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_4_0_after "${partition_sql}" + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_4_0_after "${query_partition_sql}" // base table add partition - sql "REFRESH MATERIALIZED VIEW mv_10086 AUTO" - waitingMTMVTaskFinished(getJobName(db, "mv_10086")) + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) sql """ insert into lineitem values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-21', '2023-10-21', '2023-10-21', 'a', 'b', 'yyyyyyyyy'); """ - waitingPartitionIsExpected("mv_10086", "p_20231021_20231022", false) + waitingPartitionIsExpected(mv_1_partition_name, "p_20231021_20231022", false) + - // enable union rewrite sql "SET enable_materialized_view_rewrite=false" - order_qt_query_7_0_before "${all_partition_sql}" + order_qt_query_7_0_before "${query_all_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successful when union rewrite enalbe if base table add new partition - mv_rewrite_success(all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_7_0_after "${all_partition_sql}" + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_7_0_after "${query_all_partition_sql}" sql "SET enable_materialized_view_rewrite=false" - order_qt_query_8_0_before "${partition_sql}" + order_qt_query_8_0_before "${query_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successfully when union rewrite enable if doesn't query new partition - mv_rewrite_success(partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_8_0_after "${partition_sql}" + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_8_0_after "${query_partition_sql}" // base table delete partition test - sql "REFRESH MATERIALIZED VIEW mv_10086 AUTO" - waitingMTMVTaskFinished(getJobName(db, "mv_10086")) + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) sql """ ALTER TABLE lineitem DROP PARTITION IF EXISTS p_20231017 FORCE; """ // show partitions will cause error, tmp comment - waitingPartitionIsExpected("mv_10086", "p_20231017_20231018", false) + waitingPartitionIsExpected(mv_1_partition_name, "p_20231017_20231018", false) + - // enable union rewrite sql "SET enable_materialized_view_rewrite=false" - order_qt_query_11_0_before "${all_partition_sql}" + order_qt_query_11_0_before "${query_all_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successful when union rewrite enalbe if base table delete partition - mv_rewrite_success(all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_11_0_after "${all_partition_sql}" + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_11_0_after "${query_all_partition_sql}" sql "SET enable_materialized_view_rewrite=false" - order_qt_query_12_0_before "${partition_sql}" + order_qt_query_12_0_before "${query_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successfully when union rewrite enable if doesn't query deleted partition - mv_rewrite_success(partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_12_0_after "${partition_sql}" - sql """ DROP MATERIALIZED VIEW IF EXISTS mv_10086""" + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_12_0_after "${query_partition_sql}" + sql """ DROP MATERIALIZED VIEW IF EXISTS ${mv_1_partition_name}""" // test mv with ttl def today_str = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).toString(); @@ -285,7 +304,7 @@ suite("partition_mv_rewrite") { PARTITION BY RANGE(l_shipdate) ( PARTITION `p1` VALUES LESS THAN ("2023-10-18"), - PARTITION `p2` VALUES [("2023-10-18"), ("2023-10-20")), + PARTITION `p2` VALUES [("2023-10-18"), ("2023-12-20")), PARTITION `other` VALUES LESS THAN (MAXVALUE) ) DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 @@ -297,13 +316,20 @@ suite("partition_mv_rewrite") { insert into lineitem_static values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), - (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'); + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'); """ sql """ insert into lineitem_static values (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'); """ + multi_sql """ + analyze table lineitem_static with sync; + """ + sql """alter table lineitem_static modify column l_comment set stats ('row_count'='6');""" + + def ttl_mv_def_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total @@ -315,7 +341,7 @@ suite("partition_mv_rewrite") { l_partkey, l_suppkey; """ - def ttl_all_partition_sql = """ + def query_ttl_all_partition_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem_static left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate @@ -325,7 +351,7 @@ suite("partition_mv_rewrite") { l_partkey, l_suppkey; """ - def ttl_partition_sql = """ + def query_ttl_partition_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem_static left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate @@ -357,30 +383,24 @@ suite("partition_mv_rewrite") { create_ttl_mtmv(db, ttl_mv_name, ttl_mv_def_sql) - multi_sql """ - analyze table lineitem_static with sync; - analyze table lineitem with sync; - analyze table orders with sync; - """ - // test when mv is ttl - // enable union rewrite + // test when mv is partition roll up sql "SET enable_materialized_view_rewrite=true" // should rewrite successful when union rewrite enalbe and mv is ttl, query the partition which is in mv - mv_rewrite_success(ttl_all_partition_sql, ttl_mv_name, true, + mv_rewrite_success(query_ttl_all_partition_sql, ttl_mv_name, true, is_partition_statistics_ready(db, ["lineitem_static", "orders", ttl_mv_name])) sql "SET enable_materialized_view_rewrite=false" - order_qt_query_16_0_before "${ttl_partition_sql}" + order_qt_query_16_0_before "${query_ttl_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite fail when union rewrite enalbe and query the partition which is not in mv - mv_rewrite_fail(ttl_partition_sql, ttl_mv_name) - order_qt_query_16_0_after "${ttl_partition_sql}" + mv_not_part_in(query_ttl_partition_sql, ttl_mv_name) + order_qt_query_16_0_after "${query_ttl_partition_sql}" sql """ DROP MATERIALIZED VIEW IF EXISTS ${ttl_mv_name}""" - // date roll up mv + // test date roll up mv partition rewrite def roll_up_mv_def_sql = """ select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total @@ -394,7 +414,7 @@ suite("partition_mv_rewrite") { l_suppkey; """ - def roll_up_all_partition_sql = """ + def query_roll_up_all_partition_sql = """ select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem @@ -407,7 +427,7 @@ suite("partition_mv_rewrite") { l_suppkey; """ - def roll_up_partition_sql = """ + def query_roll_up_partition_sql = """ select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem @@ -421,10 +441,10 @@ suite("partition_mv_rewrite") { l_suppkey; """ - sql """DROP MATERIALIZED VIEW IF EXISTS mv_10086""" - sql """DROP TABLE IF EXISTS mv_10086""" + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_1_partition_name}""" + sql """DROP TABLE IF EXISTS ${mv_1_partition_name}""" sql""" - CREATE MATERIALIZED VIEW mv_10086 + CREATE MATERIALIZED VIEW ${mv_1_partition_name} BUILD IMMEDIATE REFRESH AUTO ON MANUAL partition by (date_trunc(`col1`, 'month')) DISTRIBUTED BY RANDOM BUCKETS 2 @@ -432,53 +452,43 @@ suite("partition_mv_rewrite") { AS ${roll_up_mv_def_sql} """ - waitingMTMVTaskFinished(getJobName(db, "mv_10086")) + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) - multi_sql """ - analyze table lineitem_static with sync; - analyze table lineitem with sync; - analyze table orders with sync; - """ + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) - mv_rewrite_success(roll_up_all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - mv_rewrite_success(roll_up_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) // base table add partition sql """ insert into lineitem values - (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-11-21', '2023-11-21', '2023-11-21', 'a', 'b', 'yyyyyyyyy'); + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-09-17', '2023-09-17', '2023-09-17', 'a', 'b', 'yyyyyyyyy'); """ - // enable union rewrite + sql "SET enable_materialized_view_rewrite=false" - order_qt_query_17_0_before "${roll_up_all_partition_sql}" + order_qt_query_17_0_before "${query_roll_up_all_partition_sql}" sql "SET enable_materialized_view_rewrite=true" - - multi_sql """ - analyze table lineitem_static with sync; - analyze table lineitem with sync; - analyze table orders with sync; - """ + // should rewrite successful when union rewrite enalbe if base table add new partition - mv_rewrite_success(roll_up_all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_17_0_after "${roll_up_all_partition_sql}" + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_17_0_after "${query_roll_up_all_partition_sql}" sql "SET enable_materialized_view_rewrite=false" - order_qt_query_18_0_before "${roll_up_partition_sql}" + order_qt_query_18_0_before "${query_roll_up_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successfully when union rewrite enable if doesn't query new partition - mv_rewrite_success(roll_up_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_18_0_after "${roll_up_partition_sql}" + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_18_0_after "${query_roll_up_partition_sql}" - // base table partition add data - sql "REFRESH MATERIALIZED VIEW mv_10086 AUTO" - waitingMTMVTaskFinished(getJobName(db, "mv_10086")) + // base table partition modify data + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) sql """ insert into lineitem values @@ -486,62 +496,45 @@ suite("partition_mv_rewrite") { (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-11-22', '2023-11-22', '2023-11-22', 'd', 'd', 'yyyyyyyyy'); """ - // enable union rewrite + sql "SET enable_materialized_view_rewrite=false" - order_qt_query_19_0_before "${roll_up_all_partition_sql}" + order_qt_query_19_0_before "${query_roll_up_all_partition_sql}" sql "SET enable_materialized_view_rewrite=true" - - multi_sql """ - analyze table lineitem_static with sync; - analyze table lineitem with sync; - analyze table orders with sync; - """ - - sql """alter table orders modify column o_comment set stats ('row_count'='3');""" - sql """alter table lineitem modify column l_comment set stats ('row_count'='6');""" - sql """alter table lineitem_static modify column l_comment set stats ('row_count'='4');""" // should rewrite successful when union rewrite enalbe if base table add new partition - mv_rewrite_success(roll_up_all_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) - order_qt_query_19_0_after "${roll_up_all_partition_sql}" + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_19_0_after "${query_roll_up_all_partition_sql}" sql "SET enable_materialized_view_rewrite=false" - order_qt_query_20_0_before "${roll_up_partition_sql}" + order_qt_query_20_0_before "${query_roll_up_partition_sql}" sql "SET enable_materialized_view_rewrite=true" // should rewrite successfully when union rewrite enable if doesn't query new partition - mv_rewrite_success(roll_up_partition_sql, "mv_10086", true, - is_partition_statistics_ready(db, ["lineitem", "orders", "mv_10086"])) + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) - order_qt_query_20_0_after "${roll_up_partition_sql}" + order_qt_query_20_0_after "${query_roll_up_partition_sql}" - // base table delete partition - sql "REFRESH MATERIALIZED VIEW mv_10086 AUTO" - waitingMTMVTaskFinished(getJobName(db, "mv_10086")) + // test base table delete partition + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) sql """ ALTER TABLE lineitem DROP PARTITION IF EXISTS p_20231121 FORCE; """ - // enable union rewrite -// this depends on getting corret partitions when base table delete partition, tmp comment -// sql "SET enable_materialized_view_rewrite=false" -// order_qt_query_21_0_before "${roll_up_all_partition_sql}" -// sql "SET enable_materialized_view_rewrite=true" -// explain { -// sql("${roll_up_all_partition_sql}") -// // should rewrite successful when union rewrite enalbe if base table add new partition -// contains("mv_10086(mv_10086)") -// } -// order_qt_query_21_0_after "${roll_up_all_partition_sql}" -// -// sql "SET enable_materialized_view_rewrite=false" -// order_qt_query_22_0_before "${roll_up_partition_sql}" -// sql "SET enable_materialized_view_rewrite=true" -// explain { -// sql("${roll_up_partition_sql}") -// // should rewrite successfully when union rewrite enable if doesn't query new partition -// contains("mv_10086(mv_10086)") -// } -// order_qt_query_22_0_after "${roll_up_partition_sql}" + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_21_0_before "${query_roll_up_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_21_0_after "${query_roll_up_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_22_0_before "${query_roll_up_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + order_qt_query_22_0_after "${query_roll_up_partition_sql}" } diff --git a/regression-test/suites/nereids_rules_p0/mv/rewrite_duration_exceeded/rewrite_duration_exceeded.groovy b/regression-test/suites/nereids_rules_p0/mv/rewrite_duration_exceeded/rewrite_duration_exceeded.groovy new file mode 100644 index 00000000000000..283e5336ec2a39 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/rewrite_duration_exceeded/rewrite_duration_exceeded.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("rewrite_duration_exceeded") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + sql "SET enable_agg_state = true" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NOT NULL, + o_custkey INTEGER NOT NULL, + o_orderstatus CHAR(1) NOT NULL, + o_totalprice DECIMALV3(15,2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, + o_shippriority INTEGER NOT NULL, + o_comment VARCHAR(79) NOT NULL, + public_col INT NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + drop table if exists lineitem + """ + sql """ + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_quantity DECIMALV3(15,2) NOT NULL, + l_extendedprice DECIMALV3(15,2) NOT NULL, + l_discount DECIMALV3(15,2) NOT NULL, + l_tax DECIMALV3(15,2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL, + public_col INT NULL + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + sql """ + drop table if exists partsupp + """ + sql """ + CREATE TABLE IF NOT EXISTS partsupp ( + ps_partkey INTEGER NOT NULL, + ps_suppkey INTEGER NOT NULL, + ps_availqty INTEGER NOT NULL, + ps_supplycost DECIMALV3(15,2) NOT NULL, + ps_comment VARCHAR(199) NOT NULL, + public_col INT NULL + ) + DUPLICATE KEY(ps_partkey, ps_suppkey) + DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-08', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', 1), + (2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', null), + (3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', 2), + (4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09', '2023-12-10', 'a', 'b', 'yyyyyyyyy', null), + (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12', '2023-12-13', 'c', 'd', 'xxxxxxxxx', 3); + """ + + sql """ + insert into orders values + (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy', 1), + (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy', null), + (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy', 2), + (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy', null), + (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy', 3), + (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm', null), + (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi', 4), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi', null); + """ + + sql """ + insert into partsupp values + (2, 3, 9, 10.01, 'supply1', 1), + (2, 3, 10, 11.01, 'supply2', null); + """ + create_async_mv(db, "mv_1", """ + select o_shippriority, o_comment, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end), + count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2) then o_custkey else null end) as cnt_2 + from orders + group by + o_shippriority, + o_comment; + """) + + sql """set materialized_view_rewrite_duration_threshold_ms = -1;""" + + // should materialized view rewrite duration is exceeded + explain { + sql(""" select o_shippriority, o_comment, + sum(o_totalprice) as sum_total, + max(o_totalprice) as max_total, + min(o_totalprice) as min_total, + count(*) as count_all, + count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end), + count(distinct case when O_SHIPPRIORITY > 2 and o_orderkey IN (2) then o_custkey else null end) as cnt_2 + from orders + group by + o_shippriority, + o_comment; + """) + check { result -> + contains("materialized view rewrite duration is exceeded") + } + } +} diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy new file mode 100644 index 00000000000000..55394292d0b75c --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy @@ -0,0 +1,549 @@ +package mv.partition_union_rewrite + +import java.text.SimpleDateFormat + +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("union_rewrite_grace_big") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey integer not null, + o_custkey integer not null, + o_orderstatus char(9) not null, + o_totalprice decimalv3(15,2) not null, + o_orderdate date not null, + o_orderpriority char(15) not null, + o_clerk char(15) not null, + o_shippriority integer not null, + o_comment varchar(79) not null + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate)( + FROM ('2023-09-16') TO ('2023-12-30') INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + drop table if exists lineitem + """ + + // test pre init partition + sql""" + CREATE TABLE IF NOT EXISTS lineitem ( + l_orderkey integer not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimalv3(15,2) not null, + l_extendedprice decimalv3(15,2) not null, + l_discount decimalv3(15,2) not null, + l_tax decimalv3(15,2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) + (FROM ('2023-09-16') TO ('2023-12-30') INTERVAL 1 DAY) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql""" + insert into orders values + (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'), + (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'), + (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'), + (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'), + (2, 2, 'ok', 109.2, '2023-10-18', 'c','d',2, 'mm'), + (2, 2, 'ok', 109.2, '2023-10-18', 'c','d',2, 'mm'), + (2, 2, 'ok', 109.2, '2023-10-18', 'c','d',2, 'mm'), + (2, 2, 'ok', 109.2, '2023-10-18', 'c','d',2, 'mm'), + (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'), + (3, 3, 'ok', 99.5, '2023-12-19', 'a', 'b', 1, 'yy'); + """ + + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'); + """ + + + def mv_def_sql = """ + select l_shipdate, o_orderdate, l_partkey, + l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + def query_all_partition_sql = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + + def query_partition_sql = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + where (l_shipdate>= '2023-10-18' and l_shipdate <= '2023-10-19') + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + + multi_sql """ + analyze table lineitem with sync; + analyze table orders with sync; + """ + + sql """alter table orders modify column o_comment set stats ('row_count'='20');""" + sql """alter table lineitem modify column l_comment set stats ('row_count'='20');""" + + + def mv_1_partition_name = "mv_10086" + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_1_partition_name}""" + sql """DROP TABLE IF EXISTS ${mv_1_partition_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_1_partition_name} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by(l_shipdate) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'grace_period' = '31536000') + AS + ${mv_def_sql} + """ + + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) + + multi_sql """ + analyze table ${mv_1_partition_name} with sync; + """ + + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + // test base table partition data change + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'); + """ + waitingPartitionIsExpected(mv_1_partition_name, "p_20231017_20231018", false) + + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_3_0_before "${query_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + + + // should rewrite successful when union rewrite enalbe if sub partition is invalid + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_3_0_after "${query_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_4_0_before "${query_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successfully when union rewrite enable if doesn't query invalid partition + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_4_0_after "${query_partition_sql}" + + // base table add partition + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-21', '2023-10-21', '2023-10-21', 'a', 'b', 'yyyyyyyyy'); + """ + + waitingPartitionIsExpected(mv_1_partition_name, "p_20231021_20231022", false) + + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_7_0_before "${query_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successful when union rewrite enalbe if base table add new partition + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_7_0_after "${query_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_8_0_before "${query_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successfully when union rewrite enable if doesn't query new partition + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_8_0_after "${query_partition_sql}" + + // base table delete partition test + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) + sql """ ALTER TABLE lineitem DROP PARTITION IF EXISTS p_20231017 FORCE; + """ + // show partitions will cause error, tmp comment + waitingPartitionIsExpected(mv_1_partition_name, "p_20231017_20231018", false) + + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_11_0_before "${query_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successful when union rewrite enalbe if base table delete partition + mv_rewrite_success(query_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_11_0_after "${query_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_12_0_before "${query_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successfully when union rewrite enable if doesn't query deleted partition + mv_rewrite_success(query_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_12_0_after "${query_partition_sql}" + sql """ DROP MATERIALIZED VIEW IF EXISTS ${mv_1_partition_name}""" + + // test mv with ttl + def today_str = new SimpleDateFormat("yyyy-MM-dd").format(new Date()).toString(); + + sql """ + drop table if exists lineitem_static; + """ + sql""" + CREATE TABLE IF NOT EXISTS lineitem_static ( + l_orderkey integer not null, + l_partkey integer not null, + l_suppkey integer not null, + l_linenumber integer not null, + l_quantity decimalv3(15,2) not null, + l_extendedprice decimalv3(15,2) not null, + l_discount decimalv3(15,2) not null, + l_tax decimalv3(15,2) not null, + l_returnflag char(1) not null, + l_linestatus char(1) not null, + l_shipdate date not null, + l_commitdate date not null, + l_receiptdate date not null, + l_shipinstruct char(25) not null, + l_shipmode char(10) not null, + l_comment varchar(44) not null + ) + DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber) + PARTITION BY RANGE(l_shipdate) + ( + PARTITION `p1` VALUES LESS THAN ("2023-10-18"), + PARTITION `p2` VALUES [("2023-10-18"), ("2023-12-20")), + PARTITION `other` VALUES LESS THAN (MAXVALUE) + ) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert into lineitem_static values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-19', '2023-12-19', '2023-12-19', 'c', 'd', 'xxxxxxxxx'); + """ + sql """ + insert into lineitem_static values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'); + """ + + multi_sql """ + analyze table lineitem_static with sync; + """ + sql """alter table lineitem_static modify column l_comment set stats ('row_count'='6');""" + + + def ttl_mv_def_sql = """ + select l_shipdate, o_orderdate, l_partkey, + l_suppkey, sum(o_totalprice) as sum_total + from lineitem_static + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query_ttl_all_partition_sql = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total + from lineitem_static + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def query_ttl_partition_sql = """ + select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total + from lineitem_static + left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + where (l_shipdate>= '2023-10-18' and l_shipdate <= '2023-10-19') + group by + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + def ttl_mv_name = "mv_10000" + + def create_ttl_mtmv = { db_name, mv_inner_name, mv_inner_sql -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_inner_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_inner_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + PARTITION BY(l_shipdate) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'grace_period' = '31536000', + 'replication_num' = '1', + 'partition_sync_limit' = 2, + 'partition_sync_time_unit' = 'DAY', + 'partition_date_format' = 'yyyy-MM-dd') + AS ${mv_inner_sql} + """ + waitingMTMVTaskFinished(getJobName(db_name, mv_inner_name)) + } + + create_ttl_mtmv(db, ttl_mv_name, ttl_mv_def_sql) + + + // test when mv is partition roll up + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successful when union rewrite enalbe and mv is ttl, query the partition which is in mv + mv_rewrite_success(query_ttl_all_partition_sql, ttl_mv_name, true, + is_partition_statistics_ready(db, ["lineitem_static", "orders", ttl_mv_name])) + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_16_0_before "${query_ttl_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite fail when union rewrite enalbe and query the partition which is not in mv + mv_rewrite_fail(query_ttl_partition_sql, ttl_mv_name) + order_qt_query_16_0_after "${query_ttl_partition_sql}" + + sql """ DROP MATERIALIZED VIEW IF EXISTS ${ttl_mv_name}""" + + + // test date roll up mv partition rewrite + def roll_up_mv_def_sql = """ + select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, o_orderdate, l_partkey, + l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + col1, + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + def query_roll_up_all_partition_sql = """ + select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, o_orderdate, l_partkey, + l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + group by + col1, + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + def query_roll_up_partition_sql = """ + select date_trunc(`l_shipdate`, 'day') as col1, l_shipdate, o_orderdate, l_partkey, + l_suppkey, sum(o_totalprice) as sum_total + from lineitem + left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate + where (l_shipdate>= '2023-10-18' and l_shipdate <= '2023-10-19') + group by + col1, + l_shipdate, + o_orderdate, + l_partkey, + l_suppkey; + """ + + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_1_partition_name}""" + sql """DROP TABLE IF EXISTS ${mv_1_partition_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_1_partition_name} + BUILD IMMEDIATE REFRESH AUTO ON MANUAL + partition by (date_trunc(`col1`, 'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'grace_period' = '31536000', + 'replication_num' = '1' + ) + AS + ${roll_up_mv_def_sql} + """ + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) + + + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + // base table add partition + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-09-17', '2023-09-17', '2023-09-17', 'a', 'b', 'yyyyyyyyy'); + """ + + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_17_0_before "${query_roll_up_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + + + // should rewrite successful when union rewrite enalbe if base table add new partition + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_17_0_after "${query_roll_up_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_18_0_before "${query_roll_up_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successfully when union rewrite enable if doesn't query new partition + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_18_0_after "${query_roll_up_partition_sql}" + + + // base table partition modify data + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) + + sql """ + insert into lineitem values + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-11-21', '2023-11-21', '2023-11-21', 'd', 'd', 'yyyyyyyyy'), + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-11-22', '2023-11-22', '2023-11-22', 'd', 'd', 'yyyyyyyyy'); + """ + + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_19_0_before "${query_roll_up_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + + + // should rewrite successful when union rewrite enalbe if base table add new partition + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_19_0_after "${query_roll_up_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_20_0_before "${query_roll_up_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + // should rewrite successfully when union rewrite enable if doesn't query new partition + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + order_qt_query_20_0_after "${query_roll_up_partition_sql}" + + + // test base table delete partition + sql "REFRESH MATERIALIZED VIEW ${mv_1_partition_name} AUTO" + waitingMTMVTaskFinished(getJobName(db, mv_1_partition_name)) + sql """ ALTER TABLE lineitem DROP PARTITION IF EXISTS p_20231121 FORCE; + """ + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_21_0_before "${query_roll_up_all_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + mv_rewrite_success(query_roll_up_all_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + order_qt_query_21_0_after "${query_roll_up_all_partition_sql}" + + sql "SET enable_materialized_view_rewrite=false" + order_qt_query_22_0_before "${query_roll_up_partition_sql}" + sql "SET enable_materialized_view_rewrite=true" + mv_rewrite_success(query_roll_up_partition_sql, mv_1_partition_name, true, + is_partition_statistics_ready(db, ["lineitem", "orders", mv_1_partition_name])) + + order_qt_query_22_0_after "${query_roll_up_partition_sql}" +} + + + From caa978ca0b97707a2d429bd1add235a6b7256e0f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 18 Jun 2025 19:38:53 +0800 Subject: [PATCH 016/572] branch-3.0: [Fix](multicatelog) Fix insert iceberg/hive table when use broker #51187 (#51243) Cherry-picked from #51187 Co-authored-by: Lijia Liu Co-authored-by: liutang123 --- .../iceberg/viceberg_partition_writer.cpp | 3 ++ .../iceberg/viceberg_partition_writer.h | 1 + .../writer/iceberg/viceberg_table_writer.cpp | 12 +++-- .../sink/writer/vhive_partition_writer.cpp | 3 ++ .../vec/sink/writer/vhive_partition_writer.h | 1 + be/src/vec/sink/writer/vhive_table_writer.cpp | 46 ++++++++++++++----- .../org/apache/doris/catalog/BrokerMgr.java | 15 ++++++ .../planner/BaseExternalTableDataSink.java | 21 +++++++++ .../apache/doris/planner/HiveTableSink.java | 3 ++ .../doris/planner/IcebergTableSink.java | 7 ++- gensrc/thrift/DataSinks.thrift | 2 + 11 files changed, 98 insertions(+), 16 deletions(-) diff --git a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp index 8963a129eeee3e..aeaa81d9995e8e 100644 --- a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp +++ b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.cpp @@ -52,6 +52,9 @@ Status VIcebergPartitionWriter::open(RuntimeState* state, RuntimeProfile* profil io::FSPropertiesRef fs_properties(_write_info.file_type); fs_properties.properties = &_hadoop_conf; + if (!_write_info.broker_addresses.empty()) { + fs_properties.broker_addresses = &(_write_info.broker_addresses); + } io::FileDescription file_description = { .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), .fs_name {}}; diff --git a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h index b66dea1d6d2934..28605f80426e8b 100644 --- a/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h +++ b/be/src/vec/sink/writer/iceberg/viceberg_partition_writer.h @@ -50,6 +50,7 @@ class VIcebergPartitionWriter { std::string original_write_path; std::string target_path; TFileType::type file_type; + std::vector broker_addresses; }; VIcebergPartitionWriter(const TDataSink& t_sink, std::vector partition_values, diff --git a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp index 280cf8b810724c..26e3ab2858fbee 100644 --- a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp +++ b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp @@ -366,9 +366,15 @@ std::shared_ptr VIcebergTableWriter::_create_partition_ write_path = output_path; } - VIcebergPartitionWriter::WriteInfo write_info = { - std::move(write_path), std::move(original_write_path), std::move(target_path), - iceberg_table_sink.file_type}; + VIcebergPartitionWriter::WriteInfo write_info = {std::move(write_path), + std::move(original_write_path), + std::move(target_path), + iceberg_table_sink.file_type, + {}}; + if (iceberg_table_sink.__isset.broker_addresses) { + write_info.broker_addresses.assign(iceberg_table_sink.broker_addresses.begin(), + iceberg_table_sink.broker_addresses.end()); + } _write_file_count++; std::vector column_names; diff --git a/be/src/vec/sink/writer/vhive_partition_writer.cpp b/be/src/vec/sink/writer/vhive_partition_writer.cpp index 8a5e2a9777ec1c..b93303dff0360c 100644 --- a/be/src/vec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/vec/sink/writer/vhive_partition_writer.cpp @@ -58,6 +58,9 @@ Status VHivePartitionWriter::open(RuntimeState* state, RuntimeProfile* profile) io::FSPropertiesRef fs_properties(_write_info.file_type); fs_properties.properties = &_hadoop_conf; + if (!_write_info.broker_addresses.empty()) { + fs_properties.broker_addresses = &(_write_info.broker_addresses); + } io::FileDescription file_description = { .path = fmt::format("{}/{}", _write_info.write_path, _get_target_file_name()), .fs_name {}}; diff --git a/be/src/vec/sink/writer/vhive_partition_writer.h b/be/src/vec/sink/writer/vhive_partition_writer.h index 71e1e141d9c4f6..20ca6506704a0d 100644 --- a/be/src/vec/sink/writer/vhive_partition_writer.h +++ b/be/src/vec/sink/writer/vhive_partition_writer.h @@ -46,6 +46,7 @@ class VHivePartitionWriter { std::string original_write_path; std::string target_path; TFileType::type file_type; + std::vector broker_addresses; }; VHivePartitionWriter(const TDataSink& t_sink, std::string partition_name, diff --git a/be/src/vec/sink/writer/vhive_table_writer.cpp b/be/src/vec/sink/writer/vhive_table_writer.cpp index 6eb478c01b7f92..9c2726fa65230a 100644 --- a/be/src/vec/sink/writer/vhive_table_writer.cpp +++ b/be/src/vec/sink/writer/vhive_table_writer.cpp @@ -301,30 +301,42 @@ std::shared_ptr VHiveTableWriter::_create_partition_writer if (existing_table == false) { // new table update_mode = TUpdateMode::NEW; if (_partition_columns_input_index.empty()) { // new unpartitioned table - write_info = {write_location.write_path, write_location.original_write_path, - write_location.target_path, write_location.file_type}; + write_info = {write_location.write_path, + write_location.original_write_path, + write_location.target_path, + write_location.file_type, + {}}; } else { // a new partition in a new partitioned table auto write_path = fmt::format("{}/{}", write_location.write_path, partition_name); auto original_write_path = fmt::format("{}/{}", write_location.original_write_path, partition_name); auto target_path = fmt::format("{}/{}", write_location.target_path, partition_name); - write_info = {std::move(write_path), std::move(original_write_path), - std::move(target_path), write_location.file_type}; + write_info = {std::move(write_path), + std::move(original_write_path), + std::move(target_path), + write_location.file_type, + {}}; } } else { // a new partition in an existing partitioned table, or an existing unpartitioned table if (_partition_columns_input_index.empty()) { // an existing unpartitioned table update_mode = !hive_table_sink.overwrite ? TUpdateMode::APPEND : TUpdateMode::OVERWRITE; - write_info = {write_location.write_path, write_location.original_write_path, - write_location.target_path, write_location.file_type}; + write_info = {write_location.write_path, + write_location.original_write_path, + write_location.target_path, + write_location.file_type, + {}}; } else { // a new partition in an existing partitioned table update_mode = TUpdateMode::NEW; auto write_path = fmt::format("{}/{}", write_location.write_path, partition_name); auto original_write_path = fmt::format("{}/{}", write_location.original_write_path, partition_name); auto target_path = fmt::format("{}/{}", write_location.target_path, partition_name); - write_info = {std::move(write_path), std::move(original_write_path), - std::move(target_path), write_location.file_type}; + write_info = {std::move(write_path), + std::move(original_write_path), + std::move(target_path), + write_location.file_type, + {}}; } // need to get schema from existing table ? } @@ -337,8 +349,11 @@ std::shared_ptr VHiveTableWriter::_create_partition_writer auto original_write_path = fmt::format("{}/{}", write_location.original_write_path, partition_name); auto target_path = fmt::format("{}", existing_partition->location.target_path); - write_info = {std::move(write_path), std::move(original_write_path), - std::move(target_path), existing_partition->location.file_type}; + write_info = {std::move(write_path), + std::move(original_write_path), + std::move(target_path), + existing_partition->location.file_type, + {}}; file_format_type = existing_partition->file_format; write_compress_type = hive_table_sink.compression_type; } else { @@ -347,13 +362,20 @@ std::shared_ptr VHiveTableWriter::_create_partition_writer auto original_write_path = fmt::format("{}/{}", write_location.original_write_path, partition_name); auto target_path = fmt::format("{}/{}", write_location.target_path, partition_name); - write_info = {std::move(write_path), std::move(original_write_path), - std::move(target_path), write_location.file_type}; + write_info = {std::move(write_path), + std::move(original_write_path), + std::move(target_path), + write_location.file_type, + {}}; file_format_type = hive_table_sink.file_format; write_compress_type = hive_table_sink.compression_type; // need to get schema from existing table ? } } + if (hive_table_sink.__isset.broker_addresses) { + write_info.broker_addresses.assign(hive_table_sink.broker_addresses.begin(), + hive_table_sink.broker_addresses.end()); + } _write_file_count++; std::vector column_names; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java index 5e4e7c7cbc84de..6b5a519eeda8a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BrokerMgr.java @@ -112,6 +112,21 @@ public boolean containsBroker(String brokerName) { } } + public List getBrokers(String brokerName) { + List result = null; + lock.lock(); + try { + List brokerList = brokerListMap.get(brokerName); + if (brokerList == null || brokerList.isEmpty()) { + return null; + } + result = new ArrayList<>(brokerList); + } finally { + lock.unlock(); + } + return result; + } + public FsBroker getAnyBroker(String brokerName) { lock.lock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java index 96b1c7e7d397dc..433162f93fe94c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java @@ -20,14 +20,20 @@ package org.apache.doris.planner; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FsBroker; import org.apache.doris.common.AnalysisException; import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TNetworkAddress; +import java.util.Collections; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; public abstract class BaseExternalTableDataSink extends DataSink { @@ -53,6 +59,21 @@ public DataPartition getOutputPartition() { */ protected abstract Set supportedFileFormatTypes(); + protected List getBrokerAddresses(String bindBroker) throws AnalysisException { + List brokers; + if (bindBroker != null) { + brokers = Env.getCurrentEnv().getBrokerMgr().getBrokers(bindBroker); + } else { + brokers = Env.getCurrentEnv().getBrokerMgr().getAllBrokers(); + } + if (brokers == null || brokers.isEmpty()) { + throw new AnalysisException("No alive broker."); + } + Collections.shuffle(brokers); + return brokers.stream().map(broker -> new TNetworkAddress(broker.host, broker.port)) + .collect(Collectors.toList()); + } + protected TFileFormatType getTFileFormatType(String format) throws AnalysisException { TFileFormatType fileFormatType = TFileFormatType.FORMAT_UNKNOWN; String lowerCase = format.toLowerCase(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java index a4012d357e55ad..bb4786f226a971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java @@ -150,6 +150,9 @@ public void bindDataSink(Optional insertCtx) } locationParams.setFileType(fileType); tSink.setLocation(locationParams); + if (fileType.equals(TFileType.FILE_BROKER)) { + tSink.setBrokerAddresses(getBrokerAddresses(targetTable.getCatalog().bindBrokerName())); + } tSink.setHadoopConfig(targetTable.getHadoopProperties()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java index bfacb572305966..5bc0c803cb9a4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java @@ -27,6 +27,7 @@ import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.TIcebergTableSink; import org.apache.doris.thrift.TSortField; @@ -134,7 +135,11 @@ public void bindDataSink(Optional insertCtx) LocationPath locationPath = new LocationPath(IcebergUtils.dataLocation(icebergTable), catalogProps); tSink.setOutputPath(locationPath.toStorageLocation().toString()); tSink.setOriginalOutputPath(locationPath.getPath().toString()); - tSink.setFileType(locationPath.getTFileTypeForBE()); + TFileType fileType = locationPath.getTFileTypeForBE(); + tSink.setFileType(fileType); + if (fileType.equals(TFileType.FILE_BROKER)) { + tSink.setBrokerAddresses(getBrokerAddresses(targetTable.getCatalog().bindBrokerName())); + } if (insertCtx.isPresent()) { BaseExternalTableInsertCommandContext context = (BaseExternalTableInsertCommandContext) insertCtx.get(); diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index ed7ccee69cd9a1..30348c091ee511 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -353,6 +353,7 @@ struct THiveTableSink { 9: optional map hadoop_config 10: optional bool overwrite 11: optional THiveSerDeProperties serde_properties + 12: optional list broker_addresses; } enum TUpdateMode { @@ -413,6 +414,7 @@ struct TIcebergTableSink { 11: optional Types.TFileType file_type 12: optional string original_output_path 13: optional PlanNodes.TFileCompressType compression_type + 14: optional list broker_addresses; } struct TDataSink { From 4677e36133701e70bc7964965c03d9a5edd09775 Mon Sep 17 00:00:00 2001 From: Xin Liao Date: Wed, 18 Jun 2025 21:20:02 +0800 Subject: [PATCH 017/572] branch-3.0: [chore](load) reduce log for data load #51172 (#51802) cherry pick from #51172 --- be/src/runtime/tablets_channel.cpp | 9 ++++----- be/src/vec/sink/writer/vtablet_writer.cpp | 4 ++-- .../cloud/transaction/CloudGlobalTransactionMgr.java | 10 +++++++--- .../java/org/apache/doris/httpv2/rest/LoadAction.java | 6 ++++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/be/src/runtime/tablets_channel.cpp b/be/src/runtime/tablets_channel.cpp index 0e3b2c33e8a427..3879b664ed8693 100644 --- a/be/src/runtime/tablets_channel.cpp +++ b/be/src/runtime/tablets_channel.cpp @@ -133,9 +133,6 @@ Status BaseTabletsChannel::open(const PTabletWriterOpenRequest& request) { if (_state == kOpened || _state == kFinished) { return Status::OK(); } - LOG(INFO) << fmt::format("open tablets channel {}, tablets num: {} timeout(s): {}", - _key.to_string(), request.tablets().size(), - request.load_channel_timeout_s()); _txn_id = request.txn_id(); _index_id = request.index_id(); _schema = std::make_shared(); @@ -163,8 +160,10 @@ Status BaseTabletsChannel::open(const PTabletWriterOpenRequest& request) { _num_remaining_senders = max_sender; } LOG(INFO) << fmt::format( - "txn {}: TabletsChannel of index {} init senders {} with incremental {}", _txn_id, - _index_id, _num_remaining_senders, _open_by_incremental ? "on" : "off"); + "open tablets channel {}, tablets num: {} timeout(s): {}, init senders {} with " + "incremental {}", + _key.to_string(), request.tablets().size(), request.load_channel_timeout_s(), + _num_remaining_senders, _open_by_incremental ? "on" : "off"); // just use max_sender no matter incremental or not cuz we dont know how many senders will open. _next_seqs.resize(max_sender, 0); _closed_senders.Reset(max_sender); diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp index b8bebeb9acf431..4a5057b7b2d6c7 100644 --- a/be/src/vec/sink/writer/vtablet_writer.cpp +++ b/be/src/vec/sink/writer/vtablet_writer.cpp @@ -126,8 +126,8 @@ Status IndexChannel::init(RuntimeState* state, const std::vector_sender_id - << ", incremantal:" << incremental; + VLOG_CRITICAL << "init new node for instance " << _parent->_sender_id + << ", incremantal:" << incremental; } else { channel = it->second; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 2bbb45ccd7bca9..fb9c7d293a709e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -1799,7 +1799,9 @@ public TransactionState getTransactionState(long dbId, long transactionId) { subTxnIdToTxnId.get(transactionId)); transactionId = subTxnIdToTxnId.get(transactionId); } - LOG.info("try to get transaction state, dbId:{}, transactionId:{}", dbId, transactionId); + if (LOG.isDebugEnabled()) { + LOG.debug("try to get transaction state, dbId:{}, transactionId:{}", dbId, transactionId); + } GetTxnRequest.Builder builder = GetTxnRequest.newBuilder(); builder.setDbId(dbId); builder.setTxnId(transactionId); @@ -1808,10 +1810,12 @@ public TransactionState getTransactionState(long dbId, long transactionId) { final GetTxnRequest getTxnRequest = builder.build(); GetTxnResponse getTxnResponse = null; try { - LOG.info("getTxnRequest:{}", getTxnRequest); + if (LOG.isDebugEnabled()) { + LOG.info("getTxnRequest:{}", getTxnRequest); + } getTxnResponse = MetaServiceProxy .getInstance().getTxn(getTxnRequest); - LOG.info("getTxnRequest: {}", getTxnResponse); + LOG.info("getTxnResponse: {}", getTxnResponse); } catch (RpcException e) { LOG.info("getTransactionState exception: {}", e.getMessage()); return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index 2f9efc1ed1b1bf..11c2b2811111b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -331,8 +331,10 @@ private Object executeWithoutPassword(HttpServletRequest request, redirectAddr = selectRedirectBackend(request, groupCommit, tableId); } - LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}", - redirectAddr.toString(), isStreamLoad, dbName, tableName, label); + if (LOG.isDebugEnabled()) { + LOG.info("redirect load action to destination={}, stream: {}, db: {}, tbl: {}, label: {}", + redirectAddr.toString(), isStreamLoad, dbName, tableName, label); + } RedirectView redirectView = redirectTo(request, redirectAddr); return redirectView; From 97a8cd4db6dff5d03b74a486819637e17b6c1797 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 15:32:44 +0800 Subject: [PATCH 018/572] branch-3.0: [fix](memory-leak) skip libzip and libjvm memory leak #51628 (#51631) Cherry-picked from #51628 Co-authored-by: shuke --- conf/lsan_suppr.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conf/lsan_suppr.conf b/conf/lsan_suppr.conf index fe294d58e841e1..da3fdbadceaacc 100644 --- a/conf/lsan_suppr.conf +++ b/conf/lsan_suppr.conf @@ -17,3 +17,6 @@ # This is a known leak. leak:brpc +leak:libjvm +leak:libzip +leak:*_dl_map_object_deps* \ No newline at end of file From 5970c82edc72e59b07704af9bb3640f902700f08 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 16:25:09 +0800 Subject: [PATCH 019/572] branch-3.0: [fix](filecache) fix load_cache_info_into_memory crash #51684 (#51904) Cherry-picked from #51684 Signed-off-by: zhengyu Co-authored-by: zhengyu --- be/src/io/cache/block_file_cache.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index a64d339d081be6..c12b03cc998792 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -496,7 +496,6 @@ class BlockFileCache { size_t _max_query_cache_size = 0; mutable std::mutex _mutex; - std::unique_ptr _storage; bool _close {false}; std::mutex _close_mtx; std::condition_variable _close_cv; @@ -575,6 +574,11 @@ class BlockFileCache { std::shared_ptr _evict_in_advance_latency_us; std::shared_ptr _recycle_keys_length_recorder; std::shared_ptr _ttl_gc_latency_us; + // keep _storage last so it will deconstruct first + // otherwise, load_cache_info_into_memory might crash + // coz it will use other members of BlockFileCache + // so join this async load thread first + std::unique_ptr _storage; }; } // namespace doris::io From 945f776ac1d6d5607c84eb195aa9f293968138f1 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Thu, 19 Jun 2025 16:35:33 +0800 Subject: [PATCH 020/572] [cherry-pick](branch-30) fix split_by_regexp meet empty string return error (#51293) (#51862) Problem Summary: cherry-pick from master #51293 --- .../functions/function_split_by_regexp.cpp | 3 --- .../string_functions/test_split_by_regexp.out | 26 +++++++++++++++++++ .../test_split_by_regexp.groovy | 2 ++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/be/src/vec/functions/function_split_by_regexp.cpp b/be/src/vec/functions/function_split_by_regexp.cpp index 40628ee2017fc7..f2c419ec220302 100644 --- a/be/src/vec/functions/function_split_by_regexp.cpp +++ b/be/src/vec/functions/function_split_by_regexp.cpp @@ -247,9 +247,6 @@ struct ExecuteImpl { if (pattern_ref.size) { re2_ptr = std::make_unique(pattern_ref.to_string_view(), *opts); } - if (!re2_ptr->ok()) { - return Status::RuntimeError("Invalid pattern: {}", pattern_ref.debug_string()); - } RegexpSplit RegexpSplit; RegexpSplit.init(re2_ptr.get(), limit_value); for (int row = 0; row < input_rows_count; ++row) { diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out index 1fb99f58ab1c7c..483d9d89f872e5 100644 --- a/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_split_by_regexp.out @@ -68,3 +68,29 @@ -- !select13 -- ["aa", "bbb", "cccc"] +-- !select14 -- +abcde ["a", "b", "c", "d", "e"] +12553 ["1", "2", "5", "5", "3"] + [] + [] + [] +a1b1c1d ["a", "1", "b", "1", "c", "1", "d"] +,,, [",", ",", ","] +a,b,c ["a", ",", "b", ",", "c"] +a,b,c, ["a", ",", "b", ",", "c", ","] +\N \N +a,b,c,12345, ["a", ",", "b", ",", "c", ",", "1", "2", "3", "4", "5", ","] + +-- !select15 -- + [] + [] + [] +, [","] +a ["a"] +1 ["1"] +, [","] +, [","] +, [","] +, [","] +, [","] + diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy index 4b9719068e6606..394a8e721aafab 100644 --- a/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_split_by_regexp.groovy @@ -70,5 +70,7 @@ suite("test_split_by_regexp") { qt_select11 "select split_by_regexp('aa,bbb,cccc', ',', 4);" qt_select12 "select split_by_regexp('aa,bbb,cccc', ',', 100000000);" qt_select13 "select split_by_regexp('aa,bbb,cccc', ',', 10000000000000);" + qt_select14 "select v1,split_by_regexp(v1, '') from test_split_by_regexp order by k1;" + qt_select15 "select v2,split_by_regexp(v2, '') from test_split_by_regexp order by k1;" } From 66681cfe5cf8244cd62a0b49f71314934cd0af3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 16:36:43 +0800 Subject: [PATCH 021/572] branch-3.0: [feat](cloud) Support alter operation for obj_info and s3 vault obj_info #51162 (#51685) Cherry-picked from #51162 Co-authored-by: Lei Zhang --- cloud/src/meta-service/meta_service_http.cpp | 8 +- .../meta-service/meta_service_resource.cpp | 131 ++++++-- cloud/test/meta_service_test.cpp | 301 ++++++++++++++++++ .../apache/doris/catalog/S3StorageVault.java | 6 +- .../apache/doris/catalog/StorageVault.java | 6 +- .../doris/cloud/rpc/MetaServiceClient.java | 4 + .../doris/cloud/rpc/MetaServiceProxy.java | 4 + gensrc/proto/cloud.proto | 1 + .../test_alter_s3_vault_with_role.groovy | 126 ++++++++ .../node_mgr/test_ms_alter_obj_info.groovy | 206 ++++++++++++ .../node_mgr/test_ms_alter_s3_vault.groovy | 237 ++++++++++++++ 11 files changed, 999 insertions(+), 31 deletions(-) create mode 100644 regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy create mode 100644 regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy create mode 100644 regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp index 53d9dbbef83c35..d12ff59ffe094c 100644 --- a/cloud/src/meta-service/meta_service_http.cpp +++ b/cloud/src/meta-service/meta_service_http.cpp @@ -229,7 +229,8 @@ static HttpResponse process_get_obj_store_info(MetaServiceImpl* service, brpc::C static HttpResponse process_alter_obj_store_info(MetaServiceImpl* service, brpc::Controller* ctrl) { static std::unordered_map operations { {"add_obj_info", AlterObjStoreInfoRequest::ADD_OBJ_INFO}, - {"legacy_update_ak_sk", AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK}}; + {"legacy_update_ak_sk", AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK}, + {"alter_obj_info", AlterObjStoreInfoRequest::ALTER_OBJ_INFO}}; auto& path = ctrl->http_request().unresolved_path(); auto it = operations.find(remove_version_prefix(path)); @@ -251,6 +252,7 @@ static HttpResponse process_alter_storage_vault(MetaServiceImpl* service, brpc:: static std::unordered_map operations { {"drop_s3_vault", AlterObjStoreInfoRequest::DROP_S3_VAULT}, {"add_s3_vault", AlterObjStoreInfoRequest::ADD_S3_VAULT}, + {"alter_s3_vault", AlterObjStoreInfoRequest::ALTER_S3_VAULT}, {"drop_hdfs_vault", AlterObjStoreInfoRequest::DROP_HDFS_INFO}, {"add_hdfs_vault", AlterObjStoreInfoRequest::ADD_HDFS_INFO}}; @@ -740,6 +742,10 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller, {"alter_s3_vault", process_alter_storage_vault}, {"drop_s3_vault", process_alter_storage_vault}, {"drop_hdfs_vault", process_alter_storage_vault}, + {"alter_obj_info", process_alter_obj_store_info}, + {"v1/alter_obj_info", process_alter_obj_store_info}, + {"v1/alter_s3_vault", process_alter_storage_vault}, + // for tools {"decode_key", process_decode_key}, {"encode_key", process_encode_key}, diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index eb88694ff5776f..bbd94b577b1bab 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -677,14 +677,6 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptrbegin(), @@ -723,6 +715,8 @@ static int alter_s3_storage_vault(InstanceInfoPB& instance, std::unique_ptrclear_role_arn(); + new_vault.mutable_obj_info()->clear_external_id(); + new_vault.mutable_obj_info()->clear_cred_provider_type(); + + new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first); + new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second); + new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info); + } + + if (obj_info.has_role_arn()) { + new_vault.mutable_obj_info()->clear_ak(); + new_vault.mutable_obj_info()->clear_sk(); + new_vault.mutable_obj_info()->clear_encryption_info(); + + new_vault.mutable_obj_info()->set_role_arn(obj_info.role_arn()); + new_vault.mutable_obj_info()->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE); + if (obj_info.has_external_id()) { + new_vault.mutable_obj_info()->set_external_id(obj_info.external_id()); + } } - new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first); - new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second); - new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info); if (obj_info.has_use_path_style()) { new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style()); } + auto now_time = std::chrono::system_clock::now(); + uint64_t time = + std::chrono::duration_cast(now_time.time_since_epoch()).count(); + new_vault.mutable_obj_info()->set_mtime(time); + auto new_vault_info = new_vault.DebugString(); val = new_vault.SerializeAsString(); if (val.empty()) { @@ -825,6 +853,7 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, if (!obj.has_ak() || !obj.has_sk()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "s3 obj info err " + proto_to_json(*request); + LOG(INFO) << msg; return -1; } @@ -839,13 +868,12 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, ak = cipher_ak_sk_pair.first; sk = cipher_ak_sk_pair.second; } else { - if (!obj.has_cred_provider_type() || - obj.cred_provider_type() != CredProviderTypePB::INSTANCE_PROFILE || - !obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) { + if (obj.has_ak() || obj.has_sk()) { code = MetaServiceCode::INVALID_ARGUMENT; - msg = "s3 conf info err with role_arn, please check it"; + msg = "invaild argument, both set ak/sk and role_arn is not allowed"; return -1; } + role_arn = obj.has_role_arn() ? obj.role_arn() : ""; external_id = obj.has_external_id() ? obj.external_id() : ""; } @@ -1044,6 +1072,16 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr return; } + if (!role_arn.empty()) { + if (!obj.has_cred_provider_type() || + obj.cred_provider_type() != CredProviderTypePB::INSTANCE_PROFILE || + !obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "s3 conf info err with role_arn, please check it"; + return; + } + } + auto& objs = instance.obj_info(); for (auto& it : objs) { if (bucket == it.bucket() && prefix == it.prefix() && endpoint == it.endpoint() && @@ -1210,6 +1248,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont switch (request->op()) { case AlterObjStoreInfoRequest::ADD_OBJ_INFO: case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: + case AlterObjStoreInfoRequest::ALTER_OBJ_INFO: case AlterObjStoreInfoRequest::UPDATE_AK_SK: { auto tmp_desc = ObjectStorageDesc {ak, sk, bucket, prefix, @@ -1287,7 +1326,8 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont } switch (request->op()) { - case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: { + case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: + case AlterObjStoreInfoRequest::ALTER_OBJ_INFO: { // get id std::string id = request->obj().has_id() ? request->obj().id() : "0"; int idx = std::stoi(id); @@ -1301,20 +1341,55 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont const_cast&>(instance.obj_info()); for (auto& it : obj_info) { if (std::stoi(it.id()) == idx) { - if (it.ak() == ak && it.sk() == sk) { - // not change, just return ok - code = MetaServiceCode::OK; - msg = ""; - return; + if (role_arn.empty()) { + if (it.ak() == ak && it.sk() == sk) { + // not change, just return ok + code = MetaServiceCode::OK; + msg = "ak/sk not changed"; + return; + } + it.clear_role_arn(); + it.clear_external_id(); + it.clear_cred_provider_type(); + + it.set_ak(ak); + it.set_sk(sk); + it.mutable_encryption_info()->CopyFrom(encryption_info); + } else { + if (!ak.empty() || !sk.empty()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "invaild argument, both set ak/sk and role_arn is not allowed"; + LOG(INFO) << msg; + return; + } + + if (it.provider() != ObjectStoreInfoPB::S3) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "role_arn is only supported for s3 provider"; + LOG(INFO) << msg << " provider=" << it.provider(); + return; + } + + if (it.role_arn() == role_arn && it.external_id() == external_id) { + // not change, just return ok + code = MetaServiceCode::OK; + msg = "ak/sk not changed"; + return; + } + it.clear_ak(); + it.clear_sk(); + it.clear_encryption_info(); + + it.set_role_arn(role_arn); + it.set_external_id(external_id); + it.set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE); } + auto now_time = std::chrono::system_clock::now(); uint64_t time = std::chrono::duration_cast( now_time.time_since_epoch()) .count(); it.set_mtime(time); - it.set_ak(ak); - it.set_sk(sk); - it.mutable_encryption_info()->CopyFrom(encryption_info); } } } break; diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index b5e609be3a8005..fbbfbff19fe802 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -9445,4 +9445,305 @@ TEST(MetaServiceTest, StaleCommitRowset) { ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().code(); } +TEST(MetaServiceTest, AlterObjInfoTest) { + auto meta_service = get_meta_service(); + + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + + ObjectStoreInfoPB obj_info; + obj_info.set_id("1"); + obj_info.set_ak("access_key_132131"); + obj_info.set_sk("secret_key_434124"); + obj_info.set_provider(ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + InstanceInfoPB instance; + instance.add_obj_info()->CopyFrom(obj_info); + val = instance.SerializeAsString(); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + auto get_test_instance = [&](InstanceInfoPB& i) { + std::string key; + std::string val; + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK); + i.ParseFromString(val); + }; + + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + // update failed + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO); + req.mutable_obj()->set_id("2"); + req.mutable_obj()->set_ak("new_ak"); + req.mutable_obj()->set_sk(plain_sk); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_obj_store_info( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + InstanceInfoPB instance; + get_test_instance(instance); + ASSERT_EQ(instance.obj_info(0).id(), "1"); + ASSERT_EQ(instance.obj_info(0).ak(), "access_key_132131"); + ASSERT_EQ(instance.obj_info(0).sk(), "secret_key_434124"); + } + + // update ak/sk successful + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO); + req.mutable_obj()->set_id("1"); + req.mutable_obj()->set_ak("new_access_key_132131"); + req.mutable_obj()->set_sk(plain_sk); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_obj_store_info( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + InstanceInfoPB instance; + get_test_instance(instance); + LOG(INFO) << "instance:" << instance.ShortDebugString(); + ASSERT_EQ(instance.obj_info(0).id(), "1"); + ASSERT_EQ(instance.obj_info(0).ak(), "new_access_key_132131"); + ASSERT_EQ(instance.obj_info(0).sk(), cipher_sk); + } + + // update from ak/sk to role_arn + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO); + req.mutable_obj()->set_id("1"); + req.mutable_obj()->set_role_arn("arn:aws:iam::1453123012:role/test-role"); + req.mutable_obj()->set_external_id("external_id_13123"); + req.mutable_obj()->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE); + req.mutable_obj()->set_provider(ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_obj_store_info( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + InstanceInfoPB instance; + get_test_instance(instance); + LOG(INFO) << "instance:" << instance.ShortDebugString(); + ASSERT_EQ(instance.obj_info(0).id(), "1"); + ASSERT_EQ(instance.obj_info(0).role_arn(), "arn:aws:iam::1453123012:role/test-role"); + ASSERT_EQ(instance.obj_info(0).external_id(), "external_id_13123"); + ASSERT_EQ(instance.obj_info(0).provider(), + ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + ASSERT_EQ(instance.obj_info(0).cred_provider_type(), CredProviderTypePB::INSTANCE_PROFILE); + ASSERT_TRUE(instance.obj_info(0).ak().empty()); + ASSERT_TRUE(instance.obj_info(0).sk().empty()); + ASSERT_FALSE(instance.obj_info(0).has_encryption_info()); + } + + // update from role_arn to ak/sk + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_OBJ_INFO); + req.mutable_obj()->set_id("1"); + req.mutable_obj()->set_ak("new_access_key_132131"); + req.mutable_obj()->set_sk(plain_sk); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_obj_store_info( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + InstanceInfoPB instance; + get_test_instance(instance); + LOG(INFO) << "instance:" << instance.ShortDebugString(); + ASSERT_EQ(instance.obj_info(0).id(), "1"); + ASSERT_EQ(instance.obj_info(0).ak(), "new_access_key_132131"); + ASSERT_EQ(instance.obj_info(0).sk(), cipher_sk); + ASSERT_EQ(instance.obj_info(0).provider(), + ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + ASSERT_FALSE(instance.obj_info(0).has_cred_provider_type()); + ASSERT_FALSE(instance.obj_info(0).has_role_arn()); + ASSERT_FALSE(instance.obj_info(0).has_external_id()); + } + + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} + +TEST(MetaServiceTest, AlterS3StorageVaultWithRoleArnTest) { + auto meta_service = get_meta_service(); + + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + std::pair pair; + sp->set_call_back("extract_object_storage_info:get_aksk_pair", [&](auto&& args) { + auto* ret = try_any_cast*>(args[0]); + pair = *ret; + }); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + + ObjectStoreInfoPB obj_info; + obj_info.set_id("1"); + obj_info.set_ak("123456ab"); + obj_info.set_sk("@ak$"); + obj_info.set_provider(ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + StorageVaultPB vault; + constexpr char vault_name[] = "test_alter_s3_vault_111"; + vault.mutable_obj_info()->MergeFrom(obj_info); + vault.set_name(vault_name); + vault.set_id("2"); + InstanceInfoPB instance; + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + val = instance.SerializeAsString(); + txn->put(key, val); + txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + txn = nullptr; + + auto get_test_instance = [&](InstanceInfoPB& i) { + std::string key; + std::string val; + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK); + i.ParseFromString(val); + }; + + // update from ak/sk to role_arn + { + AlterObjStoreInfoRequest req; + constexpr char new_vault_name[] = "new_test_alter_s3_vault_111"; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + StorageVaultPB vault; + vault.set_alter_name(new_vault_name); + ObjectStoreInfoPB obj; + obj.set_role_arn("arn:aws:iam::12311321:role/test-alter-role"); + obj.set_external_id("external_id_123123"); + vault.mutable_obj_info()->MergeFrom(obj); + vault.set_name(vault_name); + req.mutable_vault()->CopyFrom(vault); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + InstanceInfoPB instance; + get_test_instance(instance); + LOG(INFO) << "instance:" << instance.ShortDebugString(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}), &val), + TxnErrorCode::TXN_OK); + StorageVaultPB get_obj; + get_obj.ParseFromString(val); + ASSERT_EQ(get_obj.id(), "2"); + ASSERT_EQ(get_obj.obj_info().role_arn(), "arn:aws:iam::12311321:role/test-alter-role"); + ASSERT_EQ(get_obj.obj_info().external_id(), "external_id_123123"); + ASSERT_EQ(get_obj.obj_info().provider(), + ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + ASSERT_EQ(get_obj.obj_info().cred_provider_type(), CredProviderTypePB::INSTANCE_PROFILE); + ASSERT_TRUE(get_obj.obj_info().ak().empty()); + ASSERT_TRUE(get_obj.obj_info().sk().empty()); + ASSERT_FALSE(get_obj.obj_info().has_encryption_info()); + ASSERT_EQ(get_obj.name(), new_vault_name) << get_obj.obj_info().ShortDebugString(); + } + + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + // update from role_arn to ak_sk + { + AlterObjStoreInfoRequest req; + constexpr char new_vault_name[] = "new_test_alter_s3_vault_111"; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + StorageVaultPB vault; + ObjectStoreInfoPB obj; + obj.set_ak("123456ab"); + obj.set_sk(plain_sk); + vault.mutable_obj_info()->MergeFrom(obj); + vault.set_name(new_vault_name); + req.mutable_vault()->CopyFrom(vault); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + InstanceInfoPB instance; + get_test_instance(instance); + LOG(INFO) << "instance:" << instance.ShortDebugString(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), "2"}), &val), + TxnErrorCode::TXN_OK); + StorageVaultPB get_obj; + get_obj.ParseFromString(val); + ASSERT_EQ(get_obj.id(), "2"); + ASSERT_EQ(get_obj.obj_info().provider(), + ObjectStoreInfoPB::Provider::ObjectStoreInfoPB_Provider_S3); + ASSERT_EQ(get_obj.obj_info().ak(), "123456ab"); + ASSERT_EQ(get_obj.obj_info().sk(), cipher_sk); + ASSERT_TRUE(get_obj.obj_info().role_arn().empty()); + ASSERT_TRUE(get_obj.obj_info().external_id().empty()); + ASSERT_TRUE(get_obj.obj_info().has_encryption_info()); + ASSERT_FALSE(get_obj.obj_info().has_cred_provider_type()); + ASSERT_EQ(get_obj.name(), new_vault_name) << get_obj.obj_info().ShortDebugString(); + } + + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} } // namespace doris::cloud diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java index f8196c7ea80afb..58097d3c1bc8fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java @@ -70,6 +70,8 @@ public static class PropertyKey { public static final String REGION = S3Properties.REGION; public static final String ENDPOINT = S3Properties.ENDPOINT; public static final String BUCKET = S3Properties.BUCKET; + public static final String ROLE_ARN = S3Properties.ROLE_ARN; + public static final String EXTERNAL_ID = S3Properties.EXTERNAL_ID; } public static final HashSet ALLOW_ALTER_PROPERTIES = new HashSet<>(Arrays.asList( @@ -77,7 +79,9 @@ public static class PropertyKey { StorageVault.PropertyKey.TYPE, PropertyKey.ACCESS_KEY, PropertyKey.SECRET_KEY, - PropertyKey.USE_PATH_STYLE + PropertyKey.USE_PATH_STYLE, + PropertyKey.ROLE_ARN, + PropertyKey.EXTERNAL_ID )); @SerializedName(value = "properties") diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java index c9fe0c8bcc60c6..906b4585313987 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java @@ -222,7 +222,11 @@ public static List convertToShowStorageVaultProperties(Cloud.StorageVaul Cloud.ObjectStoreInfoPB.Builder builder = Cloud.ObjectStoreInfoPB.newBuilder(); builder.mergeFrom(vault.getObjInfo()); builder.clearId(); - builder.setSk("xxxxxxx"); + + if (vault.getObjInfo().hasAk()) { + builder.setSk("xxxxxxx"); + } + if (!vault.getObjInfo().hasUsePathStyle()) { // There is no `use_path_style` field in old version, think `use_path_style` false builder.setUsePathStyle(false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index d027777bf194e0..048d8ab93dfdb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -364,6 +364,10 @@ public Cloud.AlterClusterResponse alterCluster(Cloud.AlterClusterRequest request .alterCluster(request); } + /** + * This method is deprecated, there is no code to call it. + */ + @Deprecated public Cloud.AlterObjStoreInfoResponse alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) { if (!request.hasCloudUniqueId()) { Cloud.AlterObjStoreInfoRequest.Builder builder = Cloud.AlterObjStoreInfoRequest.newBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 1e5aa7ed111a48..95753821c35b05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -385,6 +385,10 @@ public Cloud.RemoveDeleteBitmapUpdateLockResponse removeDeleteBitmapUpdateLock( return w.executeRequest((client) -> client.removeDeleteBitmapUpdateLock(request)); } + /** + * This method is deprecated, there is no code to call it. + */ + @Deprecated public Cloud.AlterObjStoreInfoResponse alterObjStoreInfo(Cloud.AlterObjStoreInfoRequest request) throws RpcException { return w.executeRequest((client) -> client.alterObjStoreInfo(request)); diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 7b50747856c265..4e5c27f6a4382b 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -878,6 +878,7 @@ message AlterObjStoreInfoRequest { UPDATE_AK_SK = 1; ADD_OBJ_INFO = 2; LEGACY_UPDATE_AK_SK = 3; + ALTER_OBJ_INFO = 4; ADD_HDFS_INFO = 100; DROP_HDFS_INFO = 101; diff --git a/regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy b/regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy new file mode 100644 index 00000000000000..990afd876f5438 --- /dev/null +++ b/regression-test/suites/aws_iam_role_p0/test_alter_s3_vault_with_role.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.base.Strings; + +suite("test_alter_s3_vault_with_role") { + if (!isCloudMode()) { + logger.info("skip ${name} case, because not cloud mode") + return + } + + if (!enableStoragevault()) { + logger.info("skip ${name} case, because storage vault not enabled") + return + } + + def randomStr = UUID.randomUUID().toString().replace("-", "") + def s3VaultName = "s3_" + randomStr + + def endpoint = context.config.awsEndpoint + def region = context.config.awsRegion + def bucket = context.config.awsBucket + def roleArn = context.config.awsRoleArn + def externalId = context.config.awsExternalId + def prefix = context.config.awsPrefix + def awsAccessKey = context.config.awsAccessKey + def awsSecretKey = context.config.awsSecretKey + + sql """ + CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName} + PROPERTIES ( + "type"="S3", + "s3.endpoint"="${endpoint}", + "s3.region" = "${region}", + "s3.role_arn" = "${roleArn}", + "s3.external_id" = "${externalId}", + "s3.root.path" = "${prefix}/aws_iam_role_p0/${s3VaultName}", + "s3.bucket" = "${bucket}", + "s3.external_endpoint" = "", + "provider" = "S3", + "use_path_style" = "false" + ); + """ + + sql """ + CREATE TABLE ${s3VaultName} ( + C_CUSTKEY INTEGER NOT NULL, + C_NAME INTEGER NOT NULL + ) + DUPLICATE KEY(C_CUSTKEY, C_NAME) + DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_vault_name" = ${s3VaultName} + ) + """ + sql """ insert into ${s3VaultName} values(1, 1); """ + sql """ sync;""" + def result = sql """ select * from ${s3VaultName}; """ + assertEquals(result.size(), 1); + + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="S3", + "s3.access_key" = "${awsAccessKey}", + "s3.secret_key" = "${awsSecretKey}" + ); + """ + + def vaultInfos = sql """SHOW STORAGE VAULTS;""" + + for (int i = 0; i < vaultInfos.size(); i++) { + logger.info("vault info: ${vaultInfos[i]}") + if (vaultInfos[i][0].equals(s3VaultName)) { + def newProperties = vaultInfos[i][2] + logger.info("newProperties: ${newProperties}") + assertTrue(newProperties.contains(awsAccessKey)) + assertFalse(newProperties.contains("role_arn")) + } + } + + sql """ insert into ${s3VaultName} values(2, 2); """ + sql """ sync;""" + result = sql """ select * from ${s3VaultName}; """ + assertEquals(result.size(), 2); + + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="S3", + "s3.role_arn" = "${roleArn}", + "s3.external_id" = "${externalId}" + ); + """ + + vaultInfos = sql """SHOW STORAGE VAULTS;""" + for (int i = 0; i < vaultInfos.size(); i++) { + logger.info("vault info: ${vaultInfos[i]}") + if (vaultInfos[i][0].equals(s3VaultName)) { + def newProperties = vaultInfos[i][2] + logger.info("newProperties: ${newProperties}") + assertFalse(newProperties.contains(awsAccessKey)) + assertTrue(newProperties.contains(roleArn)) + } + } + + sql """ insert into ${s3VaultName} values(3, 3); """ + sql """ sync;""" + result = sql """ select * from ${s3VaultName}; """ + assertEquals(result.size(), 3); +} \ No newline at end of file diff --git a/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy new file mode 100644 index 00000000000000..b864db1838e171 --- /dev/null +++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import groovy.json.JsonOutput +import org.apache.doris.regression.suite.ClusterOptions + +suite('test_ms_alter_obj_info', 'p0, docker') { + if (!isCloudMode()) { + return; + } + + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + 'heartbeat_interval_second=1' + ] + options.setFeNum(1) + options.setBeNum(1) + options.cloudMode = true + + def create_instance_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/create_instance?token=$token" + body request_body + check check_func + } + } + + + def get_instance_api = { msHttpPort, instance_id, check_func -> + httpTest { + op "get" + endpoint msHttpPort + uri "/MetaService/http/get_instance?token=${token}&instance_id=${instance_id}" + check check_func + } + } + + def alter_obj_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_obj_info?token=$token" + body request_body + check check_func + } + } + + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + // Inventory function test + def token = "greedisgood9999" + def instance_id = "instance_id_test_in_docker" + def name = "user_1" + def user_id = "10000" + + def cloudUniqueId = "1:${instance_id}:xxxxx" + // create instance + /* + curl -X GET '127.0.0.1:5000/MetaService/http/create_instance?token=greedisgood9999' -d '{ + "instance_id": "instance_id_deadbeef", + "name": "user_1", + "user_id": "10000", + "obj_info": { + "ak": "test-ak1", + "sk": "test-sk1", + "bucket": "test-bucket", + "prefix": "test-prefix", + "endpoint": "test-endpoint", + "region": "test-region", + "provider" : S3", + "external_endpoint" : "endpoint" + } + }' + */ + def jsonOutput = new JsonOutput() + def s3 = [ + ak: "test-ak1", + sk : "test-sk1", + bucket : "test-bucket", + prefix: "test-prefix", + endpoint: "test-endpoint", + region: "test-region", + provider : "S3", + external_endpoint: "test-external-endpoint" + ] + def map = [instance_id: "${instance_id}", name: "${name}", user_id: "${user_id}", obj_info: s3] + def instance_body = jsonOutput.toJson(map) + + create_instance_api.call(msHttpPort, instance_body) { + respCode, body -> + log.info("http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + def json + get_instance_api.call(msHttpPort, instance_id) { + respCode, body -> + log.info("get instance resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // alter s3 info to instance + /* + curl '127.0.0.1:5000/MetaService/http/add_obj_info?token=greedisgood9999' -d '{ + "cloud_unique_id": "cloud_unique_id_compute_node0", + "obj": { + "id": `1`, + "ak": "test-ak2", + "sk": "test-sk2" + } + }' + */ + + def alter_obj_info_api_body = [cloud_unique_id:"${cloudUniqueId}", + obj:[id:"1", ak:"new-ak2", sk:"new-sk2"]] + jsonOutput = new JsonOutput() + def alterObjInfoBody = jsonOutput.toJson(alter_obj_info_api_body) + logger.info("alter obj info body: ${alterObjInfoBody}") + + alter_obj_info_api.call(msHttpPort, alterObjInfoBody) { + respCode, body -> + log.info("http cli result: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + get_instance_api.call(msHttpPort, instance_id) { + respCode, body -> + log.info("get instance resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + assertTrue(json.result.obj_info[0]["ak"].equalsIgnoreCase("new-ak2")) + assertTrue(json.result.obj_info[0]["sk"].equalsIgnoreCase("new-sk2")) + + + alter_obj_info_api_body = [cloud_unique_id:"${cloudUniqueId}", + obj:[id:"1", role_arn:"new-role-arn", external_id:"new-external-id"]] + jsonOutput = new JsonOutput() + alterObjInfoBody = jsonOutput.toJson(alter_obj_info_api_body) + logger.info("alter obj info body: ${alterObjInfoBody}") + + alter_obj_info_api.call(msHttpPort, alterObjInfoBody) { + respCode, body -> + log.info("http cli result: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + get_instance_api.call(msHttpPort, instance_id) { + respCode, body -> + log.info("get instance resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + assertTrue(json.result.obj_info[0]["role_arn"].equalsIgnoreCase("new-role-arn")) + assertTrue(json.result.obj_info[0]["external_id"].equalsIgnoreCase("new-external-id")) + assertTrue(json.result.obj_info[0]["cred_provider_type"].equalsIgnoreCase("INSTANCE_PROFILE")) + + alter_obj_info_api_body = [cloud_unique_id:"${cloudUniqueId}", + obj:[id:"1", ak:"new-ak3", sk:"new-sk3"]] + jsonOutput = new JsonOutput() + alterObjInfoBody = jsonOutput.toJson(alter_obj_info_api_body) + logger.info("alter obj info body: ${alterObjInfoBody}") + + alter_obj_info_api.call(msHttpPort, alterObjInfoBody) { + respCode, body -> + log.info("http cli result: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + get_instance_api.call(msHttpPort, instance_id) { + respCode, body -> + log.info("get instance resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + assertTrue(json.result.obj_info[0]["ak"].equalsIgnoreCase("new-ak3")) + assertTrue(json.result.obj_info[0]["sk"].equalsIgnoreCase("new-sk3")) + } + +} \ No newline at end of file diff --git a/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy new file mode 100644 index 00000000000000..c837ecf90c69b0 --- /dev/null +++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import groovy.json.JsonOutput +import org.apache.doris.regression.suite.ClusterOptions + +suite('test_ms_alter_s3_vault', 'p0, docker') { + if (!isCloudMode()) { + return; + } + + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + 'heartbeat_interval_second=1' + ] + options.setFeNum(1) + options.setBeNum(1) + options.cloudMode = true + + def create_instance_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/create_instance?token=$token" + body request_body + check check_func + } + } + + + def get_instance_api = { msHttpPort, instance_id, check_func -> + httpTest { + op "get" + endpoint msHttpPort + uri "/MetaService/http/get_instance?token=${token}&instance_id=${instance_id}" + check check_func + } + } + + def show_storage_vaults_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/show_storage_vaults?token=${token}" + body request_body + check check_func + } + } + + def alter_s3_vault_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_s3_vault?token=$token" + body request_body + check check_func + } + } + + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + // Inventory function test + def token = "greedisgood9999" + def instance_id = "instance_id_test_in_docker" + def name = "user_1" + def user_id = "10000" + + def cloudUniqueId = "1:${instance_id}:xxxxx" + // create instance + /* + curl -X GET '127.0.0.1:5000/MetaService/http/create_instance?token=greedisgood9999' -d '{ + "instance_id": "instance_id_deadbeef", + "name": "user_1", + "user_id": "10000", + "vault": { + "obj_info": { + "ak": "test-ak1", + "sk": "test-sk1", + "bucket": "test-bucket", + "prefix": "test-prefix", + "endpoint": "test-endpoint", + "region": "test-region", + "provider" : S3", + "external_endpoint" : "endpoint" + } + } + }' + */ + def jsonOutput = new JsonOutput() + def s3 = [ + ak: "test-ak1", + sk : "test-sk1", + bucket : "test-bucket", + prefix: "test-prefix", + endpoint: "test-endpoint", + region: "test-region", + provider : "S3", + external_endpoint: "test-external-endpoint" + ] + def map = [instance_id: "${instance_id}", name: "${name}", user_id: "${user_id}", vault: [obj_info: s3]] + def instance_body = jsonOutput.toJson(map) + logger.info("instance_body: ${instance_body}") + + create_instance_api.call(msHttpPort, instance_body) { + respCode, body -> + log.info("http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + def json + get_instance_api.call(msHttpPort, instance_id) { + respCode, body -> + log.info("get instance resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show stoarge vaults + /* + + curl http://127.0.0.1:5000/MetaService/http/show_storage_vaults?token=greedisgood9999 -d '{ + "cloud_unique_id":"cloud_unique_id_compute_node0" + }' + */ + + def show_storage_vaults_api_body = [cloud_unique_id:"${cloudUniqueId}"] + show_storage_vaults_api.call(msHttpPort, jsonOutput.toJson(show_storage_vaults_api_body)) { + respCode, body -> + log.info("show_storage_vaults_api resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + def mtime = json.result.storage_vault[0]["obj_info"]["mtime"] + def ctime = json.result.storage_vault[0]["obj_info"]["ctime"] + + // alter s3 vaults + /* + curl http://127.0.0.1:5000/MetaService/http/v1/alter_s3_vault?token=greedisgood9999 -d '{ + "cloud_unique_id":"cloud_unique_id_compute_node0", + "vault": { + "name": "built_in_storage_vault", + "obj_info": { + "ak": "test-ak2", + "sk": "test-sk2" + } + } + }' + + curl http://127.0.0.1:5000/MetaService/http/v1/alter_s3_vault?token=greedisgood9999 -d '{ + "cloud_unique_id":"cloud_unique_id_compute_node0", + "vault": { + "name": "built_in_storage_vault", + "obj_info": { + "role_arn": "test-role-arn", + "external_id": "test-external-id" + } + } + }' + */ + sleep(2000) + def alter_s3_vault_body = [cloud_unique_id:"${cloudUniqueId}", + vault:[ + name:"built_in_storage_vault", + obj_info:[ak:"test-ak2", sk:"test-sk2"] + ]] + + alter_s3_vault_api.call(msHttpPort, jsonOutput.toJson(alter_s3_vault_body)) { + respCode, body -> + log.info("alter_s3_vault_api resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + show_storage_vaults_api.call(msHttpPort, jsonOutput.toJson(show_storage_vaults_api_body)) { + respCode, body -> + log.info("show_storage_vaults_api resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + assertTrue(json.result.storage_vault[0]["obj_info"]["ak"].equalsIgnoreCase("test-ak2")) + assertTrue(json.result.storage_vault[0]["obj_info"]["sk"].equalsIgnoreCase("test-sk2")) + + def mtime2 = json.result.storage_vault[0]["obj_info"]["mtime"] + def ctime2 = json.result.storage_vault[0]["obj_info"]["ctime"] + assertTrue(mtime2 > mtime) + assertTrue(ctime2 == ctime) + + + sleep(2000) + alter_s3_vault_body = [cloud_unique_id:"${cloudUniqueId}", + vault:[ + name:"built_in_storage_vault", + obj_info:[role_arn:"test-role-arn", external_id:"test-external-id"] + ]] + + alter_s3_vault_api.call(msHttpPort, jsonOutput.toJson(alter_s3_vault_body)) { + respCode, body -> + log.info("alter_s3_vault_api resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + show_storage_vaults_api.call(msHttpPort, jsonOutput.toJson(show_storage_vaults_api_body)) { + respCode, body -> + log.info("show_storage_vaults_api resp: ${body} ${respCode}".toString()) + json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + assertTrue(json.result.storage_vault[0]["obj_info"]["role_arn"].equalsIgnoreCase("test-role-arn")) + assertTrue(json.result.storage_vault[0]["obj_info"]["external_id"].equalsIgnoreCase("test-external-id")) + assertTrue(json.result.storage_vault[0]["obj_info"]["cred_provider_type"].equalsIgnoreCase("INSTANCE_PROFILE")) + + def mtime3 = json.result.storage_vault[0]["obj_info"]["mtime"] + def ctime3 = json.result.storage_vault[0]["obj_info"]["ctime"] + assertTrue(mtime3 > mtime) + assertTrue(ctime3 == ctime) + } +} \ No newline at end of file From 69912ecaafde0a020d85fe67f51e2a109a6f9ae3 Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 19 Jun 2025 16:44:02 +0800 Subject: [PATCH 022/572] branch-3.0: [fix](mow) remove delete bitmap when remove unused rowsets (#50973) (#51871) pick https://github.com/apache/doris/pull/50973 --- be/src/cloud/cloud_cumulative_compaction.cpp | 4 + be/src/cloud/cloud_delete_bitmap_action.cpp | 182 ------------- be/src/cloud/cloud_tablet.cpp | 10 +- be/src/http/action/delete_bitmap_action.cpp | 213 +++++++++++++++ .../action/delete_bitmap_action.h} | 13 +- be/src/olap/compaction.cpp | 2 + be/src/olap/storage_engine.cpp | 35 ++- be/src/olap/tablet.cpp | 2 + be/src/olap/tablet_meta.cpp | 26 +- be/src/olap/tablet_meta.h | 1 + be/src/service/http_service.cpp | 21 +- be/src/vec/exec/scan/new_olap_scanner.cpp | 2 + ...est_mow_delete_unused_rowset_dm_docker.out | 22 ++ .../test_schema_change_add_key_column.csv.gz | Bin 0 -> 72233 bytes .../test_schema_change_add_key_column1.csv.gz | Bin 0 -> 177558 bytes .../test_mow_compact_multi_segments.groovy | 253 ++++++++++++++++++ ..._mow_delete_unused_rowset_dm_docker.groovy | 231 ++++++++++++++++ ...test_mow_stale_rowset_delete_bitmap.groovy | 249 +++++++++++++++++ 18 files changed, 1043 insertions(+), 223 deletions(-) delete mode 100644 be/src/cloud/cloud_delete_bitmap_action.cpp create mode 100644 be/src/http/action/delete_bitmap_action.cpp rename be/src/{cloud/cloud_delete_bitmap_action.h => http/action/delete_bitmap_action.h} (79%) create mode 100644 regression-test/data/compaction/test_mow_delete_unused_rowset_dm_docker.out create mode 100644 regression-test/data/compaction/test_schema_change_add_key_column.csv.gz create mode 100644 regression-test/data/compaction/test_schema_change_add_key_column1.csv.gz create mode 100644 regression-test/suites/compaction/test_mow_compact_multi_segments.groovy create mode 100644 regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy create mode 100644 regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index 097bdc9bf4ab1b..8546ac04fcf92e 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -447,6 +447,10 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { { static_cast(_tablet.get())->delete_expired_stale_rowsets(); }); } } + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset", { + LOG(INFO) << "delete_expired_stale_rowsets for tablet=" << _tablet->tablet_id(); + _engine.tablet_mgr().vacuum_stale_rowsets(CountDownLatch(1)); + }); return Status::OK(); } diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp b/be/src/cloud/cloud_delete_bitmap_action.cpp deleted file mode 100644 index 3d834bfe7b373c..00000000000000 --- a/be/src/cloud/cloud_delete_bitmap_action.cpp +++ /dev/null @@ -1,182 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "cloud_delete_bitmap_action.h" - -#include -#include -#include -#include -#include - -#include // IWYU pragma: keep -#include -#include -#include -#include -#include -#include -#include -#include - -#include "cloud/cloud_meta_mgr.h" -#include "cloud/cloud_tablet.h" -#include "cloud/cloud_tablet_mgr.h" -#include "common/logging.h" -#include "common/status.h" -#include "gutil/strings/substitute.h" -#include "http/http_channel.h" -#include "http/http_headers.h" -#include "http/http_request.h" -#include "http/http_status.h" -#include "olap/olap_define.h" -#include "olap/storage_engine.h" -#include "olap/tablet_manager.h" -#include "util/doris_metrics.h" -#include "util/stopwatch.hpp" - -namespace doris { -#include "common/compile_check_begin.h" -using namespace ErrorCode; - -namespace { - -constexpr std::string_view HEADER_JSON = "application/json"; - -} // namespace - -CloudDeleteBitmapAction::CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, - CloudStorageEngine& engine, - TPrivilegeHier::type hier, - TPrivilegeType::type ptype) - : HttpHandlerWithAuth(exec_env, hier, ptype), - _engine(engine), - _delete_bitmap_action_type(ctype) {} - -static Status _check_param(HttpRequest* req, uint64_t* tablet_id) { - const auto& req_tablet_id = req->param(TABLET_ID_KEY); - if (req_tablet_id.empty()) { - return Status::InternalError("tablet id is empty!"); - } - try { - *tablet_id = std::stoull(req_tablet_id); - } catch (const std::exception& e) { - return Status::InternalError("convert tablet_id failed, {}", e.what()); - } - return Status::OK(); -} - -Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, - std::string* json_result) { - uint64_t tablet_id = 0; - // check & retrieve tablet_id from req if it contains - RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed"); - if (tablet_id == 0) { - return Status::InternalError("check param failed: missing tablet_id"); - } - - CloudTabletSPtr tablet = DORIS_TRY(_engine.tablet_mgr().get_tablet(tablet_id)); - if (tablet == nullptr) { - return Status::NotFound("Tablet not found. tablet_id={}", tablet_id); - } - - auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); - auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality(); - auto size = tablet->tablet_meta()->delete_bitmap().get_size(); - LOG(INFO) << "show_local_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count - << ",cardinality=" << cardinality << ",size=" << size; - - rapidjson::Document root; - root.SetObject(); - root.AddMember("delete_bitmap_count", count, root.GetAllocator()); - root.AddMember("cardinality", cardinality, root.GetAllocator()); - root.AddMember("size", size, root.GetAllocator()); - - // to json string - rapidjson::StringBuffer strbuf; - rapidjson::PrettyWriter writer(strbuf); - root.Accept(writer); - *json_result = std::string(strbuf.GetString()); - - return Status::OK(); -} - -Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, - std::string* json_result) { - uint64_t tablet_id = 0; - // check & retrieve tablet_id from req if it contains - RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed"); - if (tablet_id == 0) { - return Status::InternalError("check param failed: missing tablet_id"); - } - TabletMetaSharedPtr tablet_meta; - auto st = _engine.meta_mgr().get_tablet_meta(tablet_id, &tablet_meta); - if (!st.ok()) { - LOG(WARNING) << "failed to get_tablet_meta tablet=" << tablet_id - << ", st=" << st.to_string(); - return st; - } - auto tablet = std::make_shared(_engine, std::move(tablet_meta)); - st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true); - if (!st.ok()) { - LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st; - return st; - } - auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); - auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality(); - auto size = tablet->tablet_meta()->delete_bitmap().get_size(); - LOG(INFO) << "show_ms_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count - << ",cardinality=" << cardinality << ",size=" << size; - - rapidjson::Document root; - root.SetObject(); - root.AddMember("delete_bitmap_count", count, root.GetAllocator()); - root.AddMember("cardinality", cardinality, root.GetAllocator()); - root.AddMember("size", size, root.GetAllocator()); - - // to json string - rapidjson::StringBuffer strbuf; - rapidjson::PrettyWriter writer(strbuf); - root.Accept(writer); - *json_result = std::string(strbuf.GetString()); - - return Status::OK(); -} - -void CloudDeleteBitmapAction::handle(HttpRequest* req) { - req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data()); - if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) { - std::string json_result; - Status st = _handle_show_local_delete_bitmap_count(req, &json_result); - if (!st.ok()) { - HttpChannel::send_reply(req, HttpStatus::OK, st.to_json()); - } else { - HttpChannel::send_reply(req, HttpStatus::OK, json_result); - } - } else if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_MS) { - std::string json_result; - Status st = _handle_show_ms_delete_bitmap_count(req, &json_result); - if (!st.ok()) { - HttpChannel::send_reply(req, HttpStatus::OK, st.to_json()); - } else { - HttpChannel::send_reply(req, HttpStatus::OK, json_result); - } - } -} - -#include "common/compile_check_end.h" -} // namespace doris \ No newline at end of file diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index c044b8361b7af3..b4cf9d2e31c153 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -493,18 +493,16 @@ void CloudTablet::remove_unused_rowsets() { ++it; continue; } + tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); rs->clear_cache(); it = _unused_rowsets.erase(it); g_unused_rowsets_count << -1; removed_rowsets_num++; } - if (removed_rowsets_num > 0) { - LOG(INFO) << "tablet_id=" << tablet_id() - << ", unused_rowset size=" << _unused_rowsets.size() - << ", removed_rowsets_num=" << removed_rowsets_num - << ", cost(us)=" << watch.get_elapse_time_us(); - } + LOG(INFO) << "tablet_id=" << tablet_id() << ", unused_rowset size=" << _unused_rowsets.size() + << ", removed_rowsets_num=" << removed_rowsets_num + << ", cost(us)=" << watch.get_elapse_time_us(); } void CloudTablet::update_base_size(const Rowset& rs) { diff --git a/be/src/http/action/delete_bitmap_action.cpp b/be/src/http/action/delete_bitmap_action.cpp new file mode 100644 index 00000000000000..59783d1c055535 --- /dev/null +++ b/be/src/http/action/delete_bitmap_action.cpp @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "delete_bitmap_action.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "cloud/cloud_meta_mgr.h" +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" +#include "http/http_channel.h" +#include "http/http_headers.h" +#include "http/http_request.h" +#include "http/http_status.h" +#include "olap/olap_define.h" +#include "olap/tablet_manager.h" + +namespace doris { +#include "common/compile_check_begin.h" +using namespace ErrorCode; + +namespace { + +constexpr std::string_view HEADER_JSON = "application/json"; + +} // namespace + +DeleteBitmapAction::DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, + BaseStorageEngine& engine, TPrivilegeHier::type hier, + TPrivilegeType::type ptype) + : HttpHandlerWithAuth(exec_env, hier, ptype), + _engine(engine), + _delete_bitmap_action_type(ctype) {} + +static Status _check_param(HttpRequest* req, uint64_t* tablet_id, bool* verbose) { + const auto& req_tablet_id = req->param(TABLET_ID_KEY); + if (req_tablet_id.empty()) { + return Status::InternalError("tablet id is empty!"); + } + try { + *tablet_id = std::stoull(req_tablet_id); + } catch (const std::exception& e) { + return Status::InternalError("convert tablet_id failed, {}", e.what()); + } + if (*tablet_id == 0) { + return Status::InternalError("check param failed: invalid tablet_id"); + } + *verbose = iequal(req->param("verbose"), "true"); + return Status::OK(); +} + +static void _show_delete_bitmap(DeleteBitmap& dm, bool verbose, std::string* json_result) { + auto count = dm.get_delete_bitmap_count(); + auto cardinality = dm.cardinality(); + auto size = dm.get_size(); + rapidjson::Document root; + root.SetObject(); + root.AddMember("delete_bitmap_count", count, root.GetAllocator()); + root.AddMember("cardinality", cardinality, root.GetAllocator()); + root.AddMember("size", size, root.GetAllocator()); + if (verbose) { + std::string pre_rowset_id = ""; + int64_t pre_segment_id = -1; + std::vector version_vector; + rapidjson::Document dm_arr; + dm_arr.SetObject(); + + auto add_rowset_delete_bitmap_info = [&]() { + std::string key = + "rowset: " + pre_rowset_id + ", segment: " + std::to_string(pre_segment_id); + rapidjson::Value key_value; + key_value.SetString(key.data(), static_cast(key.length()), + root.GetAllocator()); + rapidjson::Document version_arr; + version_arr.SetArray(); + for (const auto& str : version_vector) { + rapidjson::Value value; + value.SetString(str.c_str(), static_cast(str.length()), + root.GetAllocator()); + version_arr.PushBack(value, root.GetAllocator()); + } + dm_arr.AddMember(key_value, version_arr, root.GetAllocator()); + version_vector.clear(); + }; + + for (auto& [id, bitmap] : dm.delete_bitmap) { + auto& [rowset_id, segment_id, version] = id; + if (rowset_id.to_string() != pre_rowset_id || segment_id != pre_segment_id) { + // add previous result + if (!pre_rowset_id.empty()) { + add_rowset_delete_bitmap_info(); + } + pre_rowset_id = rowset_id.to_string(); + pre_segment_id = segment_id; + } + std::string str = fmt::format("v: {}, c: {}, s: {}", version, bitmap.cardinality(), + bitmap.getSizeInBytes()); + version_vector.push_back(str); + } + // add last result + if (!version_vector.empty()) { + add_rowset_delete_bitmap_info(); + } + root.AddMember("delete_bitmap", dm_arr, root.GetAllocator()); + } + + // to json string + rapidjson::StringBuffer strbuf; + rapidjson::PrettyWriter writer(strbuf); + root.Accept(writer); + *json_result = std::string(strbuf.GetString()); +} + +Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req, + std::string* json_result) { + uint64_t tablet_id = 0; + bool verbose = false; + RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id, &verbose), "check param failed"); + + BaseTabletSPtr tablet = nullptr; + if (config::is_cloud_mode()) { + tablet = DORIS_TRY(_engine.to_cloud().tablet_mgr().get_tablet(tablet_id)); + DBUG_EXECUTE_IF( + "DeleteBitmapAction._handle_show_local_delete_bitmap_count.vacuum_stale_rowsets", + { _engine.to_cloud().tablet_mgr().vacuum_stale_rowsets(CountDownLatch(1)); }); + } else { + tablet = _engine.to_local().tablet_manager()->get_tablet(tablet_id); + DBUG_EXECUTE_IF( + "DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_" + "rowset", + { _engine.to_local().start_delete_unused_rowset(); }); + } + if (tablet == nullptr) { + return Status::NotFound("Tablet not found. tablet_id={}", tablet_id); + } + auto dm = tablet->tablet_meta()->delete_bitmap().snapshot(); + _show_delete_bitmap(dm, verbose, json_result); + return Status::OK(); +} + +Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, + std::string* json_result) { + uint64_t tablet_id = 0; + bool verbose = false; + RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id, &verbose), "check param failed"); + + TabletMetaSharedPtr tablet_meta; + auto st = _engine.to_cloud().meta_mgr().get_tablet_meta(tablet_id, &tablet_meta); + if (!st.ok()) { + LOG(WARNING) << "failed to get_tablet_meta for tablet=" << tablet_id + << ", st=" << st.to_string(); + return st; + } + auto tablet = std::make_shared(_engine.to_cloud(), std::move(tablet_meta)); + st = _engine.to_cloud().meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true); + if (!st.ok()) { + LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st; + return st; + } + auto dm = tablet->tablet_meta()->delete_bitmap().snapshot(); + _show_delete_bitmap(dm, verbose, json_result); + return Status::OK(); +} + +void DeleteBitmapAction::handle(HttpRequest* req) { + req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data()); + if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) { + std::string json_result; + Status st = _handle_show_local_delete_bitmap_count(req, &json_result); + if (!st.ok()) { + HttpChannel::send_reply(req, HttpStatus::OK, st.to_json()); + } else { + HttpChannel::send_reply(req, HttpStatus::OK, json_result); + } + } else if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_MS) { + std::string json_result; + Status st = _handle_show_ms_delete_bitmap_count(req, &json_result); + if (!st.ok()) { + HttpChannel::send_reply(req, HttpStatus::OK, st.to_json()); + } else { + HttpChannel::send_reply(req, HttpStatus::OK, json_result); + } + } +} + +#include "common/compile_check_end.h" +} // namespace doris \ No newline at end of file diff --git a/be/src/cloud/cloud_delete_bitmap_action.h b/be/src/http/action/delete_bitmap_action.h similarity index 79% rename from be/src/cloud/cloud_delete_bitmap_action.h rename to be/src/http/action/delete_bitmap_action.h index ce507ee9991757..284e8dbcf5705b 100644 --- a/be/src/cloud/cloud_delete_bitmap_action.h +++ b/be/src/http/action/delete_bitmap_action.h @@ -21,9 +21,9 @@ #include -#include "cloud/cloud_storage_engine.h" #include "common/status.h" #include "http/http_handler_with_auth.h" +#include "olap/storage_engine.h" #include "olap/tablet.h" namespace doris { @@ -35,13 +35,12 @@ class ExecEnv; enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 }; /// This action is used for viewing the delete bitmap status -class CloudDeleteBitmapAction : public HttpHandlerWithAuth { +class DeleteBitmapAction : public HttpHandlerWithAuth { public: - CloudDeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, - CloudStorageEngine& engine, TPrivilegeHier::type hier, - TPrivilegeType::type ptype); + DeleteBitmapAction(DeleteBitmapActionType ctype, ExecEnv* exec_env, BaseStorageEngine& engine, + TPrivilegeHier::type hier, TPrivilegeType::type ptype); - ~CloudDeleteBitmapAction() override = default; + ~DeleteBitmapAction() override = default; void handle(HttpRequest* req) override; @@ -50,7 +49,7 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth { Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string* json_result); private: - CloudStorageEngine& _engine; + BaseStorageEngine& _engine; DeleteBitmapActionType _delete_bitmap_action_type; }; #include "common/compile_check_end.h" diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 1f21795e1c9bd5..dccf56e9c9b725 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1231,6 +1231,8 @@ Status CompactionMixin::modify_rowsets() { LOG(WARNING) << "failed to remove old version delete bitmap, st: " << st; } } + DBUG_EXECUTE_IF("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset", + { tablet()->delete_expired_stale_rowset(); }); return Status::OK(); } diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 7188402e95612e..02234b326aad9f 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -944,15 +944,28 @@ void StorageEngine::_clean_unused_rowset_metas() { for (auto data_dir : data_dirs) { static_cast( RowsetMetaManager::traverse_rowset_metas(data_dir->get_meta(), clean_rowset_func)); + // 1. delete delete_bitmap + std::set tablets_to_save_meta; for (auto& rowset_meta : invalid_rowset_metas) { - static_cast(RowsetMetaManager::remove( - data_dir->get_meta(), rowset_meta->tablet_uid(), rowset_meta->rowset_id())); TabletSharedPtr tablet = _tablet_manager->get_tablet(rowset_meta->tablet_id()); if (tablet && tablet->tablet_meta()->enable_unique_key_merge_on_write()) { - tablet->tablet_meta()->delete_bitmap().remove_rowset_cache_version( - rowset_meta->rowset_id()); + tablet->tablet_meta()->remove_rowset_delete_bitmap(rowset_meta->rowset_id(), + rowset_meta->version()); + tablets_to_save_meta.emplace(tablet->tablet_id()); } } + for (const auto& tablet_id : tablets_to_save_meta) { + auto tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet) { + std::shared_lock rlock(tablet->get_header_lock()); + tablet->save_meta(); + } + } + // 2. delete rowset meta + for (auto& rowset_meta : invalid_rowset_metas) { + static_cast(RowsetMetaManager::remove( + data_dir->get_meta(), rowset_meta->tablet_uid(), rowset_meta->rowset_id())); + } LOG(INFO) << "remove " << invalid_rowset_metas.size() << " invalid rowset meta from dir: " << data_dir->path(); invalid_rowset_metas.clear(); @@ -1186,6 +1199,7 @@ void StorageEngine::_parse_default_rowset_type() { } void StorageEngine::start_delete_unused_rowset() { + DBUG_EXECUTE_IF("StorageEngine::start_delete_unused_rowset.block", DBUG_BLOCK); LOG(INFO) << "start to delete unused rowset, size: " << _unused_rowsets.size(); std::vector unused_rowsets_copy; unused_rowsets_copy.reserve(_unused_rowsets.size()); @@ -1218,20 +1232,27 @@ void StorageEngine::start_delete_unused_rowset() { << due_to_use_count << " rowsets due to use count > 1, skipped " << due_to_not_delete_file << " rowsets due to don't need to delete file, skipped " << due_to_delayed_expired_ts << " rowsets due to delayed expired timestamp."; + std::set tablets_to_save_meta; for (auto&& rs : unused_rowsets_copy) { VLOG_NOTICE << "start to remove rowset:" << rs->rowset_id() << ", version:" << rs->version(); // delete delete_bitmap of unused rowsets if (auto tablet = _tablet_manager->get_tablet(rs->rowset_meta()->tablet_id()); tablet && tablet->enable_unique_key_merge_on_write()) { - tablet->tablet_meta()->delete_bitmap().remove({rs->rowset_id(), 0, 0}, - {rs->rowset_id(), UINT32_MAX, 0}); - tablet->tablet_meta()->delete_bitmap().remove_rowset_cache_version(rs->rowset_id()); + tablet->tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); + tablets_to_save_meta.emplace(tablet->tablet_id()); } Status status = rs->remove(); unused_rowsets_counter << -1; VLOG_NOTICE << "remove rowset:" << rs->rowset_id() << " finished. status:" << status; } + for (const auto& tablet_id : tablets_to_save_meta) { + auto tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet) { + std::shared_lock rlock(tablet->get_header_lock()); + tablet->save_meta(); + } + } LOG(INFO) << "removed all collected unused rowsets"; } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 9ff7ec74edaf31..5c4770e3a3344f 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -861,6 +861,8 @@ void Tablet::delete_expired_stale_rowset() { if (config::enable_mow_verbose_log) { LOG_INFO("finish delete_expired_stale_rowset for tablet={}", tablet_id()); } + DBUG_EXECUTE_IF("Tablet.delete_expired_stale_rowset.start_delete_unused_rowset", + { _engine.start_delete_unused_rowset(); }); } Status Tablet::capture_consistent_versions_unlocked(const Version& spec_version, diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 5516196641d092..3a0ff3419ee09c 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -453,6 +453,18 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco } } +void TabletMeta::remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version) { + if (_enable_unique_key_merge_on_write) { + delete_bitmap().remove({rowset_id, 0, 0}, {rowset_id, UINT32_MAX, 0}); + if (config::enable_mow_verbose_log) { + LOG_INFO("delete rowset delete bitmap. tablet={}, rowset={}, version={}", tablet_id(), + rowset_id.to_string(), version.to_string()); + } + size_t rowset_cache_version_size = delete_bitmap().remove_rowset_cache_version(rowset_id); + _check_mow_rowset_cache_version_size(rowset_cache_version_size); + } +} + Status TabletMeta::create_from_file(const string& file_path) { TabletMetaPB tablet_meta_pb; RETURN_IF_ERROR(load_from_file(file_path, &tablet_meta_pb)); @@ -943,28 +955,14 @@ void TabletMeta::revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap } void TabletMeta::delete_stale_rs_meta_by_version(const Version& version) { - size_t rowset_cache_version_size = 0; auto it = _stale_rs_metas.begin(); while (it != _stale_rs_metas.end()) { if ((*it)->version() == version) { - if (_enable_unique_key_merge_on_write) { - // remove rowset delete bitmap - delete_bitmap().remove({(*it)->rowset_id(), 0, 0}, - {(*it)->rowset_id(), UINT32_MAX, 0}); - rowset_cache_version_size = - delete_bitmap().remove_rowset_cache_version((*it)->rowset_id()); - if (config::enable_mow_verbose_log) { - LOG_INFO( - "delete stale rowset's delete bitmap. tablet={}, version={}, rowset={}", - tablet_id(), version.to_string(), (*it)->rowset_id().to_string()); - } - } it = _stale_rs_metas.erase(it); } else { it++; } } - _check_mow_rowset_cache_version_size(rowset_cache_version_size); } RowsetMetaSharedPtr TabletMeta::acquire_rs_meta_by_version(const Version& version) const { diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index a0a2ab3d32144f..388ddc439dc31e 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -242,6 +242,7 @@ class TabletMeta : public MetadataAdder { ColumnPB* column); DeleteBitmap& delete_bitmap() { return *_delete_bitmap; } + void remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version); bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; } diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp index 43506800c43c69..32ea86b09b1d92 100644 --- a/be/src/service/http_service.cpp +++ b/be/src/service/http_service.cpp @@ -25,7 +25,6 @@ #include #include "cloud/cloud_compaction_action.h" -#include "cloud/cloud_delete_bitmap_action.h" #include "cloud/config.h" #include "cloud/injection_point_action.h" #include "common/config.h" @@ -43,6 +42,7 @@ #include "http/action/compaction_score_action.h" #include "http/action/config_action.h" #include "http/action/debug_point_action.h" +#include "http/action/delete_bitmap_action.h" #include "http/action/download_action.h" #include "http/action/download_binlog_action.h" #include "http/action/file_cache_action.h" @@ -384,6 +384,13 @@ void HttpService::register_local_handler(StorageEngine& engine) { _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status", run_status_compaction_action); + + DeleteBitmapAction* count_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", + count_delete_bitmap_action); + CheckTabletSegmentAction* check_tablet_segment_action = _pool.add(new CheckTabletSegmentAction( _env, engine, TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::POST, "/api/check_tablet_segment_lost", @@ -432,14 +439,14 @@ void HttpService::register_cloud_handler(CloudStorageEngine& engine) { TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/compaction/run_status", run_status_compaction_action); - CloudDeleteBitmapAction* count_local_delete_bitmap_action = - _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, - TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + DeleteBitmapAction* count_local_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_local", count_local_delete_bitmap_action); - CloudDeleteBitmapAction* count_ms_delete_bitmap_action = - _pool.add(new CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine, - TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); + DeleteBitmapAction* count_ms_delete_bitmap_action = + _pool.add(new DeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine, + TPrivilegeHier::GLOBAL, TPrivilegeType::ADMIN)); _ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_ms", count_ms_delete_bitmap_action); #ifdef ENABLE_INJECTION_POINT diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index 18b3571299f304..0970ded03e9850 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -372,6 +372,8 @@ Status NewOlapScanner::_init_tablet_reader_params( _tablet_reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); } + DBUG_EXECUTE_IF("NewOlapScanner::_init_tablet_reader_params.block", DBUG_BLOCK); + if (!_state->skip_storage_engine_merge()) { TOlapScanNode& olap_scan_node = ((pipeline::OlapScanLocalState*)_local_state)->olap_scan_node(); diff --git a/regression-test/data/compaction/test_mow_delete_unused_rowset_dm_docker.out b/regression-test/data/compaction/test_mow_delete_unused_rowset_dm_docker.out new file mode 100644 index 00000000000000..9896795dfd4231 --- /dev/null +++ b/regression-test/data/compaction/test_mow_delete_unused_rowset_dm_docker.out @@ -0,0 +1,22 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +1 99 +2 99 +3 99 +4 99 +5 99 + +-- !sql2 -- +1 101 +2 100 +3 100 +4 100 +5 100 + +-- !sql3 -- +1 101 +2 100 +3 100 +4 100 +5 100 + diff --git a/regression-test/data/compaction/test_schema_change_add_key_column.csv.gz b/regression-test/data/compaction/test_schema_change_add_key_column.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..bc9d3dd70ea8a55c752a3592bd509ffa97fd67ef GIT binary patch literal 72233 zcmXVYdq7h6`~UWB|L@L)TElEw$5Lk0&S?a2l zD=%cOSyECfLq%bxY0d=A5EBs<6+;0Lg;gN`~6&B&+B>K$Kfu%|9R>#a{6fEX;Na;v7;vslcJ6tj*C7@I{ed5q}ZdWq$u)?quJWr_Ff zsYb_&+EYJQI-Uo7zjiTeUd4t@L7vfPx^C_3F0e8tt(cYMZFJa0YaHIRCmX3|l2zc( zH>7ql?i30Q-&ySy9IN#yV~y?X{e;r3pxIA2uE`j7dRquxCBNkO??X2ARzB=(=t<)sUFpsWGyyr%GBv12Ipl(f)YC zb2=vF@JTAiwSQ>BO$?(-qw%%b)kRM`WHhU_LY8O=s9_ZItp;PhOry!Eqt3H%$qed2 zBkdf&6mD}K$Ydv_@n7mG9Wpc3t~ohFrKia3bgPZ0Gi6IpqGu?*ic6Tb;5&vfPi~^V zBfO`lNbTxG=3YE6><)j_z$uVf^O8~=-!a4kfgG!Jz=$4rYNzgsZB=y0lC0E5@$=?)SM%MSRsec|G;R)$iuI8F%&AzUA{snG|NUbHJGPNL%l9reXGzexQs2B)cNZv z$vlat%=#eC`CekDwMc8%Q&#itvn}Q#xL}TwLoS&@3hldwbv1I6)%!_@EdEc&kbyF8 zFSNT3C-QZU#KZ#xa^~G>(igKy|eQhIH;e?C7UW!&GC0NxHD(L@A%9 z!pwx16;2jQME)dErN7Gy|RkkKo z_ShT@SJ&cebTI*LaP?A=Aby^VFVJ&jcDvzjpe#+ejUj2q4vYRXGW8s(y*yqK79X<1 zBI#i=WRLeL`v2l*W2qb~g{Zk|UuQW;SJCDTEB@lE3>2B-(g1pCtzAuYe@;tyB}=w) z)zs@!EEqePp?GaAx~h3RLbjSh_!@tT!vCVZQ!MM8Av@eIJ7cw4NBPDRSB)e+XEkpq zhj9EQa|>fsr7^E}U+}*tBnC~OBbmSmL$?nYgQKJ?EeFXA25W++7__di9Q2mkt+Xnt z9RhLEz>(?7KZ@)Fe1$dd{gu zs#K;t$Y0O-idVodSsj^ar$fjQ__|ijsS!<6+Rx1W7wMiB>jVCz+K%p|{y^uG_=V?^ zEmo|GmL+CQnRs7wRx~FM+VdH$UCrD2o(-l&VqW1~8`wde=>HDo(m6<%eMm{+@bTX85QZLeYlKInQ@6syT z2`{R6i?)&;aLG>u_V1m) zt8L~B5zH_~JAu^`FU#?@#IV?E`&X6;v)-LHDUDxIYe}j;#PK{mLn3PP=AtENl-eg1 zV07?(rLc1WcAGD<{-0%`XC#WBj5u~n>P24n2ustuc=!>x>jCd(fp1X@61L13%vk+9 zAAWvrPMeJJhL{22ROPaLT;y5r)8BGQ!%lZt#yKagNSiVW-qt!D(5Ia+TI7DTWb<@q zK+^gt=e1zk2fS+I<@QIqkC9*%EEzrEY-$d#TglQWn{%m@jU<)O+Fn8aK|I3}!zc>v zkw@gBmyY1Y<$IiYs|lL_*o0zjAd)2dU%v4_ z(CrtXD|$>LbbCM@uEbdic5$&GZjwWN%afXu{CG&`r5edT8yAf$A7N$imJ>LP9KsgT zO1)?#L)T*Te-g-iBYWufgI=_U6tB_s+#g}x;Z@7mhv|%9j^`=a&x31kv*%f2q*R(W z&&?83%yvDqjOT?e=ILtI2)!*aD$1g{Nr%v)b9Icue(Ec5ZMCf5>wp^?}} z&RS0SaP73m395mto1JfL~2VPa);wFh$4 zD?!_syCo(`CG-fKE8K`2HeBOD$LU4N?x;ZP6x~O(3d>^-$6`iddBDh>DU^XY-E1}Q z?8$xPil+m%7nl;&L8M~y&n0jm`!Kpr?wcsL5h5j2R6Ee^q@^ugSW(FU|Cesq~;Z1 zi>K^>&5pG1E=-(H`~lXFA@g-7=Q(`f@wH>Gq~>$H14?hfJwYlFIa<{2RrEh?x`xzS zH={-S^X7|E&+`r(9ioRg-DgoI(xF6LZQ{LW1PjEVB~8z14ntALw1f_z{Z)>ylXh<=?VJp6 z>ZPXH|1Q)Ld-k5wFpeqcr&V)HZtFZOF}K;43US4MRd3n|b72$^X@}8J?lMZ z-u1h)6eg;Lg=HoF6rURlqVTG9iDgX-EHP#5a?j&;qns}^O7s4M=76LR4>q~~NSaAo z3lOQy>|D=)pOv^88GqEvDSb@OML+8)5KDi}{zhfJ#a?Eaj{jEV3c@F*E+S$zT<9NK ztbNkxkm19iEDo&Y+rU?1j%Fg)w)uO?TS-OtSIng2?501WmG&%905z}hrX+f_P4FIs5H9S$}YGQm2G zakWllJ%JPyHM*sAHC2-)i8V-`ATkONpGr$%nEPK_o)Jh%- zACXJ^#|`+e^`iUbGVZP@d?=V8yh^CCWH=+_d`acbCmkdeq3g}MfZ}5Cd)Yqq(E_p4 z2YacvfKja#5tIkTvye#iIeA!?CfBK7(u>!HU(1zUNK-DdS?!gJ<(UYaU4ja$qfHrl=n$44Ot&5aLCl7yovvH>O(vy&okHT_jC-|5r)DWhzgyczE z+eW|{@1i=_{Ff<@UH|B#EMy^bv$u9VG1ondZIuD0B5hGGBIKW5JaG@lc(CrUG0P@s(rw? z5q}zBne@nKx#h1CcH+r;0!xq zO==>wc{pREfnYm*3;OlE2+9_z1Uad~o4!=N7k<2{GnZc+GTnw(r#pzgT@t6AD#ahm z=Q;H}Kl+6w*Ln4j@Wdb5Y7zd&3zT4p{cID9y6Ab;FT4dit%F-boz&p-kg^y)c1gPGc8FoIjO7JK$XU~^ z5Cm~)*6jS4tJY^5NJ;wee$Boy-4vi21S`)`)~cWY1Bfr{u9#*ii{)?P>}aIjVpnTb zxcFsKKKB@h-{A4FL;ZK5mHf7$zl(lM5u|*Xh&G1Z{|&3I3WF3C(=+(@4*3DkE9X56 z`^lzr{`#rwI^fr3WU;S(B*hh5YgaZh@9%MQj=atfuKm3fhs*NBuC68s3fXq|o3mKE8#*wo;WjFgrU&|yY z;5#`jo-$&9TTq*r_=7=A`vS&fJpX6eCuobZY?>Vy0{DV-Yi6f(18H&y*RqCrSr zVTy<9iPNA{0~03$OVH!74e5tnz#cigXAGjpnL}ueEZY54-$f&=kY1Ca5sO1sCNxw% z2PT@NpPM{My#!r+ruJXc(n36J#CZ6Soap0*te{k8#^*qFP_nE=EtMTkg#E7rI8ahV1Dzzr~YJa z3H}OMl=UKZ+^px|sn#wD(3V-U^1|OV=a3P%^+@aJA!zXmP1W}^oYq#o@ybgUARBv+ z{VYnUf~!W3EQP-x_ zZ_+<%xGYpSh;Mq2c@_0{yTq&v;dj^v60a+yz$~NE#tI2IMfW$PwQ@=G=7nR3>9i57 zN^|#9;hQJoQgUlp0n|yoM(*1v^J^!}1gFyf2~fO%H}rGqOk!GyrR=0owz+p6v{5l5 ztOKTyF(EAS)VR_1X+3{C!}vm1o5H>^!ur|X_gT?Z+M(O*n_2>eU%f5fNtjHB2|=0m zt!aVwePi!9cQ@rp2JP$H2`5t>wK6Z{@ax9K6FYNMdIBv2_cXC+7j$8yjcy_AZO7M=7K1OX*}MmCOZ0)a*f;GP4VkNGA=p~r6}{mFkk#jS3$jxV3ApfBy)7eF z{7MF!oJ_k{a-H4?_be{$^^X8=Zrbazr58*pxYbCSC1cT|KnpP`S-4Yv3uKwhV{7NT z1R@7Mukmcuh3Ugbhf9BFXG-C@1(qyZ>m656tTS8qnYtaOK{3!o>kcY-K}gKG_Ig;+FUZ8iGMb`g0zq$`7o(hVeUX1K#Sioi)a|kcpHC zP&U7{qOnF@?)Shv;PE8ko3h$CXEcb_wQ{VT&}vF<++7HknMuni3ibHm{ukL_WN>|n z7+U~q?3>ig@sdNyTMJc$_fw<-Z6Ip!`TC(4xt2cHzE_rMSK_`2TVRCf*+~o`T#X#U#Nq%|p@BocXw1prm4#aWO8vtw-9{BA zy{^q&p$}hc&q_P2`gsN|97fgAwkje1NeO$I{Zlm`ZV5X)Xo)`Xl+>$9J3Rt%71i*U z@-Wf}E58q4ynXm6>?@*YIE=TvLABfL zW!Pj|#0V>kH?XMojFASjp_t$K`%vZD&KOx%-ep7=e%b{4B3k&+njYE*o!>0I@^n@> z)e-kuTq>Mol^VLTdix-Cls}L^9Qy=#=vI4Qu#6*vX|O`)5S-&5DrPHRFJjrXaICd} znW*SLT?-1&8!T1(rz!G zi(1%1FUm?g1stLbMzD=DTGL&@)>*tGh5N7wSdOdhHN8s%9F5@*_z#b+)Q4xAk_RrM z(s-L4Gye4_?)q?*W$$_K@v3a6JR|+Pt3p8ql`=apY*6$+ zWz1_@5wjOn8uN|^)gv*&n?mMTr=VrKJxM3NytX5IYiFZ~IV^SdN~3he(S_Zx@Xx%t z=4s&5e^bs(S-BzB?FxEjBbh96d zZ$L2uyR2x|*BH4NUYAYFEN=%&-cS>^>@yB~QuBjIR`TH%BRW4t3_(QWo} zi?YYszFe(?N;0T;=#T(ZY$45W;$qwp)^g_ISQ(s760CEzH|Fq#pc0mz7N!dWOU=_g zXd|jw@Y)l6*zg7C?0R5T%zkG#9M5BPli{HSvRy&{D*S=fF?CfRdBH3cdm`?Sj)$fy zTt!GXEliq$j1!RhQLhXhrwbWh*HWCuyv*#)QGkoe*yr<4s_~F2_C8|hTEPq?Ua<#W zLxG8XHow}wafR^Oj{2>ghYR+0RI;;q%ZJ?}sqIOkw%%iYjCVa>nBw%|{u6$cE|2Jb zma-T81r46z%r(aqJp7~B;>|#UUPO;HRLzw|cL+@TR9X3k&0OPD+cyPPSl_M>U)6h} zgq>yZhF*CJ-1*#|6|_-}_D18_{lQ|i0RI`g*tKVbbsLMh>9-MT6Pz1ps$)H9mFO}f z2OFGqQXjsXq?KOU4v+oH`@FnlzObg9u*Etz{<~|5Kb5fmTTR--dsi4rnya5{j*ndd z<2^XKDtgtrx6Ap0`tID}{;j0Ba(3U^*0+fx03i+D3vwShH;DVPH{GeoFH=gSKZCew z)6^7Y0oVfST*=?of8oVRk?6Si(_L`9JY0H+lu%T{{;rFyuIj6eg6A#MJ!k9QL#H_p z7KDw$>h_JQdT4^f+v%e9VeJl3w~a9P8QoF{CcMjjp(_-4BZZPq2c7}*Ot-`kYYDAu zq$8}=FU9}jR21mL_2FFl)koJMF1p#{!hK^@SVZr(&n5tP*L>y3OI%40V`Q4w4^HJe zZov}DyB_&4IBt@Qcgk+cxl<}#;%sux?dh#@of`!=Bwdkg`WTy@I1?X%5asHr2YS1RAsl7hX?r%UI!jPSB} z%O~bfH-9-Y=nS0Mj4y<6qLT&2Rosno>N@E7&{?)b#$5PFz+gKXyAFKQPAIJD`!%u~ zmek{B$p)ZOp_lHspMC+hHh)Ormp!M!Hm?Z;@y?`Ya8E@TpSdjH6k43n!_9>*KfFxq zKVI<^O_aO`{u}mz6E_6#^|yng!8x?FYySl2wh`{|(y~L)M12Ur(tSh+a!w8f`uV3g zmB1{q510J#B53A!!a_$E7a`a!cK7G9vNyXJ7mbT`3t;$sO}L>TdCY5sHQVMrFf*cb zgtezP_4OnU?AT7<(i@xTSqx^J9?Wo=eCmuHc|TEuf;kH^-2`!!Hu1$1M;99IG)z{k zgO31YI*rgDI1>!MnU*K{u+S=AVOp2m3uR~24l_zje*Jsi=qk#vo3 zVwFk9Zm~n;694G!aEvsx`;;KT7tXMRCU3!Btlfn++02jW${P`HB++$(0k z$UJ~`@s8!?HkezCfCgb!k%&Vq>W9agQl7>7u|nXnsg5j@-&hn}&1Ym3zQJs_*gO^X zqVi+OTk@dWmC$(*jKgmKt@%c=P%p~leePEK@vsh7kYN9rhplb0mAL~sQ7c0-?Kko7 z|R@RQQpmR^(&D<0?H>`AXAFVc&=m}hl@6nQ$&a*1DUEWykmKDjyliuVW%yR<3` z`%yGJ#TgWbt0^^J7Uf>jED|Rl##j;HERV$sM zZ&_%~r1c>yvCnq|I**MT(nSMjK%9yB>Z-45!aUIn)8l>`iagauko^(1zA_p{(}CH3 z^}-i)D%|?oo?7e$9xBZB(#@w(U@8bOkComfxLK4}g|}S4A%F$^;1ile-7OEe=MjeY zl{dr18pcQV(lZlaxIR){*pKKGc*4s&bXJr!?_T0je$jf2{vft;$|8- zaM4D%-?0?;7Tz3pTkY_4KL1Q@yuprdBap`Dn?hjDpzOfEPubNw;nZZBEo%Uv#^7y= zYODuwN_n5>JUYS&Le34>Mjk@vW?L3kCSQQXtIbxU4GnN%o36{}0+46VlQXZ5rEMV5 z8cB6?Ivt=k(_Y8L?J{oyD^@g(9vn`0x>-4%RI30JN2dj9vuiCydQNbWBmkO7QM7Gt z&G+)dt{@i-{QY|TV2QN$Um2$?&J&87w?9H2xY4FMp3e6IL*p!2L8IkT@C+}e<_oj5 zYd*ZmYHYgVqJYyj_%0y#dTNi;tN&#>o+ZKA`)h)+XGP-&PBCH{*|&|(t+w#e(il#8 z?Apg^Fmz#jt;p_X6t zmF=Ym#$R)Z`?t^f9v?oM8dHVfO{%4ny;yyQVaf#Mr0aIiAfX=mcZFG!V z`#3EZ25Ju0l-dl3dsb+Fzi7sRo-+rlR`9@B*^6s2l|a$Q$uHeZZ*2l;q}3mGtvbgb z(7Dy4hKnhau$Mu#De`3##g)!YojEAp0m5@rf6PotvB!-|wil((4cYJko$426DEF=W z34A>gGv?OS0YL&3WLMB0=Fms;eD1SZ2${b)XQ;I0X0kbn7M8SU3mp!oY=pD3wnzPT_v}P zbSlF-E}a4fNglgCGjKV!lg`Cz6-1+d9hp&GUFWf)jtu8&De}R+uoW|+G^4T=9Ijf_ z<@QE#RmR)Hp10lrF74K~9nS~raTV=#MM1~%WcKw1qk@+bAHKBGbL>!uqJ1*)={!-q<)Y?4f}07$yN(-xNTyqDXxy( z6ql+gJsH6MRVGMVwy1CTw_ZpcM)EJSQ5?oUPGA4g@JR^PIqLD3X6^)dd)MhhL)2+} z>H4K|Vk#5D*ErL+B^5l@=zC)%_*c2RR<@J0sys zQ_w&4j|!_N>}X0J{eGM>#1S_6Ge>nsuxjiq#z4SKh?SPlUzs5J{dRqzo>}cA-0eA{I2{66LtT~p@L8HGo!XlCGOjY5Y4TM zRfS}Tt+?xnAJMKsJZ=rN&ekl3)i;g4T`&7pR^GcoJQG096OI1VL z>1sN4zT$@o390bcEwvBeCjm)Wd_u8pT;fo^_}wl1imV)$ZLX&+H>*thpVq`SZl_bJ z3m+yel=X4Aq*;t-G(6;4Hx0#Ze8`{!%+IvBVwJg5`GvZ?ptc`<>&OZl z_9S){8M00~({`aK6oMhgZ#}6e>{qxG(0O^($E50IT;M0*-2isH$M0egFABPTzM@_; z#9^_|?^P915JZwqKL}V*G)O6BTIsJc-b94Q+~Dhw5!!aQ-T-VH2c&qb+0sC`X7~T3`YGnWGYl9qNgzp^JFhEW0DDy9R zDAdDv%HWI=;}x0OerXXm*#w3a`$o8S5+@{Yn2$6`r<2AdocO?oFQt_iK?%H%OKJlI z{3WuV;0MB2=fGzWS&lPQd4>|*OesP6pn)_qS} z$f4vvb8FIaP#qoQQ&63Vw;TO&7Jsu!L;t!z2rvd;E?C05{T`nN-wlQo4pdF}1)8Jj(x-^}Q#=V$1U*4wPe9e7d{&PY|vv@oCyu!;pPA#mVo5 zPr#VEk@6YqcP%xkcIW+`l7OmUbNVvoo%H}HY8tM{&6opDefROAyZk&^T<;f(w+ibE zvLfhLH(Qk(6ksg%BgGnoNB+|qmfnuOB8&6%Q^(T*?;qm{!=F`uZc=r#`-+Z-0(_>b z?=K$P-3+kV9G`@Nyp^83DfF|jeqAGjL(8!lvm_-=3JDP?Xl=Fa6n9A=^~S zu3yV{(XAw6yAZONsQw~#PklT<0d_6D{VhmcN{&1PsL^x)b>&5$Ka z9B-07wiyb~q%0i!8E){te;9>xl-mq9Ec~@ikj=det=NqToCEt%5hQ3uiN8DgJK`*; z0ZFFEHUYRH?Sg`olV>v=>Gd;Ax6ZX0t}0$+p~{yu|E_yqCAe~^GA%v_FmUovueT4>d<#Z-cHg)29z=@z@t?vUp$ZFV zXDb(|wl_m2*Ju3Q8;j!Jb9|S-FL;A7^uP=No!b+AT?d$?Y~B*|Py?VBoO)mW232rN z2EQrzYco=3`|ZqhnGfe(#$9{KASd4iD;`A=iG`k$zY#)7>kZjabO1`x%hVtH4xswV zAD~wAV3WdM-3ot(LaA^3#W5dNC#wCDQcgnP8x*X>yW+h7wwRI^J&Hh35Ir2Gg-Ssw zpYh{-4ixevujbfKLXflVUH7se#~?FHkNa(>2g*I--v|5*JhR#)0n}jMyZGYGJqY44 zf=3|UKGMhW#fnF!>!Sf#Drdgte}UZg_23pLt%cZ)pcw-RfFtdM*CnSR53TY9eZLVz ziq}w>OXA9a5K_l}V_fXrMd#WVk5y;NT=>MCj}&Va#{vHd!^A~s-S(@mr+n^v3m(Q6 zT9+V(rN3s_pg)L&yr>Ntn^H#>M`n61BK`ri@>`AhZZOo(&M(^*4J8h8MoFLaTug;b z$fKw^$MOh{p`hbZB!c$C!6~wLB1%AWymsL2H@QFSy#v{gqW|g$sV~`SlGM+`ie|slmFKT8Q)GEZ7BlZhs2}@Ac zmt0gnB|=_$Ha>dj1D@x56b@H?$*<+#)b7-Js49Y!@5Iy1t(QRKmu)_u=;bQ!`IX*LW zOMrj!hx}Ia>c|9AUHImvWqeFn)DZ&NnmjqJ<{Ly%*}F(YEft!Iyy{-58PUc^E?Bzf z3bSf>_p{awrAzs>gCS4*cFZ%Y5~Nk+e+Z}pxi5<)sn$ZP=D~08o9zY{4zscU8UV** zQuDiqNPuYu1$R-c5lD+^;TTfl>s`Jv@CBgW<;)PN z3~CYWVIQf41Rg8TN?J$b0d|rb`V*mUS>@#)R>q(R7gCA{;@v>kiq;w=n-FakytFxC zc5c^h|1eZu-!AFBKnHIpjP-B5?R8a_gU?7wH@xxT6CM92--DJFE>5B4M2$X$aYo(l z@E)K}No`Gdf(lm-Z*|;E&msti-CY-ZfT6|_pK+n-00P#tBNs*mS4LKl?m&w9OcJz}F;|Cz*o zsNqK5A!z{AuiBxsyUt2#NKXA`L;}@#|8uv|G$^{f&t+dV0vaf1PPnGQ;)#p)`~Hnc zOQYf=vc`XrEmjn!l)?VtDg&aXu76`3CuI7`>!WKXrhW1gqjSJdi(jNZ$=pmqo zemR$IH{njBJFoQ_Js&D8RyKYa%-`KAcjbyDIYueNnTfmqfQ-e*9g?Vg-bDmBueAs)UFv^wkSF5G~TZyv_}7 z0N}I?e*T#fM1j0Aq6)?|ez0=(v-*wh zOCcbh%MP^HLmYEA)p`jEAqhgK?V4rXFd=!s+7v-CYW@Rj@-Ia7L-f=tc$`a$FsN$@ zLpxWgp-IG3bdmF+OFt;Sq`REt378}jB@LPoz$9GlyIBn&X?G{Dd&KmDx)n2@@f>iL zSZ?jJ4FJ(3GHxi4>w6PxmZeP~OIjxaP~Ybl=EBb%U1d@kjJK|$&aTurRr!ZMJ{@S` z4&E9%^olI6d)7`2xS8eP5O)I7n?P#5p#@>{sEzB7aDX`L+~|!nm+?Ivx9A1K&=bVU zK4tW1VcLK#wKuJQ)yHugz4Vb$V1|81E9*e77i0%3(7XklqMk9RDmspe{$rbqdMc0) zILe9cIRrMDQt@I<1u6ks!eYXpzDBrC?77eZ%*8%+!lw|zl`Fqk*o48Xi0v9Bb^!5) z@S!uIxm~oIRBv7g9LrkoWAHN}cKmZ58Y-y?o*|!EH;+q5A%d?%pvWfp*Kj6iC~0oH z7RGO^Ba;&Z_q_Xgo`_^6kw$Jn9D6Rcw>k$7hfK51y8~EjJw~PrhM@4VH}Te;Sqy=U zCt=eV72tf``G6JmPzyb&D@JRp@$d0#{}O0CvH3Q`xZ5-DQ!pO^$~1XO(*tZ&U|HMy z3QfhXS&~yp+kx1&74S!%K*h%gyksza`L(5y2T+kQT-UsjM?k-%T|SjJ9St_L%e(6q zQp~E|#`)@6)So5%{Q84SKwpP`IOIm)fa4wZT7|2S4ShNP7m{QWk_;<;eXS zH6dSt?vMH^2lAlJ(WQREOtV6Bs}hcUe)>G3gvxD|IS}N-*|S|H%uw%!-4b&9P^%?e zth$59l>DMpvR(nQhHccK_Lmqp#r}RLl=uhU(+R4_K*%gvXSDVZld7}$7nf(WAd^b* zoJ!if5||l@_$+CBbSaSdMo3L>JQ%7@pV{Z|p?jzhF1(;zz{@aMJ72hKI z)z#EF0U^i9S+0ARco{W{+e+f7fMKLG>o087Ni)vY1a(5K!}(RJ5Q`jr?*?TXAEpbb zKk*Wcv_Vepn7EVydI$6C|Dx}Ma@Dg$eio6UX#PZGA7UTI#cGPc8-T$@E8L?f2Y+hs z?CR)?L@nkL?SX<=7&|QFSG9RGlbKbT3Kbd<_=e0wgB7ZfcIerZ6_DI=7Z)KS+S)rg zbFEt^BrP4;r|y1-TK?I|@@jCqDH;3nB@7v28Rr%^n4vBhqXk?fv48d0_K{48xJSOH zenZnt&Mu$$U!Fs@_&MIk-c~~dm}iW?sSJgFD{Y%#=V>U)6{Nb$hJx?tR<0XenFTTF zw!>ED9Gn&U8;LL_U}yrU5sP>k(KVkYEMyqmP^itI_*I1hY-lrd zRRwN9rO0`#u^^zB_SK`cuYgkF%caqghY^wOqTVz9a7C8WTN!XFoDYo!F7L|1Uf{~L zit2+39k@trPn%4&rZLQ_wF<>*9z@a7$!*pGGV-FNdR-TW-u{JphzGWTzt#B((LH|k zzYnZMfJXcsySvsvAV>VXm8lW!3zw0eYHu@CB_d{tqZG z1OArzz+~*cx&}!4WJ5IeDH;^wsnTEZ-=ocb52T?fNjLkh-*2>K0Ns)cEjLGc0Vqg^ zh+)7H@Hug#UD?2BC*J<8%|esOhj+tfp@KBPJeUX^n5Rqoa+nGTSGzIoAM;<(1oq~e zhoNG7*6wZ|C7?X#>oz%(poIF?vy~z9A&34M2{ZH$a^^78`9j+a#0T?xo*xEB$SzeuL}WV0XUtS$BcBnJn&&tc(r^b{Ly`vZfTk+CI?|Fv*5;bTGt4NJcX+b1^Y6 z4VahX8R4)D!Xb&-kAFD;h}%N^EOM$Rl)*fq>tYO5JzQvT6!LoWc>P6~;aMMt339FA zgjUTm#=PLGkRM;&X0&BDL&;%dR~10F>;w4KZowuMe5_8Bq)eirVSf~K-JC>6aKZ~z z0Q?mNffA@U%57tF(WD1vvVT4|f)9$8Xs%4=13HjYuh8a2ALYC|y2`8?$_}h)m<=jR z+UF`AXug>{BjhBc9yVkapwTna%e=Z30-O?3;-G=k4wh~x=c4giTGv*37Mwi96@FUj z*8xw6L$=w9VG0sxF)t2j1%@1FyVi3;kb%TpnbZjFPs@!e>z`UBIEUvdI2c5NN2dq6 z1vR6pb7{nH48|jVi~clku7QT50)Lg@;auL=R9_Ae9wpq!h-rcWd&CM&e0d(sJUox` z%)kMGg5gd70G<_mk+Qac0!~PZicW*{eT=^v_QRu0)B(EM9Gt}QahG~&HrzTHw5f3z8je?Q3w@t$7y^E1lOg)e>f z&r&qu&(w9}@8LkAU5tAiuG9)%#K;KgMf4V=eLSOT}^%K-1+rAuBJ=+41YNR*NM^hI-PGxuyPdnlO<4Z5U zUylx~M291;nL%@d_t?tDsZdg=o=IE(Bp@-1*Pyr$G%i+>?n1wqQyaI3b3PS+~Qu-%>;eeIGE{~!8z|B*34`i$d zwqPGiA{OldPAvO8EFVo7T-Vtm$5`m}V5Y6+^$^%8l+^D#I--JDiLSK1TmXsKM4yV= zY~2k0UwoJ`-1`UMLl~#V9r}K1pmuz03bd8wUtmd6a7@6|E>)o$Krjz0ch}{m%R+{B z9};STJHr1}q4MMy)I5c4i=zH5;Es<5 zt-X)X!6H`tQlf?qgQFgl`h|?WgQjQy`dh02Z6W32_a#I@>|;hHLtVrnMg~Xhmw!~i zgn}1g3~##%LXr|@F{dL-O~uCbECMw%9(PQy}3Jj!X|;MWHZj;H8#MbGgnL- zl<&Y`eGT&l6K8bt!USd%Jr=N*a?Wp{2IYT3mG2>F61e6oriykb!fcZA#_lQ^1Z>fV z$sZW6_Hi8YunPclq4>SV)Dk%P*~RdR|5OPx61=0mObvQz?&W30uLUoM^Pg2<;Q;hNkLmc% z5*?7T?$IvU!$hcT)@kOw6EKA zlNaQx@`q%o`L&$C8m1wXS()q5SV85Vf5s;rDn#&P@@@j+PZ=d$xeyNixoUIdr9com ziQTtrA;iIBdb4|xX(<$L-rR`FBT%Y@G8P;#0S`{v`q1GH{FfN#DC8taKqh|v01XOq+D?+F0NOW1dfl*(vvsx`va-T`2QE!G7M`DWCKQstS?oT!^Vo_G@qe`I1&m}(8wWcrq% zUv&HE)-!gsYf~j?WZn_M3Rr?>E2cU>>kc#qsjEFr2R`NaKTDd)+1B<<4$y)i%;XI4d6V zP-m_eIFHggcL$HJO5xcIWx<(gGldyYs_8=yiq|0Oy-zvA1SgKXqFw7kz=@4h=C}`{ zV{z*m?Ps!aV6T3QJ`fy6owy$}C+LhA4t8RTe78h`7TGXc88Ab}I{H4#5nzxzjikex zq#igovrUWtMGfOF=uUe6Lg`iFbx(ZA4s@_xn6QGVffj?|GaTB5VzSkI05UW>B+8}U zz8}VIwS{TFvF^t~jg-%?^dr)K?B}?M3w3da!^+`&N46|X)%ZXJtq^5v@I@6Y?P1&% zapn>55#y1GGIW*{D&TDGAtLNreP+0bvKHvsN0pyngtOwn8U5PU4kG^bf!UPop&o^>DVMH9^Ks4*9b_q zh}jW7=B{PLH*ifUJAKxR-T1^~+)1L|rt39D)w!_$HOb&w_m<##M%G`_lUiK|-& zV~CI_{;b>z)LP9;y(jjf0=aKVt#|$EEexH9x_2tiuYy^)4+rOH?+b(q_@>``VLq6^ zaB^Vt`O&*@a9_TK)|lV27q+`0^b;Pb7ncE6?7Cont4o*VlsAS|7r@w~ooH%`PAr6R zVZ7>73(|gg?lUe6>0!xW_f{^Zqn5_i@GR#5ggkw2{54dVBYVb;bI~X}>3#DNXMf~# zT7v$4Oe%J%9^H*jp0Dscjc7i1^7N*|Fo2cTEU8?D;@ZK=T#F8U#hP4PvI)p%@5NTm z08FW*TF;Sruv^KRR-Lp4a)0R5Gp5D{U4M6Q-*4Y=Yb<##wdeC#phZ||l%pTCCgK;Q z)d~S>Y6weE|)SqdB&5(kscN*)R$El&&XWkP*Ip!3tFw8W- zoFmrd7i(rOIySr3eRv+^56mitt?+8xR}po?z|UE0`UiJ(r^_OBHJ|=7ge-1*V0nYp z)||om#t;6P*-XaY69(GPgTFNwdSbJn$Oy~_@8_G_;S-Mahesy?9pH#tuvr0i#p3zi z@l_D3dUw0zA3m~tTc0Tjdc(Ze%sF@}P>RXv^WBY*T95G_xRxp?fc|T6c9O*dabe}} zv&j(5YSV&T1r!do=}MjweLIr8sA?cBvya1hy{Tf8k4-TY#qgl$wG`5pHzA!6} z?C z`wmw{aoFf1t_Ly~atEB$ChMkabGdNtA|`*kP?}upf@XUB?P&x!`x7X>o_Gx`goK5> zxQn3hWzyGHG`E^ej}lUF02a8?Sw8!B0u%c$po;DQB;+Kj=!1*JfMucI5oarMA*K%u zzd7szfu!)A4D4}1U#bPDbhhd6`9b2-fhKk{*^7O)dWzpx6ho!_8AXrLbhh#6ett7_ z+CHI1x=^k`VHTlo*a`kmlr{O1h@le_7X5mln+N(v#!J$aYzM*ZKA0F){7I``gT9F? zXMSvbCBW2;t);M#LLeFL;?Y#0$8$<{*TUXOR|z zj@4c%Ye+^QcVEUOzl`0CKCK$6jxzqsy9{Ffk)b>G$_K_Ut3=s)ppNwVyQ2*<#JK&` zHYWOF%|Ca#_Eis@VGGRLv6~(x2pl~^r{`6gilCof2jk~sNx$b>!(2QBotTuKa#FIF3ReA*Vl~F1eX+* ztwKekn38l@XEG}?3sN&QfpJO6OjcQjWhO(#4Nyr*StcVxML|VlXF?EEgprg*Sq2zj zkY$EpU>0WK_ni6tm7jo__r3SrbDr~@=bYQ}tqCukomNwVE#+<0WYBBcM?AgC!#SI} zcQ9=Lxx8yeFZW_93Jqpf#uTErYF1P}+jbK5xJ$f$c&|`sJIM&Gt3KUKYIg|4XWL_nrD({x3!Km}%ZCLhbSWGT+{WU6zEYKhLv$l7(d2@TJDbRgF+w z(PPav0Zoe&E{+s<$JJRa1v~7pL@Kx8ck@mlH(*C^7C1n;VQKpt5}U|kL+gkC!23Ee z7qVBC;0$L8pGqT;sbwl(OI{lHe6e12gR^REEl#yjHggs~6n3skdY~O(p}01@jr zA{FF!TXBxX!MDsBT`i zxxcdFSz=}(FcvFSSlpq8bi?7ehow-cS6yEJn)LAmT%yt?&-+9;7G9MWVFLq7n&wc^ zK+Jk=cSfxe=SyOjDTk*FYB#rjO4W!Sy`@_<6zxw<~ME*T(`+X+1UB{I>_h7R}7(@37LaRz1dcIlehl&*Ao5UDi1ssTS zv260V5{SM%rR)3ZVt{^E-spu^GaM$n3_WKY<)ml%3mHAJx!6;g*oL#SaUYylZtUjU zsmNcHY2r<0SC%hv?>OpslLZCB@3&>|ZXm$@|fv=UUH zA!+>y7)a;F#I@EXV{0I1^B=4TmBSz-{2o7Ce>cz}jQwYJF94g)YVW6@*D4P-SC5h~41($$8GmDxA^1Ija& z_hhyT@Lt9@;#bOMB;B6)~W_{`<Wm-(>O)-p8w%iyG9zB*B{G0ZOIha>t_K}L+EKn$(TU$jPQXplU8Aa`c zXq_wC1BD`?Bg%(^4ONSzUjWX1dRl!j`Aej7>oY>`fE#{31jP|=p1mHp7w;(g@xV;=1tLMNr2k=fA09PctXxXM-EE1dC(}t@oGZAQ7KC*XMjzq+jYBBb zZY#6HK9vQ_>SVh>2^#eqd>(??&Eubn9gpO{1|h?zo5xYh@n!INaiURJoqyo)&(CXf~zp z%zlspjNc;opZbLcs*_^7_tCVDb>OrQUl@kGp(Vg8U`fe4$Vv|e55JTE=7p1$b0Q`X zp1uW7pu)1j&S|Oi69~e;$5K@`Aiet~^@e zs!913EM!KiNz@mqywTO~N+^aeaG9aH*;_c(wBkONSjzfBzcCy}@89so#7PpyMzO^5 z)qY;4`dNfUaS9Gv(N@`}fmC?yJCay8*;rexm=}=&VyqUdbIjO2n3#Zn6&UpCO%>9&h7J8`JlK+>3`ZtoBHXs|OKwm7=Bpjz? zws=dm*(6wPqsxX_tgqk?!MZHUK#-~ph#Il@;1hUc$YHOEUx44lz8jkRu`YS>({!s< z5Hd}zevR2=+62@L2S`cvghgb5WI6oNkjx?Kb$86>8NIH>H3be1Zkyrd3|p9LjLT|D=JhCw|PZ$ zC-z}qJIXc;uH_<{#a}Y#)EIyW4U3$ry8?hM8tlgdq7cpKE9(-;YZe)LS(HS$@-)nN z@Q}~)k2);MsvB!d7~+X48L$^au<(LRij9yhiF{iG3Vfg7QwMiE0iSD~?S}v62z#^l zGH5Zx^UYles9-v7)pjAwhkNU=ODl96DD}SnVKf-t@{G+r8~=epJ$@#?&dhlzU;cv}iQTi^niw}#A#$V++9823HMv`%6 z*T-=TVje~A!jlMRrN+a$j4hT3bQK4u3>3G+h9CGu#ZMQ;_o}S}seG?WD|=fCvaavc zzn=dFqtb8Kt>KJ9ZwA%OR>5x(4nohKHU;B7m=-DnJX22%Usn+NHL@uCPs`t>z6ego ztoye~Y03SqecwMwUy}_#J=mr+^cBWVVc-ZwP*>|4|Q71D|ht;f-sJ!<@o z@gz$c0GFztd7^CYus4Ws;NGZRA|FI0M`5|iO*Q^%sK&Ny%yRV~k<_T4@vYvTvRH|G zw|Hd$T!Qdd`(DJUOy!R|MNw=>C?xJTN_%QixL@c0p}WF?>L@l5x0DFv!NPY|4r0Fm z-~4au5NsyZ7RJ}$a|<{d1RLA~0Fc;djHSq~Kb-QMs##KxyFPCcjBnj_MRLWY-duDr z=XfVT4d#Dw{v!!+s&4ex^#}zO;_wTkMQLSr9j|tUuh3t7x_6LR`a-9-BP?u?V_C7+ zd040*y(!Eoy-Ioy>4wzwS|nXM$!ke6{b`u-PX4whP<5zMhvI)GmQo=6d2`PaSaHWX z{Z>;uk>J;kf9ZRDz4k0*gS3~_%XqDR;3CtixFg4>mvRsQYFGEnpdPYVyCqJUinG+D zQ}+93ox5TCeo@hH4xp-GB@)G_YO(&~n=1nnDKn~>R>0ZO0#Kj|>~Xp;M#joyW9&IW zNXhFr+X@5-LP2X2ok`Ll(H;}_p@!riA0b?un;{1a!u5XN5JV#5sG-e}8OHZ%SDH_v z-tatpp}G%VP-Yl-Zjnepr$2rd^Bnm?`>_572VG%B92(T)bBkm1IMYblxUaI>%RcXlF=`Si($ z4Xd?~FaGZ$D*d_qh;MUsKj7i>+Qpl^@+pa574(-5Vd{x^NBj&tTZfJn`L0R}5-X(- zBgP=Nm~h`Igzp*&51!@TyfbUUVf%8A zoCOnU03e(I{wlr)YAEW|9oErQRJkJ9?Ex)E{ZL|lPtD*cpn zL>ToH+JjB$TlmKl7nH7pjBgF46+tmt_xAaxUS0UfP+iNGr^UclQvK4Y7zW-rJ|}sw zjy8Y1psbk^hvLfNU|R8QUS~aeNEaa~&(B+^QBVfaT+s_^$;NY?U&*8c)x93n_6Ar} z3_n}+Oh)30?n5&Njq~)8wNa;XYRJkI|MoKPA#_K@S!Jv<^~Jq94>n$b0?hpbMV^Q} zvFkg$YsP0e{6-c0!KiG?RWv(&oP&`0_(d5SYWzWg8IH#viHCk{&~Z$djMN*zH%{A! zZ!F~GQg<`Ic2b&y2uE1T|e-W$Ww^bu}1Sv*c;4fyZ^A_;e z^CF}p%i+GF;+VSY_>&-(jTSRragrfdb0AelnCk-V?-fEULYt#skvCSOSk^hT7gZPS z;@%zmvD@1MtChbY6Nne;5AW#wk&J6LM@s%v_dnj25u*MoUNW2J6a-WxiM_V33v1!a z4%R%N@HUSRMJl|3>YQZDcsiVc;bhP_usSk!;mI`QPy>#G#RsrUhF;#-A`z-ODo2^{ zIV8x^s;7E4R2?cmPG4ys5%euc~lBdR~psRU;Zp&!$&LN!oqL zUwEt;hnhewFh&7*szKts_irI+tagM`n?NC9WXgSmnk8KHC(f!JwpuSL0 z{fKLWXEny@rWf8I&hb#l8_81C*tZI+AJiJ{K*o4MX@X->@Wl|K9!ClsszQ^ zMEK_5(=wG`I61jNZ#Y)5S~a@@ZZ;|K7x&sSK-;)iT~1L#H0U_F&o7?D5>Ka0O4r@x z18S%QA}VAaFB(89KwRS|PEkrR^tg4iEb;Lv>+0Gc#ic~{tD)Vr-xUJ3l;IUX(G zWqrs2h~txlzn1>kzMi6cPvKPpLFzR<(Gu-gTT5(mMiTYg{OT}eBf$FxoYMb5qIt-K z@TCUs1f0hAB;g; z23J!!ujRHY;sh^Dxb#Rj&K9U;-an|4>%KJHSS^LDJdh>$;yV0* zq0g>G=N0bNj5e}CXt;Qe)?BQ1quJy;cnhT1ORvI zpGUeg(t%ASJ-fr_X?nRX7ne6FF)WTDK#E;XV+)r z1Zodt8k&h$P-WzGK*~7mTfM=`0qD+z`@?(UuGLQ?PeZ;}PH{w*7VFBpKuR!yuuXcj=Q>R!;Js8hg& z<%2ETJ1Ac`?zZzeO6g9bk*xIRfoMaCJF+%8XdU9V-Pbz2_;~N3swV~r$s2t~kM~XB zPjnR%7mJ`dve9i$GbvG(mPO)=_}b(Ox97Fgs+DAU&$iBpKrpm&3;P0`cSqbTiINnD zHNG7Su|xjJPNC1Qo#-u8Kdst#!w1LBxFocI)Ocz9OXWv_j3G!4Sjh*2&~NdbNrT9b zSg4j#og!vh-p`~<M9th`PA#cMGI20bUg*cH{XjS2O=8oB)B1d($Y=db_Eg-V~ZjUw!BQumWo3 zu?Ch)QMI@e{NC7ZlRlqi#v3Nh7$#W8->Nl9ce>xcTm>A&Th8^=T*NmUjftEZoRXAO z{XHW}C?2uijSaBH+BzEvrGxoSI}#H)l;K`JVmL_6?hW?Jr6zx^V{ns`5df3SjwE(L z&syGBlKz~J&n#Qc{fW-?F8(r^V!H*bQ!&z7l{SRr3T`u~w+d9U_N2i}xD8D`g+pEv zKPoLGdGn5w9!2r)tmNCsf(^lj^zVM6wH5&Y6Y_jXkVl*sc;`Iw^F>KauWR4--?9Afj($I$f z#0yamkWFK<)uVhV{9LX=r18oC?a;#hqTE9C1l}nkFu(@5W-3&dBSmE@NqCnrTnv1u zb)6q{k_^n?aV+5$r9>^=iFdQmw0uD*2y4PZcy#~bQ%AZB{yo-rnaC5_#rftE2r-=) zY2mMAlF`QPs2zZxNz32ZWuk{-Z7T~+9Y7O9T&YR5z4$4!Shod&GnMt&^6s5?;IE9| zrv?7!l;2r$NMoPZai=oaBF|l=JnUZrg^9oa=sYbFPu+{&+?Q$Ez*${DRaP^V%Vs)> zNsJo#{?MosC2lGI@}uMoBqFVs970o~g!>U<g4`+0O98CI)8NKf3uGFC4%Q{V_@aJ~D0F7a3DO01+gcFKXXp zP8|2-*~?S6kgo^DFY4#-fHb4--mmPhGUBlJ*0DXa%}H;ODGg4re2QGA!t-X74Hcy2 z_u>j4nub?6s@+eaFC9Od6_pv|gvWk<$HgZTxvy7Z+_2yT&TH(0(4RvJ(O%oP^I~lN z*E%e$)X%m*bd-glH`L(T6AcQgY`e>7A}ItW*LQR-AdBN>_fC*ztfS0%Y}|^t1%AccV4Q3e*3nR3R(jz*1C@2Ymi#tqB1){>=KR}9!R-nPgSbVHadE@=k zs18`fWYFk54<@N=y0hGCwK&>CrSqd1aRg2JzA@{RlD}&2OW2Gbcxu zXJWD90rG9XC~ecz0yKCq^Jh0Wmw*5cd-t-Le>|L*x>v&Z~&I z=N5kqyAk(uy}*dDI!oGUx71IYs=sMMZLs*aQP=n>toGX%d3n@c`6zIhhkj?OP&BgX z)Y_Q8?%_5Vjzb}5d(xrBe1)vpd^p+U&eqn$7pV4}c5Ay2;oYRQso8bNLGn8P$ht#Z zP|fMPCzM1anJ-kecB33GbouC=+cZ{1=G89tM9?-CMP`;ss2<$F8mrraY>V0OxWpL( z2;KeotVgAI({gFtmg<-P!Ko}<()oKl3!@oQ?n~T>7_c()4(R@&6YVQ`O;-xTs0bLv zel8#`f5JJ!y1f<(bz|JfXS<5fXNSQ?6Z0|!mBDsbtgvb1#@;`sJV0U*UB1G*p{=X1 z<*ENP_P4a%@Ap)&OnO%oub4jqGV=RveTmSxxi3B>wH-JpsFT@#l^mqIv{7X{7>h*5 z!)@%)2wMK74*Prk@xa?GUoSmHLkH@072@vbEYM3u=EIlnp>vn%lpB`iHidj}(o@q| z0_sGjoylx9U<({0mi==F@9oca@mnloz@^t6WQF1}4WX%dCrd|&^xPnqnDU94vlu-$ zFp8nWE$DWzi5$3fRqg{Hg!b~#duItrS^uoD8axLQMSVQZt_2P5eaSneiZV1hgc?H< zI#2~_m^_@b#xRLZkPA<^ln}Mhue_T@d`?50?Q>s}Lk*q$3NVGzlFD^i8uH9sw`NNh zkQ`_|*S=PVIHmSjJe)|slF{T{b_mX`b5kL6@C>|@bA_G9A{8YuIrsRp_;92CvF0Ux znphOe1~0Q7Lt9(uG%Ylq=6baA6?M73CQ*|A4w$SgzB}1B1wXJ?H*Nbkjnax8-Tc6I z{6^;~%X(zg$??;sCi(*eu;%RfvYx-F=J3PVvLXhYhhmld5s)bqGPXWCILzyUI*$x0 zF+}`vE$RgTLN@bgxO;2m8F8nW%X|!VblSEp-DkmYq8M8=*==IRD)))@hP;7oZ1SC@ zvsEE6uRB~5a2(odC|6>AUJOI7+>(_Le~ail=Zb3k%588-e7~(L-Ap5uq{E{|@;3&= zYz`8IfDn865ek}IUF#ki&w^3dYX?Oz7^~XEG2C@Fc<=(C=u`>MSBHgv^?rXBB9OK{55YcIy9s9-KFCta-nf7n6e z`^>ddl#)gKx*+r(9qZjY3R8$DxW?(f7}QWkH@-u_{CyGlG2igTVM;W6BQz~(Mf>TOY7U}?_IX29&N`!h0a2c)k zV?diBOD+*~?gM&b9t%A{^Q(rUb_I`<7X02_i@I{Sgn@O%GOE4p!YFI46LGMyoPQbp z_}H`)t4>Y}jKfLeH5~(IiJfS$FUoG?0^`(JvNZ?5`q~)2vROAa8wN5lGrNinmWx^~ zUNud*artknzZcY$bykaH2p(h$3K_}%## z0Y&ZjJo^@#4>u4Qpo$z0d%A5etUDmb=65MPkchR;MgT`4dH2{%Pte#vn;vs@ToB9| zAIyJyANEWVD^7G%c)3`CmifpcwTiN_|KSpWQb_N-u4#eVE zI)B1CDu&Xhd8u`^`!lEftO@sHoQO^&@7~jKe2^aWUHfwyW`TlIT?q-tLS$k`2 z$xPCHi;nuLvCw^wwohFlerH_0%RiVjw=$ht{+%4n8(6)FUmv4C39X$TYgJ811Z*pZ zA|WCV28(#R=$NW^>}7pnQ$h>=#j)&6B<@#+GJI|4HX53k8GYP$3knK|Llhz&d z4>RpdnUeC-i&afG;qCUBO+A4t0?rWo(M*r6!c>oHO(_upy@4C(1ZPg1Focw{0arYt zBpKAX!m+vR!v7NsqGGrxV!`Iw;N z_mQ9Z0a*TdQEkyCDx|xmx5j4rB2~0?bg0CiJZBa$ig<)?+!hlb8_xcO@d7rDfiD@1 zf&bv3@=J$)jai3X2}>w$3HQdE51F&=K3CuzITcQICR~1k1jorYr`rbvn!C!;wTimP zg}S%qpiK$-f_t3D0P-qhMHPX(Ov4OY^SFPoFDl2L-;NHUGR9YY!Bm_?+Sa`L7x#y( z^ChFB{F8mi=-X|yF}0{f+FjYUj4Q)K$yqKP7NBPeLwcAyNFY6Y zv!H;1Y(sIcB1*CvT!UJY;Zw93zxtfHz-UUDw2!nh)6UiLF2}Dx1;p$adxyV+(C|#~ z_A$~k)-2j7l9NjIW^F+E)2;!WkZgxWhu#igQuhjt*@=;{!6fHH!M_+l$<}DA3?W$n z62`evq&n7#qQL2oF6xv;Fjr#urm&CSs;YCX%p!hfruzAweG zt?7Ma#6eX%+O?SM=NtTu?_u3uoUE`@@?xT~9TsJ_Q#v%9+Aj^OAa2Q7;oK2{jHCE) zbyP7@xFElexM}SP@PFD9!uP1X0e5M}t73Sjc0^yzoRid#)n4EpT&UYP+IGyhQPhJY z1zo`WF&kJ&rl|R|olv5{eOv!E0z};@R<~IEPHV4fioi3-;`(_9Z4fX{GA8yj)o3%x z%`eKY+(8h}a98m}4j7>dl>C_k@Drq8g1g$DP*+!J#Cc-J6llacQI2@*S-Wo)Y3vxm zza1Bmfj}v;k)$bJK6j_TBP+ouh`j>8YnDVH-wgiQ&|nImpI@EyKx}8;pLrD1t4bAz zJj6uXs->kCR%YqsDV4LKu|yADuj553EXEyaF|)e`JHYA(%W&j|(sWyVMGo+#f6m(j z@);R)NO@6<1MOSOe&9ZDzP91{Je+j@4a3evizbN<+K*6$#qDZ_kp4&C!f*RcnvMJ* z??#_Fdeu9%a+!3x5Eb$cfj`6K0nJEI@v>B^(AW~y=+LG+QJ zIEaxZrWO=EBvu~X+0HFNI1oc~JC%^N>c}r@p{DA(t#`~B)J57LUK`Kl;~x^igQ4zroMh{{HKz^Wjdlok zmT4NOB=ME4ClU2%I(MJ}Ay#be(e(<#1l@=1{pqlto-DYViH+q)EI)1p4Vmt>LVDaZ z((dpzum1fwnfzsTPeP%IsJIYTKwP^>z}ttN5*G)2PwUV>J=LcY^WAih)Y9rQ6V*EZ zz-y%4GbEKtVQzP%|E{`ciy0QRjY};h_)A8yw|iUzB&rjAlejnrWi$vg;%L08T}$}^ zBG$((g@a>A{tThIXKe%zEyb)|P}wp^ z^qW4w&$_NeZ&G+co09n20qoR}-I4|xAJw5L+2`<71mhhMr5Alof7HCno`(t1IMlN0 z4}T)XhvA<2&mMtPF*)eK#;7+8v+#^8lH-^G8H@K-oX|;wvxMx(PuylvWse{ z$POQ`pFWt$b%J#5p7i6cV)9s)<+|4=e#heXE!dd&tI`v`D|kEm05!g;<0EM9t&F!c zv62*t<$3?M(c#p(v?@rn?v#BP2UEjQ z??0`4Bv1kDiDN}#q~7IgvPwyy-#<}xMQIJVu2Yr^7JK1XNDexdS<)-NAJO#`qis}^ z4Trl)o}|>~S&n7HZTBUw^f@O5597)|_T-2h5iv+VU*SA}Bb1*rbTR=PhAL8{9T>!* zi>KX^YIrQ;u;|9)VC^?1rFfq&`0>^#m> zJzSH3A2MJ@YrTqHNNTrjX|n@%QOoyYxb{VzX4@yPo@#DK+ z(kPlvwWTUD3#ixW9B-qkuav zvS4zm%I5ZCq3U%<3O^v;<|IR5HJS>aFd^Ti-o?QJA(>Ne%>nolRx1kx4j9hjy|3mp zOgjEB`>qn442ADzJSN@(mQS$6?D-K&jc4yGFUTQ?Y zrWdC&<*)?1VRTt?xa20n>QZ6grDWIvdL=h|+ZbjbL;Rs@Q4qbT{^Aq8JMD7+)OSUH z5O!3gj(QaHVH)vGNj}t|d&lPt+mwcc9qJ1H6qvM`(f;bl>*x*RMI2Sv1Yumvj?Dk{ ztfetqP>+YN>_83{RPm}dow71!*2_){sKhnFxPNaq3rNfOlPSBg3jW-h6L;?oaraAP4mM!40HlY(Qhzo18i;#4}dbuxSf6&~Gu^xz)9R22l zQ;$Kwm)q!GSHu3c6`GeO#n{smjAb%)k@PFdZ>XO`ee^@ZY#1$ z{3jE&GfCy{#84(=tCqCNB(>&`Tck>tzc{tDVl3&YVI9C$=; z*%0wP#ydJmlk^o5q`*wp0%1BSu#|PnQ24{6j@O*2dIVHZekpn6a0E2462Y;+wJAUv zZYx`Jw}OWAMdj-EjAz6{14F0Y(;*gIxCe}4&m|pEqWN?S!TlcGHrnb#~^Ly*M5gc z7N$wLQ1%V=VUIgU9ud)Ks-+_JRTdF5GjwI4%iwpJpJcGrfDl&l_!j#NC2&rIeb2PO zGSv1BGwrktimWN^7CDMe6SJZHk^~*T?0IG~4{&KNR=aud%)6X5v;t$wi01J+Bx03i(a9a*1lvs zhh}2+Wxe0?#v(MMOqyk9{RV@4lQI*2cH1>M(p zcw}@D?c@nyvU%klnnq5*@&Q{H+`8!U-wdo0>Yg(*r_95dhR#DI;r#D2&_3+FboFXM->A_csp zX$q~0$?#h|eCa`Oe({t=@lqUvE9$?sXQ^y>)c;Lm02rEe{+``ErflaXBQb165!4;nvnxXWZdAanU0o+j7cjXo< zo$xuVW8#A(8f(qJoV{O9J-3_De#83nGp7wQS*A*??m!ycaO4NQAlCc7nZ1}yILX0cP86a zo0OzE!QVAp0c0=v*WZXW7gP*>2z=8J_g#(_2NM@lY2TvWUj~22LIG5bYIK)Spl??{ z5O_8qB8Pb?p6tyBI8*2!IZDyN5|t&HZA;8cv0>0jg>>7!V$xF!eyc@j!Hl3uh~t() zM^vYPxb0%pOXXhhqWszBZ_r?+*s0ydBHO;gC{SOWG*&N=RhPXSd!lRLZtRVL1JbM; zyLQu=&c+h`0cyv5!Wp*B5<?6yo*Jnp9gx9LpSf|_6@1OFtZM_q^v?kJaq*djm zBdYoySS`8(>QHP~UG+f#^(5qq&O4A0J*mWlda4m~Ftr6J4ZV7-XkuT6Z??|6YP$L$ z_x|1=Sg!!Ga*BGWk8oSylj!Iy<6opVD?7%HJBu9t^=Te3@T^f5e!cgz}kt=|XL&9uS?hH_BV zh7NVE!U2oSo6G+Z;E*4&C`Rgz71XS2o6|=@N?}LbmZBN$EkIU-R7y~Y+WD@QU}gjd<9-;2dR`E} z>%nJeGLxL}R<3}Kh5KoM)eyk>*yQag8YAm2n6UrAuD6iEJ*kjFWrD#F5D-Gq~$W?Hp)$__hMY*dYJy2bM{Z<}Tt4 zc8$xzdz+B<2vZD5;%laDV$_)C3rer4Nvo=?J6dXs43Np*o$X9Bsjo|#6;hh{)Sp&4 z{gWvIM2p(@${GO*r?z#k?#l5-)m*XoLmxnY-<&^cXe?bXZXUS;q;9)%s18kOs=(lU z`z(85d_{PrJ`=t$$ATr)1;BYst1_(Rqu8j~(tL#wRgKba>{rvk0*+15=5b%Nj%A&S z_SR?7ZvbdXn8^L69E4v7#L_^bO-%m~uYbP&$ zwMu2;EF6XkiMo+yaEBmPOvcq%KWWdw)^HI15dnp@upY#iR937HxLatv7%&!Zr5aCL zC;ZcCEeb@n6>H-9!_WhP{LJND4NXNFt9xj=h9csRuDbasvxVE*sC>w^_~eyd^@my{ zrH%85fwQW;@szuWOhYk2D(cPNHMJlF8d z<8W!p1maNe9-GZT-1`KFW`;`F`9S{nN%YRB_zBUZOLP6(oySW4wc*8vbht9ywBH~H zZQa|*4tW# zc5=Nf!(4Ow14$5iJ^v~vfvWJ5H?CN6f2YbtWLD_U2s9HbRx3tLQ-ddb z4xadrsLI*KaGJIo)b8$gq8{B16}_$V3SEDP)F4MVTv%<7MRcy1oc^i=>nrKJnLdkh zsp80Vm;LyM>U4?CJMkkp3*INnljAO@6#y+hrbV}GJdLXdgxjo)aAg>#j0kMmI(4m=v9K#3u(>UH66 z3h{w<_7yuR;00N)h`Baq15UW*dGCu-u*P|U-50l!5}+?+u^CCnvr8|_PkLSim)hj( zzlubUpE$+rpoC|0Fh(NKNihFouLk|SnUx<{a8 zrBv5c%l!$#()EGNnQ)W(HKR<54ic3wyKwbX5Fm2SHQxp_DH~l%hFbCHQ=ti_OH_&j ztL;}2WX2%;%{R=jg|SXm`7#<9TF%_z8gqZFm>~HiafI15!9S!+LqGDZMCj}W}To) zp#H52>#Kt_A|oBYke3kuNQ;o-ohkVMXG1e5BezAB46f!ev8~qGn0cvBR>_ z&v&9&ZIk(6a(T|GTk+K6g>{)MSS#jq zuCvL7F$_)5p4NzJtXEX!GzYsIkZD1fe99^aZ-%PL^x&&tJdvr)r`uM7ItLDA%Mg<> zjE?oDdNPZaHtpyD2S@+<+BKQW8lK@i*M4Gfq5eZj;6dL<<=0H-$vBhJnlP(#?Y=nb zv9@zkO2~6JS|w<@X@(i+3g2zQGZw&cb(b`gguElSdwoe59h7Cx3+zYjL*Y2fHG&2U zZnK*qs~!i+zFC#Z>}DaGD9)bV@n5t6*|o!+tYorm*H6Oy&~AW?w?CXfHq9*SAfbK)-Pg5bi3tEy@jFp=_*{Y!X zk@%4*b4Wo_;#!$w>h}?jW(sKd7Sm!e>(xmly)_5x%1w3Fref9Xv9qwFt-|5dddjof zUmbfvcM8>-Z?~}~F8pM^^+bvg8ZK$z+k50F4z}bU>S&8tOe@hEL8XEg@4B)G9#}oM zH&YGu&;~VV<`#-OAS_C@`8fGcVh?y`D&<3rID{r?a5+tXqVX52rKlp<1s+M?U=lxk z7;G7Fm`s|FA|#HKu0Z*7Pb!vz)A?+<4Ao9G?^DBJgll3nW3~{?iJj_;J3#z}^|YQ+ znkHYb`#dL}99l9%dmkbuNZKxQ7SPa+Vc!hlO_Jiei{9>}{=rP$X{|gRgOQ;DE>hFn zPPX^Gcg+97LYDE0DoJmNBt>@v^ntOq>6xkiWSko;_N>LojKYiEEv5zsnzCh@P-UK^ z%crPBWg%*+RD0}XRa27QFq?mhWR7CP(UZ@~mw9UEh(x6L<%+BCeVql9lFhUd+PWcm zMyq7$H5q&@Q*KrICyi=}k_p~FfNXEVan{&f8l2i`)-4LALTR~tli(=bM`ie{ciV@U zF6LDGlF&i4jy|`={BxVzFeZZ$Gw1*RUWxv_8-;(v_4T^0ys;Ke$ExMS`@1gSaH{o7 z^W1?4(|s=sn*jCDv#fnenM3oc?9TdrxK><3R8bNl`Vubdy>_($H6RB1#v;hGmh1{l zc+y0Dm1Viicg6uNS}c+)JP8Hk&z(M3gGH;CxQxC`^4UP)tpF>kDqpXPo~XG2zl-_2 z;`x~Z93fus99p$N{@m-Ep&0gT~f1f*>Q;dLX3vGyJXPR~aI@dB$I; zRtJ(c?iz{;@6eRYXM%k1)(?RdI>!vhp3*q3`t@gq_TjuqWJM{~0{eVpUBrh+h%wQB z`N(l>BW$ZFb)c2x1kC(XE^Ddnx?b{@_y%cUCo3QD1I1{3IFW8sk0Ln)Sw-_&X}GcA zH;vu!T;yU|;g3bctA$yuR-_K&*|3gs8WiGJ(9!^eJ;=9=s>xc$)hlF@c)vH-_cU=J14MQuAN{D)I;NBiB z(3lwHQ#yH9JHU3GZrRRJ!nO^kJbsiR6)Ti&K0NawM3^AQ^+zIGNYmC`7?KAvSTWM8 zHceo)o8=!L4P1sXJ7Suwj?2@kUw@C!$PZ|^I`IuKWl+Fe)uTRYrM5`6d&M6ll+5dq zNJMyS#HzVz3<~a)_S)_K&9F~p!~IQ?3eYYiR=p>RgArb0%`^UVVNm|s{hyBr%urLw z7a^AZ40tu2@@E#BymF&__5QiG>POF%Wij+!d7*v@?^rnQJV&5yln1d&&L*WoHVr+X? z+DVif@FedaIDm_z?98lYZbPJM;q1vAS_7+cRaKu4r*d@0BK>O_ImG`2kKH(WVhEl# z`M2nX7v!qax+j^(Vv&7-o{e`Lr@DGd*NS)&bLzXcvF1{xu-3Bl9h$}xZW}-BK&nH2 z#^rwE1YJUUW*-ry`7+XWjTyFDK59Tm3$T6F*&PD8vQmSpCG8L zxs-zCu~O%Xom9BWH-EaY0U8Kr!F540k22yJqdIbnXjv0%D&0!5mucz$QMumhXsn}J z2FH|hDE<-6NFO>rH}Oao`N_irUyaPC{LMW2O`^cU_H?mL_wI$hH~RNu5mILRLk5RQ zC@y<-ElHiwUQ8B0bfreZOx=cq78v}6&2sLK`oM>rS-*!HwLeZ+oH4tIbV0|}wN1Q5 zD};MQ-mP8--}r>1tfeHR?%sdAI+DXG&zF4g+#gZP(~N(~G_>h6v3f5J0tM}#BXxx- z7thk{NEDs+m2-c{m=DbE9(u%UY6ab6qASl)Ay%a)eim0m0k!6JiJ-?CH668K9qR-g z>tWx4jD8OU-;6~)-z(BE0j%lt!QmLnQ$&k;7b&vf_R_12A7midGM!>Vgw%h9@?qLV z^co>r>5B>PwRD<$nY-^N=+J8&DxQg0K7V+GYEo8~NqoKrW)s$NOhA%Df7!$GoNo$C z@UOhm_PI2YGOVHT(trR4+wz8RHFY)a6S)3d1qWu_e8=_i!FYAywyrEB z0)=s+TO@#j1BrD~KSWtFe_@pL7#!Hx=jm=}oF8;-sO*O;Kw&0+Wdn$_K}VPEbg7wC z?^%89Vp}RMFcF4IbRD?p!|1|09K*zU4x9N*d=Ae0x^tLZmr66jlNlKS!zO-pQ|;^| zT9+Ytf9x$}`?wEh%1FajTvKl3d@VbR%%kFE!9((Z_G;yyR7~|$xB2T4v(l>88ULW9 zI>W11`~nkgpjR3Q8Hd~eb7d0;FgCt4^hoOt$U{LCd8z*tAffGdcJ;c0YDmB9Yh0Qd zfHQDo=1)F=CWV_0k~mE5QNpX~MlWXh-`={XutZ-UOcD=3rTDtg%BBtOupzEOoiB2M zI3wK7x_W<)8h`zz&9-rOI=t=Oub9&jGX@!0w@v4LY^NTM%OT)i%xoIUAyjmX{(xhh z#7x-rg0it|_6atzwCiqHLu{TgAxz?b(t+nrkCHwyoyLnrz3nVJrDML2)?C>sH? zSRT%I%|obh7Ib@5-9+Mx>-SuI4(uV>#N8?DY7tm_-JFO!_p^y&;ny$<{Gy1@O`ZtY zpXMvl*bZH#u;X0O>X0+^5y}nWN2?ByO1$LHkW)odU^_hA#|);)sfs-ro7$sH#Z-^A zltV4z=yj$+ziBxIWhhtT8bJ&1RbmhalRs@w>*t}OqY50i@SxnH-#K(y?^#rX>b|~< zE4R@X7Bh|I!A*umNiyr&h%?P9*%zUP9yr!G*K&lk+ZS{JPqN?>eFF=cN1$W2a`?)f zK&el|)FibNU?zvH>B_y7Ctn0Tr(TNywyuft_d7m8!!@*o=S6nBr2;td7KXZyZlx(V zW^VfbUj6N?YeKpridB4$DkII;6F7tT)=0_b`}WCz$w4-oiQfbE1$+EL5u$nXn6@D7;-%V^ z6QX697Q<59?mdVP&lYsMzdsHSUM}Co_>5}%kcPOKCZ3gTU#TN*UgFUGuCIxixMZB@ z&6vfD?VW#jQy8kt2Fw@XU@WjsnYh3(2gfx3l;+<-IA4L!)Ejg;=L?4JKN?>Ni7xj9 zQF?_>Ev5(kfItKp+&{%sj?*y|sng1BkT{@->D_{KrZ42rd>SJgJ9XHZL;RP_YxI?K zIxOIXJT<3FkDwK?u)CFjgqKg`&<_f<&eYw>Z8gyU!O#w2n-bRrV$!TgCK_u?SH}-J z_kbeB^(tn%i|{i}%-_pw2^HCSUFM4MEJJB-*XyYrJmQB|W*mTK>|h)B##&g5aU(mF zG$W{&^nMmoftCHsT#yv~oW@K?9^BUBfsq^!!uL1RcxET&i`f=(;IH)j#+w4lx~5nY z-;vgnXNCQaI4vHi_o z=gw{yo`F{^fBN{dN-zkH=O!Uis_}6;kWZ>xb>&jQU@}4!rz6Q#YXl-mDDQS=0{Iv4 z(nGW8cp5ol?l&AC2Fo!ZHr(Y}8XVQVogH7J0Oap<+xj!wQ^O*|4Oc!mY{NiS`lpX* z&|9^x#DWkprv0x4B)RIZ+Gx298Wm{y`Ag_> zM7zX-3yDZ;oAXso(Iod0?;qsj;U|w99utXKTshiYWC7{Ql)N{d26{7%SO2x#9fzzu z>bHm%(t`|L*dgz+g>))#NB$}sa*q1iN?CUtP@$T`olZ#| zQ@qG_g%km?KBrXqO)Lrv^^AlQLo5V^@$ft5<^=JYuUFYgjUbCKr={nl&uhF`4Ev9; zra?St;VPeQK=L+p{{uGNZ`%@(D0q%3+NFw{n!_}aE2wOSF`fqB$FlFr265(eZTTgk z$4H%OB`8_09T8ZjP|kv1=4DP$FgukGbSRhbs?iO(WjmdH(&vPok(q^p}pc z6x9ca54n+47LgiHaMIo4;p9hrQbOq-43*oUqv@pS=4h8CXJgG}dG<1>`_sZ%qoPSr z%29(e-qV67)y2eg=}4!<>BuOdD;`jTjh(4}4C8*k+Nl@yJ9k}Epz(humBP|vlJ_ay zWSxr%zd*c#DFmI))?&T_e zJ%AgWlkq|t>}u%v%Bh=j_-71W%O_0P{H&Vw9gQ3rUlQ8bF9lKojp=B*-!Sdgg<5FH zgqlbZ>Sj~-oIes4;1l|zEk?*KsHYDZ3AO2h+lx`b^N)Q`XQe|geUk6nPEi^ZT`|<= zIi=to+LM_X#QXKj!!7P|5PBr#-7`ZmCe3~Fe9WS!pW)y!jWhU_wZ*U{d4fkRH5WHC zjB}I_C*xYR$*f1f8qNY{l+Ul7Gc$u2Cvg`f$f5p>R8tb&af6#v(oTZy!m!$CAK7*MNzi+WDjNT+W0Wyy ziWs<-sNWgjCRBmLbL~`!YlXGGFZ%1T6Phi9%QpjXKq^#v*;Kg84N2afxDI;zZP!!w z09Xa`^O}fRpKvzr6QyAd>?Th$YGC4-I@ZL*1(VdO?xa~i#0o97d;R(&U@?X-`Ao7U z_lf`96v&<1&APQ|(R?axUP+LEdR7Uw@7V!q^*`B=_Od@fk}6O(-Bg|G%Q{^<=t`{t zpDR0iQP8xKUysi+4O?CNz>Xjmh!v0AByU7s+!YuE&(uGs!nKn!m9CY+e7qx}W%x;jP3B7}g~WRnKk5AoA6xQoZ<{TRWHe4Ujrlg* z^GPN&)}d6-R^P>-3HO*?r-h~o!fLVmR8kv^QsLU0;4Z?!@R(b+W;g>Lnm1kWuq>T) z2vxR!QVV@dmD>c$FuJ+=p}ed>T0cqZ6KEkA**O~TJfy7Gd- zb*jM-*>X?A-V6-IDYn}nXY^yPTzyVL%R!qJn4|mRfrw&Sr&&DRavV81Q&eydJ72&F zP5C=-4ZwB8Br-#dE@a=g0mQsv|5i^%>{F3AFE=!mu<uSf&Sw-F1{T4feD?33c4UNxnIBAQ( zqR$=jy+HliEq*y)m+r>f)%_xF=Mp@7i~AEa519IJpi~!$3Qy{K{aHEaWI{TmQ`xIcdZCCWq7r3%aPEhcGU#@fXXc9 zL2wn&e%+)a3 zgc9{FQ9LN|46l1`#dMiXm0X%Gh;C|&luBE1ZG?ED#{mIho6doiA4>tz*f3%pc1L@1y7i9o< zP8x}H9-g+j-c`f7fv#&n=g|#mkWVzk?de#=fz}y!LE7CfS#Pp>b@?#fXxP`Q!HUM` zJqnafd>DZJjbOf;F$Eu>-1KVMN+Z&8jS5|U8D zDELXq0MkAiYYLej2j7JRI>@ zxnDcCa|>+U^wcjXlcsBoe{WS(Q%%J@V=!h+@+|Q3c19<%Lp)~jWSVRr`m?&BoQz~7 zHmedSrBRNH2b@w%#fbG))0aXYQ+ZUzCh~l%jX@L^??f}Lc4ONTfKq+ zFqAVQ-Dx|y#$UUI@af|3BF)-t{7BxZ_~WDv8tqVFzu1iDdVxV}2uNc- z_g%SiFCebTk*6wz5%+%D#@_=!h(+3(#D6iHV3&B%$8a2ZjOWOfRs~I)gJtsYD_(<| zjZK1u6q#EStNjYiVWNY~)`t^@$oOpTl8Q-RA-ss{xwFD`cx_luTBhWHCo}bYdS=%G z;ilrXd7hlkgGFuQsn5fiy{_TV*M5|K#@$FoQ{z^1Z_hhDImcQH_a0q)NYob$Mf5UbuNj8lrIG%j_~Or3l8>X|4MzJ`jMBOYOw8) zPvG|3%}sUt?^jNzXzQMOjHt#1;E74SKyXM5hkj}5YukIgg9Wwqa|$+3qq zNd5hnkry2`KJ@)Q8yXEQ_oAuVpw!g6ypQ@D5X{N8v6nX86m+sXYA9#Hq|` zyQ}mSZi{;_j{=AX@}x)XqToRGPK(^@L_LbJJ9`EGz<~myv!p9?jC&|!yA}LN$rY1ZuU1&CuJAA zne&!nJDx)KRR|{rL?PBoTq|}W0=nnt9_P)JX}(S$onP-b2Q&Yyl-l2EKv+^zw#K^{ zr>$--jMLM>rH{{&Qz=U_e(xOr7dRc>V!3N$I=bNl)5p1t;y>tTtE)KB6i}b)Xa}Wg z>|9@$LHdeMqXX)dZKDV=oGlNc^(cK;o!i$;y9y*;nw6~;xD&=RtTq*>wmpI8O}Hb) z&1Jt?wxR6eDpfjKCQ(i2nULc#wszpDYFElJ7+zKIxjbywtnF-UwxIqO&*A_5Qch<) zdOR*GoOW&`XPl8&AhFcTH`TvF_eU}NQhlKf9c^gC(tkZsyfC4>6oYY@reS^g`-7fv zK20~S2DEvgi|<(I&YK}rH@g}7QUPr@js4>SzeWaNEO)Ug)G9|-rQyTR&xZ;UFQKh-LDx+P$I2{Y5N@a@TE#l% zR)fFP|DV?3xdsF_R_TmWB;bV`=kjNnUeg~9I0XG!973hmdqw`V$|kvYPGl>3jA=Z0 zQ68Pz-Y+{s{b`c0dz$*Vn-DD&vgvhLL`$8pe{?*7DN9BZw^UN5)BkV%!>c1GKC!>; z7)RbGpEXToN8@d&`@PQ+7LSr#9lwEEkdl{IFe-fzPt<)%cO?&(L*B{}WE94dq1?^W zPML>ixLF?RdWgRQbkLC3Vrq%NVc&d|OY$ZcU2i=38@Xgki+sA1dJt@^GU2bh>3bo4 zm%@k{w*}3W3j38Mf?gyTl|Qto!)MbFS?4=t+kdBjZj&BUHKA?<7cAD^&~lFeB>Rwd z=g)OAlXhPv3Q{Xp+5zylLDs zHYWRBM31JcFS~6`KT6hB_)i?OkXwygXX<%-Q~L(owaz)+siZ86NordO z1TSBHa;Ig8CL{|iv!ygsg@rRcPGe#+t&Ogh#K=o61C|X6(JS;Itgd(0#Vc?>-U{F6 z)SVovSn{mlA`5e{5m|ZEJhn?SN9#+8%J98aY3?y}ANpNWvrQ8Ls$#MfV;uu_Hypyn zFl4O7&?i$?!27tJ8=M1dgPoIpr1^YZ<`+^(ShdHwOEK0~>}Bb`-p4P*TV8x4qY(T) ztf20}Af7u>QS{e_e*8k0oZq~r43pL*<3!b6$UA}T*dMr`KEL@)vDI`dccAkI%Tf`j zuaWfhJyS-;2og^D69{e!L-JCZwf;C=P1sQJx2b>=O;)MBWaADw?6M{B6n;z_LHJG-S11oQ>w%@JXu~ZoC^o!^_lA@bQ%t>@f?hH1Q&m-@X??%9u?^u|Imo!A0Cn- zCK8wq%@S>Z-$0XSWEy9fh=NeT=c;wDLuj<2HQ$F&`bh31Q3A|9%k)jJX~2XGEaENM zQ}io&YA)_bbfJ0NSB$~$Y`oK8u}htJb6bG`qYfP2^#eww^dG5?mz<$ivhWZIdjic) z$JO~Jyc2`Xq3pnC=jc+#OnMwzhkmcxr@AfH*NTZJaksurOBtK_k5_dpNyU#hzhA+i z1%<8rb4)CP&Q16IEc-K>z_{5EoRwc%h3wMH(oj|lN|4W6{0jr0Z|AK$Kc;VR-7#8~ zXvalCElb+f`=Jj{4-HmDQ+XfYFAqvI9*vl z&Q;WD(4ID;xKETW+pco!5D*p`1>UgOiSJ2#dQL!Oo*!#=p^SJXwrBS=YJ=->HtbhA z;r&>w%UY%a@zub)=Y5Z;A;>N$TFmGSF?Q!&ylFrkR{<@U8-=K|86C`TP(wm7vsXlD zHGTiO?0F-YX2e=kj_O|+IPV@}GmAu8E1FKS5qhilrA^g00~YHc*Dem_uBL%4!fN>;|>Fx~d53EE32E=sZzG~?~v{fu; z-y5XHpqv149xPelkYHUkCVh34MLT^)(i;X|AqS;C?@-zom>&9R=?5f?*!fv4MBOpP z%MSc`F5O#)z}q{}G6l^MUTz%^DPvW-+W>XRzRr)0ry7(ZgIU4E8Kruq*RZK)6wQ8B zcj~~h^}Zb$;zxXQGH1xE0$XN`ZfI)ML7AI~$5z*fJNbQn*Dc>o|b zWEEX`F&@BsjT<-0+c}U1ml)(y`KuzCq5UDQ}u!sI5qXld(zPe3;n*lT(NzN}OYHRM?AdcJN_^c$-e| z^j=3xT=H2hY{sZJ3i}ehPY!yoAjvtM^F8(1NbAQ7VhaJZ+q|^ih#mgTb-y+pAza<8 z5hYBd9^XvA`o#xoy>V7KWif5w3Tp@D;+Ggxzy{NWE~g>eFj94SnV|PQD%)}RM1@xB zHkxNhc2mgM;ZKr!Pgn)rFWS(08f7vX1TdY6X~TL-uh=s2XLeO zGb=d3PIOZ|m;tip;6T!!x4H(2kXfYdU0aMvD4g@N*px&QLR z6kIafDvv?|u1Ud~zJD8cgESdE<9jR{U&z^_vCS0F7^Z8D?g5f`SJVB3Eh9~!DKKYC ze~?m|<+0I;dc--W2Oi#_xlkxaEg53zYb3NAaa zdIA}RgWpf_Appec!-k7=swvs^0sAOIOChdCO{NWk>_tp?7~`eZ+!rzc(Tz@TYngb< zqp3G`>3}+2T(_h+(8ZE(eY%_=KFfCJIjYX5Z`@~I$YX%_zUz}uZFLRyQxxNj2nU7J zY7182$b(&;8BP16QQeG6>kgz&%j;=V8s6~4DyxTUN2}ArAY|gw5PoMj$;umbz8Q2$ zUgn89|<%rUBEiTjRUs( z@pI^YS43>Fs*YJIDJ?CUs-1_FVFZ&q{v=>9%((OAOMA?|W5v=ifu_!P*}U^muJwsx z?JkeTlxc+I=SqmfWAe(Br~ecOZTNmSY0Pl#*^I65iKUry`!*45e`-^)B8JYFe@^og z?Q#s3R5-m1T!YLiP6Y8`gn-@~j?aOAnf~=Al42W7i!xKAxsCa_3;5Q*I*9Vbn-IE zgW5l={)jo?Jx24-=JnAxH7_y@T|;z>YPlM*c`KF4nys%?iD>!RV`=1!TOiO|yw?S|7YGsU3eVFLSJgFgH>L%EoljGvew`35f3L^=bk8YcvP2 z&W9akdlLM(%pB{Q@2&KIT}YA1^i_$48rN+!wOZoKqT+1Ok^D|N^r{8A*APLm(;qVI z-h-Asl3m!sz-*|y5{m`T1V&`NVQ6VYfz&1LGFw2pDT7P3(};%&?3upwGEu?Lt~)X& zC4ydi{q!TBG*o<{&e-01^CI;Vlx=pdFjK0sXlTinIAmsq&X7-of1iQBym?aIMR)&>Vvq*MErQ>oXjvti0Nw*qu!^kJG9i5-gyeN942XR&N1CX*c&`hKh6)e zMG%c))AUd|?woT%n9QS|iGLh;-XqvTUd(hIIz3ixT(0@N=L9eUltKcN4H`IAP1x(( zBn<2+)r@YBLyn-EQnB~~AJ9u;WBj!gjAJG1BtMxoHgsLzc@DTD-EoiRTndf9rE)(> z)_j2hHudE_N=##^S}o2c(N3IFt9e#0VY4m`DXKmn?Dae|8p$!?+N%R4Gl}!z#7+~B zqwdD2J1_Tz3Q#KrJ+p3M?0}sYH)uetJ z@g|(sJ8bKGdO^EVe-N;#y{v>cG;9l~_=IJ+S59I?BhFgx6WS-%7|K-FkMWZ}?}+~; zZgUs~=ua)c+$x{14piF#g=kMjTyAlrMRM&~_ahT0NV2qF_E?MJiN_0UY@!iYg%|P) z?mDWx++=O;BK>pcMJ{jX0|@ZN+`JWvF`Z7sxvp>Fr#mMEGyn(+O2hAl&A}@zQvKZA zcL{@G#!g#yDXKf{mK+|sFNO6@_oy@=Z?Xv*8!pwaZC$7ayscF1aTDNO`LSZE5&=6g z?bi_M%7e>?E@91M2WxsR?X+Y$mOv;NZ0p&QEojh7Z_y9Mwd)Xcq;1@&Z2Um}++j;v zF&}JxoMcjfu@YInt$s&>l_kUJA4U}VVP?(zhS&Dj8%Q>&^QjhEj!}j@JGmR68Q96R zqNDgqVcxe-v+{0wqLg%@YAiku0kak5ekK29 z2~5X$G~TP{0emQ2v0W>vd3tKnl!;KbsLcI3(3N0KPfopK(K3g6<~tG!EU zxS=xQYr#qVSoeAH<~p2>`RVpefpEtjj%m6FNQv~hzUyBH!V_2LIGx{~=wg}M;Zaf@+8z>UxvqNqiO$}TTZWaOoF4AMd1Gq&4;BVh`M-kSNn>`?1bQ)3Xu>csjk$AO>{+4aAESf9%DI z2j^jf2f3ZFp3*m%paz5a@3D^`4DK45PgL_02b~t`SIR^u zTHNrE^~ixuo2-W!AEcOQfaZsvDkcr!pj|XmE&nFgC+bdP`d<|D5(T4|2*+gW#hKG6 z5LBYv)rY&WHRq1oTp)Vm;N-JBB-|NN|Km!s%|8q>2f zq}j28g?zC#Wez z+n<`nKK4DdLZo+$Z`)QpgfyNX5GtuI9V5PCv5y5&iN;xRGwopg|Di$TTTz6e{!FY?m$;M>9N9D6nd#}8?ea9Rk} zx%~IIJ51o8sOvO9uf-arM7 z!W)XOEq0^Y(Cgo#Yv)l#|I?Gg^~f)gnec{*6XVB3FCB$He4vmPt?~l5+RV4{ilkA> z&d=mdJ6P)o*hH4NGvG}V|Ksk%_mnasT_OYwp-GtXw;a!``=a9UG!neryFs~Fjqi2V zBuNcuB|}mhYz7D$ju^g2iT`|^3Z^Ven>;aqt4kn0Sey87AnxnE+)eU+sP|-7SS~Z;H#}Q^ zaF!d@hq1I~(HUyFdZzP;{2$ufdWLhh-}W7ll_8|3_>D)T`5142J+ESos@I!B2S_FS|5G8Ydz&o-29rYVS#zeURp}R!VZ8 z3$KiSBOMDQu?z;QZO|gzS>%{b(wk_TG&4vi{9%q`U8Wk!2oZc`C0g%;^@n1&b%9wC z7$wcfghWJ;UqXkwO$3pT@GA$xDgRU!)gys z<^l=mzKJ|fJroMlR{U`Y98v35<*YFkBYCsB+VB@KplK1akqLxsF#p?Lh&CZ@Y~ky% z`RA}Z{>nm3>-W2BKcqxD%=k}8$8Q9Qty}I9G#GNaKI>!XHDTw=(910>xB>pA!&TG> zZwjxR;u#6k;FTZyQ3i6VA=uQv5*_h&#s`Mi321Sz(|z)A3estHPJIn23cD+^)F-h( z!7g!C@OLx|1!mIzGNOP>>V@S3nr`0J6A=D0+@9y&@}kCau+!!tTj9`e$P9UxOCLky z!K9RIx-Z5p8;;W(VY*n|TC%AQEs5Axwy}c9y51}ERG5Mg^#-s*HsdNmTD)YH0qq?k z%Ly^wh+Ba|UZs7tR{(7JLj5=)k+y^GN*h{vw$-U2egQ3C8_xN2{LKj1Cib_a855BZ zdky^kkX$PyoNurX;nV8gGJHxV!{oMnV1nyg%Rg$EHUB0LQq1JTR%{rN@u}{Q(F_C`!*X^8^)y4+%WIe7I{xnd#;P?` zFt<+qeBynoqnLj#yGz1ekJIvY5zinfjM$`JNY_4VZx-pq=L!#}u~9I92-3i=Cnyxw z9{kac?L*l;uP+8aA~!6-hUeS&1!5_`tJR4CJFKia710UyENk!{GcO#k4y$+Mm+_(I zw@kB5n2FNVJ4Jtmh+f+16X{z#Hqtu8?dL&2B41QH2}DK0`kwh4t$(D@{naVheYi=1cV z_9o}qk^I(+(8mIr*Ocg0M=9&7WNo*IwnJhTKXJ&F#x>BAIhd#Q1RlIr{zdDoy`CiI zQxVDlM`FA;zTuD^u=CKQ8s|8^CCFgMp^ub$DF9LG#;Y=#q3@mP;x;u4kB}{?OtQvx zwNX8c6Zh_=w-wP2RUIa?SAIq4p)qpv)@l3i#z0y$s2f&4+&6FVEQycAdiA_DmF8pE zk1LX7Dla6QrN4XZJxFa!iuy&Szo{W@G%=31A!jE(HF0WoG)>ZmsGEOjD5nabjnnb0 z=t1;w*6#jQHQAhiw5m?m!NW!bQMCWf)Rrwu^1lUfhIKSOF7Zm)8+V-k3&%1rUyX*+ z+~vv6#Gj)D0O67f(a%)7B9Br(b)nO#hEzK=ZYDDGmRNnO!HOM}nXuFkvQw>EBMk&f#YhPSm2RGDh9D@UyqM*_>lnL znT>bRP>>gV%xhT#e9v=H=Zf6|P_Uz4*YAvRI3cqynqt3c?3eA`Vk5ahec`b?4H@Nl zTYXJ0_&&r#r!eI4w2+CM!p? zG9x7T36-9rE?kkzec-v}7mftn04hu7CKcCtC*tjeGlwm#z!iFWk%{=;9 zwlzg{h3bGsJN12}pYmdlAy$irx^8Ogj!pClKV}aQrf0_07GOp7gp$oBYvurxyrT5rK7P@6|g(yfEk(_`gT$RL@Q>$UYkb zexlI1$GH-CDI#IY;A|LARG>#QlNyyq?Fy{(!Eq&)mRx#PgkAj*`4ca zD~C)I#;;mexM8{|uVsU!7bWUZHZMC0FcRCk(Hu}PwvKmaVcf^?i*}y>8+1Pt4Y{kX z6$Msmm~L|ILV+gQ9)YmKV7>tU0lOgT5mHEnCWMWSaOQOd31{)u(7hzVHd8OYGOIf(3qmWPYdI`@3 z?y2~lGJe@jY8AM(HoEQt$`_{B(Ya%bTX}LmgW7$vqyvlH0E}AIhQAxFU@e6XC!KB? zEz#lV^M#?1pn&{9O>R36xsUNs{q<0%toBcb#fgMt%esIog$c=XQJs{^kZuzV8)7^HD(< zZsJI~VULmx&?RXOaIYfs3V(Z;11I~*maxB{(5Js)@*KM*%YS3eD@LITK4GL^jWW5p zbi}tOn{G*9ndudRqP})Y$cwFbU6mrn>(zLD4Q-0+l-T@zRmib)b1|G?q?B$6mkE z?_sx@^hMz0tck(Kwvl7-k{EHRF3LX0!&zMVfrEuYzvzFbm~hlk`(A7!FA$6N7~ z3wQ4S>drJd_!tzW^Fj+!U~X;^j?b{J3AgzZD{>b3oy3}!+JlFxlw-5Pr*9}|B@o0k_;)2X=p%gHA1MYMJ$TT(krd*CjbCLc zG3J?@u77B(qaWFKv*MT~t%t%Iv0%e1T!PBdjjj*T@-cGGBQl0)6p5GRM_WPXLA|PC zzq^vQdIxO#l77{3RWRf8cd1isan%nqtf4*G`r7%fj-WOaG3HofxI|QYCWcR?^v8kE z_@ASzJh3ZozAp)hAWzU+Y3>5q8-orvW!B(hd#ekt;0d~OkLKNmcVg&TeER=ISCz`= zi>W#~Xl>c@BRV`a*!g4uMZsr__wZkWn8x_!i_Rh{sa>!AG`oTJH<;e;IZ(D9pX*w; zGNcGdHat6P(2LR#_L^rKw}s#jgpnL8YL~@oNFjzO@iF0kt@FkzvV~i#6t~A#ZJ@dC z5+15+eQUIz8LXm!HRs1N$`Gm|X1X>}=X~p2(?R?%_FjA1sK@#i*T0P4M9W_|vVl(v zK$D|CV_@k;=ADzR)pQPfYZ)ABWUzU3vhg^^~+V!lC{uuFkZSx>=ln+H-nC=^5M?{1=`Rs zf~=cSX8}&Yx&6CzlGz*eJ+iz?3KGb8z9KD-(0VGwjo_QpkFE0BPkjT9Skh((;!3;m zT0`I11nTCO4x}!NKH1_tCf!YkxF4>B{U-ClqIBbdk_PTx-Zb8#*83pl+n@gC>+ zp)ob=2VuPnXXBXAPq+Vnk-*L7&gZXa(wOsQlxV;ajL`70tQO(Q z@w7r4ZuTTm-3PCwA=p#ikTou3Y%?2QsqY zy5ky0+(AUi;jmmJ8r+)b?GJwfhzoq?B_=SM=JEZ{VPFh`?0W9_lLa>~ZDBX7i7KV*fn^M5a7Wi_Ci` zHkFPlC2=;FN66@I^4tp0?k#$u?r%zX*7o$Q6H9?LMx51`Xz6ese`T-&ep?k}J*+o? z*K}Xr5m981#8I7abv8{@JQpMle?-Yxj%2{&uU_2Y;|$K-Y;YtErfaNh3LU5R{&F26 zX;1T+ah%8S^;JG!^1QtaxZN7(J=gSGun>hgD|H$Jp%MbMdmL$p$MJ~8t=ZH`f|7Fs zG5^IQ)08R)!sqP|UpxK@{g*o41m>Dzj1fEyta1w@6kaHsRd*iOrBG#E=x`iI4Bdj% z!7Hd2qrN_qf)-86Wa%-9?asey^-*RLqNCum>?o=NB#+Ngjs6@omhVJOCK)_@CbsG-AWYy(nBNV+~HXhsJH1^MktKb*phnq zD{2HvR#%DVQTD8#9rN4awFm=)KPEV2lXcZq?+=x%0afLpOg$F;3C=k8YjcVbnMJN2Qq5z)T$r79e*rNN8gW5kRlfAB!9wiOU%_k3QToR@`R*VNqut%xph zc~zg;J;jYM`gQLf!K4t@xcStOIFiPB==gvW0HRSh`r*yD2xyF2w0*sTID?XxYwX`q zg9dz{_{^soHH#ie&cz_KvVu@moeYF2g=YuvtGY)5HiNsnqe=D=sT=18>h*TqV*POf zbXYwfd(R&pvtQc1Rf*0`jhHnkplqZybs%>K=Ivn(J=!Z~d|Jq<;aOw*wDzyI7zNKz zH`+4r4o#pnh}p9Ii|HVpO~2`0(SE7g1y>uLIB?aF7Mt}H2qC;Bw^wtuyC3PV=rhx4 z_1E;dJPD1k7&}bgrqZZCidc_aP{`?8Zdfhf2FPtKd_w_l2&y~s?`frY6>D{=oRkZ~ zjNh5>>EMI=r42rhvhZJWBwaO8DR0tM(8|%7qcwVH-s54_ zCbDr@>%&lO()DNPr)98jp_{;*_^;xI3#9{@DNjtDq6qWL!ENRxbkRB9ObV7di{wze zCH30u8BzXJ=ZRuToZ){@9?oyKTZT8#BGta9$H^pEvh9>j@Q4}t&pO#WSb{7ilX)_* zHp0Bu>MOT%5g2*l@0(hseqWNMx-32u^_{N2Nf5=%7J@X6n`Q1ReH*{P+WWZVeTki} z=i_Vv`w(c|%0C2BjW<=a?#Eoi9L*Dt!V2f2ls0Tr+p2gyVmFn#(KQnFMHl~bUiB*%S&sHY=ubfD^zr?R`Ijf@s zu3xl}X~*43{VOjIw~kN?f?)LX$i@)z@SSJQS#9BYs%AuaHs25#V%=ufvksXD@PWMv z!Ig46T#K!{75piGG-djdQlxja?#ybx->LgP@rdyE>Y7fGBa=V$#n{u1zKrOAOwX`G zmiDS0bW8i(txCe{@YN8Gd=kb#i}BM5=BsQuDk~Prre4DVS)FF4j2DGJ5z9{=V&A4+ zBfodl&JTpbll)|54bNGUsJCFqHDwWksHHcUf)une*ajP{X=`Dao*7`kKj~%7&wi9> z!z-3?XE^-=2V8itah6+7r$}wTzAE;D^eiS#-)J9wl!Jn5Q{=V)d2t!(pN#4j1RkKu zDYJf`7<1yXDXJ_(FU~-|`I*m$l4}TaZ)^-m*8h!ww5a<1l6+Jgu&U^L!$v&5!R33&FF=9%dA|Ciw9}NYA%GNd%C7e4f&wnw=#>aKEW!-aIgQ$J!=g~)Tr#Sta=q* z_vHhw?QsWXrcHVA#~tulDK6|?WD}il-A&VETj4ehRB=8xF9j*TrN^Z(1-@ExN~rXn z%hNaL4IAnpL!dfSyDwn_5QgrIl~h{}eTBg-euoB=Wd7p(*k4pS07a?8uyv3OPuRZM z|MkV8CQfRjsM0-M#@obml*<#!BLc-`R{A1kn~1M(;TT)ddm%fx))OinVo79qrUG`o z>yT}`=S&D|#0~6In9l-1ZbR>Sd|8BKM1NU~do$Qt?Npr$VmkG6yI2`p*MC3o**$fO z6`1~~>%1n(=ms4#tspllJ`;Op1#goNi|zP z7cQe6#l{cJ_=1~$s%ZPN2tVW*~_2A(NWI@lz z@%!G#U8^azP=%s_M_RAi$Uv*Vx$HlQrQvs6&j3GzR}?BWQFU#L=9^E;c*CuSGD~2+ zg5AvFT>uw8b{fUMe9o|^S#!$iYo0!k)AfN+Vi+luzDkCRLhIBOCne6V%BQgPmZn_! z9r)UY0yo9`_#@v=AqmW9^!l$>{ESx=o6YZ9iSOvs<%Dm%;Jz76F$P+6EZi) zny25L_1m}xs8pk@{DyAE6NH&w)$Q;J>gn+1?+ru`=AD6K+kEBM+f)!TPnF&`l~YMD z;T^*Y@*>~qf4-0k->0+Dbn>Q1S~T4Y{0|MEfwx$3O_qgE!?1d|Itcod+>=dbD3-Qn z?B!QK#r;{gkfEaaGDTe0E~6fyQrUgJ6yB_(#v)vI12!)9>(*yqgyF7_WVj24(8Kjp z$+RdOOGY{8cc;w&U4O+1`cw4bKodc{`%c5 zhAVJIPYSOre~W2&AFrCBs7`zj&^otX5}H^ZVO|=@av1$$EFgEzJPmVs+jC++ezVjn zHH%vSP*%I3l7a9dt~{lAFbh5_ah&JVfAN^OvfQa=U`ns|Fh_`@u1dFx^=TdcM(bN% zbNeokEvAi^yWtmxGe0s&Mo;IVncVjJmJu;0!{>@w_|Q@NRak4w*E% z7SyHRT)-)dIB%|L-s=K57*<1X$&IkH zU}+8-wq^gK>(5sVo{8ST)8A%F$8!+%>x9FXN`!z27mGUrv;#1ZOjf~{m21KCYFr9C z74QdH!;2Z^ZX&#=;+5GAJBv$3S(<*MHsk<0y-#}u!Ap(=SM_fSKoQyO!wwQ}Y@^F^ z%Qj5gA0SH?m~dB<{l43Kd58l8X*NmWFI=S4_`OnC_(k(j-MqMY1l))AgeaK!C~NH| zOMFz~O8Y>LrG`yYT^ z+<2V~_zjH(YW*BOu$Ilhm0J!7^X7Ezn;l2-gcsM?3ls|}aC54>8`?z90pdp2-{EA0 znbGY5{$!fN#t(ziflNBzslJd4k22`qU0975fKRW>(c?#{eV-N+yzpWsjBh{`>6lhf zETTV0EEIrAzCdSP3qqc%Rnv8Gf49xpSHZZoOHm72e7P zPKlV6xEp*AoDO@xQiKvM@A;$fAfD&YMEyva1ConDLq)k`66H>wywxhWso|VC4fj2e zv1!_?TzMZEMrsX-hF3y-TYA+OB}dipt3!(huTUUQRwP;V6;8oYnF;Ebqlc)SC&5Do z+!$x5T;Ph=*XLgE{h<{CRPP@r{p3Z6n0D`J&s8V4i3FZ3`F{+s+Jn-b$0N9T2IH3o z3EZXFcz#NK!3%_X)(L%^Cmce;rBq{O$wAOGhq#UrHw+`WYX0xiEj;~MfkR;!PTuLK z(zyDA{PGCqHOKJ|L;0|d22HzrFkVx#u$<>+Z~^(hf0zB02L-3l*%QsWo)~kNGlvv> zD_aZd9wVd`P2D9}wsr_eC&k!PrE$@Bya!6TAxRPgenKvMcKgrZ5Mzpk&xvvi!$@Sy z5sUJ;LduiQ?c-~;a8}71e_QUQP1N>_YwH)Yz2Ko;@@0?Yt3g@ct6)lwXZJ! zng{74Wd--z@5#~dy&=p&bE+A)nRCZSfYzQWmvy#NOoQtI#O6uNNB4MO^e}&@^2-Ez z#)4}W#uo@IebI)M^~dG{0fxox(1Ze9ht%O`Y51KYZeD|j zi&XE9+^Wf+`*qwld~4=RQ(Obm*`awa)w3K{U`=ac!?w-W@zLVif$_ASSXr7lE4>bH zG(6~?=HqFq$&$pTTQ2cy;K|*KDr(|W%8(hhDg7mZJpGB#fP}hQ-DHVfp@&!O$d{0k z4(qFNUf9XvYWtg%?7S_;r?`MF&nCsCk7ZBg3;iOH_ltNgQPp>mhVX`s-)&ol)68)7 zxWUYKX%j`KZkLSeK(3YF+k1kEtkaVd#x%&CdT|?ge8c`$@CYim?K_qbG?Xu`u^Ro* zgp2EZ*Ejy#D!g>Zd)a-s!MvF4h<0bZgQ$5d<<>4H?xVey6|NdbPLk23=tPc)dWhSV z@J{N4_5H8;O&pIhmAbW!Q|@jpT>K()&CSiR9;b z+^i{gaA}mh)#6b@X~|x(CQVdHDA;z1!BrZH5=Ki)buPK5zA)`m=fApb@%{x2Jy~}& z?_sg-rmPHOmZLQQ>Y&f|r6u(i1DnQ{OsShKN8VNVNBq48BPxC`hJR4jA}F`G4OpjS z%3;|_6$NjYT;X5xnmnps!z$UH`?ay7lO)D~ylkZ!13!{Vs{lNur~aHFs67)`DWmv} zT|02}4OTu!EPib55IM)auUD@OdIB6D{$Fy?_drjbi_)ArHL&I2y`_vC%@+KGxn=v} zWQV{OSR|KoS0H9PGe)-1S2r{m%f;}x#UHa@IB9;yL_%F#e2Wt*^fogiz7nh7L#sji zE&C+g@WgLSB?Ar!C{PVav@ ziv^@Ge${fq5za@$3)$QI&m4g-Sai*PLzSAE7%?-}LY~Q0^Y*I3gmF~ym!Txt#X{Yq}5w=dIzQ z`r{kqytmLQkH0_?=J9MG=gG z&3K&G2UCkq2#gTuF$OCZ(%reJnVDS-Pmvhin%$(fC%!6X`{5=SOf++=Un2*ZlGmDe zJ>@|W{`Jv&tq@HzKN#d?$LabtBWbNzaE?>^Eu>O~F*?_jw^P!{RN1P?ey>y{b*}pP zRPeDY?mT8+=%cPaRpYD@ zr*fnZ|Ah1)w{&I=nta5nHc^DJ` zJNlq%J7S@EuOQQTzZTb~c4uav1^K_xvj1i=+C3-`NRH$x2tBG+Mf?K*Gdv|N^+?M# zKnhihu@EF%G;6(;CYE(j#(Qm7a`mD&0)#TFXy^Duz>eMNyyaGudqw%RNRiuwt;}8- zj+92i9NDdaNf^ssGq)i8lnRFPKggzzyo>hlje?*lZhdby%oX>hujAE+V2FgAyJ;mm zfp=Sm4?2!Mmrzu-(w|G5NzIR6dizKbIlH!Jb?@&3Vk;FE2!ipk56c%ue2k;)jNIW` zo(<&0F01eza)CE7eq}o&#vP{{b#sIHG{+Lu_2j&=&w=Ohe_HGICm>rMn#h^lxBwVV zJu(n7P}(M9o(VYXOBKML{q^i;2y)hZ z+aS!ug&fKswKO6N@;hu;(ous4+H@i(6CYHZ_g+UX8L<8?w&8(U&+z^1U^@Ooi<`yJ z{!Bp2!lR)Cm&D!QfQIc-!;MS_LK^%1Lq+_DQ^C(6 z5+3W}Fo{;w;Ws-o(?s3Sc1UarRlO><<>`an+y3*%>5Ob~yQ8Drsehn0G-(;8!RB%q zK}KZUK^hBFs=@>1;Fay;DC?BoAD=!#jQg$=DkHTKa{_u3{Q zLVU~WO7l7iggwfL@gP+Og}A8mGd`ARHt-KPEI=4A4JQQmJSGaeP4t^|Iee((f11?z zOMrix<@#^|2)3zR?K?OF1!W+2~rlh>SOvw8J8L9qLmmz)OoUofWwbw#H+yFHw}JopvZJ zz7b@8N3uY5ngAg8^Q?KVPeUxjM^YYl19*|=okTy}i_*rQTu1l3^YGCqS2RxDd zPD5!`$u!3d$?Bba#+!z`6Rh%`uZdgg`xe&obMM{aB z+dCsiDBFp;P;HT5EG-#*KOj4hfTZ6f*H>%YrO~p0Wt;-t4`U}2&7d)H0 zl2r-HP5)Hq@ElZ7ZD(ZZ7Gz}C<*RwQ8UT>Oi;*837b35TpWUZxl1wLoJ z?$djiBGk5LF9Y4OVj!8~)JSHa*hM{GVtf*kWPXf9$e1d3bx#Gq5qN}eL119`;y&?L z_<_Av2d(${Y=AQhZHtQLA+)tl<<#gVA-P;S;ML-U3XW{P|F7J?@XEe^=~`uvGREm9 zVd)DH-lCMhV?r85VEX(fa+eHkf#Lgg?HUMyE&0Sxm+6V{*;uGtq>4B#Q;z;4^M@Hw zvGY~+PReIzsP_6G8fIy#wHH8C?N#?(bFSKZ5F?&Hw0f)YRz_!x*uHM3_Lr#~1|_5F zQ`h1uf$*RvJo2VASlfR;tvZ>~s$SNwX@{~fx-xt)-R&zevEaEwr+zZxfHKPf$ zEJDRwr_==MjORRpN_No$v~;C?gbb`Rr9H5nWKg_3`{JuZQ8f%W709I#a}~q3j$lrT zIP&g-L6CN+t}ZZ5E?xuUwQ*CGbKW9!*RyLX5N}J-O=}Z_xPA5aM;KA?HQ4r|Z=>?d zu*!z<&@>4&nIPVY3IvdyOxLU?DyP?^SUdLG{U+K)->S{^W6hU*Wz2&h86Pzi*#-7mU#FOYsea0p2|6z>b_Bc)%Ls6wO-Be8b4Mg+@*m zyueAPx#l(_5Z9nh#ZJ8#h}Uc8pg;^}N>%iQD7K#R8}}kH!jjtL-?NY~8I*k!x^-v; zOqihPVGEK#$}V@u_gC;uY)`^OCs`;UCZsuC^OX^94>re#TENY@SPYI-uZ1C7 zePxgkKi3yHkj^Xdqy)Xet0aKpNELfPNw_}Q`e1Dj?wwuYbluU>WsbO-llcQ@Z}y_5 z-QPc}t_?m)-}76}QNaZu*_gGKANwOPc1&ykFuY>~Za%xt_-8tZSh0D5q5v5`mR?BW zMY-oyj&LZXU!=kNzv zL5)9+VfV*NgwTA=Rd63V#*%qTvr0bGa0s*Rv==;Uq324$$lFbJn^+JEZ zz}OT}5D-Fzp}@w=5M;Q)#&|ZivHSe^uPDZz-^=&=d3%1pc+DLMzFGalYe2NiW0V9|Q!N%7K^@T!dT#_mrGFvcPswEZxFodeB+DeP`MR^*x|{ zLA@j%LPn;1Ufz>QZxG7rW8K1u5m@40?nC(&u;H?ZAJRHejG{%>t43isYA1y*={$Y zZtdp`QUiMc3One#E?w%6@H$cF{eAuYXHb$kd)SkoLDR+g;FEh=uMy&vG0W5-`PbJz zvG5|yfZeJu#$UCCqLle3a|7|nFt_bKs!3qYfaf3X!dCF%c}}c5%WvkXr$?e{;)t7J z*eV}L#!U#4%YuWwVTb@qTT8@tU|i0y^J_DNLoF^f_&!5pzO`O|ALax^39%#IhSCCb z;+O-)HnR>yOc3f-JwfWd>=U_h2ca?hb|-I5N3cyHPcNmvL2F?4S2mlW#5ynZeq|cc zyMN`8*<_rWt)5xBVummu&6x<*3qvT}j_kyF5cI6m8O!bxe>7I=lR1^dA`yupE(miDcb4o0+M73XsohA93;g`*LHkNupf($I^N+ik1$Z%RU7sPxQ9uzvvTWNx7Lml}VsOgshP+i6cUj zqb|aGgBA*j<>iO;n?z`$M}G@lg}h52ofm&z4^r>{^!DyLl%C2qx8CQ~_%3Hgl$l(> z)=KubqYB9Wj(#6$C$YaHw|J)mkb^d*eYL6*I8EDCWOxpcE^RVap#t=Qr|y|egUkoZ zGnaph{K$UJ_>~!ib)wV~&4R0lQt?D>!a)J{q&aCY!el&$09^l3bu+=SD)ln<4nYzyfm?e6C6xz*KfI9Wy9)*WFg-BWC8t?LE3?+Ecq-gp`ng!C|PHZ<`% zMZ!>%fxRA5HLjMlW)+`?(lDNrzzKwtVf3yLd(&{T9`hHQSGR%*35Ey4g+Q1?J$EDX zi9pgewq3S{C^NHhI_}7R%K95F~TH}SfhL~L)}sU-*2_O>56*&2*~Tpm1W z9KDS(bDR$8N?Sv;3*pMG-55=I;9Y$YiG_%l2)|GM9`H-~3c~=dfhxGip zWHDDz!@y5+c1HIT15fT|@4GRILcyD$R)?eUQ^%sC*?9WLhc35&*M~)gSjy?;8^AMM zXIr(KV1MD()lz&R|Hj$;(T)Lk))mg7G{3o z-~z_-sp0X}QW)rxTdd$7Lh_GYqm6iGtvaJ+AOx7ynx8^HEOH;HWb|;7@@aG*K+uS$ zis8zQyqkJ6(MtZ;=hFL|^^a()cD+R>s28#n!)*A)V-*N{6E+o_Ii!I+E8D>9xqV zi>%xHaAP<2%N47Qbv6$h+V-;8fb@HG@r%7}Upfq9*Li2v-}XQlQ>mS5yGcXd?Q=jd z%_Wk>WYl;ihIVvs`*Hqz2yl$}{wRj2(MY{j=;5iHBhv{~5!Aim?i}`ypvvX!he1rT zPlZmkA1m<93MBlMl~A%Q0kJG*LKJ53)Eegm^x(`|uaFLN1@7YVPZwW`0l`#X@hWft zND7^5=AIw6*Yvwtpkd{q?|5Y>(F7m+GE^LxOVf(v6?UjO`qb$mqSGqb@5tBc@!`P= z>vwsf5u_nujDB7vuIfyRfY!YUxQBWjK+!a<($T63$ z&xVM|cBP=i3%_Ack6=<$ARM^Zkvi9cq^%EUS8{(FTHUCk2?uNvPU&T31n(rG;KZ+I zc|<~5>*+DoYv_?u$_wlY^!}o|)_2eXI>fGl6{&fsR?9xh@B@qRW1J8;_(XGY+1ba{ z=S%>;bhn>y*{rZjb^jrxy-NB&x7Cn`s3`B6)*NIBllPcA8>&cHs^{{XB)G<>DqLF; z0q)cX#l4rz!FCX^?z0TQ$)S)pL)cU}ohkK|1d^^aJf9c=Lh|I=K0&vA2_blsJx8|VCo^uB9*ouC%9%eH z&1s{Ex%x?#P)Z^#XmMDB}OGBCPa;48}-Kcdvl1N|7j)kHT%xOf5M3grdK3Jp60_=*VUx% z<$R7E#Qt_kY2z%+FXKw4=_?3CS+Lmz@$i8V?-YaMSst1puIjrB3`Y{|;doqSe+And# z55_lDC=aT<)MX@MhI5(=da+#f<`He!ZmcKKrug|J&6m(uVH7cl@Y-hK7L!npZG>Ey zrD)!>BN!s$<3mDmZDbv6(n052IxIZHiiGSDIJtxOF8MIUSK-U~jGy->oqX75GIR!a zuu<=~$ry&ez*-eucGiGXrzwwC^JstozB1e0@HVg$MNPVXUWlDP^gOMiH(-r!z1bNa ziZq`nGf_jpj={Z)`k4qqB2G3>hYu;zhiHy$Wkc{%zPS4`y%+yWd6{zqI7ExvZH1B0 zH*xN^5hj)z&}aPC7QYZujJ{8(o6`lY#-py=T6n(VqXByj?&u0~1=BD8Qpcs;$M8f-iQk9J}b+3MQk|_P7yo0AIm; z#q`zEd#H7l8M3Rq>wxVw?8qtDc)|Lsg_p8GvvhPqq&$@eTm9JtwF~xCP0E_-u>u%+ zMQQSJR?|xIy-bHIB5DwX@Tja$qq?!phLSERStNe%+2;}RFAJcy$*ZBX%oqSiCzcz% zfT(5K`4Wxx5o9XU`e)t*CRQaiY@rAURIE<8NH}%Xt;ZI^as!oPdWIH<2Und+iE&GG z#Q@%Ut03!b%twLXZIC?tLo9UaUdZqMaUbXw+3~%pi&P;>Krkhq%FbPI6+39Ta?)M= z8Ja=vjvpLVKSQrN(_?AkeTpU&u1jgE$B_c$#S8ZG)ix;x`C0OjdEUg2OY&C52RsAQ z9scS@vksj?qVrE>kmoMs4;^#|*3ogNb7`!bajwnK@)Fst9?rX|!CSz~;kxs2eY*L)-RTRhwuN(YEGQ-6YSiJ!Q+sBCe6Q3ahbA)xwR+AySzerqzVGUaog>OMwGbDEv3un}pSqSSw}t`qoZ??M+27P6zPM=X{PWIH*$E zBpNT^iG)6(e}_8}NxJfq!wfet8r{NQ)h8j}1t)DDTfWB^ALu7IZu>5eND+3{uTE_N z7@^3F)T1qUhC2&Bjx2vod$F*`L5V7);8qMb~$p92CsTB@8P#_pqbnU8ULPS%+^FnE{jp1@bxnFIm`Z5{2v z!Zv+6nB?~aF)T$MD?n3?(s<>(q-ky~!~}J^5C%KYxj~F~Wo)fhtrgyxVO!GAa}WAT z89(ipEU<5`X?^BTKpZvkxJg~+Pn_sXyFSzvXOLvL)pirxlsZ=+6UlM)uVF>fimlpq zlo^9&V^$H#O31=AF**lYr6~1|gAlERcc%7{Bw*$ePbbY8Fq}h2Y4a8o+C$1ZUC<43 zhwS2eDV~s!D~pRv0ptvl#@WMtk1~k3++3*%0b-M_4Tx%rLP=GwacCP^$JDb)g}oJI z>K|BS6)q@z9j7kmm?H3Zhs|G(9EO_I;+4ydotyX6^4v#wxu`R?^tRs?#2w31hhEL-jo|c_v49f_141x;vTfsGBr(y|eq1o~Dz-vP^nUuEGYJ@j^a|%${Au~{ z8E3X0KCfwAF@rq>6F;v0#|M;Zgk*SU=1N&K$1EC!(C(3UJnidL5_VACI1y`#hP!TW YqWw@MI>e5rr4FaE1y2sno98s||DR3q#{d8T literal 0 HcmV?d00001 diff --git a/regression-test/data/compaction/test_schema_change_add_key_column1.csv.gz b/regression-test/data/compaction/test_schema_change_add_key_column1.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..83f1ecd41c002e753c221ced963af1ed3b77bbaa GIT binary patch literal 177558 zcmXVYe_T`b{(tv+jmm_K3>9IO8Qp6_YRZqnmYH$O63dS*Ls&|NN-`1I82gneqN0#e zk_k*CL!B~z;efFnm0uFbK;`Gyjxb;dY-8-l*>-le@9XS-|MEP}&UwFIKc27G>-l=U z4^r}9ctQMq^seV-9oU(9fSH-HYvZ*(^Gdcw}1aVGi@iAnZo)pYj66B@RUsU zbDx(+pBagIb+(Px_QKn1PGd;lzuMhPB)P9|GXMUsqoWN$7H$%TA#B2fGZD<&vbx#9 zqUq{(Bi>%A2^SiA@iY;hy^D1l!7&5Qx`PXMn{LzP3Ix$_%j}}7*P?6bz5{Bl@o+QubDClx|BH2 zpug>sM-l?0US*5B^BCn|m`26@nke%q<%p_J<$Onk{e_r}rfgSeUQ><6ifM=>97Bp{ zFe?^v!s{kH*RVg~0hZpBe8{w)T!F(!upfI_pD$u&%1oXrrBEe~F!hg{dhbMBfRQ7}oN87zhGagiwUl2B8Y_39MhoWj3I``vydR z^%=Iuij^Y*qiNC+>B6TxRdN0-`7>4pqB62XLY))e)SkCdB+fV8^cvL*R2xOTM$@g9 z)Q9(YY?quZ>K)V}ygKzW(*Us-W6>~d#%bjkJa!W%z#|1TB1w+yO+v7R6kWgX83LnC z?k}N|_gN`gm%w*rvQ3(DLiDiqL$_N{j6U8)v2s?3O-w{N(R&^#GEs#lORN~FM2c*< zf*%3f@xO`Z@ym)c7FaotP&-bTERWE&BE8YX>LJF4t{@vkVHT%GXXm|QuyWYS4Jc}! ziMDcP1V|j+;<0kH?czAf*!Ps*t(==ogV{|iF-1Q!^$`Nh!Z5IkJOvt*r&8fI-WM)v z*;KH_xxo9WsAV`#M7X|~JEs2=Nx@`N6b*UoeBbw)R#6p!^pdwZ*LWtn27FmzE+G^& zEbmj1-z0_$q-a96=K?%{MSj53M2lW03{#BBNE#9}?9BJDl_r-$H4&hz9cPJFQiSSq zRXI{>;}Oc<1*GBv3>%o7B;3jQ9{VW-S3fsyx`31tTyap!2jT<~6~*@8L@5In!t}+? zyC#ZD-7nRm7WE`~Ic_Qoo0R>dYokbQG)Wuq5|=`qUWuGE;Sb|FswVS1cbaI1%OtMQ7J1g#B}7sg+w3m( z+{q_WO=`ikhUww~R0Rd@cfv*3MI;jMuI_y%SVf2Xk5M8-eFdP$>r%F4BBTJ3dQ&#W zz5OFxS-5Mod6?*#NrB_kCH82CgXiMOrh@6w-%$w#Yb;n-pWP8JVvS{*cx6#yx{FsH zX^%A-><~XW(Un5gS+UAHZffE-atDHS-Yzul2E`-vp7TG^5HE`8F=?ejNh`oYx)x7n zO7V8M*|B8Id7w;W#dZ)zGrwh+DuhcK$|LMIP>XUr!Zpz^!otF70~`_4+Rf@Q5UDQw z6RO@+rZUK$+XY#~ts&g4^@=VAJS)gn@w#Zc#nF|R|FAqoCln;x9Ip z*utzPLW_60;gw}QkXeQa(q6~SKNb@ok;~va+*?U`)-x46 zk>}(r-3et*7Xzuqc*+>BZ=)M30*`$4HmR+}fUoCei>%%9DOL(|$6zOE`hiJVt#^>z zKi6Lrm&&qsOOykap}kWwSau6DBVl17gb--ns~NANlKKp9E2Cgneb!PcSnUcnSZh7g z?C!-Mfbg*Uszs%D9p_2s)F~qSNo%Btr0D4#p%>uZ7SxhDf`PwmMqcCNu30|eiH$>w z1oqf0QjrzuSQ|}5`gV&hbYL57%_*^rhYboSu6(7-?)J|8m*roiSIQI-0-nEjz=t4B z-LN^C>qA5xh9%i}@bJuFeTmPqw(}NIiyp`s$@i^Lob5tvKxh5rOy`CHf=(Onp_a^Gw}MGqI`Gb&WeQ`Qkq3LpHCMbA`OTo zBeED0P%?ejdJz!^)+LM`cpOU&Se?+it_NVBTDy5XX13kwaq>aa9DCj5El)Ur)?m4a zsr8*q>lQIjKpYMsWz912%nQcP{14v>uhhu-n3Okl@(lY6a>+eL-ou14ONh^ql`HH~ z91)33R4Wrq%}dNxb3ot0jH8DcJF;9zz|@VQsKNTlLmf1FBJyI*(HVcLbf3o7ZNwBf?X49 znKuI5cPOr0WlnOhX@|y#Jb{U~LDSdC2yk67nH+Tik=E`CVvkB)TipmbbX=0AYA{Bt zI~gkKg@jL1=s%82`~<80?EmUOLGDgQZ;v;v+vGDlF48a;k-NmECaR~ghV;SeJK@A$#t9IU`^kZ47l}$x+`!-g z+_papB3J!lDoIhcP9)Cs63f%IfuxQ~K{#3^2RV8;_KQJDU8EKhY2*4{l#FJ%;ag1@M~KI>oSx&Huy zvYuJ3px74yE)FR#Fo{g`L$Jv*gM=_+RtDZZXqmVd?vr{?I<@^O*S9Z)Jrm5^=ri7A zrfAGLE{PXZt0q?ixJ~;AqhJf306(Vu5Zci~0QA7qvRX|tq|b3l06wC3K~5Z(+_1G) zChsMEm8~-g~>wKVyOH^5vki0bnx3B^Ch0Fa8`{u3u zAZzzyTf~owYo+zz3|n_u-cIjsa-Hrw{v#p2q$aRqGX&}VweK zW3Z2{-}@A)Ml+vLe6QLvB7}9SN@PM}h#04M{_xo!O*RQKSYT>DK^C*mQtBnIY~}LX ziEDhacAji=`W`ghPoJV4Q|3Hj1lYeZKJ%Gc8%J^Jl?Q!3Icq=9^XZf{__$l``#Bja z8uH`zh{E0kTY}?OJq;?xNyx2Vd*r zo@~(v3@E%t^N;+N0_C@2tm2wlZ$Oi7-=ZhDKPx#DBB;L26qKadN{ZQbiJ;ICa?7KE z&@bX4f0S&7UppCTlXZWne9vt!7S=EiSi8RhpzP~m#WSxla0{ZH;Ss63Wx1_5iFnMv07|S z4;axlm1%jTdep_(AgIQ=#k(#t$0fE)k#@&q1QecJrR#S_gnm89eNi;j?ElR4OzPUB z4(QGAm>QQ<4@$2yB1lPYGYwAXa(fY|wG-JP($)Lk-7_9&_^;0(j;1>Of8&|Ui9-z; zPcd@ci4b0p9_+E)HlOY2HGfP>iGPXiWQxbyUK@?9nBBI|CJKgP_SZ(YHOuwWnMD zYL^_;KHUZe>F)5fFTCNq6URaZ7^Hr-p%y+r!RLGzr8Iyv;G&W?si@eq2N_0Sq}RXa+8=miEVM#E59E&aT-XK!r7!`gQIJZV|` zI${%u`h4m`Vs%IG>rtsecC=|yZ9iyyUe~6(;rm+u@kB5yiscZgzBo@-tv?;*V_Ft zGs*GqI|(9Lz(B1jQ;JMP7qNAx>~3zch|^Ud)LNXYl{?N$-*H`8KD?p^g*V)TVp)2I zm@+Qe?cUO0Y~(B+ms|$^$^9yTt9nCMFg^ zK?ry4IRo@V?8kF_$b5LQtkL+y*H@&z${gXZV?NAs&yfjLU|fLR^}rZuC*!e0B9L-# zxB1Mgpmq##-tCGYuf@Ch2w`t!vKQ z!SsP>V$vC$YVEGKQVbZk#t&>+uPtd~oCK`xK?d}A1;GaZKU;>)1rjwX*p5dSU2%k> zAWp9+2CoRJMf7BBDT#*a5YdcK7;o)<&-+Hklc`nUb&}|-m^SUl%(*)C*}RBJ!3AkE zXR1iG;0|Zt7^{UMyFLrB-$}2*WS8UE7ohZNdl9g<>$u__C?pg?KBh zL5AIr?}@!#9l~6u?uXKi*6#O}r8PDAHrQzv6hHj_0rdBA$wB3Pw0GcUDPXXfCTh8x zzJ~m{JIp-Gfr4HOL`~6EA%LwTW(H3X7sDny~5-;%vw$a33$7xipXBM==~EMj!Ge$RH!Oi6 z;E9xyY!e&j_-u4Rb@CRZhZTd;>CEjBa8)k)1s^z|DVIL)ddaJK{U|gzI%%IM99iPp zbK;gzZ-LgTnrYzHw5OgWpWgShusL8Qm`Jets3`{$n#+%aPmJG%4Kwr|s_f zPJgLov0(KHR;2cvCFg?llU0;dRVs1Fw@F`dsO5!sNcYulwkVs;0P0Tqe?$A5h+QNG3=w*rRRBY;m%Ibp9!Yqd7ee6GOG)VqI5#^?4`5!wkO~Hh?!+zoaDWQ_raA;T4RcsiuQ@vxKq#^7;fDX z&m5{;g8jLjYy76Xt%@3B`Mz9QXErpx8QD?n+iF3iDa7|izGJ(s6*OtdzQ4Cp@CVSY zMWXfmXHUz3BZdgF3>n$tUf+OEkipCbVktP`XB-p>5?pDZ=bswrL?=|OEnHhrgmnvp zIV4t?6F|Rs=87`Kc1{v$j_;OKb(4>c1-y#ck|DhIAaAj5(BUGSn1k1^qV4l3=6Gbj z8StKUOD~||V20*@>MRpB0DT71s}tT=f+Ykw-ck2{MFm*mN}Qv#mJYg#MHXkRZ?q#A z(I*Ro$lY%Q3|0fj);QS1^A#cxNUSZaL5K1{paP}qXu>zezKP-3=9&apfx+BMZE$zW zV9&fs4kpa`yqw@AnXbpU93~A z4ZS!?u7KrYBl++U#U;m-IT;0@6L5=R*~!O=5#h-no_W?G^WYf-o*k=CTfH9)GXX3p(k8ac4hL(1C`pHe3P0oM7L z(w1FtURvk5cg#Cm0~k&6KE2~*l;OT#&0@(Z)2&X1;@VVj%KkD~mz65=STjLL2J=Sv z^Xv{ECTh^jvtX-p^#3j{$k4$K=RB{uk--YMS4?znTNdwTijllx-fD2%mX1`um4O04Uc^e#cw1GbDH{e&!{UmDyku!TP}hPd}`2 z0;y}2Hi~D?OaI_MKNVoQu6Qgo#uwW!!EZ`9l=_Y?>SW9ntk;gJC$N0BF7i(30*5Ko zx&>wwjxDx48cdy)dlHfAtt#g(z4AKoWbaBbh5UIt9ld{w?Mj5+n&$5gf?1hjVA77) z6vmB9?0eAcC?v*U?RBr6G7xu4J<4XuRK}|G>9x$&)-5es4@o-GXqpQMoS5}yKR**c zJqhm}QPy3LU#PO2mo9Q$K?bS6i(%Pkl#=EzR-}`^zU5!8m|+brC3T^ zO>(FnV*wWu+vkAp4;_-F#fP9%t&Zn)&|wze-N`V8=4~H@Y1N)W``iPo@n*Pk`PCxy zsSU8!ax8TW&grb2-7M6TYmAL&uC~p$e?X-9_8v*TDjEW*b<%Xl%Tp}4S1wg5nAKx- z>zLI$mE8hk1i6OcMq+WDiasv++Vzq=4~b`1XBlzTH1z^#GSP3T$6CPIo~MoUCdC-6 zMjRSh|A!0^GZ1UOV7c!P8vo&0I~;j}#UgSmc_H6(UtWpZRWkr&CFQUheqxRb!wAQE!=^ti>qhAi}0hD%7wZVj&bo`M7-W*e* z5{5~CF}2R0;9{A3*SgYa_zk2z>Eyr;t*JFnlN`_-&lTh{9cMO>5;+fzq`lW0;<{ps zu+(#U=ee$AS>&~7Im-utYuAGHJE8A;=w53r@v-Y z#C`_6<+LeXeDZ9~B(O#u4pbpY=zH)-%EQcNnkTSyWO@QSVxn9zCEicxsWmf;d z*YtcjMkPJtJQRWlkbpSBLiOwAwSIuOzlh@k72X7b%AE1I(-rT7{=JAR{07)JU6-;o z-jU|Jw_G{~Yhzyu&Pia&UEE=w=$mhQWvsI9n|KJcCj#RGA_)(@}t&lCk4F(Z*51x9$ z2n>Begso?;w)hSiG<8$DAMMCUB z^p-f7A1lk^_k9~h9^Db4_=qlU9QK?q$}QNz%;{pt%mxbnoW%?GfcO7j%6>7-3S`bh zC*8ZOk6<~9A-X7b7Z~;=l5;FzyHGhUNyN7-i8yB&CiP{Wiho)sgb`dy>@5CoEl`HW zC9IVRB`SDKkW~?Q-49++J0!d^_aq!%NF!bBS?6|pgZ++{rP3!mpJ9P;35|6#P z64GZPtwD(>Z9xS z3Bd+Jw`QHP9~2&4bvHFp?CaW3XFh;dD;*9b5El&~-o4k|ZDg!snL)Xkg?BcF(>3ry zzUo8mMHs|oMOxN%1w&^ECBd0hu*x;z$w$5&Anm%>^M-TR*xGtL6g$|3p-EB@A@<@k~H~R_wQoMD}tzq*BA*p}DEGd80e|F`8`cN0e+m#}Nu=NO&Y`#EDeGtkRpZu-je>DWn?tg5utN3h_AO!X;C#CT z9$P3{ue}7AaZNlDX5A|IT?SWQF%`s!i9arZQxEMVrWA;a*<+avOs0WiaQHE^w@q2# zssAX)d)|!H55Nv1`S-Ek`?xe~cdqhVysUa8MhGv*+539=hhYze5LM8FIp%U{xbx*i zdi~hHMdK2QgOL~@nhJoGauR4EoayJcR;@d`%)35gJ*=}-Jd~EFc9u)4dWpZC?WQh9 zZwmjtK`4dr{71}t^Pe?9MXBp#SXPvs#NwGrO{sUoInKo{d{-f{G1d-3m1WrDbkTHc z_fF5#`y2b9x9k?K7a9rQU^}Z^7N&24r=YWb5pA9g#z}+GiC$`c9e3(u}C*7t?o0C!^SPoIh=L(yW|?CXLfr zPj^k6R!#5l)79hfqkY+d4K{$tGs=(qcP#|)UR1u7zDEn=t&N80#^{=4c*e7K>aGlk zZq^sNg|r!r3r{WezLdUqNwRevGigGSUxVh*Aofm~_GL01@PhM@L-)G<5=ik380!x= zyrhg#;-SNROl$Y*K{%-V-rdE3le_!=Yup7S3hc%8Cq`kKo$@i`gXkAQ%axuh83B8I zoq11k#O62)D5q1m`aMYp7l6|mhaOyW!B%Dik9p0F!B$zihspLp;ERBui=}g^kHD5> zfg5Ha#2~#lta$@t|Jam8X|#90L9j0&SrqYD1__5? zRx@o15?R$d^fTGdcTB<_<@`ojqGJ_9Cb%+BfM9&l>?-Vy!I; z=l*|PgX;fkgKoGjb5)fZx&TuVf`s=%GOA0UO(`!yJv+%_PH*gB*4SYy}P}m^YnH+O~r2er^(ORn}dbC z49zjW(CJpwv>F*N@fX|u5l#KAdGgK>psUjpJU1*-UYXhFvy-kv?-oKcTf zJy>7AyBO_=%?oQKM*R_D-+^HM5vrSg;K3c;7ulOQW;Ie2CXd`TFe*#)`ziCL=^V#l z7-_w~qA&EICLGa3EpNoH+GGwxpV`hY5@qGVny7s!wPnE++i4j4@>R@; z{YNOTn4aUdwWcyvdPG)zO03^ zVOdJ0M!#~+CD<;Bo}W`_`o_(U?sFubGt8)Y*6l4UULFdL=v6r6O|0IH`5~TR)88xx z=NaAgV=ilsZ3GJ|`RkwQKJ+ z7S$vlM?31Y?9*A@cpHlq9l^d}OOFuBOCsH$(U?LxH+TGuAg~{H?@Ai)GseP_!hJoL zEwv8Az(4Mk8nR0A(e%No^~$$RTitBq7wru#+^<07w;T7aL776Ten#iA2}Wd-SeSIr z`<^CNaY}$`0BvK%b2RuibT1Es9)~1N#gTdFucwS&ZMyPv2!EcvK0?w{V?COils zj(MJCZJdvkdQRu5H7r;r>G_)Cf^}wfchZxfeS=_%%ra$G`s;`$W_G(Iq?eNiFPk@L z2bUv?8sjXl>5dc(xJrCY@)cMsVGutQHw5B2>#;nElkYT?AdelbpUU0#vzgzYGG*qW zw3U>j&yh&NQD>A}0d{!si%s5A0x{k0~`@-6il&(WSOD!m@B%@!Ub#3 z7;4sx%FZzy!edND1FKm5T;Yxt{tykZKd-X8AS@z{1zcMn1e<&*U-JV*rsaI=lHX4x zn$^*T{|PM<&W8^Xj=Kd2%dAdV{OpV5p%5WnnzbSVUkW?kKSJ0m00qSoOZhAUG$KCh zqeLY%uzJ&9jF_}QU^2U$`7tX2d8!{69zqvaOF zVKA!p{SI+HQJr`?=rNfK&iM)M%};gbL&T4U+(2?sqhf*a6@4}&p6cne*n(8EI@M}8 zTm)$@oZ;u)kU|Dy@=#9^F8~EQk5+w=1-1dl=KM~^wFsuJuYWG7j2@bwJ98j9q#DXD^7%zkM zGj6FmZfHQfQdJ=J82rn#zFd35x6iziuZC|xQar1|<$=z6^=Nk+SB)DC!eloa22!^o z>)vuh*O!u1LxCiWM>vlZTdp|_&8LP%C6KJ}_IiOi5^NjXyEbkS!ZX99BN@STuH{19 zTy$rQM1A2L!aX7yHNUl)^1WPKOAUYw|#Ry&m=LEGB&ZLAi5 zmFEQ0L?#|?FBLC@L~Yb=J~~+gadCm(`sM)GEFsjd{V<6pOL4tp)G`VmS3cc;VmIJd z_cAAR;bI%)08AfYJ_w#{jJ{fg2>>E337$0u@IaO;T))Mv&ZmABdqgC27+k;EY18E` z_gTFrgz#_N3Knd8bKzwjI}cL3zxny`3)AY8OO_U9I~F?)`?Kzp^$Hh28Ssl*HC{BW z)`-ze<6$&?TaOldC#TwB7?NNzuZHd979J97(iM)Y@RLnvb`8f@RS??ZIrBcYM z&4_W9gI)ufj=0Hp-S&pKCC%T-+E|rH<7nN+21u*@>$MMI@r)xqdr8OH@PwXR)hP%K zdd8hY_FfRN~EoY--Eg%Ob{5YwD6euzx`zuy129=Q8*;t;T z1XF~*?({E!#Beh6y6_iM1d=3J{Z-yLq`aDbY6b@c>Up+A+?H_$twK9_63d?sgVv8X z(E`?_Y)lC=V*W_d#+g=2T?0tEv&BNjQ?6G3LPis*ONy-Qs_84YJyelORDkmg(X5#$ zh|ke?6vt-OliB&cq44*z2uK`^U!#(-8^e$a^O3)!plH-ZKan>IUlq!?y=R}R2lOQj z`coUyiy>`XD;%o}2VdQeG(QL1G~bv)HqfTPHYG1bdB?rOGAz@(_em|-*?S>BdFC<% zU%zybJ_!S})l#-|78$)wEX=dPQ>`P*`=LIbTAw)rMQ$jcjh%$dz5K_0ML;~ND2wT{ zb0PqKF_-W$;4^8+z~tz;aT$q^ApBJw1XgsNI{Zug6SMkZ;=PEi5I+Tq4&osJOkx$D z$e=@caqXQ}2$YaU@5`A_IDq$Jb?5ZC$5JSEEJ}N7R<4_E>!vZ41Vhr4D@wB?Sx3%J%+iffkO~-6V4}?79 z;T9o~bkj8@d;NaG59Fc3C<$v)SqY&^jDJ3r2PsH@H|o4TAFU#;a7|C0kpB|Wd9&+{ z;9A|`V+_6W6+jk>!(onPT3r~Y|0*IEEl}Q1{#UORX`4&aG zPRGNJ8%4~T^LB<1m!RIaIt=ANjJEa9E4T+9U3_}Ju@U~wavj~gpMMTMp7U=ZHAnZRisRyp!YOY<@&aX?vc>aNA5fRl})V?68Pml5L0mq|s$8WM+%LurwK>%lGqdT{WjdQ=w7;Lhv= z^j=GQKB&ta0p1f_4tO981}pzsnLYz{v9q%`6^a-_Dz>DyU)TcGRrb%Y>o^ZmTC_5= z;4mt*j#N~^qORlae-7`4wN6S7x9tToW)%FA9c_#Ury^ZX)$aqcn5p!`cn;8hkNb`; zY%Zj&`6chF2-rR`H8cQd9UEq6qe*6;h%hTU94E&xdqb|I#+O z8dBjPhJj5j6>_7m8en_M4ib=a2l5A>d#DGgnYWpRA@{A4{YvA;BRmkIL+I$ ztlQ1%YUY>XU8HMd8(-m8_r2M`YDtbyZq{+$gF((~Gl3eQz~S+-&CH*iU%S~o5w<5! z4ZxtpQ4NwYNMmPJ(|V@+cc4kk?@GEa4$0D3qU*QJ;7`UNb#RTbiDXg>{os!ioQ7uB zcVQWEP;+f<_1%T93#n+3=X(W7KkOD+NB^*lVqDlRgG`^q3(5LNXVH)>1+8hPYY?cT zJb$OaaR`!KR+z|H?mreHq}tgN$4g&zkTZpRZ&&UzU-iT;}W?a(*$L^!`L zOB<5O={oxUvezI0`!&oc9vMu9ozeS#AYpZ&>jVE)fV&mn%+&-7@uQyIt0BPQLr!*C z3lZzKpRsw~vqPS)4M?8e*9lCmSp8w#cu)vDTg;t_E2$g;a#0_)zwiY_W11t9I{^|Rp%eyhsJXUi@|se5$qDZlU$G) z3+5jA8yPkE3?cqD0#YOVnUb&=sL_gip%@i=tZ9d#@|EZ4T8Pxlu>b=$KJu%zIs zo}Ig%?EHcD8j_^=hyF~|B_O}SPtx8|L5o*vOG|#g?pjsv+Q2ZBR`>PRHM@VrtB^jO;q#QH#v`8tj`dz9?rw=M(p0KxPl*Jf7!t&{z$$ zWj5-KC0r#4OSm_QTCiWC&UGmQcA+mI#9&aes)Zzsjru`H@hI2;sg<4fg9a^K`;u*0 zKQK8uHk*$DttDFdX-@*{8(3KE&tZ4tJ^C~6A zJ|X#0eXBfWa0DRQX}A>)sW~$F;+K)XV9@MPieD}0foy}JNX!A>a{BrD-H=aD3RpLj z)$j|UvD6M%+-I4h6WUZzD)e1F^Z`Gxp8e$PUMyUQ$8ZTu?*NMBNHxxLdyrVUr~>(+W)j6j1H;j@kJ4peH6PJ*k-Ng%+kZcjU%? z74$@K-q>1+e+5*Mh3Y;heg)-C_!CZw7(&|;#Ort(x=2XH=VvS`wXATnRm`cB!DUc6 zH>PhW1p;B;te@mfV9=1XzoI`P+X{NwTwC+1tAVmtAZ`g{_!LB*@W57jE|TbG z+f!?l(=UO!RNUKvvtb67?z#WHZ~*ePe;D^_2h0b=+0SpBfCL^wxL;vBgNDn#nX`X6 zA|;!T=GnC{C8~;JzrtKgmP40es2S2$+r>Y%$BE@Y1Rn7h>VYb3MR0dcx*p1cMtm&A zT)&Lev?j4Et%=2oFHSb{nt(zz#C-!Znn?JRM?VDQHOU*H|5?A=tVXB8q|VjkK;g3G zLg{jlS)bs!2B?HCE8z|fJ=?b8Tjrp`Y(8g@*-^r>t}zNVqKbxyKB1ZYGIaexNt=ar~^!$^FUP(-%GqgRvn-F}^X zmQrqw_ineV4V%rRf3Ijt~a4G3~|C;LJ!uYR}vB5ASB;x=;TWm)|@y|D<`jp!fR zV%-~%2iC{Va^D3sN9Y|Nq`wXRmK0i^z_`;MmXmcQ47h(LmKJ}3$jHlhVG?8`j>FuKt3w0$(3d7gn=l%@9EBH(Q!0A$d?N z^xV6K#n_fW@Km?t6#f9CxGGV3rE(hVfvt^-E}H|}dOr6Ak*TF5zENLqoJIE7@BSsc zJ+0<7JJ)!_0U$B^-Y%g-ZC&@#;@9kX!!n8Vos4=~C6tw0nVvc^q?me-X=EOx@Ywe! zXDeOQ-IG`2@b_B!Sv9wjo-kt$YNhtv=8q}B8dYQbCae?a@&9K zNVzNA8mN80X)N6ADySnkEvu$I?_rM`3zY~6 zS@A@c<$;A`);6|WSq4^&eSgqXm{Ej=bAFBKcybQtJ3T8U@!I2I=K9NDgPF{X#F1_Cut7rtcXk>uquGy2T9dm6G+-z*!{$PU5K zMNilDBxvJMfpyjqly>29Xw)?1?w*CUm*+eMTFUbjWKv#)ysCLiS!yl@^{wYp74Rwy ze7-IP6ydAP8%lt8rHDLF(fvkHf2AmN;w;QVC+YWDMyTIF5S{*0DWGm)b@CyIF1h=p z(E1^uc~-FL9t|P_&X(dIhcAN0#6Dk1kZ7v7CVE;07HaG6kGU2NQ#vExI9Ny)KHZB$ z_75QOgeY#b7{2mCuQIIe9f6i8Dfu5*`qV05^?XTTT>zY5NYefB=k#B~NFl6iKH263 zs}i=!$mZ9n48#k`!TtW7Eko-M1E5axE=|hF#_LAPxdf1a5cr#5!e&HD@YTRl=*}o~ zl3#?<;zYH6wJ@MhcR!zHErE#u-K=e>nFOXl5r`~z0K}E=Id;?HA$X`$mmUJP=*|x6 z;R2F$UE@97OE!sZj3ItwA7TL%aMrOh{#up!frhW00xC$T)LyW)v-T+LMA5-(X5rR_| z-Y_eo30ejCWAg`Z>S013S$5Fz3KT*6m2>NqvGyC}Xp+YaX@dq+Zl60-9~*xhM$`QI z_B0^J+26VUT?KhsB7KlLq=Y6(-=)-t+xCEY8ZHDRAx-c>T#x@xN{B@huY#wEgr2DB zKmGwK7@!XQB#@G$k!aDmjI?^Fpbx!q2?`yqcr!F!SH9xz=)p22{ec!@D8@GAp5(UT#jvMLW3vSO04s z7>CcaY_21#Q^0S_cwOP7zE1<1{2?tm4~;MMjV}s6E}5@~?Z)R$e14V=Mb|o?%UA!h zR_5aXHaMj(!CDet|#tgEScPTDzZe=CDNe5! ze>LoL;2r7w_sp-#$hN>?S1`Zqogv$>`)zd=a#m$(YxxcZzS}45$qza2Fib3)S#p8_ zBggU9Q9(4AF?c$QLNZp0`f|&5(885Z>B$1jdtRFeT)VS&dE6;2WUo zYqb(G=*>Tj9E>4*0^#ZhX({NtLi`w#!@{5dWQN|8Y^_Rj8aAhwXKW&mXn3Lb5b`)J zobD%@oP;z)?@FO=It2a_IS-6ALDO4$K2CIIvIFUH1kGISpD;7MMGs7sP?iH7&`)hMTF5emURRS7Xv`Y zCBRdIYQDtWfCWd%ry()Tpex5!9T>E`m=TdM_7>eY!zB#UVNNJ@k?0<2MO7Ij8MC%= z{S&ZpQQ<7cEpk9L`uA|J5hi%{K*StNDsPUkPEITDLz&b`S%XsG{>b|qVKxDTczT6) z?Y7Uza{t(s4z#aVyz(mqhh$?)bN@%5;xJf?o%Ex5keAj(Mnca(R9RM+3UI{xplyQ*9{ z-vWa(UFIU44o(XaW1ON)_oI%5P|BDSkF!8T z)s!s2MjkT-u)bfwoG0WJdbmn5$XM|Ko3LpRz;n8Cs_qP`yn5?X#dEz3NPMO{$-(5Q zghM`bw+-6e5J6K^njZ{B{ny0!i8eV5LNp{kE(EGT+r4JyF%dZroO38jMIPQ6O6 zq8Kvm#J`fpmO$(Txu_*4VWvlYG5!I0sLh?-V|z-B$_kgc_CA0yj3#DNUpI^F2TU76 zWX1S@9fp~O8%qmdEvrSVLNN01;bahzkG%ko;D58sK#@mhZPwo>>cO#!x(B^u7+H2+ zjcGAK>qY))+U|fC;bLdp+Fm$LiYs&t{R$N;cX@TjBeO{@B zJ@qBD&@y9F=zU-v#PpWX=sW-wlQR_b6n3j--V2oTh0x}}h^ehk0Gyc@daVe7a*?Gx z%928{BL+nbLj%Ny=wNn$3QnlbQCzRep5hN61>>pEk|yEJ8)Cx{vLCp4CMOv<2Ftdp zo_c^slGlc05y(76Ak|6MJMP`iy zAi9e71wUY>gQ$fchmLx}4L4~8c1bg>)+RcHCU4^?BclCD_L%Y6%*i0gxkk=3^ly&2T#^m;gMRw{oLOK zHL=8Jy>pR7(xg|<#{LY@EAaz?9FiQDVg!#n*+nn7Kf?i6(ghD7W$gsLu2tIZXftC&E< zf_$X%_iq>m5EP@oSb_u#ppom(A`VGb~gN6 z!VqMgV#zJNv*Qvu#i)(nqA4$uR+0f@p~)b8UD*;OsY89ILTQ)4P=Dj5yk%OL z+#*x6IwGBhf$BcWbPAB0fYhv{4%iKernJ~=Nfh@it3O8H4ub+X^E+Gr6pR^LH=Gsv zIue2$;(j1C{TrgaM;X5X+jg=FLY!==nmZVQrG);CjO)>aG1`m?{5b-=hc6 zMT>&!k2d=b(i&o(NFsw09WOz%K^hMPz{$+l)g_VCR^_h ziTRLVC>>6DN)wrnrf%=Q)X5;vAduodryVr>8n zCTr+GaxF(4?M=WL;XspSjE@DTSAz7td8~IY%ohjDM}zNqpuNm`_qyK&5!4A{IKtu^ z*G~oD<-SwIJg0!AwzXM*^zEr>3S3g_U3J>c4 zA4%69&}7-ZE5D@DKrMZ+E?77XP08^*YVR2IK&RT$aTz4vrqklh9foU7@$bYM0=2yrfZ zi;Cz{{;q0*`u50>A*H}M>Es~6_;+>on_>uI_>Xd~z zskL6from@j+_gS$8#E4og(Yz^(bpH483sAL>&L^(kV&8hKw-Vmha@x~d-GrKh zef_{nu?RY=J%f1qbaW*8t21-4rp?1%KeooH~rxHu)-_ z&>h1ncRF4Q?pNSR`-6X9VLXh}D`ihTU-b*^{+64yw1%^R$4ObC zO6>{$hidA;-x9tjagI^kwxOj2SkRWPiC^bbemOmslKr-;5>myZn^)&uM}c5E?_+GG z+OO>O$xf$!l*MJQ_jOje9adO(*be4irF@3H%zmFKl%3Y&1&r9Jz48vRd=i~X+!LG4)Nz&*eoX8G$!%0j$w8znr4yz?#tMHn`_wWiv z^S^4;5ru9BQ>MHX&enV=V1V>gUYvFjF*NNl*G ztCImmqpQ@(kAz^=t|h|Pf#>I1{+n^0bzyCrBmM-lhJ4<^@PPrTej)SHhSwFOCuKyh zu8DqyDhYOb@eK~=G^_OlLQW~a#Ve;DW>B#?!1R~Bs)D2*akH zDe)*cwroe=+rd28v^2Rkuk;eh6mp)?Zij7mX?*Hco!U3e`sYI1eN3O=%xu5ONbCU5 z%pDyH3LAzRVaT?=Uv4|%z(M+Wy0ZpGB2v%QWsjh?aFDA%1EM809<#;ISaB4iw|u@E3d@l# zkV5fkB|+s$cA-Me(*CR2m}z<+FJzqQaq*DBaC4sV)emz21EjmoBrc^OSG&)_+(k(6 z%&yLbdT7#QiBrSH4zmf2Jjo+(vLh?;qdBIZuxAg1CE5hmQ$S!C#*=OJRGfTkE`^&_ zCm$9LRe=>!INCIO;5q{ZOYHg=-Qd_t*}Q*=@acJ)orN7Vu@dhN4-i~7zV^B+-p@yP z{#Y->Sf<`#{~8Ot`^ZP;Z%9mcm2)kFN4bdVy^GCn!1bwYmY!?V02T9@hnI%VfWaS& z#nb_{kd|O)rnwKvm#)QJ+5zF{Iz!PHcq}PzjdKFYQfrs&>E14yFX0U)-~3X7L((uN zadCKy!TpAU=^M%*^v~AU&5%_hDG%MA)5S#85j~pzlywUBvv!5Mg{sF0y{neeD0T47 zw5mH=I>&*=gj8reZ6i~OG74&6JXpOSTww5CxhX;V95iQ-8(yG%qtS%s0#amJ*Nl8x z#)qi1`$4ORt8XS8lVMHS1Il-@|QXu%;6S{5$XH#o9V#EV;e8!$KeYh z56CRLi-@Q-?qivec;=cO(>0wtMin2+$zhZ-1HH1oyq*hbL7)$}35sn_m@TYervC=y zSKmkm{hdIomh#6*4w7s6$03o1ax6yYDEDa&zyOBUYnI@ArWbml3`A3`wneZ{Fxy9rgY zRccnG5$5YweCc~v!Voq%pUvXE3VBTE#=oyo#Tdp?HS4w_2XM2~ujdQ}gN*0b>u>T8BI~Dfg zw5;uPmV$_Z3_VYh86hVSOfqoa3sfI;E}Na{3%gmhjTWGOC*o!BBv!^jl`pv@5n zQ!gTzh>BjBFg5}MR{H+FX9LuQ)^FKg*AQkhBM?~){(xQ+uSW$~kbH>Li(}u+wNZP+ z*N)vF8uicljh}}YLn13*A9|7ez^T}x5)N@UwFE!pnv0Zt+qQ{DFp7?P5NT+R3q}byr;-$_| zd@uKIZir!{Z^5)UuH4ZE7^rqmgKP>b+1S6Vs1n(o6U+Ea$~6RQY@s>N`o}oL&u>hO z<8RFYjR~i+$oH}`!Xu7TeRra?4vHtW<%qB6Pz>crJ>N&Ef&Tg~pZIxrWE?5zB?(Ul zSC+EMrx_Y4l_I6`hVdD`k9Ec5x4n}9D;{zpI^A(%Y%LG-?caxKg~XTjwYT7mnx1x? zC)fxxdGL$J9h?)0sG|eM&tY4p3=ivEaC(={>CR0P^5#jhtSE`6myUW6a%+0PNz*{9#r5GwOnrvk z(AEe@bRh5cpHSX|$;GfL#tSal00KIHF?qiv7__uy32W2U29B-&fU5mwUQ!II7+P383F zyD1tng5bUFt8FW(4?^HJ_db#g6DLRRB1m)+w^B#E;@VYk`mquTro0 zd^bt>FRm!PofQF2*3ww@qTRQs9|dPc@+lL&ZQST_9T96AUz8DV9%p{F`A}k4FvR^z z1!wvc&QYLy)j}cIG0UqA>to$;KCv-9k6g>8+FjV`{aiX>6W0Gk@nNX<+JvGtnXtpr z;PGo$wXi=!^$`(?{#|X>WIqjRH~wMnPkHl!o@9m9p1^?@YkZ(W8G#H*_$j9vAws!P zkcs+(U#1A)mxsBldW_6q-+sByg$6& zX{Gy0*qYZdb3D}@xR-78%~;x9l?T7jde`@{5-u!z<;p_YSF$Zxu*P zCiVUx2#V$YN7+XcC_iLo_s&78S0dl&)dEY+T6LcYrtGl#~IyY$7lYvxgey((gNI#Om?%C66^b-@wR+MN&I zQMCn?M8m`npvJn)*E1@DlNEg+nv@-PKY_I`wlK1a4=MaT1Yhz1^9idoKj!@b6n5gn zVRthmZ5YhlHT|FlG9IU_+ny*_1PcTu6NSA<;x4f~>G>8N3U&6uGSq5WZpCk6>y@1< z2I#do?M%ICh>vh?yrk;PLcVCN(tVbOihA(*1(GrG=RoIJ-#Z_N>g@a6#?7v9&_$M~ z{8Nbus15o?J_xUGiW!m(`a;xV@-T~zad*%4(AAt66n$_#_mTA~<06f0qrzF{)cx_;w11z;VK-k$Hgd)bvF5CDiNh2bg zGr!5Zjj4}I?B>Onz;A{wRyTAO;DuZ(dfuV@b;zfm6DvTL{ImU*S{al8W$NHufT%3v z3$ovdzuFwUx7mOtndokAde08kfhFFcavJWQ&9r4jKfrbkT$TICrzl~-_LR&g2VS4} z)(^n(q5e84Yv6%WX=Oxj$x;Ra8@+GnshdE06y$u)kxyq=&Gb&Of@}1%^W&J$nDyY? zJzYle%0h46fusmJ?@EV|!S%2wOowtFhoQ43f*VYHIJWxr=5Gzg$&P@%`+boXnaoc$R5B<(TL4ZyFZ^I5|19dT1r`IqN58Rs~p74QnEn{xsWWy8d8@ z1gOo?s^&FthuN&QnkP}$a;?%#NAOn@uZ?Q&D9JP5+_zbI1z+El#$$U3khYiuj#WJ) zEhYkVhCM!ylz_`(i>g7dTL=AwPNA@H<-*wia%9LPicVXj$@csAm9X+XplAsDC27H^ zl8(y}?RGYJRqf6yH_7Ypa~qRfl;{c?Our%DWBHH6@Smf*OG5IC+FQQ8V(3t{iHiKn z<>8+)U4LFxXUmg%ILQ_4_$i{y~jgsz1g2#9Bm>x-dhh8kn6Z@cJRwgPN^G3?F^x z9%`i-yNjY5W%ck0Wdkjj*obuvzUC?BAylU4*~Jp1({9=`K?YEp`9j91jhYG@>z34! zAXNXA`BInRgd&~yjEi)P2)2RyU9~<~)KRBX0mv@P)lptON0BA9WVR`r;0T>uD`WqT zt0KA4z^Rbwc#^~xga`Bi+6)wbQQ%+Pk3}>TuFEN)%{_L-@Z-oDIR0wee`#BAQ0n+! z#NM$(1wZJqEg+W)OMS&LLpdonbC>lcwz=Z-^0vzsO;aY{%Baeh`a-AVcQka0x((~{ zw4b^S>+rucO`Gm}7Ga5@llUI(jTTqwYu+ei6|Rh(XAp0(^+gXU=hw}h`1t~?8Pg@Y zbbi2JgY}0~NCO25@%lna_iX)z$qgu9(nm^{B z*5kN5WJG^F?u%q{sQ-A%4KEiE!j?aG*+|uTB#GG^iqc&+>cNWzkW;B&HQgbEm?PI3 zxc6Bl1pIF#^~AUWwlTHjxaD1%9?LME=?I)8h>btFIc($+gvlWD;tyI0Nc2waWcv$2 zq)116y{VQsP@8F*b&diz_f>E}g$Fzr5iX#0bf12Mn6z@+XOb4*V-T5Vy1teIb!DJ$ zuRgx2n0hoORp)GdfO0GKOF)!1*qf4tW zbJx;f%B?bDE)qgE>gA|h5hq9LpbWXGdd3bAGNQ{Cip3ba(3H0bNndeUu@_1HjT@Fu zH&!DS*0L7{?aGNm8L;@4{dzMx^;**`_RJONOZV{YuA`Q$A=_t+VjyC^^3ABNU)JN#E;Cu z#x=%A4&=Xu@-aB*q&zkEHL^QJNjI6n)0Tgb)|+~Hx(sd-m7J;Z>w*qL5VRog-=z1b z#V0-YrV-vjQXH*=n-@u+Nc>1df{CE-6bA;LJg5-waevc|=);8fCztsmQ%$!dvXq5r zdeP@azY_Qz-p<&r;kZDsf#E1!mrn#OmxLDiD5770*40OmWyl5x+L?aG*V6#0le+77t|EnHPipolMDiIc>b}y=6fM4M0Z~qDRorcSxb1%21 z<}QF{6z*DIQV0gP((riS+BOm7niJ;Fm$iQnGjzx}>p&4kb4L~=4w&c=u9~?x83&W* zhi71h6p{~)BI%2;52nXtO)Z7g))1~3ZeIdK()VoGDZ>SjVsgvsA|EX3MAa_KbCT*E zc^PW7vvhe!-c*`Uh!od2oo>DgEViNJC21pJ5bfaxuT>m3I9QXzZeo$T%!lT>rUJb7 zQDr5czG?=hrw-Vo^8&)a6c)L5W3NU&>RV98rV{dh7ajai(Enpfwgu~cfmbO~M3?=6 zk5{eWG@&!obUxpE;Os#lj@>uH(VYIU{q?kT`@q&yOvjw*h)^SS|h6IM@&mNe2hNU)j=usgr+yTNe7c!#`c@u&v~tn#Nv_GRNYQ)`wL zX?o!*)1_5M)^;0)0Y~udrn;2~M_XBrk=@dG6#~4=l-P0Y$HdKt0l7t{xwRdz;vKwSL%AEamj|E>sjqCsJ$>OFO zPC)#1hX)HY2zgNbZZVRea7@!G8rY{tTa??`7(IKpAAl|U`Y#WYCM8krXRbg4SH(P+ zg~klLfcDEPDO3cobbHl<1VieUXa<7tT=Y53899cYSyi|_qMbcjW~RmdVORyA{>u!mB4-IqNZ9H@C<=2V%Z#20@& z+A%)t4wQ!5a-dteFuDqlSMz;xm<0Q+7%r@o!ysT)`d0{XjQ*4Ba$~!il+YvddV)tF zny5S%QNrtUQoV>0Bp^+Vo{7pt%gu;)tsvAI=dEW}gh8A7W7$_3^cW6yrtXsG=*3Mf z|2l<`W{@e16l6WLg1FLlI_5kI8wj^s=9(CgDniUZs+z8#VIck9<1ytDz8F@HQJM~Ubdc88}eh48eu>O?*9T&}E#hB6t=6uIuYwM&aszt!PvU?%A%rY$Xt z>4^>9R{&=qj2XqldO39@3~AR||DaC%!Sok28@bdFnd(p_#=a_>_8WFc0Q$I{_MK+{ znbqjIeR9UH0F>3_XxOM8JMQ)V8@{#ltD5~AdPwf=d2h@O$~3^4hR7V27WqqiuWI{syRT-}DKTc^1+*8664JOZ(p z5#8*df=w`#(wFjJ=+^bZjNJ2AuvA-9ZdCX(U}X+j2RcRMlB+i9g$>AY)qYLpW$2~A z44I$VR{@yv*X02mB+EMg+s%Bm%?CsJkQ9fvA07W9>ogFzP@itAxNRj~a?hB)qWi5k z$VlS8FZ)9Ws_3HTaO4*K1|Z)3_Cdus{^H4;bpfP8{-5ShLNmSgPPV5Ef~i~1UH!&0 zb5JQ1|4)`u;ft^AHwb;~A_(r9%)1{mTT(WAjZr?Vh*woi0Q?V?ylL!8Lpjk^^-@9) z*=bWIQ#XAV{SY2PhRUfOoL%6vIiF2M;px76PTL3v)0SAq2qDOK@bld)X%w`!e9tsh z9N@3e^aV?g(qI^7@xFwNTXZ1e63nTi9jFOi{CaQTEH`d_;C-gt1xr|beCDy{hd5A` zY`^oWW&lbiwWQ;`61&2bN+x*ZZQ6SS5+qnE)r;4k=2Dz1&yUjHMQ}9SS@fcolo#ce z;E^G4J1tHF8Ei9hg8RA+mM8XZLVEV<8N0dUzK>HtCfKDi(GSNUUt=P!MLP=UPFb9dNck@sFFgM z*5#YN#^Q{+KQwT4a=|SKkU}NnHPfgQph0wfN)SMY&FM~rJapC_I2xg)!&H@s$r((f z0WfV^rlkL}>I07E`7W{{PrJU#P5}(K(>XdMF9TW29`OY=STOj?w&l5~g_)Y&e+GVo z&rQ1C#_7t!;c`4S^D1py+M@4#%TbN-18l5f(?3|Up-ZgIk_#v>a`p3E=_mj2ya767 zWv^@Z6ZXbDIft zC<|k!Zc|pmp7LERAXnUftz0OEO_>%|Kehck7@It>a*R_m z1QT8L%a3lONIy32_xaBcc;o2To&8sY5G<*r)0-b8;_tIm)^D<=Nh}d|?`$_aAKMeV zBvoiYipI)WGN#1mjE&zcr%6i@0`bMtdQgc{!=Vj5{!1wlovx9=o-I*QUFj4U;ew(E z*Ic_`Q$;7aZjDcl3xgl-ESi;JF8;kv32@!rvB8o_)auc~kawgA zm67(*{?w<^@YjGicrsPtJAS z)akR|Wh{+oXND?h!c3k`ta-=5w3gHms(l${m2`yvTrm-jBfKYoGvZEZNVZ&}Tpm3S zZ#t!otrN7@aNIJUNuN;>@tF8kR6FrYLDfwalG$z=|DVfR@hK(`BV!o-xETtY-UY2s zv5n_SAI4*xqx<74^HO~2NuzyhWp;{8NCT4%=X;$|z#_Wo|4iMdonNO^*e%F75+p>!+G z0wJ)?I)^~$`>lu&{p!f+Hz-@L-SLk4)l}p~0aNBG>hA3JU@2rqq>25^3#M4SzNsXC zj{x$@&bgz$H{}%h8d_+vhR^nFejjOV}BTiD#iYB`@kj1WJeme zz@>Gs6^=o-p-QNi5vG-y=tJIWC?-F!nmSZVINO3;s7|+BV80eO1}8dPUpM_6pIC)} z>3j>MLRpI*S3c?!M$@%018*#5pk-IFsi$I`rdQ=MJQok6ZKi}(!xd6kmt)Rb7owqh z=FqWwILm5_qp5cgvcCw)4g2>xzegSv)%~*dE>=N%n(Lmpl!Ad<*OQ-LX(%+9SB4!z ztGAxfZ@$LSkZGQeH@`~=n>wI*m75esCB7UazDQ6&GP|$M&k=y}{plA4z2U35lN2 z*E;|2yINpcu}?nWkQ_0_WxKN3MquLs-w-8m`I!uTbv^eRxV46R;uJu8abAnZm6a&_ zef0Y5gr6SbjHv7+=Nm|fqGHq(l~D@V&#E}@hVT&(6j>d=IRnu$GUV;59VGX8sQD#R zMmZ%arhx|K?ZRMI&5zdvXtFeU#Qw_FBBcouWb2OuLt;d!|1hA)R7FFL42!-ivo$ByQ76R>JhV_lJF=Xk!A}Fkah?_~z!&bDpUX`DyQzm07 zj`i8Tm6ue90T`P;%Hu!>VzTmWK^9`WPE~hvrtrRgs);X)0W!~6+8nxyVWG~o4%~%` z4Tw5jtiB%;utwn-SNWN$E3;&eZxjOvvgyz0EtCdYR{9I8TpXnNJtIEk zxulLu-KK!%wlT%br9I?nwhWw#a%J$j(D`nQ2lfL@iJKXhfxx_Bzf%tv*@)>Sl%ey;}((paEIV4N*`6dtDAN6b%id@AX(kx0&r7I@?m$nMsXKH5Lx~cbob8k4} zqNb5e%!~tNduM0c9D3PhmUW(rzy6krau4h3kt5|ol3c}mj;k>$ft9G3=fZbDU*4Lo zk0vkE3mn+uIK7z^2{o^_6rqj!JLfOVsnh~3S6E&m3r*+U^UfxGHm%ZwL6gk(jNP8Y zqtOZ4;i$<}7-}+m7;BFEeNN4_%sGVoEzUQlhOQ-&=x4+1pe|E|;l9jNefc6m(oA`WemZUw6dmfgL?{G18L zPZtxEOPNdb;Zbfngq{&oBg`@lv=eR*lD8ufRT~>-ZgB7m6Ty3SP}dkxxTE(gLA%VX zni)GBfY>4`8t&tgZp(M4`qr$XSL(RKZkq+5JDH7yrng?0G zDsi%>9eY+uX;M*s@wu20j0mgjJj{#aXX;ShCxyQphzvN=caHfL;`By?9i!RzQPs1( zB<^FcfiIB%#5QUDpcEB+mv@PUP0b6b4Vpv=D3`n!`qJop@=YnaSBWAjzuvj_K?JO_ zd%n+d1EAK6$G**bhj7O>)vujz(qauqjEw&)M|I6wIvQWbK&DgSRqISKH#}p<(w;1o zp_tZk*S?e*jUUUwZw-Q z#~9Yvns||LSeo4m2|J^@UFU$;ig5Jo?X) z+?ScxYtJabd_2FzIHoQLj!e-cWKsj@sH}O(8TSap<$B-5JDcG#ZTpR9rm!TSk;?@@ zLe1JV=ig=*F|O?mn!Bh9GMGZUqK#yK*UBQB^ui#(Q z*5pG0IF#}*CGRdtozR6)PI1aUB7-T`dylm# zQ7!tOm8KKjixGG1i}}dG6MCLx;Uw|mG^DoS1XR3;Wknq|N1VM|l-ip|}JtMMq>^^DQ?L1SV}mtp(f zuaRMBw~yF?q&Gi1dV07UFtpoKwV^6f3<^3T+(&L}k()}bue&|Ro6R#Tb`LE)2s$ur#n%vDuoCkZ z+@TmUHkOmoMVNrY@47zJpXfm3N7VtIX*HhgFya7%hq`b~ju1YyCL`)sL~amp-^-r{ zr}Ye@o&0P=BDW-ehMTC9MvRpHk%`>1Sm>#xmYn`2J@1x6mKMqXkE?zNb7?wM*G;NC zjCp>I1`rlV+QNGe!SgrC487k0ExlN|ChUy%9}3syUj_IFgjIKP%I*^6BzWLDo}T3GsNu88E;BtV~IdA3E$+ zk$*&7ia-vi0^4t4_ooGd(e@2pe+C3Uy+IW%UVuX5l}3+h?5@fhnCW$KoBjOd?c8{(q+Y=oUhRz29z z`8u*{Zp+Hu>~W+Jrx1C7s($I-RW|sa3oQY;1RK|Rwh3`&V8g~T6Sy|?c{*pH7(0|PX``x&#Uzj1HK>$1y3cVayKJg zNx5-is0z~&y#Hyt{~HQco3+eR3w>FUBzxvJlKqu47BCBQSy*vx0Gb|f?sOjd966p^ zZ8(&o0mDQykrHCC>)qO}<>!b~Xs!P%%!%ar@k2(Fmvpj2k*TmeO-ZGB-h~zYr~%Oa z6YoOE(AJdwWeh#qc)Nu4D+))A1udd+rgxTu8+$L`I+2&6*`;M=Q zaIo@=kMt+$HXo!eXv=+1Bb7#ech*vm`4Hn;uYl@T-agNcj@geL4ZyVzpdLoOcz@al zl4{=Ym%EqgD`;dDb!Cs`0a36V81)zxB7IT6GBHBKIWQED^@{~BU+&$uozg5RWS^Hi zph3dmhvwKlUC0HKSM|LZQZS3=bh4y{n*NJNZK~dG z>^sK)C3m5vU(LG|ZnrHNBK#_5;wMp3#PuP@al5_LiC)X`b#ui`2CuZJavB4pX`0iO zjCD#AM4sPu7n5-$-&hM55m2{F!>S<+w(rDvX>yJuY*tfMx5fWG>gLe)u$Kwso7xKd*uC-Sqov|*J|G)5LmF2 zWj#+6^!V1P0>GI5bK+jo{ta|IW3Mn7%^GQvnVx!W9tQEcn0kV)V|XV>jE%XTG(Bp% z?N)U?A{p+b+n1$4JK!B9=g;oVC}Oy;^SxnAdWrX^QCq+aXjZBEHYve0@(*=AbblvwuZw#E$p~Xg0IuYf8EWG&GxE{m0=!S zmSJ7aQPS|KgQHeE+xkR`C3dP&pqslSm%Hy{2oE|#wYO>IrUuUDJ>d<>=OV^`jXMa8 z^{*G>R!nUE?QMc{*eASV)(nVL^?~NPgLQ3$P!{_e&|l`2aU}R{0doIZk%PL4#!D1P zqUP!3=>Euy{fFm?4={tny{czU79WhnNRh^J?Z&+X4#_KyQ>LQ#_VlgAyc4N) ze#*}?J7BMdRecNYY7q!B6MJ78w^GVAPkCI6**g8g-hF@6zyn*C8KNy)sGr-->C}ht zdQwHTP10P9TmV%-=pH)g?J;dB6Kv9gXCf*o!R6rp1N0v?9q4|Z964~`S zN*#?(Cf*Nhl)%2Pm{W<7&dxYc-4qK==4Qh`^%oJ?J2y?gNkgi2wtpAZ;Q(YyDqDf{ ziLyB@z1g$ggj>VZCj(`O4e#}&*zKOtw=oxBX64Ljjfw`lKxzI2b?7j~>1Y!@iA`x~ z8Q_nH!V<;%GpcE8Fn_Z zt3r9x3qM1*jW{a^#6Mqn`Hs)5fc}PyKi&5bWY&I`LMQ0F}+ zpO)qW)m!}bXv*vct}-th0sCLV%CxVZvW!LY_AJpyVwUw6>e;-(^Kk{^8DMyK0I#-~VN&PRYZ);wJ8;&#EUk$@IB}A@uN1AmfqU^#9MlBl5gUO?m z2t=-zF4?EtwV)7&2K`T)VLYkdEBxaw9~?lb!bh%-D%@ zSa^$9j20z*y=g52C)9Fe=Jj@@JHbJp$!^g=(Sr1m2p2$Wwb`3wzb6#XNt$5qEWk)# zp6-T7f_xIaH3t6bOHKe%d5b?S^p2s8+3duiT1R%urh-{ zho>q(zDW9O@5jF_o9#Yte3*hhEmLZGQk5NKny%j~42|=E&V=3dH4~J`<4x(eRlXTx8iXz!Oo=_n;gSUU$DKSty zraw(~QhTPvr4oCbw&-O(f6^anvt1`N*DOds)73B37SPPunU6}c-X*1H`(KT6Jt#G{ z7o#@)jaePY@L8@MvHk6DIO(G(yDqHHXp_)g6IZ#PN=CIPJ)4#fDEo?CK~365a-${B z0}cwf-+h@luS4(u8tOzbM`iC3J=z$oGkPRDu+^ z3|6jqZIbT-xCKKVR%=<9s5#tLOD3ljdHTGKz zXxdQeYtj9=sHgOq5e9Y0(JuF57w5~8hJ#uUz>|#MlE-8aP37{q0>*3GXjf!SCDWU+ zbu*{VRrV4gz)#Ug{t57f3*##e1X9?ni#+N&5>t)I{kb1?Tl7Ct8C2U?H1I}xpUA=T zXWJNEEuuE$PG!wXa#L=Ub;3;eE2lmSr!p?jUVjIQblgh9<@ZME>Ai# z#}(@qkd)H#CNiFsPi^+xc6x@jUSB)n@``vj9kMBn@`DSO8B_?d0Yz$ruwtK&j^WVIvr&-8n6Zh@*#Vo@Wh1S~n{6X&H&XMa z(D(HhkI#;JdEMmFPFcg#L#2&UwZ24bm5b!&v=>Of~z`2XJ3c`JTK<}8o)L_15EbpAwUF{(OuWm)_4P z#qV=oR|2tCEskUjRrBs+6-`;OGYf)zZg2#$sxGg+Kf)A3b{xnY1?T2u2u!|3GS-I} znLProd2r8yh|6QOFq@je>1II7PRby^WpvL=TKK+jkLNKCL>1-rWg(pi>t_WnToWyt zVDfx99I(`Y5I}7~7(!x#2MCo24y*`3r7-yJR{^PnfyM6sO-40Z8dr0^6vxX}Sp0GI zsZI2_CmkPGQ_Xe}e1hH#y~6V;9VuW`$sn$_SMR~Pp~@!Efup=OXguO)N%69qiq zh|;gvoifjqaM?#9_&H%!X}t0kLYa#cr>AY$=PcvDJ&(XAqjred+YOq)({A#&GDlKL zzTDkLDP4a+JkZ;YDrUD+Q(>n>PxsFdU zu(n%8_NI4}KEatc)K}dqf>i|sea5`lf~yd+2lffD>&(Pg3@Egw%dCX1_*^vX3k?`l!dO#NB)_bid~ z<(GYD1L*M1VRcQxNj=nv-2%f)2>?ac979GUS1wCl)nO>q5Q1?``sC#lvSSf{C(qjym0@fajMA z4s+pRMUu2>0j0R6+FQokQ14ep$LC~>1|vdo+6HQ=Fu%YaIg}=Yb6~3N?eEOe_Le=0 z@^S}|8k6hUW9N{V#xLXCAiZ|EcYoWD`1GP@!jfo86?(Oo@BN0wC2B+6lj%mD>sLTo)_c{2|BIT(63JGs%(YsCzqVEHgC?SnB-^qHl$vq;Zt@VeuREjXiR~Evhif(x7Irdj5VF$xB#gI| zrHS2mVVY=E{vqD;{$@n8&L*C#sQW%5x8+W?E)I=dzymz12i3`b~mp&6)2#F zh3%>x6&RxfCgo@vWlhzL+uRAutj)%=a@;VSTX48o%6knN1JCrbKLyg9-hYp|)7Y!Q zxf$bXnv(eLC+6uQ3S6F7igF>vU@A>7%rYcZTWmM52yIOF=^V%uAQ9!g85Ad>Vd^<* zF}k7>zZ&RzGm=1)J)ES&2wQxicAX%_tnaI6Xxs_-$K>dDKa|j=tf{5cM7`&Qi;h%G zKq1vI;Dr5TYgN~dcd^qQXxftJL8v~2Pmno=t;XK z$D;2N9u!O)kx{0FFE6@G|D%KzQ~cn$R7!~K>$7LdalE8#^IpfeQ&1oY7G$)LcgsDI zJspO_fIv3NogJgAyrf$m?T?b7*tA$*O(R4#^Bi>kgbZj7q#4&z7PfR=Mf_^G9=2CO zH;n_peHrENdcp*734*PhTCy?3gS8XW4AJ(|w`x3#N|N~7E1@_ZxmdZ+_5yBQK&RLB zm?4*iq>+#QsSvRP@ZhmDa>BTUQjCB_C$hv%e_TEsz0mrz8 z%>0;$Bt0Y9kYfH1OFZ+lX_+_b&Y8kU1LbGx|6XL=g9?Qe`~0drG{|}tHkZ(p4=L}s zZy(*T1>`qm*;EXUYQ{6Gd$E-6%xNLb%Z=S*=leFbkf9SaV2H>^?|+7)rqG2l5}{v3 zkb8f3iO_*?k0SQuM!H6gnGxRMP(U41{a@+hC3KA#zPG~z%yndEmy>%RV#m#{iybrp zVQlO`!W=Ac!(V1CAd|>WpDKHE6?_+eUGjIx$Xx^bJ33`(a5AJS$i1_?m=E5fG<;@5 zuAzZAJu$<2yWpU?hg|N)OB~p08bShcDRs-)JS)%>nJw_ zIKCZ2<4jb0?2_6-d!%uq?HVW55-NFPW86nBL)TzwtSIfqTS9oVA^CSx7)=f9Rg9p4 zjyGgX^LCug7Gf1W4)B=|H6J+84Fb-V)X;{6pQsxvQ89BKD-!D_-Zc_L-@L)vu0p}i z%K7;(Az&)ii>Dc*^#84wTU3ZQ7aJ~({T1~LyhgXdbd)*~d=0-QSdmstXgnfjDX|jH zNEkRq$M~s_JA1JF%}-ty*CI3W{)emhI~{=!Lo%$l_io)2I#98h;dEAK1tW7>Vmiq{ zgsEe%_j|SfEebU^3rjI>j&JT_L|5T0blO&J^DY;lLu#ICtcZ?|^@m_r0%w#9t&bT1 z)5{DN-va6sKlP+2=Agn~W_&+49glW2pyKKQ3@nuyW|_pHcdDAtL;~ZGOU4h$SU8mC zQ@V$j@_}0V)~pMeoytp7L8X=O?#Jhk%I*}>^-@finpo_s*ZY${C&H0tFe2a0dm%2c@#UA~J9R$a$z)_68( zeYFI3r3vS+Bsx@UycH<5Qfo$%+iDuQm=*r!?RnIQ+GY7+p&hh~E%$K^4m0kb@s6X8 zp(+l>&8ZWw=$=gcANi;ili5?@NC6y^u5XF}rXMM&uZtF-DP-x%E`^MD0Upych<685 zRvR?ruV)#%Ptp$1!7`Wyz7aIN;EAQ#Ge83_dfdxoQ(EfAe}?p2GseJbDO?UTW#9L1 zM|87I?!BG01kTuQBkx-SisXi;UzRKmg*Y>5&!(({{CbQnU9Jzaz@cvqo_dPC7t{q$ zk&tnngq{Y4va@e_#cU55{Zw z|D=A6&%ITx(H@)Gg>+Ff8s3f$CZ=_X`#ze-UniJKC`MGRcy0Pb7jD){@BYc582xBK zOS+Ke6T}=e@*5pRx6YZ77%UK9Sryj&#tXOzjJ=6D@>f_dS?7Z)4H1B<)psO3fOEK> z!+oR{6CfhPYUbGP`)}hWu}4y!~JOX)xzi(Mu2x`NbbyN z@TsF-8g@uD<+uM@ci$0V5OAyGuU9Cp*-mt_;N!v-pQdcfT16!3!O{_|w`IyuNwa&L zG`#TO+0kbSn???iO1gOXo@T2U)+*PeDbk?k!03(pd}O8ie$B5eRQy?U)|;J_80%0w zr{>Z19E5_%wOOod7}9~6byH>)!ijspP8Tj1`{9ogb|j*0LNzLy@uPc77MKNUK;2y1 zUoMXlxESO+{+Az`%q?g9V!wd58Cruo#BklpNsLL<54}1oHj>O&2GU3M&0JB^N{(}- zS+JA)hLb5Yv`V-n#V}zxhT;7x|49ixBN_0!4K5Se>yD}g{nVT5J-w)8GiZBTPlWRV z%>P7Ebjb3b4)uj3dFiw-?aSPo9=`O}hQe&?YJ1Wh-d<;2CL?{xzItU}C!&;YL+%>+ z2DHIM=vvYC1R34BkbSbOV4IR%Mt4v&TeHF?WKVcq6OqG8~7_{)gsHU=Z#hzS9CY3QnHohs0I;)vPs}! zT^}ss&Qe{kOuR(+xm@BP94GDBeYWQt-Rv72*Cn!vb_JyGsWR;uV4EW8P^+??ntZ#n zJ*$VH$IWK8*9Xznl^EF``7f13`goP-Xg5#>@5pf_Y05o>@>q2|5{lw`L7l-gf7HBd z+7$-damhLaf9I8PuEs=&F@LhbrT;~8t1VTV9lxhzf86|m=We`}zA!-Y2!WrfPougM z(}DR{1S?7!h^1go)X*IaO-|Ffy>F0fTJ(Xh6J%1{2C*ekfEH$(<{)bv1JSy{{lC)# zxlyQkNxzn^>U>b_KGZUW43^cjZ;uOh8dpt`ScQfP>c7}7?F_FwjHraB8U;h`@*6; zA`4Auc-SR7eKXA$fh3r7yx=GBKF*dZrLsWsGXA)swy6X=!TPZC>tf1v*(a0zM(;p| zxg`VF2_yFrswZ&+09~Z=<^v{zR&2}*u^g%m1TAkM}YW3fo?*K>cZ@V9%#1`>=66a5&&Nv zsd{@_NvD6?Ji!7=;@q^!rzJ~>*8NIPfcR9cPC?6Qp#o9nZ~CQOln7ay2JA9%7l9r_$HZV#@fsYfs|I zx2E*1blry*Sgvg5>)D~{NA1J~tN?@=FD=l9+&4R!GH!$jl7VGe0?aMHNBbmj?vihH+wk)5idZDAv)F)7j;^60(g7(<(n)JmuCp9r`(C8`8Ft73az7dbVO8I%Q5&qT?(PwuF zQcSfq)o&498qg`o*1h~ZoQP<6pR&*81>nDVyzUhk;v|TYWvZ+3VN;8}f1tphoe}ew zeKR(ydQINnd>D@0m_CF8N}c?-{}E~d9#l?^AB5{_O*^#fU^+aeHaP5OWH~fIU;M82 z13s7OA0Z6b^FH9@ z22bK_LL27qg1E~6iV!XHtHT4{Y}A^`FSFZ{ z!A1vF4I4Zi;Q!1X(N%=21Cz;)1sPEtAn+f05}?$`|GK&lkceua|2HPU&#hehfTDu}ZIFF<^MTD%ujqx1z4`-1g%}_+JadHGoTVZ=P0XOfJ9z zDK=KG)1zwe)M9FyWT`%~Dx-{N-V=@hJ{O~>lmA%O$XZ?@> z>r;Oy_*yysLFH(#9QUH@A%Ffal~C{A_gRw{2qyT9`K91XWN3v^l74@Xu8LskOL5t- zy5Q4y?8L>yKsT>CQGlYUJ|Zg%$XizC?vS(O6Fv}*J|jV_i?EzkR1jS`x&HQsNlc<} zZA!cETZe#MxW^fVf{7L4(~5?t+MwS#w~+Fsi~GdC&Q|ws?2)p!;DD9l_jpf_9Y-e_ z^O9{R6M-Ok>C`hSZ&ZPa{2&s42IZdET1k{EtH#)y^Audf>l3f#t)NKP+mR52wV3(2 zqE`**a%+J@U;>o69gU(FWw&*x(Y2M05r4uaZ2f$u94#)5tQU*#!47OHt41N;A2`u4b{%l7}uN2y4bnUQf=MMg$~Nyd3g zP0>uyd~?k=HG?)^3&Gr zbN&w?y2erERxnfmvnDGa26(wxKIP;gS)`ws7?FoW0i>74P`LO(Qc;c5*&hcA2j4e+R~JC27qK6YuJQ&G;(pQ9 z%)^c_ZcijF_shD;Y-ga>w;NPNK3P)*KhY6TOl^3kAelnipFN!2vN^jD{INT*1Tc21 z2mGL@{a%A~m~7=pourCHZliAfzeN$ey*@Ivs~5;;vXV^NpGY<MK&FngD9oucFhx{q8G!qt`Aw6 zOUz+|j!>IokWH4)0fkB0rkUl(kX(3p98cSzIY_7+xmjG`G!-w%TFwlDQ@%%UdNpti z@|MT%J4J#^fDy=p(d;||3oYUQ(jas45(4>Xrre&NE)x-ZIYz|hj5x5Jn z$w_wr`HFj8oOcikvh-T+w2uW(Bb>y51n8e~^j|tHRLz3tO_UNO1tz0j{>o$zAzV+l zNm!bXVy^GYtf)Z1L|SX5ptaP>x_rr& zok&`Q)-x9rEF#lyJINX-)sK$C&k?v#I1O|pW+{jD!!)=Sv5QLNh?J&D7VY>4A9Rqw z`y@y(68ThifOl+^^v!m8y&;y8_C>qY&RRo=2#2XP=ifHu1cs~&Q4*$Pwpzw{qo7@Rg>x^NLqJyZ_ajq2diXL}7Yij_(&;xt^GxpjHfo|I?6&nv&8d z(muu(3xCh6hITuPNfZ~<&(*NJJQM>D?44Zg5b}$jX0nrIk%;UVnvBT6xv|dJc)gI2 zOY=%9WK>HR*1l9OOKeXk3^x_FX45nf^PnTHtRS8D@jLgJ5!759Mb?f0#}@b`wQVnD zJoyC4{YJO)(!J%dN3tHnskyoQR-@yE=zsSjEICk@ks8VY>bJNvy4G+yOE6 z_;YRgiluzKzH0%u0gn+$b+IfCKvu1le(tQeifPrtkTO|Ze|ipq%<-u)JMmYl%Z=+| zauMLM$BzmW{=kwO-}EmY#kfcgE7N+>e}p{QxTc)z7lpLL@7J&*0tIci`-S=^naK5{ zR2cSPcPwx68Anfe{z_^0gz62Y%+&j7CgjY0j{htZgB^ zjqrE6Po-qS$!*z{{t6kDd2*q{As7VhvK8=GrNm5$G*k_!7yz2r^zAL)M zVYVA*=opk7ML0_x0u9G5dDStwV%nFEEs(Xao~dbE?mNMM2 zinT%6Os1{pN#b$$MU!9ItYV_}IC<55r8)@$tgCr)@Gugf%IqIzcKgU+Rm1!KDZTsg zB}6aEQ>L7D$g-$p(s}0AU!>C&A#n^@bl9(nI zx?!>ewP^FLnxoLqU0Jt}*7>7~6DK@AuMb!wo`Zix*2Ot|E@K<-0GyTS&jXP+N&6D{ z|2dleX<_(q(P<9OmQh%J4%dvT`5o>NI|uAj1i! z9}bvKl8|aZmS!Ph<}mjL2k|39y|Aac4ic`ZW9^zOMIyLR>N~L}iC`g*o$)TfTdKP@7#$qYa=+Cv zVP3ryLtmkqVENoNzW-neCLoCI{^gTQRrs*W{Z}GY2Hxc*5;X zLL&esEl!Vo2!tw*Wj9Dp?w%f(6&EK%?$44b$(&;Wzth!Y7$yO`1gNQmFC)tN4a?Ad zR64|*`_<27Du5`Dw3?3FK+u|oI*eauzzG-fn`ooL_e9l-kd(#gxOaN!NM)q}{RI>; zqZ$z4-ej0oHxp1sx`SzJND{gK-jh{W+2)rrPc#^fhC@{Kw%J0$Mu6(iOgtGU+$9Q* z`h|#D!qVfxyWoheGd=PFQs?K`%qU3TXy{R|eImh>WV%*}O%FQC&2g8N&<9w127uD%L=411a_rE7o{~a=u>~|eTC9r|BuZ)XUe@PH^+GpHK=VORCM>Nf`X8HNjoRu26HKyf=L=U>Q;*m02y6RAY^FtC;RrGB5EF7Ry}nU zU#%K>BY6w?jK^i7?_3Fb;j4rfiTg;2=vo(*OoPb#$=eHq2xj3}u;>_a5~R#2yl!B} z<@UY;YTzLepgD}31SI^Rawy@wCuZW{M09)?XHcwlUH zg&G4u!-WqL2xFp`$BC-s$t|c5HWXVYhvwK~b}GIlP||dx>Rkx|Ra^GRbk}9RP)$kf z(JrV5_qH?L^Z+kCT$1=9$@!5vtWP5}&xQO-bzLlAO@5rO%6tWco3S|fVcZ`y*t?iN zXIALLVX8NmwdbjSBiE+q2D+k#8=x8751HdXqW+OgVuyqc$Lr$hB#SFBb_Ws~$+zyw ze9;g9gimij36v&p@%;)-Hi5@|(bl*JuBSloX0Un}WPNXuh>SCL2z*)n9V=kMQ;$_^ z`;8DnkD@%5bO}1Z-BszvM8X>9NeRdz=IgY54IB5gn0GWVnY0(Xp7VI7{D2ghrX2pT*hMn za1Yqjy;^}ROXKFVZv<&TaQM^~GjXs9vE`A)N>of{>lI00cW7Xvo^4zh$(YIpD0EtsP?`ZMkEe@K8Yxv4P5WW!{2f23U_M&dWBq}hkh_(9Cts#AK6 zBwybzdS-ZIQh{i}vPN^DpE#WQ=Cgd`oU}F^|;lE|hO;>?COf;U{<`_!Ds{D0>B!CiqCN%=lx1)pH~s zmHR~ulDtY>`oDeyN`zm&6EcN){#{=3`*uGg3M`i~`{{G18P-DC1G`&f{?$Kz(w4a9 za>n8Y+DWX`WOym&)SJ_sQ|y*nd-W18zzXu6mo`Cwd(FW#>s2RW@=oh>%FS z2A!xFY-zZFJ{@&1_eU;z&+MsaJ3OO{eDq&uBx3SKod1~rA;#}2^+9Deq83ptWk~xw zF2FBg*K8eX>s+c>LmEw`?(?P}H3M@82$4`JBYa@ZhmUc^Y=1&BnD8}WgXfgp&Wibn zkS@O_uVecl4%>QK5EvQ=;V-@(*QY}+6YE{-6i&ubT`4I)Qqit5dW24rK^S^>?O!4^ zi7{a8PB5FTA*r&wAQ>V__ho9aKLa%gUaX@Catt1rjVI>eVF#=|GR0)BE!m_cFn&r# zTK|@RGIM{B@tZO+=FG_13WZ}Smu-dNG?z_(gcikZUGXu$0s;L4h`Tw$xve9 zGir-+3M&|zs+g_5QMU?~;oK{(N2bYY-=O4gPQihJC+c}0F3AcXvf2Sc2Pu5-!B7JA z6O}NVHp4FjG8PFczae(OaokP)1MZk9joTOKG~!l1X`l9;8AC{({PK|hfs#fm47S*j zu1G-E@rKV}<-mN(OMXIB_lM(kNl?E@ExUHzAK-RnOsIq2uO)FGw^+2AG52atVB|4S#ja0U^~bn5q2pezA{gzpzHb*}Q-R62=#Esxi8Y)Ic$laV;7#%(ix zj2$4JFVB@BcAD(zD|K-F(*G2}Uuxvf~rWwo;lss^mDE_b-6G&yIf#h~Urz6K;)hfF^l){Cptn|cK?h@g6^ z)|TCz>y%3B-F%U(Vtzek7MztKj71GE$|Hed$*t6GC$x^;3ufLRO@0-du3SX?l9oNT z_9_$CF0A1k3J2~nT14$1!`;T-rOjl`iTRn{7p~w=MV^+^9dks1$~%^~3DaEc)wg1j zPN7ZJoOi_SUt%AeUY=PL4jZP~&U=1u2#GT9f2(UBfy)Ts8jH~$ljhN7SvoXAhTc5Q z7)FMIP-4-+e^3!irFJWx=@3~By>V4rh8}iQPIJO|H4FzeE#s0Jt!rx3)HU*iEz6T{ z*dxgY>z@DWaTJZa7?TV_Q4q*zbXWs`kXWDUX>E0#bGH1ra}7xC+C{;RzY$EB>H~Q( zA71B&>PHvhN->qJ_I}`rJ^2b==_Jc|Am|n_DhHswE&0jsXtn0?qb#~yRbz{H7skcg!yo>Q61Ev$! zNbIll67>0UFnTleXe<3Hp@wSP^EUNY!n3tUG*(3?fNqnqCE;kLAL5KSp-2oB8p>WA zvekSRBvqme(2_b57^%2Ns;)q4RWCP+ zj)mZ1d6)jICir}i-?41tzQa>lA6xbN^GU0rV1>VN^L%LrBiCdb=DLw~? z1W|3I!vMpOy&!=WJEB_}yhR?45R`w{BU=b_j+`l*!p9hW4OL{LcAje^Z?vePAHBt) zwJJalQt5}%>&zIvh>(@tC_sTjlx6X&J-Rg{d{R)3);%WZ2&HYC zI>I0L(6QO-`g|-*U{5XMNFicgxm5b=qzQXTP-C|D!;!Z&zeD?#P$XF%Z=faTLvr+Q zb$m=*3%y%8$|55!UWzlMeT6N^mem>+*>q%UJ&Ny-qB(VcI=LN&SQb=Wj-4f%-?dz7 zJdQPXW1{0BIf={|F;s$oG&VMzk^4Z^X)m=mk)A`s`pD!cy#H9iUsA%am|30UTN1s7 z*f!~@z!`u;fKvoB?vys8UD4B$YL`8BGJOrE*PQwb?6hoMR&L=#VCwsa?7 zG|+R+rz;=9-MKehK4EKKJ$PP>uhZ)(pg~_1)x8vT+Z>B%A;bol8k0cJAvtsiZtTE@|6R7X%xk z*)f(`Yz5zwdE_8t0$FBiR!|vPQCWiN?;$cBHeGCV^b(@OKjxNv#U3Y=sJIv~mv_ib z%NVW1Bb0mm*bs*@JDI>}QiI@rd&9!6cAO#YHf8bVa!7R3-D9X zy40PN3MxX{PaKBRIB6#^O_C1IV9Xx^L|56@6KA?%#8s5H>@nYX$|7n(5bGd%UeHQL zhsQSQUm>Q|hl_~2$CVUNS|>%SZG@q2X#U3t@;KfFJUvQ85&R?VEj@fB6vFpn;+tSD z+$Q#W3!qa~FIR0hy@-fAaBt1?y#%nCR#0tCBI{{81veXJiIlX(xa8KvoF;*MicpHN zRBLRmF!}*xG&|7zQt)25?y!;|$CD`cpxYHS0FHN_!>OF~7{V^jO?||JImvugzlOlN zQ|bM=_fv>{S)lYGD%ysrrQuOHyFJ~)ohV*}akX*%3*m*VBXx8d;nenS-0qi#RDgHD z=|yTR+JD%u^XL~(Xi-RT?UordeuB6sDA_jwd(GhfW4CBvIRces!L3zI1b=bExD~mc zwXw9{gsmiMV~?u`EQM-a--SG^J&5E+wG|GSaWEV&tq|tAh#p@f+eJXO;ptH^-Kb@N zWvXC`O*$(!MTacd8Y&RcS$ybp8MzH$$YWJGgtUnMp~4OV{X9Rg7Xh>7b=cEuJt*fO(?z%ZsUCX=46 zAgCdFawUM89F5zmUQ(P_Ukcu$exP3x;+G?&*}dZIBD6F1AZx6LM5Zf`)A>EUz1`tKGgHP?92! z#_>;RiD!!XLbf0FK*YJr?ri~r*^?{9Hg15uw;2bZ(C)zttOMAqV5mFyPWwLimInI9 zs)FgO@XBtV$w|wZ?vr#e2HBn9RB$K!KQN>f1&Au0xDVH7XBj~rIh65gsl*K00mZMi z1%pB=G`~ixBRCNAPs+`*n>fXBiAP#VAHn`RPo&1{QIwCQF>|Ua2us3H=1(C%$}$C;)+BL+&1tD`59PmhDb(qvD|@lrhHS@q)V`OF!_>Hq zJ5S<6|7I*okH|$s7t6IqVZ+TG2rE#_wT=z8q2s5C5I8&o3k60|Lu#Sj&-lc(^F* zb#Q#s)I}5%A>wM_{B0iI;uj63SE0;VhTrYBe=EVCG|{O(Ht*OF^+54b8vkTNWjR&9_v!2?gHN?ZGE$CmGh63aXfeS@ZD@4m&R$6Su zByKB}BQD_L``VbTq6nM~00s^Fy8#|(+tXom=q8QAX-Vw9WN@wq)5JF|bJRY@`UxY{ zL>D=>x`~R=H<@Gl1@$F_U1i&o_L6k!s8e1l6BGFXo1}nq0=@pkVTwsLA_WH~{-$6U zp`X~$N*O@|urlGz8#aR#up6391J_AIu_^L9Cm$ogxuS=obS)WN>j*V07l7<1@rw+I|m}C3J|VB?JZ8 zjxv57sl-2u6mHf6P!eTYkR3ib05>GM>b@Zf)0W2KhEJ1ANEOwl&vumH0Ie@LiU_?7 zU^n5+ZWyw*B|Rc93u-6hAzzpB2O&4#b&oTGo-Qd{@F4X;EC^U;xRyG`xr~#1(k8ok&8d zxv^Yxksz8Z<}AZUsg49!D(|P&l4g66F+8yT8fyFI2mXfjhz-wAkG4!B0g!b{7^+BE z>PEL;_!3azIhns?k5yusp)oc7q#o3#O-I{2K7|Jq8-C~YBl+RzURh^6fd2Z+|1gH< z%7c~{GcvCfiMU3DJ=GrLx*Zb**t9;bx?sf_G7jtR zc406~bTu;E2c5U3?bpH&le&4e@?68gAS`ymTvAjyXz0Uo9n!)Gv){~r`<$bp*f~az zw3h||V^BU>(&UfOE<-Zv$_F}Vy!ak{MCr~+<jWlUYZt)?p7951@znSt{BE2u)0*AcEc`6X9mCOL@;T@0DQ;EFz!Ht$Lp3Kjqyh2 zD)~$rgF4S85!3P)K@)2-YZhn)3BsGksZ#X{Sk%`U*gBbH8e+Cso%86i)^k3i1L zNq)fY^FT=w{3U-9Ucw>9?uIxTy1jq5SKo9YjZAB8+{|L=MegY;&+(tpg51eyTY~xo zxLfRgknP8_O6_}Dliqj1(W0>LxvhRg{LCuG*iVIIdMos%QyoE-Itae61e8Dm1gh=0 zs-4l!SL_V6yTU*EQ1qd)44p?J4>_|Yz1YL_Yy*IV-xC`H66G*Xmvy%?ELl4_Q_yu;2R3l)C9Z|a+Mz6Wgr&M+vbTL=4g zFTwvQ86t~+ur9}zxcOMg>~_*Cu|-n03?)K?o3Zh&qNnIwWW}DfZ$#G@s8U)h!Jjto zzZ}#MC_;MVY@?kNCjov_ak1hW17&67@ywKC1U{HruSz8tUa#QuDSs8AAA|0O{U;)# zVlVf3C6Z-pqLIGhh-ZW_L+Fvt7;$a?Y67rg<+#J~~fs4JE2%mREWowLsW?A-ojueYWpWE7?_vb}B} z19+w<2|UG|^gd92hJ;64JzAIbsU8t1J@ku>FW_W>O6A^iz^N53zN1}O_BDQ4>o(Is zW^TQLPL%CwKMMRxW^AyC6tLF{GF9M_ulvP~cT|Y>j+$e2XO}38GO2VBb z3YhYfxi<XGuMbR3H6zf*4_IK)UAS3u-z=;rP4P$k}?yW}rt7~`^?-q*5` zr?nS-tsY0X&KcWgHx^U`L+JN&&uR}+H`n%GN^C=!Pwn?n@;5hmkZgKs&GwX-sd@041A&zhV^T793+9wv*73=0F3mqTE@4E$e7r`K4#3;V7^ z(RvMhx%@|@Y9WjjW$Va;O1YxJGdQVEeo|m`qpJtgr z1=263wq^4&Vo;-U7L9>f1nMl9YNg5%E6@`Qjj=dl%U-Ib_dXHB{R>3mIB>}hcP{1L zN6-}2<2GGk%}u5$7b=xVWQ|!(zMtw>!tlY z@h?&FV@{%D@I4uek5SekTA_3Pbr8vM)yCd0!aImVoZlV16n&QBr>{}hT6%~OIGJ{i zU=Ff)*DC8YAD3J0ap&GODEh8=OFB0i(v_Q5du=BKR}lTrJ@8JU>=bRi%9$mfw#t}V za}9nl)ww49hX4|#tv5B+rD1KKd2A^BQTZE~d`$Yggo8r+t{=lz5H=E#_a~;qcvR)# zRPHwvM62#?()99eByF4J`JSMFwJ>qqA^LkJ>G3uD)&WT?kjY2%OpL*{2D2MF=zWUMWGI78=Y+OtNI$lgEaKvwlF^p?byFa71sK5uf28%1x#kXKg! zF3*EX28Ub!8-nPZbtK^F^lvg+q2~(?j|d)r;BMp71~G2@QcUF~GFa#utl<==6E5pl zsHWHP399%7RkpbCrq-SJm<012NcZR;@dg?K8>|9EWYOT_NL}sfZ;=e6Lf%3OikwDX z3;L$Uog&>hsml0pjK)MlZM{EhqOpkt1dG0HdFT6V1p6Adj^)HJ7(J&$$b!D8o`u8n z$VTDml5y6|naEPG#!}iRw+G??WZbSz=Z#+@`RpmsM2mn{j+Q)AV6w~j zuW%2NUubnmSC!07R5XNgQJJW3{+^zMoIA|@4cT;GCF-~CC%yGUz67{Y+afigy6)d>H~6YmwLqND@Z&cNTAG+dUi%S}exK2(ns)8#BaK6QVG zua!dzrnx8pAnt9KW(E<~B2YDaVH!M3h(^7wnfw%z#WV&#;bH*cQi8#okH9o@e&%^} zMA1c$^zR~dzb0cEff%OBM3w?WJ8o~lSCcdiMp@%DuT}Uu*yqjM_;EUXtv^>e)qn_= zq1o+!f_MPAhrV#Gy5n?xy@^gx!ahq<)1t`ail@qY*cHdA<$PX!m@F%w09OiGO}^MZ zBcL!Ck%(D$R5YqRi@Y|)RlmC90+dgY8NecC)!~fS7!NTtU&-3UC23vkH`?lbS59Jd zJwJ%b`4`?j>JNu~HMYJi+K(RV$KqRc%Sn&Gq2YTrDWza7 zY`@uS7TbP*G9hwB$x${bTid`IlxfK|Z|J zFmsDO*QcFTO(D{?Tw`bnDHrzUevzhzfvH+D>q+YP&lJg24UAN$;CM+7O^E0t%&2nW zfKs`OT0!A-QSj-am7qIMOzz&bz0!<1;;pLP^R1)=O% zU<QI7ZGPskA*L4$~!I81YWe>_Mg1WV_IBi57!zn4ro^VB^#Vw{eT6dLN*PeAFjxN{El8?N1&)s# z^~s!kmT<0GwJh=qJV3@e7w3O5_?4sF+$}!o1Pn)nXso^%A0n1*lOIPO`@4Q;j~Tvc zmZ#pI4eOYGm@P;_F8-{`Ub|d~XrTG&m2iqS6N8%K`5#^(Z9QvS^$a4hp|Mw8dwq7{ zV!gbzPH6rw78Or)2{;GAJ6m6HHz!es!jQ$KzaXF;QETWj5!q*4XC&Rl*3~|bKaKxbd&OD|-yG|9D&hl?V#$%&|E%`gpm(7&fmf{h_T97t*3s8lzeA(?ymV$NL zKe+Tu9ULb$Q1%lO73r!4rsX_*4_37}|7SV_N<8d2U0|CUO9;z9=AI`zJVqi}DFp0~ zZ6%?z1BfXLTm{jAh&A)Y?FH18Fj;Yf`^0r~{OEN9-y)D@=z4!8D=y(u`L=vAuNOOh zBJVpY#$q)E)%@T` zJ!Kbifl%3~Hk7wVQvsV%+joET6#rOQG*b7#vYfzY ziwV^6h^!_x5RudF8b(4mEOYV##VM@6ZFR6`=W}Wh$y?u!*+N{k=*jw5j}S96A-TDo zRC~Y$cY+7+g@iuMZw|K=^PSgSDYSk1L)|9A!Hog1U{((oSU&TGG{-UTo1|9zuH;8= zF**6ZYqZNKSBBZS#*EK(^R1S=9@KkPFP8>A<~U=1xlUIk!K=H{H2)(*En>O$9s@@s z{a{+&5kdMm?<*UP$*3WRWS1nV{1J;7TQ_su4B&id-ie5O zShR{_Y2_SnD9%((`Qo;r-13e`2+#bIs&nAo4`KKMf zaOL!3qwl-tQrad`CSsR5969~f75(i@NJCXtxNUg<*Bs-dgInbAoQF&4NcvI$t&?3A z=F+x?fOytMR5n?m;(`Q&KV8>)1?T|F@rsDyUHS>lO9m5v9{o>5H+M%OV+ri9>x9>B z$$6qOMW%P{kWd@4oBU_`E2>hAvu9*K^nfZUt)7}(g+{55#-p3U$J}3-pMGn5;_ot{ zIpbP*1tP$bianYW;AWuo6et!Y-V(L|(f@HUAh zh~b~^7WH=j7xwGP8vpa!kDxb84Au+s-$*8MOSgH&E41ZOEj)s0Ni%{clAtIC8V zGR))iKSYX|F2139;1Qm-Y2k+SV?7w4Ok1kzmQxU%_<1kUtr)_?E-I}er!|n^p`9Dx zq+a3em3;)?_gYWwkUt?v|9i@lqC^oQt@A2(p$y9e#I|&U*N_j)tEeJ5Xc!>a>?|Ss zr@d03)Dvagmi52Mgypy6r}RZlOh%84F{UfRX&<)q&zp^?078nn!F*YUwr<%D8SAv= z4+uehoZak7(te+=STB1OUlvyVL5vS^bHx(fwOreW%m$hV%Y*y|+mI~L6@v%m*+FfB z6EF~Z?Jtodc*m~%@9s7aq=FYlw3e5uvmi84ZAfcWW9I#-dC#^i7TAuh0a=)|3$y+d zEA7+vl`=DHR0g0nTt&eFx799)eRn$T%K9(Ic#|Gsp7|Pf(syhCSzS`9nOj^jz& z*eb41JE^!WnXd1`V492#Qm6om$UG*56N8Qi@)`L>Wv0X#Tbr0C-H<;EhCHl&Qdto+RNAFo%AI*40bhmjBknKB8}$=ou2j<}d;%GJvWZ*D3>_*}($+NT z0Ms25JUDCxGECEaVdt1D!dmOeF!mh&Q8uEvAQ>vn%2aTeN)JJ<{uQ1yIY>WoPhh8i zhm@7{YrJ1$IV0?qa%_nc$FUi{xSGFN`L72^~leJSPS@EQ93$}nboz{7Rqvw zxs!9hPKX0g?}^U7Q`qD{?GyFwzA z>V!&uEuyW7WQnE#QdTN>S*MyaKx=2v-ldM;rYGALzjZ&K-Fa1`%@QL(bczmiF1({ z22I|2{^#PIxTv&|`3(afVo~es7CPrUV%Higd&HOz8hSHZSBH!6lWDd2ai{{NJ!Jpx z$wCo0s=D(|=1ubGgT^Tepg~Q~Q+^a(O{4wVJ?95lpYK}%7!@{ zkT8wRaENPytMtAwlw-W^v+SDGVNCfUgpXZx11?YfN zcY^K{zr3G9N!WvkLUdK~w&gSZM1iq=zo`n}wqs;%4~7NWWuFO-2Z1ZYNTWx;bf*i! z1@p0}BLO4@T<-!wJ0T<#DW0ZO%mCtH;}{FN`AYdwkK70|2d~e4GFnqy1@h-E!KZZS zt?ELVjodKP6)2zBO+%wwjiIF*L_WKhu=1c~_(33prcHzNYYM&mM(dI>-ERFdtbWvj!lABpzGS5YU0A&-kMk7z zS;`zg)CCgZ4sL<_5Ks-3CwF9TbLd5QUChLJECUK1zf zNYR2~dmVRhlm>>IhrbKKN*oB!qC zulfN-%}Z+C%x#23`4r6Fc+YtUfU>H(@_E%e@ztvji>6g@Tk4IH1+gY4rI4Neb0b%S#f!*YG-ef) zB1bLqGkm2xjVIMPWIt`6vA_#Vgh;voJp)e4OH+xOBIiiY`7Ru!igmYD&VneU1Q@Dk zx>^hTTUM*WYMC-aqkZPXIs2!YKW>lap2)zT`5D6F<*H{l=A*A^HAQDWh`Op(YtvKe+c2FO-HvQ|J8-e-e45Lgk-dDORCdf$E}52U{1OS!UO7d z6SX!VgdLnH|uyrL<4M1Zd{yjm2~R=^*F$s zv}clj+?|B&{HMmMw(W?s(=7dAKJC8& z2fd!TDI0pEvTaF^bw>tn=gN9l1AnRJx2{O#!rBx+pu`5@!^B-R{*jg@W-w#z>^tR1 z4Fgshj_}EdioNU?MiN|#l9jvDkJJ8C394uG_&eK2y4Cz`xn4 zzhJlrMgYRn>72(ixfRT*+pf%17VMOD9da1zDgk z4oqM-GO4>nF?+*FKC@mf!0S}9UHd*gM}ioWGRw$SVGXp_r9a_RDZaJShIRcu(W{*9 zwEGbIVNdq%dI@wegn5kJs|OOcN^^yl=FC!eM9%ue5$3TXhT)Ii5Zq0&YiNvV3i8fj zj3;#iBsAbKqM7C)84lhk)$BoFS7gz5+1n8#)BTi)^ikD`kY6iyC|~cELZCo1a#kg8j7qPfjrCN|AEJgfuSg z1I3}THE^aAbuOh>a2yPShpq?nr5^LxPdg&f=MXxuw*fBX6kt_YYLn(R%A}o_cLFXG zGLlV#iK6O56qO3jHL{ArS#C@w9za(u!NN3+{vC><0zKK(qnKUT4kN8G>5Cco6X_-4 zdmXKKvxYT-NyAwg$E@D-X12>iyuhK{)MN*ATmQG)63*YbfrCpHuvxgAI+?jHXm<~t@$sYZ2s@-iIE#$hkQ z>RA5R-t_Jyavk-pksYCdeTb>%mxi91>6#Yk-0PtfrOxMlzYsysMj|K!*?6&{>a~>N zWOznqKHtuKk@!q2CuW|Z4lZ{W|ByvRu3s&fvY842&pG#=;VbJ_*!CrIJAEu5^J*2- za3~DFDw;ZP!yy@o@VJs__Lys5#QATKIF3v@Ec~nEuVf+(pM_w60AL3&kPoTg{*?7^yN^LBW8?eWd{KfQ;_Rm;Tfox!?4Y^{KM$PfIN%JiH4gh< z+Kksw@Mv$_8Zjj^y(77>4ndknWWSG zJj5qq!Vk;~*_a$@iujzn))~fG|6u{U0B2vAe@;G%$_G`w&aV6e1U-?-K9S@&>lHfp z!8H%T_V^d=#Z-j(D+EfZ2`IFQ?Vm~Y2;2~oR{vAILS&XI<5&Us#Nv*Ri2%onM~x@w zNFFtv@}0;jjAN;OgGE?4#_s%+CblBDoMngC46&dF#G{CpX&T)5nqLy7>n>ouI+~THoGP4sY2qo)I&DCY!b(4IW}#R zOGohE#BL7c$%HHytpXW^6L;WG_d2qg#jwUOFIRjH^Y4cB0v1%VQMp~<#b%;X&hPg* zV?-$+z>eKXVmPIWnfyPJSghhz{Dw>Q*lPb=gYo|yshfEXlvnNQ!^k^{%8%)7N4mNW z286L&K6C`Bp!1vJLeiYnhv=-&`cN`cilXCLuq;iYwSf~H(2dAl`d`X|dla&^vT<&b zqC7B17nn5LQiZ3w@D!$BEw=U?6rN7mTYgDsBP?A>jB?0<`pDF}si!1*;%GEEI{FYs z52mft0z?=dFPEP7-i@Ks$Yfnx2_8f0z@?B>%b3<_{|A{5DoQo1B5r#omT8D4Zl%8K z$v%@Kk4W{aOueZXwH@`5u^<~5{y197@gz29lwcw=lvS+PywD}!g?tujHclqt5)?P* zsiEG*lGy}v>`7SyOFg_@DD#vw{`I?p=04WLA)2RnE7gJ&0*PhHqQDCLeVkZ7{X-R+ zRZ;yW%M1r);M~0SYutX)2!G%5Y45~Be*wM4seT5kz%{kW{~Y`Lm#o?R$;bWX7&z>SL5koH}<|`G{HRX z>14c*oFGi}R;GCZrlgCW^pD3B-iUR_KI%hkYtj^O?24O}IMfo(ZPqn%Ku1Rcjv<0@ z*S-?=Io_hQXzS!sC5cP?t4c%R7af%)o^&mkPKRPoyQ;yAgdulctkD^$god@Ey(FY) z%ej(X#j--@roYw3KyQ5t+IIIqpOUtyb`mj)`8NJoCiE%Zj!#$RKC{6iI~>mndxk4) zo#@CXt0EgHG16R;9+lcT4%L%e2(8Tbe}pGA&fo4OrvdLTf6?SVu|bFYt@rsXGjbsB z+=`t-)QHEMT0Hv$F;N$&b?2Sc1Kp4?Ig*f5io#|(|K9FoF(f5Yn{(+rdL8C5)y)vZ zqsC4-f8@+0Zd8+(p7saiE;Z_^?79WOIl~1J-4FL?e))z?4E-$Hbk}m9#$frMadby} zBDrAI27x*Q$~<=V!DgvH6mo21-_=_l@ZQpcCN&xHE>tkGU7~QFvV=*Bz8|ll_(sUm z2}!Dz5IJ}&mr8_s{awLKn(TtT6+a&dq!ai@3nOYqxy9Ni|a+>a<3pSu>MoKReb=soc$5s|n0 zo_8iTx7=-n_eTgsgRvxR8D}$s*fVEj{U~oZG`KJ)5$EM7n}nOBdUs@Ac~yM^=u(6r zNdGNDBx`{=d$DoOb;d3ApCS?&wRZU@N}?6kTLq`kmvCx7dya*QLUOR_18F3pZaKB$ z?P0{>v3G@WvlPFr9OF3WCt)wD1Ll#HnS<=XR)^AV#|JncEkkD&qdisXellBU#aedv z{3pJzql6j~vdQr*8v)LM|E#u(TxQ{}-HbR?1~a+qsh`bu;Pu*KzD$0RDDmXUPnhyT zlV*S1k?@%>$csy^^N7N`gNCM|(I3=bMfB$?dtqOCXf7q@O?Y}Q*Rg{VJatiK{BJhN zxWw*(>l~6}SJnL(K7jwLo@@zQp@bZpFOTLgkl<{3D&AA8X!($6zu7gGeF&#{ss+3* zEy%grmO5vt06893B?&8MgW;Uz&vBWnfXB0rdH257O)}WN|BCdYnXV+Kq06#TX0i;(f+DhM&yv zy*}33%CcCR`EK^7-XjoW1;?b0asQVpyN}g3b#m`19Zi`uIKEOO%xFzi9db?Ay}X zmm=ASzZAE;2e|NYB^+L&ZV;tzWB8Q<)67o%k*2yo;1#O?*iz+9%NRoQc6QEG_g}DZ zmbP3G|Ck0hpEF%cNlWG<@O9af8!aTF9+@^;nS}Zr`&i!oP-r&lUBcdelni9%ALuQ% z_^u`=bH9d1YGKB<=CLUyVi1w<7lv9~XeH+Ye{LIQ)zlq}0p%6z1@(G*8#(wtqu;NF z_-j_@IloN*A4&o)>=Aw{-YMs!?2aL*or>K$;f-*$C+l$r0|8^ez#i_EAS?NhmpM!X zjO@5u6AlS*5!&^Uv3?j$jih;41jN6{UB=u7D7P-zepR_AOR~-{Nj3} znlxyh%&TmM5P5MDRJkl-TWKl1voomJQ`LFHjY5diz)|f<>Q@|NQ5=`cLD6JlUCb4& zhV(ZY&nCGDx1o&4tNJAdc_@FvD$4dieAGpMmNX3A)4w90n!=?kKIl`I7vWo+{TdmB zUAb-B4A&7(kQE+Av{4%MdQpo6o{6{lgms_-Ib6<4mYh`Dyri7%?Y(gi3nHrq z15eRWXJPgEarYo9y2N-p{D=s3^)T+KZted$u_QiucI?6GU;)PR#G1M^L3zhc=ZOg*wRUhPd_OqL<}qfWYpS<>S;^0cW_Et= zb=(Qc7U%PRs1V-cYwOg~n zNeN@9uaRYn$S7X6Ned#~qk+GJ-&LVEWqfy=JhK|t`shy%y-yI~83%gTcj|@LN>YW5d78GQF-sF`>jrv1KPKt^Sr_BXsR&_ILF6xpW(7uDXn*Vp#Il~QnpU`8vU|^OuGsacd z$^0ZpD0Ar3i%lq67VdvLH!A=SnRdaF1$n?oK+kG(KFn)jT)wbl8a=5N_Z0&LP-UA8 zb~5oMT1I@zTxmF1hQU@ora{Vf@MS^PGdTQGjpJ)gL_#Z)#!D3FD=6N&eo|WvOaa@n zjZ%wpWu<=A*s(jCNkIN^e>O5_lbg47I1Uf1=Gn#njvI}=a)X{A!hj(+;f(05;JGBuOLQ zUorMSJDe1KEaW{JDMAz+q>?DCX~mzkLuFf0eiu((C>K?wm^StA6Bm$6rk57ofPGj_ z>As{4fUB}T@_$142)?W5^1KHm*6UOzm^q|gWg0um8zQ9;?}?=PI5OtBGGXzfE?B*D zMn1Z)mZ+lf$KlW;&JmjGA-;82>8b}!D-oh(1n2GwL6A0CSv-=*#z*nDjCo9X;yHbT z|I(MCr_es_pG!q9RuvuJsYGN-Kje@V1fEUwdO05@#NqbZ)OiAnk2~4THS^I^)I!Wf zVZU$Y0{Yq-%;D1D4Nvh5H%`I(`&498dC;6n>Nd#?(XH0@E2X;`=+pV`O#Wte6M_vd zdRdF74z@kk=XfdMuC3HRKAiR{0{O78*)`oK5OGzelqjnwGl~6vPf@T@iUQ4_m@(K;0Z&tK@8V;B_4Pz?s$O_-GC5=SD)x(AOb>M zU!is=QGoTGlJJ+_>xBKToQW0>`H%r{Zp^I<=rJ^iQY20isN;0)wLh)nP^9UM9uJ)z z^4LoMyM3rgaraB+G??8bf1W-_O6q45au9+e2OCWz2|wJ2D5@Xzx2}*O2SLm*dH*N$ zA4$n)*uM9@%k!4QlQNTO>Bw*;X|&$>eenKT|BV4>2ys!l+w6^LD=4;Ed`{=!a%7c1 zg$xam)K@%~qsGI_8JZ;0xGzWyo-{x_#w|1Tm?SL9?R&b-k%RZ%EbfZr2F=`p#c2O5 z`#&T+3A6ulVvGdeMVx`lQNMvWvPV_>5nm3wsoYK)J+p=O-D&tF0Ui2B9c^ zh}5gA@P0jhCSTLUR9BwJFmtPAiwvLZx;eMU3<#Xhd)dZpU;0wmK9~3Voaa2x zbIx12d9b__n*vsgK&>?8GLNHN>YSgw8lrfqp-@yfgx{K{c_(R?7(8-eTZwBXf{4V< zE^&NAODE5)1X$2H9T(>Wn5f|B6ti03mM@hqStUo2)DEgvP)$RH8gGLUd7`q`)@hKzFyW#b_Z)-|%==Y7bDMmU~K5M~G7Hc1H+N zdZ|Rd|54M+x2rgggLoOG5?9YR{ayh5jvGf}G_(&2nUdq>IOn@L)8Vp!A`!*xA+bz`zpC64?Iq=PRagk;W)69Z3{9VP(VJ#X)l`{{1O)3E`|OA7x_Fd zFw6-%N}K1~oM9PiG=(GiBP}3r^Q#N<&R0Rk=$Y5MmpH>@QFnpQGJKfBEYFL=pW*Jd z_1v$NcpT;4DPQ#|QP2lEvVgioH%QyWvK9PIzD8Wq@~jJgV9Hsu5zA zI(X~BV8Dp1!lWkXbQ!f1Ytr19I27q^7n6!}N6b()Qz$bXxhkgN2x9?WDN~1k2fh|M z9+pz^$l^}i;n(3W#v4;(ekU5C%2i@(0*+&*3yPya>$1{Vi;BSujD>CSzFUeasWR(| zhqx+Fiq4`oH$T0N4LOwpaG`_B(XGu z%bdt)bD5^yUixDwxbrtxjfsIG4ZZQ@R=irL?>6T&<41ztTtTTi%i(Ipv;_x zo0Zm2&G4>4$D4ir*{6pJ@Rsf#dp!^=qR=vz9}Vn+?u5at`|na}xBQaX`hfD=wi_?c z%mC+XG%RG%KeTG+i%s6RjCk8nv1va7knczS9@51)!=s5J)a#+u6P!Ol!Z`Y$STy2G zA3F3`mw}?E3L4z|ab73L>)L0#3a}J7&aN;(hYVI@w!t6d}oJQlCl-8?S8;V|2dTV-50Q3RThMAf~1lVN8lUv+FzPQ8VlAzbjujN zyq&X|fIzV%Dja>cSr=xP?<5vIw0fF9a%i~YhO-%EcyirB=k0F9&I*RmeeN3teCLk- zLxwjKY-yzXxZR=@)9&`HS0eZv5u0K zqk1S~Z|!O+qO`-)mAt@I3`!5yZ(|>RifPR7jX)mB|li-<`+W2w*QiV zPr(b<$(~7iiAI**`z#x!fO)W@b4wRdz5(pa`XyT_AaE}nt!O8aF^ ze&3^NbLds*(u3i_`^w1EQ|0qA4c|-zutBXp0dzV zw(Ba8Dq~fW#V)2q@q&D!pWlng?oqzyGhRW@_WQ2Kst;lJRMzha@CJU=J(fOBHCv_V zM?s_k*x#rOs9r$)cGZqe9S5Q68ecXPK=O%q%}Kx3@IBU&&w?;w~(zn zgWDCK5-V-b_nE(*KtSCK@&@;6f)cjdliaMo^{|fv8HFpD!m=SC*ys0DOYs7eBeot{ zHj41Vi1)43q06d_v9GBI41icKnI3Wscir#Bbv{j$;O!+n#`a3wwdbSAnJF#fovQCu z-9h+zgCcasBV^|5E5BNX;2$43E9HIXk#T@hMJK}>)o1AKwoX_^YD~|zR;C7Gv1jJl z7fnDcjpc6rB;zu?KCoLVJHMpr%+-63--`#-RISWJ2yK+*##Tqy^JMY9X`$#e&}6wc zSy?ntO+7Ka2{mTCD~SJ+xepOM_;21#Z<^)0!)rePkn1l^NLet}jA^Af*!~GFw0VEu zd?|u37scLfyz~B`u)5ojvHIQ>+m`BEVKWJi`Kk}^e#Evj?B$MCpnuA?@h+1NZQHRU zSw^z6J;|g!eKY#8(SR_g(Wyptcdf{O?n>JN%~^w# z=}eYF8KfNl_k7a6RRIYn2*(}`WfP=YqzI;RvuedOjIzK{0Wtj*s?9=B6&gGHBuqbQ z+iu2k=bs3CoIbt@WNA{ZM)YPG8)BSkWnEWumBP8Tv6&xahR?f*{hI-WM_^Mzpv(g$ z63#?R-{^K|$YT9%7X;_dj-g%91p#Cg=R+R@7IlnFG5x^=QhQx-o@KQXffXB7thRvqaQE`kw&7VH zR1L=dQ;FM3YemL}?#Bj)=!Z+iU#G#x10a42U|;tE$jba0U5yvQl0FWj2Z&S2&f(~Ur*;oA3e zJRxFG=f3Y1-ovfb*h_Pq6iosd54z%DT<+tYP;DsA0YRfZ95da49+pA*{hljK;6Y(a z1=%(eWe+%;;uD~VesID|7lHCHYfewV5EJ-E%}n5@;AN?*P6kos23H-khCvKOXK6Pd z+?9e^y~SR25A0y7LlUoqnLsCmS$Gcxg~O#=;PVKSuiw`RMGqxb_1nYmV2-8bWYU)e zC0#$eW?DzczMPx%7wP1tl}@J_X*N%})U$8fSq%M2jk*0ZCi*3 zQ<(Jm+4p%!a>9YnMg|Z>YiEX@cG81sawkQCEvQlT8|%fuZtXnf!{+Mhkn!f|W0G2+!`fY-KyceC0M+2!lPbJ_0V_)c>B0?G8Vn5w#nh9p4KetofnIN;E*3r>=Ws zybxi`|Ki4?M0}?CmZ%RRfgS5>M*lLDp11BOZ<4Xxy-RV?eiFYTb@pVAuWAOaR7LAi zjs$BWE5#_{bdQ~Va&QJ7mFh*;?$J&F%|v-}JKhUNdEVW^N5pB$+PWo$47830>pe?xcTp+VDKWlmqqTxkCcnHvAl10W7n`}b969xhMNzDR8$v@Vbt&y6DVi?yd% z!AhdHHBUIxfyc#hbuO}ie}uueowOqq%x8EP@ae+0W1Ne+03XLcWoM_?_|t?U@AQo< zNo`h(W1_4=fkH~}+uLaLLk;ME!01b>FGHANC6Yon%@yR11%B~mWxwIylRmk(x>t?X zfjLqA;9cm#cwxt^)Z$6cKI`d(Qp3k&7UJLD)C=Y=R{0RyNXM0R`mRWhfRhB1N`F0r zaUF^BO9iniyeY@t!Y5?_Ec&;02KBnhqbQ!3OHBhh^E~muBD_NRlVJzxOxjY-54>FE zIz*6TnmWD?3Z9&4xTpIjv!iuBh#M zSt)EvOW1NzrE-Ui%SF9@2_8d5)ktAfqx7T)&`*E;`%i%R5|g zbCTt~bH`>z(!4kRp6)(XFYevGrcGpLeLi^XL^oD)VpE{=riz7xuFY5_^X80#WgQt< z63_%nq{Gm|Cu@e`Z2J#PO#2DU*de&ri=YaU!@UHLU?ekFCN`T92*(1gzYt{`DBWhQ zUr#lJa=l{?IkW{92gh|4@GX5u?@49cFr|R4yR8v!9YSxI*Wev53$8i6?!Pu3uB#yb zsc%LpZPd6v?LdcbviLI#SAnjASWf7J=24o8_>9ZgoDC_X`p?qzZvmkO{~gfUHZ~Gu z(vC@p)d!vA_rL zn{3sG1ll#P`(XVTHqX#a*!tQEEbQ*{<6W3slWenP=NPb*dT#b7G01$-p7IH=#u^CM zdV-y&Pyh$Q36weo+cKDFy=l1ISm)sE_9;w!%zAa@!6^bx#< zDly*g<$Dps7%V-wU6~%`cMZ=ZZeS{Vy9G?Y$hYv@Ef?sa+_L@_jDno~ZBvWPKq7OH zAhBg5{-;8;PuM0ys$e{lW~#xdrD>14J{d<9GuN<)xeL-ma*=n0ZkGXVR4LzVDciiR z7#n-OD4L*W`=XhaG`y+u^6@_d$;O&gD>oHfh0Ch1I_}(Gq!0hWmxc)@BC^ZPsyeRF_uo(MNTmc08 zzgA_~;K&M`7C7b`oY`Bc?9Tfl$LJ*Uz>`PrPudltd>!Z9@Ik9hCf46TFT*}NTwJP} zLVd+sCF|U@qlelgRu$x(PE+vi4l;1n7=Csh0Q_F%(ftriuPMG;$5}SZL8V;z!|~fm z=t=p3=knjY2NzxGJ#1avtRSFSnk%^>@1ERn&KI1O$bd`sdFr!KCWVygLW-dZDh)q* z8qp7+4tbSt?m%H~3hw>u^%(SUYR&b*B2Ynz;?v^mSZKlzuGjRyh`RhtvM$; z0NZ)wmB)mb-|qcG@~;B=f@%xXXL(&c{+2#6`lhH>2iuBTU8c*TwuxdHc_ETqu?}xM z{TfF6wGw1n?%?MqBrGTPXUoN_oebShSk*Y2pAZwh-}l2Qt^;W~D8V|X^#*>adBySa zTecY#MnZIi;5ubv$=u&?BkA`m3Qyoq{k;d=Ly&38M_e}yCTP`q-nku{sdD#xJeV<` zhkZA{xbb^h7?Km)8a1T9|DvduKxiNvh~fcuZJ75KZHo~N)RGe#S&Iy|z@vwkt!HDb zpE^AK%oKNWN8PHRdrb&k(JRFXPSkDJ^8d1oBL7Ot%n{Tho-3knNyw~V-0-PJP*;SmGH@!K;Q3%mu8BL2$Jn!O;M7==MwwMFb~q6CgYPz2l7Ej@~{>E-^n_C55U!*4pU2%mW~tBS zTK)|Lf6UTL$H{9T{H#3_AqI~;Ez@%~h`u7MXbefxoN%8bz$mwKM<|BU7U+qEI24LQ1orFeQGXJ?9*nm0S6`eS`7 zjSG?FJ{%1`JrCA+S#M!>A5lqXj>@MyupVJuV};_E(dmyfYP`WywTSIt*7Y$76HVO^ zm_%Fa4vksyM4MRk?89=`^T$x-{R5M9FmyDRR^5}}}T zH-6`OfJ=^NGd^w=zzXH-I84i3~HBTID}=7{hcFW~gR2b~=!rXUnaYu@F4+JR8e zAa8Z0iU>yvj(#wNN9`JN@J}kU0n?Zf1N!quK*6w>JU$UivATyIjNnIvGKv2dH+IC} zzY2bMB06OPy7G#9bs3Ox4});-Wqh~uY&ISnZH zIhJJq29!#e-SDGk#DiWdzhe;sD$r}6w|z)tExdRtlLqZSV8a4D8Bnny8=T$62H&a? zCh}aU-BtGZ{H#%|A?>Mwcg3U!%UJf>&3UkUZ%rtp%AsIo=>0T2Ql$5srdq;49fn;p z&o#V4ll<{82AT*QYF+4r4oS${9@{Z`2yro#AvAa*PoZT&&+RtMi~*Gx_@>TJfqJ7+ z=IgG%hP5=L+&l^t2_2bxLngJi<08x&L#TX*ADaAJJ6RiP<2Eh{E(pP!9ds%)`+c9E z+^Ye*l~-z!ugm>}zvPx;fmHMSHy7et%Wkcma8bf6OUs$I#p(pTtZGQQz6x1cX{uxx z7Z5+gw14`A1uw_rs0*GrhLzDPzNVzUs!ElwuqlCRe#_Jsf+rBk%FDIe*EGRFfU0T! z{Y+3|ejZyvK*wH~WD?Zg_VQbRyOn|8e9AEbbIP7;e@nLunJuO#hvt((+a3FeNLB4A zjc@`zaO`iBy6RyMsnH$HalWF#J61&vluO|@D{t96lJ+&x0OIzFQ7rM2k*ECMZ3h|H zzpU>Aa!w;I9g*~bk5LZl*rT&?vvUm=vAh!!k*L+f_h}I#;9Ke(U*ld&(+-VnbE4gF zaC?Sd0|LIg%_q{w6o-2)w8ZG6h!8Kf@CKi=3aNSX{!DTsl@?r#S+7Jt)!!cRTzX$> z@03K8LFP8$kC8&arL?XLjRhAF+~pm$t_J-B|HM38&D5Yi%(H|`{dzY+>M@kHqsctSn?3p_MuJyv*nVBFSLPP4B?ia@)v-NH;Z5M+U{2CG zeI~l9f~RH7q?qKmV|n}jGn7NUKCZRvCbfmVzTt%+OXq2KzcB=It$F`3#_M`;fkQ$3>Dpsac)gOH z^JWBNy(`iOf)l7mMY~9OK(~(*Q#ecQSVlo?ZBbK6}cln z8NtLs6q6-l9!0R|BI^R^%px?YNbUsK@eVtR>g+1KcK0HAii<8}s3PJ~=Vk0|gO*)C z4`nDr?R(+|VtuJbs4v!U21;m~s+F8#?t`eTTE#_+M%*1#N&6Z$ziOgp#O%`g%jAR^U_7#F93R^aKC7wIp$qOw^&89pT>Vtj+>QT(Y zRP=N^0AH!*nR_xI;J}~GS0(IMo?9ap#8?5M1yrOQO4lj3do zFVR|R$_ix)Qs#syiB_B^KJN2~4i3tW_FV~sIY=TtS6Mw6)s9Y_ZGKt{>Za+(#lLKQ znR@E^IoPshp?~i}l5fjdPdgqUL8&vt?Z>Ebw$gi=`jQ@dBPhYS+TIjNV^(f(dEaPI z`6^xyl_Y}qN8^g|y!%h=^uhNp2Nff~jPQRPe}!jy`96r9h0kMA0M7F;WH-_dz@(Bb=+7jhkL5cGnk=nL>>*`{+!U$ z0Qa0atY(vA6*LO=Z)usEQIob;9_^-#6UA2~x*(Fe?)9ZfiZGT_G6&Ty5N}`-ib0jKZea zgg!)@5#w7pDGrR~JXu}VttYOD`S@%e-jVsS(KesDyq7T!vQPYNQ+bYc610xK(E2L4 ze&1c`OKa(HF=Z0J7+)IIp{<-rkbVAdk71+^Hk)l}v5sswXH=Wpj;k=e!S#fq(pCd% zPNJ~>>^x1o+v~srJH>;squc^BN}2Cv9V;TLpqs4ky>$j*+MqZ$V;0rEeJ`F5W&u4z zCYSYVKuS9J6|f?ifr4q;FL070!mSz);NKV_7MTA zc@LIirH}TM{t*-~tkQX7?(xA4*}aW*xEiGX{Q6M*fTi{wabP?8&l=MP*5vv_qv{zR zSaBNpFRd#_x0#wsH0e9WzrnKG!#y8#8CF>06Ir9FZ;qm4=Z-Uc}q5x;KRm@ zmN0=EmPY%J!Erkfla1S^xMpFfgfMN(u#Xi`Nj(soe02@=D!U6N_+p^3F7FO+^jA9W zDT56l5ZrT2GdDj#&evRvtl$pfNA1&lRQ7KWn~!aGG*rRcXj<&!UJMct%9!bE0m{Pu zAlE;EJx3;!cHyJqf1z;5O^;LoT(~Z)#EQwOHv_d_Jkm4XIndW;C$~%cJ>u+t3^#v1n~m zN*QQVE^QM39Q;NQtc<*2)iKd=M2k6shla8AQ=OMQVJMn{mCvz3f}mT%J@hBFD3q0G z!%}`jI@UhUd=|ZX<~t_V1RgfSq>oN_<2x?)I7QU4(6~|&D~2VszaneZL=N#0zlwGD zfH2}RGBZR?AUWQi!D~fFm_j@6p9bPIGZf#scP64X-V}W4iOI+>UnYC44&43twuZje zkI^~qao$-Hl8trHl4Sjqn38MK?%-MEH#a!a5>h<))V$(wF6{TFjREN&kPl|iogSM+ zlVBL*(JesJo8%`J7;9j17rpN{U^IflNMpULn%aR7Eo{=Sf0n)&`tvfO13A0-VMqIp z>ObhVm#LO@RUyT!-EoIf3+%9wL&GSPM);%ZG4K#;m91~hhZ(uFDWPt3F}a!ss9gwlIx|H{Ws0*M*ekI@#aWL-hJfoxN)vFNcPMOM)1Y!=mZvb7R!z9q8 zpmja5T;p6TH={ltGX6U6GOV(20V-dMm}HDBINL?WO{mi?Fy6n4?!7kd;n|5Wr>a@W zLksR<4>%rC#m8V|MV5ibIGs1iVwl(Z@eu!98@~ONUYW(6y)FizN%z>`7yoc3tv0sM z-sPYo^ez0H*&{@Ok9|H7KhaGUe!{EHH-|n#%(kU+W6=~SN@ws5AnssU&Qicsg;4#t zy^;E$bGiJ4bC5xW?e2SnJy_atSF}B7%|qL1nw|;Sye4bEO0yY2TW$MjKBgA|k0kZp z;-N}ZR*Q@57>JZ8AMo|j5wh=C`EDGUSt=#-T-!zi@E4xny|3s(`o+u^Z7uiN%Tt{Ev|EH&i^Vr`l+^17Z&=6A8kk0e`hk_{1i2bIFE^hf z$jFgDVtE7W$DrWkn8BetSoPl@)ttwcbT72e?m?c`^FER41mN^D)$iyG_}D?2m+nPU ztdfR3WqHtMjymqAhk%>cjG5n}-fY!fMinKx-IilKKM}@obhJq8HdkR2_pOQ;){^ES zW0_N&m>|%`bzRRV_1gS?*o~wtJbS&=9;Np~tWs>4Px3v-WvsVT4@gi5gVu+0$>13| znjj@Yx3g46vttH=A5WiKsHcb2azs(Ap^*GD=Z8FmkA`^zqZws9>+}u8e^Ju`k{7Tz@2HnRUu=q#&5w%zrXKmf&Oe@p!&CeBK#!j}sb7-4w(gUEAiQr-x3JRN@4#8Z@TW|3v zi`jVEiCY~QAJ5Ua%Z_sVU^n4e^b4rQt=c!SsMQ}+H>!60dWJGG(vN62kgHPds9MUbLvLh=N%2hf;&e37mYruQnUGOHxXA$jDPv! z&UxOtm7X4aWGHW7_@Jk+@~GxH73%uwS5bM~w>q$AxC_%e z3_O+W+cwoHc!C@4H4PZ_|ixR!ijA||>aF~D_Lz=J-l z@t8l(G+-AEPCY3})POH(Ev{evJAjbXabr6BJjvzUjKekkp=d7Z5?%0_bWA08*VwZ0 zn7>(=@Yf_GP%8d?G8Z1mQIYQ8WZHAOX%8oMoBU|7hD#?+dzAVp?WNMRYWQonwkSu3 zBfT)!h&`3d@x#k4(2F8+Q1D{rM%V}AODSY_kH!(%ena_vaDxuu>qckMysaa3hqBPeTJCpD>mFv0M+9uJE#y$!!{r;JPX;ydC~`R%jO zM>uR0oI!&NPNplh`jdMsq@&ybfGD2AnkvF6F?zXsxf%qQSP-NiCv()h>;*ow8jdtf zJ-GnuJNciix4{RQD~wy8!LwEig6Drn^Ruk;HlOG%Ln$eeg>i@Zv=6(Ky0d7nlxOnB zmw|C++nvVEzX1JL8w7ifQJuq@V%yRZP8;x2d>qOuNYi0Qcb&zFt5%h-DXuphaf)TT zrt2g_eT#C9B2Wvnqf9cwNWm1-zM$)d`b?Bb1$tH)1KcIszLjElW5C!vAH;o2gmt(g zbtQr@`(*S@>{-<2yd*zdqGrXG4q~iY_}vRft_dcYi_wWp_IPJ~dVnfhMlO%B`4h)z zj9vkAnPw;Ty@u+7g=cBJ#^sEB>1j8H6jZqyk`nMmNuFc!Bm8nw-NmU@d{`HBia-f| zU}#R(js2LjA*nmk73T+N%Ckt9#S8Itb?3WXuqD+ttfWCWQr`dG&cO7f3Wq5qO-$J| zKjVH53BFdj0+qFmGgwK@=1}FnyTpS^?V3})QH$3btMu5~CfRQDQJzxNK}B4KZg`9s zR!73egC@FOlPp(ClNFQef>Pq0aJJhN8pEnT1=zY+YO&Aez^-P$zQXfFJHVM~*eWyV zX}XB&3F848esfiqFM9O02PwGH=pWdE1(`>J)(DNJaKX7$;c!vUh| z>|3HbQlV0nD~c4Mm6SSJ3ljg|Gu0sKrfE=uxq|P^NX)}4FNyvWJ_Ciw4j#{ir#t?( z(iCXGoESfG0u5DbUNUMU6*Zr`zHT1#RkEeLuRcX_$J?mIdU zya!H8J>`e1*n@2`|CXQOz?b9hT{nPNrLh+tE5~pyea(#+YUN@Nvg6vf zQ7?t7$0JW)O+h+zjSB-nhoS^gnRJ$Yqx{6O!>VzV$Zkc``-NCa3WKyV>|Y7=F5!uY z&q=w|*94fRQ9HRh>C1H-%wp1U$2Yd1&TVTd9@e4Sz<8|B3pN3@o`^3ROt`?4p$gkw z^c14A$T+-@yH4b&=!hS}k%YzKQ}qKsB6@i+d5~Hr+jSAeOse13tB=799d5iMQryMq zX~A+st_QrTM=(TIl z1vrPyv8eawWeW~CNeAetSo5+C2sn@_t(F+#jKH_{3yl9$ z1gR&zcdjm#sTmT{yvoyVDQvF7CjN6>^sS7d9LqS&tI1vGGixB3faM=6y$r@YK3aP| z^dbgj5ts2X#ZTYQUoV`bwe`@Cz8;%G-_;A43emJ^WtPm{DbpZ;iHr00`EQi z%-7)d3pU8TuX{+0;J(vsBo-=fpRvZUROG00`0iFk+5h+)L5gF$bGaK$x9YI32X{YB z5~riJl6l)Oj&BL^k2pobUWV+aAkQe`Hrvd)z^BA^mpZT6 zn99iNjLf?};Fryn#!p18aN@?TwVIju+?w=%2OjsKbi#2R(5Dt}*W{^L|8tLGg9JW( zZ@n(fI$R?g`Z1vg|DaZX#^v3(K20I?G^Wc)aG9*DI=EGVbuP z9bQ6SyPiyyymkans&Sh1=d-u)tJMPx)O;o!h(?3H(Xy;=um=_BY>Tz$wy*GQ4TZjm!#^9%@M0%>`8AVy zz>ocC{B;$MQzdA8i&-&B@2Bn!Zv#N5KIc%*7T81rlll*_c9JiJzuNlEEj|*)*{6k{ zo(#{Xxofj>#H2wqMh5n*pFG5LV{S(z(X1UB#{?Sj6xy4EbGuL=2kcV3t3=krmAFxNs zk_Xkd+n2r}s3_4ma9j4Tc91v~L%rTbiLNmbZ)^ zXVOHOWRKk766}wop+9-jw0s(Mq?%U2*Wr_UGP(YnZ!i$=iq5vb?6n{7xu)=?p6!L( zXjfiV7_dr;Tu+plXGe38o%D<&LG!J?v(khWwq3~ zo1S5P^)C7A^WKng(H89HC@*H6!?j`({9a!NFi)Q4xac6}b=J5XinK*>HP|-nkznEH zxWeHnJMxy((5xFa@oy4@SBt}cSc0fr-omImHL2K5i*e-RdKvv|6^7+S7ioiQT3spL zi7Yd2+-yuuFeIOiNHXFMZN2VH6H0>a(Uo?^$l1cL5EIZr_r~ z)0Mb?m*VN_(M9l6`hTTvZML3bb zv5rS=NDQGzH|(6M6nO25lOg>QXReU{|a&6tU0Gcl% zyK5Y?3ZX1S>zWLF;Cim9v}s{7_@omv+c2ZUr-6ml7e8u=-`xTD?U=Ze z)QyGadP3A9v0+Y`w(fo#gYc$%&iv=iW5infn6yIWHdQ4#y$We{8@%Tf7uEVrU|Cg@8*L@9S(9o zV+Jo>wWD{96YB<}dv*V9#}(D6E=7_WkXQGUW%)ffW|wZQJNhcA4N0!wcsmlE@KW#` zg1xi~Gdvr){s8%T!f5t1Br4=onQnlbzzzBrY0R%mIT3SSi6`q9GIoTfD-U|=l1ZAa z(Cb4^t(>&Zv7VAQms$kx$(SvDE!+&|r0hXIp-V8$lu@2Otf^J4>;9}Xus@Tap;m|@RfbXEYd-dyRh3e)_d zL5TkC^`bdzi83GUj}*_v^VRyvX?aY+*z5njhmhlFKXpQa#mrS(I!e_@CSt8)#B#W9 zI<;>`%;=y-iPFBRAB3o5E!?4Z{$Gaio6m>EDVO(;;iF;Po}*S7Jq&9^`&&*O4z3bS zcv>EWfmyJ;eD*;nff+~g-%+qyVT#gGVXB7q!EwR!J-t|+zE6o0&{gKh(?v8!0-RRY zq;#WhaCap2mK0$tbu3lNX`HD3UiZx!pw-I2XNtGM;N_QXI$_)p2O(J8D7J@7AhPwTIRM8%NdqWGJEdvHps3wBR3tWQLOJD+or~{K4-CU(XYZ zcB+Cp z+T6z9xxj@9y+uaDztJ|Bkzptpze*#3g$dSYL&)z@Sb0X9h%fG%?eZBTT}Iuv?&xf+ zsV0`PUqV9#q`OQH=J5JsNV92;8Us;_VQ`*yltb+g3O6ScWZTp6@!2nF_THWtR<9CV zm-$ZmdjU1%QS3>V3my{VcN}28{G+a}6lAUXTf}z*+Ts!%1UKukfly!hyvxeDgJer3Y0j{T5)3cBM~i zr6CDHHM!Q?GXp2KIMjx>uKOTk8-)*=pOP8?=!}o>W0X)9hoS=PF*+!(rJ|vqd@*qo zO8Fl59V9hYwSQBy?FFZKLiqC(iSQ*TibY~3bsP4IPcl^5(~$ojOL==J7Os%y&wWeC zvY=8j1tKam(!U*Y{te6(y+esQ_2rYz z3%X$N%xg-Rkm43ONoTK2p+9x6UAcwluSri@#zwQm(XEa{vJRvX_rkH$vukixOEc2=mSSz6KQ(vo?DbE^s96WOFed^!<;BtyGs zq#Mf!^XYd^q2YhKrD4Cg+ zGU*@5^JE}vT*>{^o{I&~Hg)<|E3n{0AHQ0Oe$UP_?K#%h0QS;N7F#;CS_BsxZI$)l zvEWi!#Q6WgNbhtNT45&@a3@aJ^Ff@T>#C$kO5$7CqL!Dbwkxaut+Sq7kF`R}8j6s$ zZ0l>}6b+8D+{Cp*eK?X$imusMOA)}3*P+a5>pH9 zcaFv>7Bx)8V@@w=8R#rjpvQWXd}lBT-pTRou+Q5;J+?n4>yslfa!rZMExGu$!MP`v zoc9LjEYKR$b}s`Gw`HUzPnJ_|%Tfuov`R*%ZL|Lkq~l!De)0P*G$J{dSmXQekcf0# z@iQOOvmqfigrJ|G{jo%!JGgAAHsb)CbDF?_K~6L+EREE9c^GM1CwR{p=LKr8uN&`@2O?f)L{K`qWmb_iAnEWZbWS!dkie~tz%2o7+YY7Vtc zQ3gDaCz?^f%py-6jjEPvOFWIBV_bi0E-feJ2fbWZ?V#tucGo>$(uFITPaZ+oHJ7-rdTrL>FWCmJ+>FZJF)Mm-8mSPWL7V4MDDbpZd8h@m zxjW)6TMyhccERhlK$0;mr&vW6l>9;!J<^E6P^t1C{S0U`*mqMN zMS&((b5aXdZIk#w0qXZsj-VfvWn z<=7Oxn+BACf8}+n#cnM=UYqXa?xnLGI%RR27vozpG`AD`FnM51K3cgah^D6waf&Zv zv&5|(JR4eqe{kErGPlYi_yCT&hnd@X7V2E6QFVlj!5iA4{6e#xgZpu}^Q1Bl;)G>~{%aNvX)zyV zj75?Gu3*8IjcO7rUzDuic2K+xYP>aRoifx<6?judEkkdk1RsyU!BvfduoK4{ylOT0 zS;r^8`reOQOY>Fq31mZ(E_(s(0BFWA4|W>-P4T$KYIg^$pJt`N(kp=Iq?cB&r}2UP z%T7+Kq3G+| z*YLax8@EOHPLeQ+ZZ-pCvAKwu-8L1B80SDpuJL0W-NuOg2SX0c*N#P8orgHBzEf#< z#d4DPy(1MD$*CwF?+QBwq2}(g#eQH%5b#8ctN~aUXBIFo^YN`PU}lJEq725;)nIaSpA{ZDVF^ek75SH}%5Kz9YWsr0gd#A;T$f95&lfy@LJI2&?QdIh<~R!l zs(I0${-bnKLO&GN)g-(OUKKIcracAdBiCf6LM?rPJK)p2pteK~>VJ zn>(@%KcHB?E36$a&BPg9ei`g3B=gP~9w8Q;$bWD?2nNXrb0xZfYvTU?(f`4HtbV<( zf5=o8d=QP(G#_DLGh72bb^5O`(xC3#q*>s=_OVtscfU$f!47k=PO(jZ%z@6>jG zND#Jp1?#?&T4s8i?|=E%N*I=GA0_N(y^Jqt&&kT`cObZ3W7ZCJ$RfdI3x}kV@G%&e zGd#lU9j5|;_I>tcC1RaQr+pmpivYI8ev2fv1P>RoEZ4sQ+goFwANo8#4t3lcd8DDg zJMmgMmxjg%k1d%T?jk#>y9w%CJ$51H9N?*6_C?`w@;jDpeefv<#xT9qUD1zGOc^ZC z?>LZNO9W3i!02SlS4CWwgo?%tO;w`co&T= zkv>Oeo6(7au3ER)9*?zd`(*J)NN1puJz5+~q0p+9Y{f7RoV%cjMKeAQCQT8gfHTDa z4>OY>@O(AD1|u0g+G0PN@2MU3jflkAR5e-mSNUTjgO6k_=wAlo412lkJ=3qWJoeIXEgJFOQG2f(wbChk^?2+w*hjRe~3er?ua(v zg%x_`maG*6@7k7*hff&5LrR;^i<ivb!1Q`ApTuSY}L(6$3Q zDFW8KX1f9?t9Y%X5x^G>jf$QL>@im<4k|gw#w!yqWcIJAVxOw0Gdhf{PyHch)D<%AX|?&hej*dJYn;03x$cPZN8BD?O~-N<>3@(L`)F zglF`K=t4(p$lE4%-@)XfQyjKZPIM;YM1?RG1ei*l^MzmUE+p#WZIQhKYzqg|o(7^L zTw7F-?L?<}8&{Yp1jI)Vt48B$eV!W2SJwK8Zp@>R ziC(tJDbt0L>*?8`LW*oxZSNPs71Q4wVwj=BKa9=d35o#BDm?z1wKOmZqmv z)cKnS@s!&)2~yUj_v3v{FYXo6)PZcb-{0Y|Y~rwi^f#@YR8wdr>A{$Pt;yqVXTy!) z+*9G=Bew17xWTz3gwWoY?YYZE$a3A)U#tc|#JJfxH8Fq}{9j!c(jaVZ>dX-|p{jCu zIc8yPlu8yEPO_f^a8&hf*s~I^LC}~`huTMB(KI_Tq`;o9Yi|a*!S@6h#)zjdZW{=o z5l{@1m$&6St)_suye48SH$|~oG+E^gXYVus;l9@5f zqTU@HF*%zdB?94u3dTX^<0*$JN>7V_TJ$bO;~@hL4J_dPz^JJWq}g1#32N*oUgY08 zKb#(bA)q%he#=~iYV5}w^;U#2rC#IBUyGaKMO9u^kQXs1<*t_!ME&?Qjo|YRa;EO7 ztbZ$>nqw-IViE1NGLK99IBlsz!taUZE0w&DvqT_97N%w97NROsZUQ+uid`yM#(F|Y z!`hxISV@a)OI?GE$P$ayh}FswKc;G!9MrMUUByS?YLY8L&b_Jll+L`%{7)ks+rgfd z>{o_RUS()JDYGydB)<45C*O=6)wk+uyaa;+j0aQZYN$o;AgeZp2u07aWMOp|;;1b3 z9Q(^!Ab$CEVf^59&|ZT5j?Dm$Rb3FoivfbIOtUla22jwnt3UJ8bfzm zR?8=6Oe%AmdN7G0nzc_+@1sNCt4lGVIXFo>K4xc=eO;5~&Yn*=@LH&{yACO@Iq8Rj zZx!(TRGQ77#Nbm+8_k={phFan|J)-EE!Jn6&!eDH(4bH#qr>Nd8CO{4L?nvT7Qena z6xfWpLygoIbVrhqx%BQLY!jd20YL~?is_b)hb)vFjV}e74q%~Zl+7!!(_pz6dgiWy z5RrCRleO0ecdSF3soDTMo^bf+{rNbNbEGtNEsr=J-97ZRb3C1nqb97J0`|Z1*-Emz zM&+1ZrvClQ{CE|!nDoP_C5T?FFhzXmgIff@*#y+TI&Jv!? zbknAq2rgkZ5<>f=-K0V_jnR(QfG#dHWk+uZaU9U(8>k{ZI?eN|1)REjp?j8VFJeL8 zF7GWVq+ z2BUGngA4J0y69($*ZMUlGA8Lgqf#R{F6jcCEoT-qqQYc<>oXg(4AE&u|c*H>;U&o7xN0Xa-+pab8$l%g5BAaej*OG@GV;ye+!~IT&^~ zbfd7Z~3nT9_F94mghv%74cSV>x#btm#;Ak$@jbUF{7EH9= zeHeV9u)i?GnSh`tW(D)$(XbrN(4WNxi%)O$+ejyr3~r3pp)sai3gO@6q5o-cG;#5s zGh2M6{kzEkG2)&kmz+%|E86`_m!&w{$unrZEbj+Qt?_BKgpz5&5it;cH48B+vVKPk z&8(P&fp(wnN|~@_WgUQ8r);4v1W!MhJMnKGmW_Vdo;4UQP-ALoRXSLGl~H2T#n*T5 zCaV?K`>%ztnl$RKt2TWDyx^(3;c+yMb1**R*{UAy2Kfo&nHOLn(l0wHzw@P)y3!v1 z>r=S4SVE2Kvmd|^C`O)Bq-=bzuUgBUo*Om_15{OM{QU}cr^Hg5Q6foIUp8#tjZCC_ujd5k6^L4) z3ppXjMn37Qj&Z#Zc@E@jhwvFRqN&e%jRn36ADdabKEVcxnWKA7K6E)Rt~@+zfBjRb zkKTlUl73ZrdR@BlpEoy^~D zLsM~@fhz;@hw*C81=2qe$_nKvYu2Is0XX8(FD`0Nt)`$M2X3TE&B2-drHVK_Vc$jco? z_I+EPhS4_d(e$KQA0iNPU$c15r2V&QWR0J~90AZFImC`>LaRW9W9T<@Z)-X<759XakB50icv_XZE_Ep#vdK z1#n{S^;ORnpWQ>#OG0(GJ_3m_xIG~2nFVvk2Xi-_|M<6sKnA*om32oD)Wc=nuMX{O z=&vqlYE2~1fLbSZlSv^v`2}9~Tko#Mn<~@Zow+|%{b1VGgj`tXEOE}2T&)Ro%g6w8 z{_+J8ebtP{DSKw%U=5k=w&d#az1S;Xve~*8#Geeyndetwc5F{(p?Hx$LR50i^Go5< z6GqoA#G!F5mZFT1%nxqA-Qnsfbierm-kPw7J+J#TJNoSOO>!*3+*|+hs?%sT)ok|< zb1^ZXOl$4&|Lp_fBn?G-UP9&)eYMXCJe8AvDq0A}sec9YtmT6cxL>0MhQe*T&te;i zlIG4us10mipu4pbrb5S^qE^e?4U>{?*e5~WAn9cR!p}%}QZRM@$vujU@bEIjFu4{C z)4aN_9Eaa7SUo!X>Zicm-q-BO3t;<=8g?3@kH3>p2Q$#4SNCD4WY-MVKQr%A_vwT1 z{#+1<86bO(vv-NF3)1w_+^Z+K_NzQRBzZewKcljLGgS=STv+v~8B z4ssal00lEV*G&BY3kOxVAHEDN%Mhaa^fJJTxF_QBdOYT^oJDIvE9+ajCTd>St}U-g zU-sK~$ZPr*_ZRO2+NCaLWgiKE7?SdhJ@Jq8WK*c#wTR;6QjKxt5VCK?wCWEp6NhN1 z3EzUv&{9-8*!fg?)kyoy<%_YN{H*7^3;vI!>yB&k`rfr_sSB!>FH!_t6f4%Kv}GrC z;%LPktw3rWI9jHWnMA7y$Wjqe8Bx(9qEJPP%%~uUNFvHm84&^m2oORDA^Z1Se}9$w z3FLk6J@?#mp7WgNWT((Q(k_*iUN}qUGR-g;Bx&BcF5<_+^k0B-x6&MJA+6*@@%LpO zB=)1@ZzEk^IYB@B{do|zIr+2tR04&rix&OrX;nOFW(<1F2|djINN9 zH55FK`v^LLfX8!~{1q;M!!fBUJzgk$RJG6Nq6IQ}cU9Mq4+Jpm(i?2XJltS$%6V2D zMaGzbrYuOl{OR2D>#uA>4g0#~g_*c{UDsKA+o6si)@%)?!~mGpV$6z_ON7<8*c;bPFL1QBhsJwh8qr-j={KOvmJ+a zZm%Z%4_bD!oMd2vk@efRKwMk<+z7^R3xJ8rO5tloC(Q|9ZB|t~C_cMJ{fqU>(W~{4MIuK?4bj^Jj-7NGJ z=1Y?;;}FfszVX=>@G|6{-AG!_-_F`mF9&D0z1UFNdu#2qWDteTDW1pd0N;rtCeYrm zJ4g;k&<(<6daqNTbKMPYoX+!L5mvnUgs-OusRm8BKJ&lN=V^~WcBUx)B>--QS#;2E z(;)!EGJoB*Rt0$0*N_@fFu5RPk_M@Z;HOKo8H;5$H?vDj9mdwLXsR@W2}GceG-y5;K8>`o&rbM zw8i7d1h8fWQ~cC1cad@;oEmrUYZN3@5zj0G^`r;-v~$AtwHUh1oL0S^@bi3bZ91a+ zWcuK^bn3#=+}p$VDmdGAV5LfhlcnUGvg1Uql$&P{eC zjrBXeUjjGXea*C7F+9a8VS{TWGF&C)N}|l_#^)=8jDw_0%QmIS)JT0=#Z6xP)fh!pyY!cb}j} zS28q@)5)xKIqvFn%#&23e{0&eCZj~%>fZfe-tP#45+WmxE}CYZ7rKhCSo<075J?pG zAR04KSn=L{5}6CruJSN?KU&9bYTxW!d`Xg9_IzwDQK+DVCFYx+?m50s zcUPei7@}`=2O`0nu8_Rgws9elGt1sTI0ZLmX7p+3K4LbgQHA0ic5c{wFSGJcWMC~@ zLYvyGGwu{E9_I75Uq%e9_cTs4eUK{gOahX8DpS(oCqFP$w)JpLNLK<|KJ*HB zF0aqzmq}}I(FGr*G3;>2rE>kPEVTX!m6HmpCU@b6mINQo#u*aN@oSlUwDKtXE6%Dv zP!R5OzP<0kJVz8^!acHp*ntCPy?>%R*_UaJk4#(Ql>bSS_5URU>1?>Zo6)N?r|6Se z*&_(ugQ>#a@I%kT3J0UuDam}a;tSj>Jhm1JLK+S{?R~VIbQAA<;l57$C6A@C*mwig z?N>hPsdo+$xO;1qO*S&5eHweUAIb&lhH}!D(9b)i{hRP9$|(91=MgMYn9q7%dlCrS zP}X8F zu6{LZR-UI)w9YN1A||laWH|W(f^AG%7;d#WxU_Ab@?Npc8xVSqwYp_QBX*;f?~r3+ z$zbR?*N_)hadg8==MAKLw>V+`B}CB7-CaEdzlS=#j2JN{f2b_w3S>&w6?yhF)Kg?R z40SCzMV)3}=)PQF4-K@^3n@!>N6LPqugjp9Vr5gOq(|djE4e&bAJ9l&>Kf|hCi6UI zWc|drbGv5csh)~j*jtbZnJyVwG`Dl9x7JRw+cO3TEN3`|Wek??+#=uVsiffw1#yTJ zV1CkU(>yXXXk0@4O=xruB)L4xCeP=f{PWH^0*c-lX&Qw;Y~p=WuUWSe8xnMil8oFo zr`TSwBqmz+Ppzh`Q2Lc9-e>2ts6e#(le189$`)1yW5)6+G^y`ObSM#UmY2 z2))VE3LhDI=b-;LL7X-t1CEJR=4-c^-~-~U1zfm6j~Q9I$XBCUyC%!xtDo`!PW8SG zZArq6qLss+x8VYl9J_6jd>OP(&)p+!GX&2tW+9gAw6bzv33On8-taTBe9N&f{TQEn z7Y^2orrLEQxGCx>mxBzL!7Z*rX$2CmHgZFpM%IW})161PXS{ zYj(z$;K(=lwTzXMrixnGMl{8PZy@f{qo%<+*=uAw&XFAd=)58dE+3<2s^=&C!a^84 z>Eu$}_I_O2@82Q@D%>>`x?ukq;U@OepmP;i=24r|E-4URLwC35#Ne|iIu15@^kdF( z;Pn{IdOl2V(?kX30^}~5xDj>xhSJD6=#03o10_LADVj6s5`qo)WO-&qJ@~z&!uEH0 z40KP?hnKF>PetvDqUfIS2jU~z`j$OgA*wrOF?_!j@@DAYpsg72Myj_@Ca+z9no!~F zMACm9UtrH!kc}tQr-U`kUwx=DwaiHvTz}(?kg8N2GAA3qFM^b%M)<7344?|($N@%5AQjZg$fqs(Gc3Qo3fyNC*$2Xe(@ zJUf74{id!Sq8Z3jyysB$_we9?|{ zZBNu-(nZY4J{mTxNw}fjZ@M$%Lub)UBk#4(5CIAKa2Y1T31{bl(W3{R|r#N&yFT{;^cLon>}L|DxbMS zvE!{Tm;j33XLzt==gyato6Y7UR6p4JzTn>%8f>!Qus z(?zOP4omu`DK0+v?Vje0NgkO>u3u1cz=uR(-5(9>Ho#vwAL@9Mdky?!HMdI97?|td z+U37tEvevr?x|$ta__$>_THKp+;E2jUYPs^%FJBXkA0nn90e|TmVySN}U$@_k~-*0u7UmGXl;aNRx8pKTRHmjWFWVec-T+ zZ53uDcns7SNawo?mqw$J$rwl+E7HE^hXpWZpJ7LYT2;n!6cGW>rwd%$t}%C=Qaifs zEp?P@{KH9EAk}1z;9BdKK(Gv{?6qP4*?9(!Rc?$#sF7L|u>LYo_EK}jmmY2drn#EG zXC93NA~}}SxyUu0 zr1G@N_qY%^|FAb}5T~nHeRmcBbkAj7l7D1+xCKuA*m(%t1TIIjW|MbZu&%e%Mg(q= z{%w}P5>h&akyS!cVRToX*!Cf*7u(M~R?E8J)n4?rY5o;RD|Y>Hzl`nnU7mkyt^0z{ z(PYx6{E%MfYAN!I2z^*?hYFM0bX6DBNgGY6{%u;+pzOI$DeHeCC5a8;UDZHlS^Cm@ zkC7XOxH#{(hw0bEsI?K3((uDvr=FZQFRJmakA<%=4+2#Ty)PfY&pk?bc8bG{1KT_4 zE;W!=VEb7ABm#yZx5 z@dz!{n!}Os-uaqmnOjjCuVb}u4wdvLOW1ve%PYRbw;ib+Ivqa&W|=p%H}2;dkw4LH zGlCu7JbuZf=VfRt;uq+D(M1!(7_f-VtzPE^HzV*5bVZJWDtcMg1zz1o=t7)M2gb@n z2i5jF;IJu6v-6kEKiBh6<=e-5wGa`h_sFEY@^!+g{Dc2|N4%w(v`VrzxgmDR zu*Z}I*w zo-Q6AwxHvZ&+_R9XS!Nn&tLv_IzB_u?S6%(S=8=A($6A^z9JZ^xUe4JsatsgZ$S_) z%(TjJ%2%&qchBfgE^%)<{J=GW!S<^JZ>9rt?=vdE#qwUZ#1!l}#-n{%R#?{5<{?_} zJITREBewT#c?bX5`FJodt$hDKv!pgzTU!4S9R4p>I>ke97M-T$e_b1_jsrwi?8kRZ z>a~|tg`O<>%ngZdJY!2i4{DjdmOn&z*-wO#csm}xFat$S-inMDzxaNNE|jn8q2@0U zn@j?UIY-HEzj@{UkD6X~g{o%ix(82yD2wY<;Dxdl8moK9!t?Ks{+P7wuQLU^Tg%xy zM6FkZ@J&T2%mw*>UfP|zn00si92DuSCp+dYnt;M?iHEl!ZY%na_H~t~n)@{tM_5Sa z04=w=R3d--i+XaCWp0z8=oF+>S?u~Ps3S4*_epl(15#7XrrcB=ip+vffVA=GeaT*w zVt4|oh0SN89h`}N+n)K44Q$*rzvWK@=ssyib+;@|!j1TruS!3^A@0uN8%p+FSAy=d ziynQo8MG|n&$idlLTDKt$rr9e#vVAp?zMl+Mrs-LIrX}9?wuK289C~|}7A-RRhXKnd#yK*mz z|Jkc1{<*^~a7r)tkAujY>vZb#u4w+!H*A&O`5PXsaCRI$vNP{ama8Fj?fK^+@r>`a zJ4_L%TW~vFZv{fn6WZt28@hG@rmaBv5-QZC*y+Gp9xF@>WKK zVCGRQ8#CHx?cwaRTREbSx|~1vmtbOvb@g_1q|vrGoX@*>H-$<6-;awg5b@T2!MGX-SieSE&C?<7=?pO^U9Qw5Er$ucjr0W07 zsL|y7_b9{C@+yIK6iEWZdyeO=LHI34k1uh(7aNwFuxNLBq+UE4{p%NS18Tqd5`4YL zV(4$J_m&k$x!I}yrK@k7k!#8G) z(ui71gWoO>pXV!bX3Qw?sHzU9FIzwfT7LonUx=}==4@Qfjv3kWE&LWuM=};Xx_i>!mHmJ>r3{DmLKaHj&Xlh}x&Kr>1Z^_!mJ*+BUN-JQgD9s^fXZgH#k!uita z7YGhKKfB|5j%LfEQ>6Hm>F3cEyJJx!{{Nafw3VCcdR zki$Sbz>X!rMw=3frtM+DJz5BetZA$8Pk>*+L|5mU~)CJ(82RNE?4%! z!HDLRycuuEsI_D5Zv<{zFsfbGg~<_QzRE?(PVPY2lbSf3uZrgO(JE@mz2z?bLYC?v zvUl#vmRBzmw#Vi5uN^q8FS*U7_G@%Fv%(xXis04lODsQvNrB-L^ed>+#|81K(snF8JWZ(mY7K zXZm1g?_k+F`6ipZhO%O6*A8&Zi(MD?-|WPnNf)RMZBeq9K0hb!?;<`*b1zRx-Koh{ z+m^>6is=J2D(80Am-95IW$JDm~5rJO|E+?vu}KSSNyk;Ghji!Dkm2wo&J&r_{3v=MaC4o@1-%H|fC ziZ{|Mv`Iki!`i$t@AOR7K#+3H1~L$KbiTSH8z~iVL*K-({}2Fp{9&6Og@sR#N;HQV z*Jf?Dwv_?gQ3}_)ULq5GOa3-matT5EzNWXo`uvV^;rWlnSF`XuWv46FtU8NAoHD&+ z$1{LD^*q+9Ia3k7z106(NE7MayDP$JYo`KpG{ojsQg)!DQ@maW@FT;h*LwOhQ<1JZ zNOOym)0v9^fsJDFMfHU;fo+i|;pX<)3Tqj_^l}%69;rS|`k}UFEx7;v!)5#_6{ZBB z;;VY(9%s&f5~}jQ_3#x?E`IYW0WM(8yr*ITtBqslsWfYgHT5UKT1U9$+-J+YRX1kR zHY`U}oThA_v~DSaVKC^K1lqF@esono0@qDvBecfR=(*Zqzaf=h-PnCfosqxHlC@po@)MUV67^>g%cOjqy%vM+4pqAMgw zlVy4Yd&~P!T|Tx7&_t7$^W}?eg`tHTX^1Q(HobD2Nk9ww^R;%9iU^NY_wkJ*QvU*d z&of#(fVw5ZS3$S&4t;qq_Jp>P)M%l2dKKzLWeZz0=S+c}C=YP9as(lXwUQb#6iP&= zs}Bo`)L(f4yI(B+jL;h?jzNdy6!b;jqA&B3t& z3^we$`pED4@Ezfaq`cB;HoLsPDmprV735Yn`RCf|a}n2T!pu|w`(J|S3j zWP+DwW?!Co#puLIxu^?ayk$;f?I&-2R4l{VvLh{rZh|)Axq0!<=|e!&%pa z3@m5Q$rMdylezSr>h<&s#HKjB0Jj-!xR`jf_RGt1e^nWKbpOzSdY?EmwWvg!A-ifF z5m!E@jSJSwYDeg}oicRm82i?DV-qzqwvIzZ(4kFVYCVsgiJM>Zta=i1&N7jc$@FA2 zhU_RF`NI$R=5TI|NrY7m$-Txq5420o^Ax8(+>#n2DOcSeTO*7ncR;DiTj;v}N2N58 z^W$alsXJQ#))?!y)!*XxyLYqwy+YtOa$?uI%R*PIV?s!kgH**m z$yeQ2JmV)tY^^ToeR2rDy6_;v1I}cv4-W@4ur1nz;cb0o;6BalN+$ArW~N7I93G-f z^RIU*Y2T47?9Ebn(ei)Gbhx~IULu^$+n4V}@L(xH35;XIYXB6RsQL{nx-`Ien5ByQ z_~rgt>Sg>>K(75;&BA^f%s-;Lv5y|*L{xYDHcYhw@*cX0+a*<-ADzwleM_%B&`C*w z^}>gEFL&J~6j42nM|IXMaeaaX`>$Dij0GExdh7Zl`kNDkYf1)BKXydfN^@h%%3iYC zxd&*wiMJ`{44z(qKUU=%o=`>_vBMKKR}Hd?Qr!(dy?c*8-sc>x`Myd~xo51qU8$cN z01tid?R8E?qVVYVT|f0$Q!!F>wDx&emRx?fAc-lnBKM2JzPIYw5PHOgy8YH)Z^P?# zgqV$rNAzLAfo-=%Z%N(Aw(on+VImP|(z{R3lh8}6(s;hhHS>`IWNI1n0`W`QC6W;@ z23%mmUXe~Ty9LpOX9XA%3_n7kyr%= z&wlxJ{FQFGplDWiDvs!HBcTxw#|L{@3(GvXbI1gf{u-kIR}!#|UFmuG;x-t;T`Th{ ziLv<%j53eXq#6G`8%O=;_#A``d9NRh&tGj12wOlx1^1!y%JtkTm8Zr8SfZtY++;1(?X2f1UdrKcL?@WnWmj>`qP5-HoM>eSbJC z)|njdL=qoxZ^|a94`r?oq%Yx9Xu0;@lF7Rns}GT#AN=Ulsa)Hj-_~T17j2IZ+`IzrCd&v~LY1Nvw24l^5DqoIjk3$b-)jtp3FVl3-9*G$X zsOH328G73o7k|g8ti5&7(~YQ( zpWv?QdVw`pMO>_LM&>)HNu@8O!-%hW?X73Kye--cz@~MP5CV}K) zNm=!`fu?0;cGdmrR!_smUJ+DueLRT$QsnjQ8Q1Ouli4&Tr`yv=zx?A)K{zM>vtVA6 zjHqilqr&D&wmeuL++4^d6#ernTaoJ!ha1P)1FpHA+m&*@c> zUI@k+1P9HBb^g1s;n#WPp%1=#}z|pjrI)=wenLq7O_IA7w z9U>C}xc$>8bsBqAud3c@5OePdcF6w48a~_Y)RcH3Ztc+Wll%(ufn6iXY$DlQvuqAG zT8QY)b8$y`&no!LZcAhSb$wg^zf8OOIuQ?POz=eB%3@$M%;P<^j%K9w9~~%YKgT>e zqmEiQ;8slxUl?%ZKn9E3eV*9L4ViLNo_$-n^W7-+5<-r@D-6)xLzixW!PTiQZn!zy zqJfsC&yao7^taJ)PzK^>rrw9>9=YpAE!^66T)IlA85jzA!H7kY!Jcrx=(!XXhQtfu8Qiq ze4N!_#?8nJM_`>|`TBq0<8t}!htiu6d`fGARb8RmcO2%nzjAFONuawSi7-F=oT;3+ z^gobBxQbUM1_YrG$Dkfrv3=Iou+E~qrVt9DUNT)qQrRUYeN?8jK)00y>#4=V*X*&| zIW-rGT^=WVGsJ7zA}qiC8-;my<}K_~x-L58$Fn+|b4g0sIvR)9a4OV5v1*DwLif59 zu%HozxXSkG9IC$TNM(8170D1$y^^V@W>dkWQN1K}*Y;Vd|Y9;`eOj^+R2W zC3)Z*O*F?EDa_;hFJb#LjD$QXR_wmvADvC12WgFPEExKM+{dp!HgqcQ?BC6M7uBmg zdsg9Ruo*@Hz^CM$-Mp1J>zVpf&MG{hM#{RB3UAQFsibtxMl{KsJz_r`zWe#LC9&+k z&za&|0^MaCPj^)9SbDQR8t`?w9L+aXcuV>Vlc>9%Uc`k)di{ zZW$SG1ySUn;=vnN+gP@?q3zRw&6}z=Z2IAouW{v24Z9zy_Hr7czO00^8I!NKobYM- z@cy9!7Jj*z{b8$Q~>^Xlt8J zY+|+B>N9xe%I0bwnavzrLN7dzB`ciP`JK0PpJD7tto z*>GK{tF0Gmb4bmc8t!uYN&HoraluK2gf+O6(_eC&EeNT#DC_ZVX~O*~OM5E^CK|Y| zj3p+TMS2IW^#ExK9E?tyt!Gpor5y@+u?kWX#iBLAoyuR6<2@4!N}f(3>?zD?;L)N8BZOBuSsL=`X?!Qi%ocd?h4?OhygP3my782xv6 z1uxg9UuQWHiH3okXW@^iM>E=@@WK7*qG7g*N-RLBlb5Y zPZ&tQohXM44}xHU>u}o6^6Y(1Jy9L=a-w7TWdAWR>_a|azWtZolp1I^}dtH9g^0rI+7^U_1t@IgvAL!}rD_TBwh2`w734I|g zCz71ws{Mg}X(zXtC11v_j|#aF3qDOzX?sC&MEg(I|1MOaNxY%B?_v*kT?-gCqnUg4 z>F5;GY$^7!Y=jxQI)?VOv+jPoGpmC7H}T{eX`=jU)4k>Y4aSv4jm-u8Xni`dmJNb2 z^J3$A(!zroo59pDG9juW=nM6eCZT;>Ph0z{qOz@zYSmtW8$5FNnEFOsU|X~EfB|G( z+Y@B<&!U#1n?YO=I#2Fdu!7T&MHbpM1`Hv@s$m++=DWacd8Ln6|1g>kYR!l0H3wc3 ziSOe}LQavxb2v?X1$e51Sne~lpP0y&N76~q+HNgkha~fj@DDbtxtZMEq!`xe`hsXC zv?@<6sndXOv8yw-dJMtwy`#I*q$DhN;bupD1F!fS{SfVK6+R-F%o9IMe1|8{OqHew z0bdO_xZSq>roA9bYVSFH_*;0saP-MQ-*>e3zI6|{bD&h0DSvt1f(49N=i5ERbfCd5_HUH^-xG>4sakRHF8xK^rXDJY>EyM z(Sw@x-k+{hAL?7{30-jOmWIh*rN=SeQ|mMKXWB*VX$GhLi{H_!7F^Qf9Kn4vI+ACl z`5%!8ZxBB*`kjNqM($nxyJ}{o=0(iB#D7a@BGz$XyxFZ0+!1$qBj`FN6`=k1$$^p&5bbMFCH9@f+umh#EJ#$Qbc7EU)kU3s zhF5UjtCkWcWiDM+GLZu&C9-5owh{uWSv>`5FaWE@fZwAY0X&mZ88t!p8-W+<4t!tr z9AL=w2Qz!f9r{JdAHvkHH$yKZ+~h1tlEo7pUyY3iK<)8tt`q5S_tV`xU4VCNkUpMs z2TNE$9n~K$Ah+KH?(Skspo=U?sHK_6t54D&T&(~JySzi0f`w!SeVk>H1+LdvsCTKmOS}ns_u|ygzXN#K z;}c~ZuK*4|s%o}wXouyp4Qp+mV5lXnuWHGQMOefBhTtXnr-pEL!y_5X($PY;$H&M$ zl7(i2!odpW`|EHHffus5MOrt}quj4bC%-HwkGn71x&n%PEQv~30}UR{d)3lfj7u+b zV#@>FKoe3$vzM*j0Q<_m3xD@@-?XzG$wT6Uy5X99D zx-u3zk&m{0H1S`89;uLouzf-jZ`^n%l#Q=JzxNkD5KckiSLa5+g~Y*aDPs7}JWE07 zzqj!+PB~_wM<7uAR;gMfjsRUsZDWYFBE^s!L+@aKevdPUDpLgIAp#7dzk3SCC7-Sj z8Y~CT>{+V90Lq$=sYc`sJWa132#F=ELy=Rk<{KnIT`NT%>QZF3b{o^rOAyizm=^s? z2A^{uC2cI-3=hiTMT|4Oil-0>#{9;xQvyBPi_A@luWB1gGfyE!jT)1^Gy)L!uc0kQ z6PJsab~vfctlnGDZ^#;1q1INn9FXRaxxG^IsXgk$9bgmX)7g&OCuj zrXv=vADOGrfp6n-^)VNhVxW7&mP1?wF`nt$N?9oDC^Ez6@HV239^|F?&GLKzeD+Y6Of6B3$C$b zBCWuEN-05CQNh)&(j0`*H_QxMG8#H7uVstl;l95;pj>21DD!{Z`L`Wqq(4UQBl6KE1@c2^m!2zt*VKh;{T_7sY@4cW`OBLDrQys5Wt! zGgIb6)KJ55v`>WR@?1Q+V!1muR~LVZnh4+wz0Ma>_M!m|@jctWyBO9f=PKm5QD=LF z^6SaROn9KFRI>EdAZFhYkcdeOZ`}nl;Y*b ziO98L!2fe%rGAotXxbQ}L!Xk0r*X|N0qfhR#ulMNlyIo6UZ&yd|I6CEBaq1M1q;*G z-)cv|mD{?zO%FFIV@n76n3Z+~Qc}CS=Kc5Gp2o?jiN6ZB8=m70k z`5n@+DL5+W0W#rdtjHP!SBdw?k1|1l?U9GZ2Y!R~i%Vg>9P=TzZq1)hpaK`ph<_Be zO&DBf^isSZn7gks0MNRJh}mfwU4VO!C%fPv1|dGj&(?a}T%5@=!KqOn0&^TLX_hi^ zcSfwzT}RHQY0%Y>I}Bnn+WlW{!3uU*j5M)HNIbgtbaEkiRJCu5qFx|V4rC6uw=LZSLbZDKh@9wr z(v`Bb8mxKHCOsL#<|0&HGj3vHQ~pYLeO8C@FyV^dti=eGTDGWv?>JAqyOGB25t7P% zTwailGK%Vbz@n#xk6BG>o4P zu+x|I-p9wA0GQoICLM2J(Jz?})?&mCEG~y4_z_N<_QHuz;80W+?mgt9ux=9B{NF>| zY9bG;(4{OAqZj0b;g#eOM?1P?_8UoT$roEOpV?VisSo>eA z#&9i*AgVH991w-FLg&!~fF<+UwBmqOD{(-%tv)_nxn5btv>tk-<$EWSC6lo$bIWE7lpUJAsD-hiQI)Dqh*c$TlK`Q+ZT-B+$4BPA`WO z>Gcgn&W~&U^p2nqFtxALNb&)Rg^0c}quBwGE7UHzk+}hS*iuA$iXU)DT|eZ=-S6k^fKT+1Od_Ky&2JB(v92uJH#|Ok%LV|ci`qv zoTtnm#JMU}+(=}8MIJ3UB?pV6ELobJCvz4(lG;1ZlOF&=%jGF@j8T%(#9cGuOUQoQ3McSu&L48sz0NG98$$ zWR-n&4(1a_XDyb2Fs&-Az3YE!_` z`T%!f6Z30Qs16oak==M^dLKhtA(>><;z}l{2Tq}}CQhy-&b@D|;R%G#YJ7gSEqRD$ zkz(uX?t$rewi3(IlN!+4BZQH{m1KsN!bunrLTbBQj$DXSS*%*F&apz=%CFs>-i^CX zskw11$E}z+b#N9opzGad?Zf4QINW4uMq;(V*Ml5W+@dPSe&w$QN z)WEwzn`oL1!|Rq82=8qSNAIfqilWO2u4TZ!#kHG5TDBe-A%}fn;ejH^MC!2)b9e@e?IY zI7m*E#p@>G8Th9L%{qT2Nc(l zBTuRP*6KM#S2cq*Ln|+kf1o=>za>+FX%epWtJ~ykpX~Ly2s=d^c1d;uOh&t29Wx7j zNRYWboE8DBYZPtH5@}3`_tKtc!xHVxa!m9s9I**6)WSHK}@K&?5EnKq3ns9Ed zA2k<*7U0Db-W0qbBiUrL)yYA61kI|cdCiWXY(tr!^4bHDSBQ&(mF9u_Qs!*85GxY+ zU9VuJX93n#~Jr^KP(7MdXZ*;ErzGUtFH*40E*8rSr6oCux#ko%$WK_&}y z_h_>xx`FS+LasLmJX8&B^KQcxZW zmZaj@Wf4c62P#1s#8-&#jA>FuMF{S=<(f~iYWA`30;rK5Xw36zL zqWEL)GIRmQFyd=zsBlx?DE}Y3U~GBZIA#LSgQn&mm~jBh+Rsa)vS@QX?)SEVqjocIKz5bG>5p5W$09>jV&l!jl;P@1W}9)wDk;$I#Oqa$5} zX1wVp`>~w*l#)48lAX@h9o|*IhNXo6Zf2VgIm8+MI-ss$lznNPmWGJ<9DJB z2%CiCcWjJmEBcYG_)YvAJ{9em(gm3JhJ z2L_S$e85=jr{k2J@AAr?Opae;`-(IrsNWe4I*8CIA>A1984bNW{8KfKrsy;D6^f0l zz6=G~RAFlyR{xY>IpS05i%mdM$`gv-0OyKLyow&DxF{v|@ zQ35_&RDr$go78@9x*7CLm8OpjAK;rb+Gb`bzBc9DSoM$|T}(#~i_HP9RC0Sc0cs9{ za-q+p2IB8d)r*~JpFXMkO$QA&KV>c5I{-EO{jUr6V#C}MCm8h-w0SUeh4DSyEPc&1 z4X<-0UV-t^k6Hl(#*janKTsAk!(?6cvWMeI{S?#w2L6MSv2)y>Qt}>^meR_<2xf`4 zE_plJ9$CujU(*j_nz=Tdh<<5ugT4E7IFTho&r1bwF_HkulIDvgs8sEf^6F{{P?z9r z+;krPp58-C`Goqbck#wpElj7_{75 z)BVIeGZ3XjzN)FDVI*{xbgRI`_O%k7zPHmO$V8Z<67vP;kog%Z?NVTaqJcE&Lu5q4 zX6e;3EX{y0(cZVWf(W=MZ5Ls&9p%H4m=Bx~S)9HNw>q zNICp?A6>O1uqw0UOzn~bck_PQAb$on=2$~8dv*0vaBr30d@n;k_>YH5SYV?jY zkzi&zuZj15SDTA#t~D8C;XE|Xy-q^1D{JlSV@Jt%9PeJe+YLly-Mq`lpT}f!b&Orqcs<1Cg?6XKU0M&K$Y`i$xBj0t2}HJ(Je2wLdoKi!1qcG7D_QeXmu6!lAC(N|y1%by*NbBU6ngBP z5=935AXSTPoulygLvKlS*B+;Gyy#R-qVZddHgE~pIC7sx(7@57k6$b9uMJdJC$R)q`+_8 ze88ZEnc1{qYFzMpT8iUkA|fkttxvYT3z*Tt=L?~xn2$E>hnp6_6%H8bRJ18mh;{j_ zbNiS`60On6n|>z(T7qAPve{(RR^YLAyS7(2aU0X$oH$j18cu7GpDK+c+49BpHe^vi z4T&vZwU~=@(b}(h%)tB<>tR)aNyP=cny&w~7XyjZRfn6DH|@?S0fj5N7{*O&FpX?( zW6V$nT&CqCKj~9fB+ez{ncl0=-GVmQpHn)Rm8tC4lmL4o(+xCl&#z4|tF+j5E`_K< zzOz%uY<}-d;)Tqn$S;?{#s#aJ)FJ7zzTl^+k?UNsCVU@eOcL_VmMx8~#F~yth2g7g zq4QYa`?YQdtku7j;W(bKGjzF9))M|iSYCfa%%+2&R4EHIdk;YFVN=`FqE^3eUlFE* z!|bJ6@U#M3-G~1p8FnK}#M_IeDsjumDXzbUBj9uu*_)X8{f=O-u=k`P#tlVGX)HA< z{SXEqb5-vfrc4yUW|*w^6ry-p0v6pTO4^Ro44j^VScZq^JRoIH#N z?#+5dB+S#tht6z7Njok@*P&nm4YM_;^^H1g%31`t_h0kgVN9;ZI%TT{7dBJqE@|<> z3({Yk&IVSgrjNMHAH*6%0)eqK0tP58dGl+e2;Y(GqCRqQJl4zUccIj21(wH+a&3GC zV5(MUrBs)hhlg)Adil5r&H+UvP81>3SJtpL$Kg1-nbCUIlangTdqJCF3m*s-d8!k# z0K83Td~7JZhIpbn_ImEM;&2=C! zQ1D1v{ZTwgZ4#Tky&rvYB7t>KYc<-3i~4KI%McPXkNe5tdO<0ONI^yh%-ibNuf*eZ zEl4sbior5&x!?#z=bDg2sC_g_K_M}=g)j`dokR>-X>>UG>zZc6-%=Z%?j37wLX&PD z4#G1R4t%TwrQ}1NKQ@oECgWH)KsSAZm@dpfCYv&UWH4y83Nle#Lw-vl8T68vKzG6L zDWS3};$<21=!9l^*JMyFMs4WFe{A~^*3A1mQCv0~^J5GybFJ{bkLdl|(o&!#Oo?mF zyxfQ8jGDDw5?n`2=FwjpM_7=A9#tP&72a0;6P)*)OfP zm8glO&z)_;j{)PrM4fM+B`-lwfaJOU1o`MnAB7+Ot7OOv$%&>9{N$8?6|Ca!*9Fdl zbIRE^k1vVBaIj{)lpx>l0!Nv2xC2PI`wk?6k>la5t8(?sqn4hr!#3+pev~9F4>*!fNi&3 zoL7k`1a$j$LmH|D;oh4K0y6DG_Qp2&NFGLg@ot#ZNr*eKteNKX3b6CG^y|#8FvOfF zdP;V~hF?y}h(4hD=`=uY zAZBvU|KJHy#~Q{7@iJg*(U@Nnc81qny+0YTHG1HqOB-B7_q6tvK`Vik<%HW(oIELH z;A6ICOocEs*zlv^1tb{?DCKiNvP9ETh_&SAWth)Q?#21n4d{f-wJ3E`;TQy`8$TK{ zAf(X@W*)g!2}fRUmD^SJ2|)~ds1ajbb@z0I9%LHRn1G`AmcZEqqBFvc5h!zX=d;$e zTB8CG>Mvnoq5InE3_Y-)GH1b#G3;x+hqsG0Qva5Zv5H5J`RLH{YRW*YUu>8mEGJDB zPVa1sYGL@Jss)L(5EzGD8w2$qj0b@(qp+N4#OxzyK8wJeh|?+ zr^zIr_^slbf3P62 zXb{R$nvf@G=V-nRP>@UC-`#}78}7V~!wk|`7MXgMM_~8&)c&-$a6eL0#n=Z2pcKr= zC_QwByOq@`U7xJA&vWx@9AE>V*Nq9oDKjH2M_mA{F;;4h5j<5WmO7a%x`zAqozIxb z!YBBlTS?a#;aA*s$N9&E$ZWMoUQ@=%5*a3IT*;BH8JC$?QHvsB75C^@ z@ddmA5__W0{SFnkCSP4T5?p4{Ee;0a&2&#$PM}EpKux-Gf9DM&=gK-+h#!~`yH_EJ|T&V zrrb;V4M|Adfo!oD&YG*3>AfzItYVTw+B9-)X}bNr&uD+bDmj4oHY z+3IJR{sjkGtMg7FS`eC_&o`A@;o>S)H?|&jCDSkF)zK4}@ByRmsRxMuLKDSxGQXER z>{WFskXCBy#>3nyobb^r(v>9;Y-yyhtQ|vPF>coJ13vg$)F{?m(!t(wzyAF6lZeSh zG!f;z6QaMIa9*=f5J}BhN=gP0#K3PF;^jzhQ^t=DlEJ1Gw(o<-9*_lyr)YPfsE^+K zD^aGnSPBxSdd?vyYb>dAqAzi8eR;R-h}ff7`t^vZ1cp+5-6u@9#cfqe^A|^pl7)lp zW6#=2wWI8GQ-kws5OW6Q%*|U6uZSa#y^QH7M!C7smsSH=zhcirec!^5hnsV{OJW%0 zaH-Z9KH}m+#|edb3KzFSl=8#_1jc2MEF%k}STq*jSx7{Gno94sU_n@c=v4v^lfG28 zY+ye;f3@2O+W_JMOJ(B;brMR|@uFY|rX-Ana|iJ^L2;t1I!OTCzP|Mbq|bnxi}o%5 z77NgAuXCO|1A%JSiMqUB3CoYQ-H7Rfn@J`+phV_3t{)G#Z<<8;*f zf7;{8=sa30n$bJAMX&PCMFCt@R_?r$lG7QNRYF+Cvpg2O)%{N`W*3&4pfV$0NiTOs1uxNzA%lPK z`st9ZANn;r78D_0@0I^;j>9_|%9o9W8k9J7?~;^5C_sSrwbK11 z{H9QL{Es_Ag$gP|M!F0MsE6J8YM%~Qps#+0^M+w(2oimhZUs))Xb!t~A2kQZa`|h+ z*9NGjwHj@BR*t1+wO+F}$NVZVO523U-_jKu)A{X>J4qSY@A?He9{r#u#2ad^*tl05wa*7bKX@W z&&w!tN>Y#+09C?kQu%dKaFXof^i`6;_bcseqTy~muR|_YYZp2=zNl^c0$F^JWqYR@ADTS2y>)^ z+bA!K5NHUS-%q0Yx*1{LpV7Y7;#V2B7Iq z^A8}ll*pqNs!>|r!(GX+C!HXTN3Th!$ne~Ls%m>tP4Z%wo-7eTh02mLwXet{?y3+t zRiR8=@KxJ+f?O2(`0J}{qJ(-nJtWndTt(kysUV5nFus|a5`c#IbWNdR6aZRcWY+%K zfJa9elE2am5crD|Smdh36a&603(+(Xy-ltJaKco<8S*jWP>ggIrZ~r_YPlCS;=huz z_K;5EuAI#DLj!U`q|9pB2rnoVNL#&PYw$2OCyvz4lx!yASHF#1e~W7zBDMa8Gw7TQyx;lt^;HN` z#=iJ}0%=3+hr}n6IAxYW`=^22BR_AECZQe+LH2{P2oh`V^9ym*Qn6!2^`bGW{=`$9 zN9@t(j|=rA5G+cM%mRKwr-VI9?1On5`npR*x;%gxGnLf2IXF(iTGj8}@P`UpA5A{~ z|2-!={6{PBoKzP#vMEr59O_9*i?1lcFOnjvpW#ua8AF$v6t(ZMKxPX+Tvm={_?ZhNq?vJ)Uz8 zUmNe|TD$~sFEG%f#Fz8g$7;rtv3(Mc3nnkIXF5B~SciqQCiY~mCiGvi{O1Az0j)w7 z>$3heV9J-G`bW-mJm2Dw{wpw*=B&?XokPwdon&K3I+23uJIP6NU|Ch#nN2CR-f#zY zndaaaMFayBwhiHk+cZte(5s;Dwnn@xX1c(_mA{rILzYx?%RAEX1Ysg}2dSsYIHU^2 zxn%8vg_Xt|j}F7O#v1nVnBBOH8aF;u(ZFKt`s2LE{Y{``H%ZsDodlX$teE1!vPN(( zs8fZOk-o=jlRzrd8|osgjIL^|LDx@TQU2f|metwomcK(b?L!r1T_1us#%#;)9<&w0 zhSukMR=^qFvCO9o;J#5`q*!g#>0yAfwxdnKQ6Th9@lH}8n&o+A)mb1dsA^+x#X@gn zwSHtR2X;vp{G9$d8v%J-a7Y7$Q^5dL;9d$fr{=6ETi0yQ!H%lLj%17_nGR04k`pXyS^^3CIm*p z6IDG$Uy}5ktq5q16ejRi2q!b=khrHN#>N*a3c?iichTgeiK$F+^{-?D{>wqd%iU?8BqD7%fl}#K~YHgubEn0!7sMK1C zjIfiqQ4w&|Qp<>|MMa2+%1%TCM1)9Xs1d>r5+H<-y?)R2`}?a^lQ-|Z_sr)!_Z%|N z)7_;Jvpx`QpOI*7keogcdI8P%ltrrWRDdBU9tq6SMlk#U1gWtiCWe+yIXz&og73Wr!lL3`MVbu8jVzj9q)O3BZAA>$>*~adF z96XUIW$8Z*oLhpnUb&(xls}_!ef(B%-whE-1JwZo(F-~e3Rz3f2?cojc72l-UG^66 zhw+BNnY4Hh)g46VEOZHSxCA`#}c$@d6r%$@R z^%NnNGo8ro|L8w&qzY+{a=i6xf5ZA+c-8MXh3BC6T9|3lzX4y zO3bxls)aHMU7nm`{?MnsMZf60qWQdDI8C%kVt$?_hQSr7$5s$?#rdE;IquQ>PC(va zq$sXzKU|v-$A%$GE^T^`|66IlV#73RJqAw)(f}k`{3rcF$0DDMHMWB3F}A*Q)Qy4= zNohT;3z$rg{~J}38#;jXZ+w_^pz8a4k^#*xe5Z0hwjXF z@y97B_*V8L$1&UI?Z+``AcdWQHFdk82lS7eu3_pslA1#fVcayRT-8j~4OC1paQS9^ z4M0bx+&YX=h%iFb8TfApkeXrn;VG_2it$m(zw=`e&E8x=ivd-5EPwIa)y&XKFp-dIi2W+3DmFfI++Q%8K?YI_6-Ov152W=nW z5Aa@`J|VI*w$LR7ur`!rkpwY(QSdQlg9f|^jHsWhm`YsV4}${_;LdZ|#-WFrJJp1pt<^d44_+5G#DrdJH zdYQk0G2h1VGAUt%>YNizss%1UyD z)ICS|qX_8rJWI!zJqPQGCn$JvwT^=i)kks6<6jP3c|HLm{m!q3tCM-?gai#1bguI#6up(b zmzF~ag(@2DQr~;Mxnp-M4w6|uy=Nxk%Z?pV!)7`rvMDtW(>^CxbaZ}?t$PADBMCUE z_)87j*+ZM=_Dci}d3WNams-#sUbA1O~Y#%7c{Lk~0q8G{6=oI{(+A(# z_$_;~A_3Nb?lpXjz-FuuY(I``fkU#eEAuHrvMM)yl?rB)Tq6%Xi^@O-<8&lrxoUe(ZVBLqmt6@D}(Ip037;5s%ujNwfMi0aU2knuNVU z_y+}}UcLriX`M{88(ztkbMHly44eaztW+1#0dX?wuCNihUt(9sU4Hfe!lG~+SoM~V zPZ(RAqn*(J)!+5zp+LN*bDd^fmzw*&MPXCHol#+u#(q3VWcxFu^vJE`I_Bqez2o?Q zBFlVib#BB1KXm@%_1OH($|FKi$KEur3{FP7L01CPl0tvp{|?{WLCdMdU)Qp;@01A& z?+tpPRhE_L(fqQ{@;xP67Perj3RS+4##XzsnY0G!xx+h|tXk5TxgP?bS0#)TVZ8;ewbv=<-Nwmgv~ z8VSuKI4a%?Q<)AGte2d|iqtL5E7qP1hNcUTv@N^LL3lB*%X3C(WPD(&5?0ES2= zH|;1z^BLDPrDJFavUT65+9l{5UlyMITKflxThY3!7T4crshydfpI|*2-e}k9saQ^5 z`F;x!u&2e0%Uw^(VFx6O0u&K=rv6-phZ|w(Xj}YsSMka1ZC2v*Ng6zr;W+pef$#da zcaF|^Zi$xsWQMT2 z>_+N(Pk@pbb#3NTPa5-M1jh`5hA3@LR7pt(+06BWO()?}WM!vZd9Qsz0ZlN4i}7Rp zWUceZTd+C0AMdnQ;oVyb=JdH>Z>*(-@#aCWUNP-=JiV@A#{*N-ee%lT>p0q(ZSQbx zvfXR8!O9P14MiDG9Czw}_?F|-m2dMl zgAioX_P4(z8%ruPm#n8%pjFCmp^a^z36!zEP}uT7&(E_l=r8wy79T|IRkVuSvtLOj|1!Bw8 zm#~vzUqz4Ym<}av_#mOF2bxz9kfwMl!Vq9k8`{-f_(;30A&~-1hOu2#{Py&p#Er@2 zwNIPaZG4moIHRB597k4bBi_V$4wt}+5yibYh6fFs)&;GUlZiT$o_yd$E)2hM94~l< z@v__b4sREbdyHzEX4AFKAYz|ns+)=1(dg6kI_(QNB1g34s1p)dG+0|ibrSR$a97-t z8l+NQvsw4ZM;P17xFWpYi1nQRks0;M#fJk9Eki2vI_<@gUr{kfQ|j}oxmd`oaECAJ zi3H8=THPf;H}Rku9vesxryWI+1tB=*reUuLp&hs>399Rfw;b?B8CL5ZB1!844X@?| zA>YcG+pV}HrtSVuzCTNfV=xWQrAbK(fk?i9Bf%TWe;tzpY`c`aE^CGLAY{a2q0(6d z*B#NAml2YU5T0{XTDI!&b~5@k=hqv2at*chh&P|m4wcrMsCCHXSq8<`xyx~?MQP9H zv#Jqz1(^9|ex#$p?nEqE?g`5?L+KJDGFElm4+-Gxn_2s!6gNIp~$T;AKd0LKXLyzg{05HL(><83s>q{dRa-|SCzIMcyT!_ppI;LF=3+5U3pOAN%%xoQ)GZ4Ti za|R87AMb7@u0T~&71XDPEsJR%h$kL~UnYH-RifJ?o7 zNy_$F+5gR)sb`*wf)F-1ZsM@i;3Nhf_)S16-5BigGUE+^rPK5m>Az`_K8bWqihA&W znUP;GH@Q=h0r6!|xjO-Cuzojvy6zb$ql&W3bHzk{+&ljsLuRb<`jvn77_f}uS&F^i zL_xp$H>CLU;9zk@y0E-A4WB*RCU+KN`eK9UqphAKc2XIAJdFTyJ$8+M{+#nZUs(3j z>4!*8a;LO9L#_?!<MeacZF@MW&z$gDF2hTCJO5ts`b)lHs62k|S+ajqTtuTA5rl`G9wE3ZuD&*& zeo_mbeUPs*8Bq!>C-c@@xOwB+X=2_pW(e{*#k$0m|QbuK@0e=U}q-?iQuqe5_3uETQSBYZ=#k!nw> zl*)=XrB8tKv@^n2yJ72cH+u|Z^rI$r&T?Iao%jlbjB=jViMA-7ak~bu&TWU7xl@>8 z?n@M5(SB7EsH`0oTJpOypC$l*_xLsHuOgJgsJc}ZE%o^9c0RRx9pPl*zIgD67@P!k zfOJ2f4}C-+wyu4a5qG*W*@Nh>J+vs7nX#ri;bXW6Yp0t;;pC5dOU59i`c74Q-F7Z<4L`-c zA+)amX~GII}$LI+SD_RIjl`YK1%1_f`iH{3^6^!ZKkDWHl`1BxWTHM zo;a_26DeWY=jhNk(0BL(^CO&-)<&K5%0$R#Jn{JnunPgM$=Tw4&?;$O=)O*zrH7~fdqM+jeSP&}Ed|?Hy0gR^;X^v} zhO-zZqaGwHdI@8KLFjWn`FJ1kJ)r|H1EmN9(?89=D#Ar2W@NCYmoTKfrPp_m4of3( zY*qC(WEHL+H)~Z0)q-Yej&d*oPMIfho4XXj)@ku0;nfQ;W&~LOn|PEOp*wws3B-h; zg)FWTn--|zC<>9L=C`{l|3egn&M4?VhB!cZWVtz}wqU%-AXlFX2q$>dlTgX1zd7{H ze1EcU61&tQ+ao&g>?ktaM!ObuKI| zIp$=>s3rG5E(^wwxuI#n0a2)KLzf9rh-ll`5;dN-BMH|P4t<9M$WD8jbj=L8$Fu!U zEXc(dr?z&zy%fd1lI!XTYTS~jzqM_A#VB}=7=KfpLx4nF;jvHBA!loScV%;*C;S() zhH`y(JTZk|@arby2O^CX1Jqb8ep~(5u8`t8{3OL3%wK{Hx`BG zpPug*0Y2{=SFvuy`N@r~1d`%y`m_qNwZk>x$MeXY(k&GS_@|-bK9sHrgr&kb(c0xF zHKn))szltkZTOCO^`O(HyEqjoFDZbJ6GpqqMjh3JtDQdjULPsg^6b$Cx2%x%$eEV~ zTUNbKS(@urDni;A_x#hInAH43b*^@V`azRrs@;*UrdCe*Bo2XY(2Vi}Lt6=ThoW^_ zCc;Nkdw!?gCVI8YeG(yrdsCRAWLMZY*X$6^TrdH213Xh^qpJoYdyLnH`#lJ!4k^(? zhXDGYY>I|uDG0C%tgT0XF^1HVsm5)nfUv4MmI_?1FQ}(dvkd6WvNd5w)_SPv@Y5XtE!}lSETYY2=}92q-;^Z@Uef)t~mlrKgtQ%Z7kp5~iBh6$&uLqL5YR`Qu^) zP1I87D<`&LYkCBO1tc`Fq??Ak-rNTCU-2+3TAv6lzRn+eTZ<5|O|+BJH3vXL>1#T5 zc`z7Sa!pOgZbWP7BWq%7yw0k;Bn}n=EBtA>lrR1R9SDqp zP`y5ST?Hxp+du9TKfn9f8q-zb*xxeDGz z_w>$C8lq8F%SD~nSyxS^bM`GZIMJK<+ACpUh9i0G(55D#^y6ctYlE zwEyN$h=6i&v5;~I%V$U(rZ+-gTE(y0BCuMHPh%6pO)=YeVm zSXt_@+{$-3eLl%t#k`BBNSr%BxlzC}pFpWzZ_Qz5eh|x@l(F zq{Xy&b@2zp1!M`DcN9>d+vz>W1y!hhfJMtIdJdzS=NLsLkztVEXJTg#avHhsxIl|& z=3jbai5w7rZs=iZO*{%a-e1LOs=}9b9J1;C$^{9fxATuy$JijEzMV9l|9|}>*kW+N z@*KU~X@{H^kzH|Ko{catDChbZ03D~IU6pl&bL%uKaK&)6X1GzD&Op0UG@%x+lU5>O zY&6!F%=iZy!VKg4QBe<6iqd_)Q~yJhD!e1--ac$t`@pIk!Xrr+jvi_pgMZX_{*fyn zqeitSq=BEf(hdB78y9SdwlzDd^V#V1^`UZ8wYJ{2GQMp2_P@V7=XQA2Bb1-#Twsph z1iTX0Q{Gd&l%>mH#&sBZfXTK74_=6E>;`iGc06e}&55NezGt*J^*0?UR{2*gRd7G5 zOBO?nA#~z+v0(-bv>{O>$rsn6CYNlS$ZabqI^Z0*SR5b|_C4!%xI#g&$-WS8Ee?_i zy-iwXMQX($_l4)CbOYgqRB{cFEbw$vt>v|r)4eG0m_KDds;O^v~$4qCEM`!7TwTvTrOd+3p}97tcFddG**qPUOC2$*L(ixIT}Rv zKz(t`f(o>-_Up-KUR4h_zm~Oc%Emj@FFQP?QUjub?z!)0AJefrrOx{KZ57=`QnfHd zmz^wPX`c@L5_$$=LPJB25a~hYYjN9UT#Yvz>gaf=l@v#$k$-a3vx@LEMiKkM$ z`B{8~x7LE*yt$n6Y8Z+5p9gngg?VjPSdm(G0n*HPr;itebOO1&+0`PK9o zuO9j#A3XGq;0N}iUj(LTTNC8!x?Lh*L22Y1UMXRSnXOFh&G#PHStKb zG?h}*Y(!F^5;%H^(V3PXpV<`);S}lr$ybv1sHl;)6sqEaj4k0TLzRb{V$|Hh@GGz%iWIrR1*0S)`vyK=0Y>}N~D z^|fI>7D{x8ZyXO#oJIN|#>Fl7#1THHb*C(fBk9cNf2EImD3w)BCz=ert|C)Qx^>bL zR)S(-;84uji`_WNU31LYX%)kVhQG_1jTpIUE#)`-x%yyiXZmi4pSJ}|`q~56FVNC8 zW1kUQ8ri`g-$8EFt>I7Tz6LF#x5Tn|WqM`wUfwIyz%)KHRP=U>62i?LqGerHlTzVS z*P8HcsJ$H|B#RzG%$!)8$*)885G5{7l(XQo(c zcR)F}O{uuA^OFg;a|b5e^t0D1XPO$Ok6}z!q^Z?5sTv(%%&fvaA#|9o=kc+s0m>O) zw>J}Ut}9EiS@%1hZozA=D;3NFBd+l<{cI4Dbc>%txAc?~d(n4$7coMq7c3M8E2N_K zBJC%OwQ$a1?4-4+5Ee`gp!FJe=#_uULe{lF4~z>K=_JsWasM{OMA#!K;$oE65oI)tqj1XDhrT2>tf);&D;}{ho>IW}7^WN2PO54<;u>}600z0a%uxL7 z0Q5>s8`RkiFJ&cAB^k05a#?j{@rI$Ct|Cu1Uq-h*08#f}%{AfVhZg$KdBs?m@4M?i zoRq+Isc&djKylur-`!+26ywa&ou#?92ylbL?yuPT9>&lE);Z0oHOS9`D!#KQg2Kbu zF~(3}?x9}vckLkglf2VAGS!ioihZ1qVO12z)HZ|XF|aQ&N@#Xjyaa5EP|=oPKEb`l zjGO)<)^nujn{n(=O(flcj$($5=`Al78;-JX4IPh=IS#INBJFzgxet0vWF2P^0cCmE zwu!s(48OSDaR=D{$kgnvG4}#;mw(2AENA$YmD;dRi$Ixq!l8V58$aSssoeThN&k5QV{&Rty6MHf*7-(OWl8 zCt7!F^Yx#GrUAH3EVGMe>K`+5{0?n%z`Z_+JU9QjL}Ha+7N4LpK>Jc3*N$qE5s7{n zb@&aeLDdJ6n#*7)^sqbn*W&L*;_jcq@a=9&kqfD45Vs|>IGhZFh zpYe$)Nv&W9s)QaGLMN1oTXM0Wir!??bJdqDPi-k(|MmJIf|scLj7<{q4Ru}|V#qAM zrFnT*V;e1@bC@G{prkVX_Qk8(VL4+HZjyQS0RS1aW&)Q_p(88Pu{F!HHnx<7Jh z<`UhNp1TP3*Gf0HZGX>5J2>wz>;|Uo`uxI1XY)(aZ~c^IsTInFi5Rk(-wguuTFQWK z33|p}_F@?95^B4iIhA-I_{oueXS#>2moFX*vMxdt5gfHby|kK63DBFuymyl1zQ_Z0 z3mKRleNWrroc^X9=RBNas;0vfU!jEX;5Z>HZ|+S=Xra|5MFkSyXC3!JHw$OMZo0a( ztq4EQIWF8#4r@{!`iLIIT*T73o17%4f$CdK^$&@o;Hb4@{kmu`1Zus8z(GA3D6`T{ zndu0jt*R@T0pK=TH)Dpy6H;<8wlsM3NDGRJuXCvTGanMlgKS5$3A6#VdvqrgaYx$y zPCGRsAf~5ciBJKZd6mqoC=0W~392Y2BtwdQ17>b9_{R9P=zkON4O6*EnciKrgFkwM zB7rEV#@8L4u0mX=^WpL%cM}-pUBG~PG3*n&}GBQHy~x%{kxq9EY9YziC#~p9ioZ_Gu`Frx;r_>OUs{J83s1 z{kft93$YqX9**2CWxCbXYhNIglm2UAPJj${(y$AOf>O8Djy5~}}?}l))jr`GcFDx~mmsQ$SuM43o8s=G~nh3k^ z2zu{1kg)!NPj|P6vYS>}!Eabg#(a11^?vWVP+j9J@LTgQvA(=TiC2Y^9xJp&EB}(% zwk-Z3uP`Z})m7@M+rJz)OX)JopNwdDNaKpMM+r_ONtRZYqaXN_5Rs!{m#Di)sJa|^ zt#o8mXi>{6al$wQ=U??)?Kwmb^_;+W$26#$4Q8b??%~g&tIWpK2^gg3(<@nS4qY@n zE~0I20-c@}yZQ&CKN>Zv+@-5-#3hTIXrE@Adv& z<)QSeEtJ@Za-l+;y24IhP&F%Tv&wsJkXUe_t3d^e(=R>R44w!pH0ZWGLWW@C_LSW; zV3&3i#nbgdcwqXH;njoim~Kxy}kCq}w~v z(t>c>qPy8g-B9;ZZauQG-*idjliIOk_am4BFKcbHn;%?CVuWo6pomtBzuCR8plB2% zSKYaeo&=d@IpHOv!nEwJ3v1z>G7M4m{y?$i`2`qg5jAcVL;QVV#SkS8#tcd|XI3YmIPIKy>8eAK!7#=uN_Ys~Y483ulvWE$` zDZcLP_iGkNc&(l^UkYSDTKjvL1P4lu-+5^hB;6s{^{f?%@)tVv7U4(Y&W>g}OOI7> zcm#n^bXW!kMK&eAKXm|_GOY0Ub?Y4gRjIb zuPpx@$!{{DAg(K)$tM%Rn~p7o?=`l}r*-;l#gz}&wQ~KX*SomJONOs)IS31>#m!Y; zAz^ajk9a7S!1=HhAce&umL999v+~5AHtL5X@(kX}H>&OJsk%)hN10B!+lZ^uy4zEu zq{Oifd%wK^;47hKo+OUUgS0j}l;47T>e)W>4NPMS?P}T1e{t8-f})=1+=^{ro}Z-) zzy-^J3NvdHzWG#5&ts)kjB&#hc5hBpq{mN+eVBnTqfB2%=O5PNE9joXl0`sTB>t@U zgsA%Ad(>2|lDW|IUuTgOCb}ioGkp@k@spR!R$=V1V^71WACN6m<;fe*o&FWaXZ1a^ zNQynF`QXLGY1uGfLz|PeSJHJbwqCOt{w$(Q{KKsIFC|jvz?hmm&j=i3DJFO~sNg31 z6@#a3?gLiuF!erZhM)CIiaaAlPWr~P>oKI_LH;@YMmZC)MWm&n-De|UqoMlI;BD^s zpNIfn?8mwY@-~(uZyV%Kky)`RZ!)B?HPvg{+L(fBblYM{{&!wU_H2&dXWtzrbhyv; z%jUCSsT!->Y`iy+xr1ei5&i3ALeWr0^A#pSOV*1g)9gg>Bz?9VlwK=6n!X~5 zC}w|U*QDPOL=T%alo~u(XK+*Vba5YoXY>~Jn|{84{M2g(E208YnzB+o)eEsyO3yOq zL^`y9Gjx%cNzh@@z)o0X&$BzOr62OL0uR`!!Mp`It=C2N@w;yTmYI(ppFobs{u_P> z@k<#2^Ua33uXZ~x>hRAhV(RLaIq2XvVbz;jR!quDVm<+U<&nkW$j+RhAMBERvY?wI7;Zs{-}Po|p~1gM^h)aY*XiaSWUQDb0u3T?85K`@%cZ z{w_9HDL5gN8n*&x^~#F8tf~#}`K8ypsqY~^a_(H4l0-pS=9=$ncgcV#WF$(4r3C?e z$wSU;VkNVayDYcpSaYOlZ9-(&3S^jqR=OIbDp@eLwAU8_M{!Uar$LPku;BN zPng47P>;j+$DStqke1=^$5v!WUhN3HGs#S#+sO@I+P!h86Q3-S%+zw|3-C~xOOx4* z5=?n2Geu;#cTf^?Y4a4F4fyceP2CgDC1cmUd-#PD$V+cIr=q>Wa(K>}TbE$*Li%Uz zT=K(8(>06cqL7uSuxe+nRMu821Q^g=!yR;o*)Yq9e6Lm!Hu+C1@wl1?U-Zz*lu)6u z>gF$5v8NFndg_0&e-3FHvhCgbIa)5ayCtR6pijaBkqnx1|cJ)G5yeXB-M$7P4(v9Vx@jb9vG0hF`SQ=~a$}VmNmShl1 zeX1C%5P8RR@BefVr>z!@-=ZVVk1p}Zp@+brk6T*{35cOb+ebwJkP7BWiX;Zg828Vg zu~0oj3die~8|?7`x;;%`T8exrB5ZcP3$UTR?T79MIsmZJql;NMv>{K-h`GX+RZAg$Rg>0)?to|y-P{mM>vcdO{ z<3wCT+q_9I995X|@j;Xfq`I*R9vT#t8GC(X=g{C@-pN4m8`t5#@y9A*qEoP)&xpxKEEb_-}s6{>S z+SzF@goRW9uCYdd*=n>G_h!Ah;qwndU*D&t>x@9-@*JN~?0vA;9&5Yz4MDpLeG2Ip z&hLj`qnii*Fkb*smfv`CE70tWh_{?SNizZ~d)Bnin?-6nF?UmU;UU(7@uZ3@!07=+ z@;r+w5(CJ;8{LUKyND`NEvSIPq|nd#j(R@@OnkxNB*&VJ^UyE+(iQ@7HdL3>9J!nB z$@pW!jEy)ar&)$K`c(1)?YDBUR7`QokpPqRr_}Hy9?HL6_1*4SXGjc{#Z*GpNua%u z^NfW0H4aU-C28-m)bfWtNX?Xsw+$q!osdw<0@{XofSy23j%h9pc(0%pEvzjtKE+)o zOJ|T(C;RzvJ| zcy>G8i?(C@qVnR4dZEs~0*`W1Jh3{)Ri*7aD1 z$f!|Y&GFriHB*g+wyz+>ttFeywg^htpfwvVMnOx{uY3N)CE*TjAI*!venH%)&UdA7 zL$sOQ^s`lP81>o?`9U6LW9sV!{CF>9K&eg#UbD^0@O;&Y7?~QHHNx5>+QAKxE_c>f zbKN%JSgZw^wQ;y~tF%n3m^TceDu}7v^n@55zcE$9(;B$nhypkMW;R|zrF$GHfe#nI zoA6-J`!igV+en)rS0GEvy}>)|m4NqNy_wRh1_&i_F+S8G&xq*YG3?A?DY6uk+h==` zvt!&7oHhf<+pe1u&_GPJ(!ReI(!h`)oCijB#m#=}#nnhViZY6+VOz z)85NH(C!Fl@$8fT1MsE>j6)EV*8KRG+O0C-aKeLEIk7-c9E@HAFOy89hk<=mc;*gd zk;2hDi$&+5m6@s>ueI5zAGDTSG~Gtp9$wH{N+1ldq@<$oV4_GJ4wsn3JnU^=szui2 z)9sM96;(KG|M%)a;&hWDJDL{borf(y@@-;pVehu>8^4TiwB}B+oybCjI-K!R)O;Ez zIb1)QC%kj+{nMNk;W#+7K~b*Ozk(4qUF&)O0yeXdf53ig?6rC4d~x$pFO4zz?SaKJ zK1pk@d5LgYaG2BMF&7qGFby+!AbRa-r>Zt& zX#*D3#(y*5hp1^b|ETKd{A=(Rjhs@}@R#sHlb9ZQqWu+*1OprAx9arlW;Rycx8^!0Mh`&JT)CU0Ky`w3u5^FT8B#S}U9??Qi2&?j^JW=4S=4ba zKg??wnxy&T6AFn_n2}G*R&Ypa8QHrtmNJx;bg}&DkBC)UGJep`*CY%rZP%%MGUTPi znCo7$gLpZ z-KUNP0Bf9TjsJ`YL?Lu4Lu65Y4;hxmkl#^5{z6>op6Lva<#5(6BM~ej)R;SOU~Mvv ztg3+qyoz5%EWC#`TxPj$qbQZW}N}LwJ-o-ZPAKPWFhj zJfpd9JE%r6O7C4SNe z^ITC)DVsMBy4AO!%x^cWrn_mmUGp&1BEXk@FVZ$)Bl`qxJl-Ve_&IQiTNPJj&BqAT z(qH*7`VJz)P$>_fVO98V+5&`8g37ez=D3UaZu5&EFe(F_CyP01zhMd|>%BZTg6#aIl4%MsiQCZlTnfqIe;~+Yenr&v zklm^Kcu~>Ws$H3YecQwt+ISLm^*w*|60OUiz>d0k*V4gGRl8_D_9HG>H@>MzXy8Mq z?$&wBRj<&|%WKwhTiEYxarwL`NGNi*6`QVtZ&lmT=5c_$N*XULn)s2H$$R~1V*Lkh}W0|BB;+b^ZU#M5B6$Gk|qKb zpDe{0uY^1*0xY=N{d8 zyh>NHeNc@7QF}%Fz=3ywU8F9C?oBlioh<&HB9d%oNVCVc?wSM4C>EEso}!k3i41DF zwu(rx=_Q@~Vs%_#7TQuL+33F(qjqL+9IG!9{evLzlli!($$tee`P)kk#9eaNBYdAU zt)kzo#X*GlUcr6Ay|w^}MPaalp&y21?Ytf-M{VBtM0C-&jX;q%cubU|Hlj_UfJv(A z6;^FbJHp~HteE}|x;E!4?HECjI92=K#&}Z!W(SS_`;dTM{cL)7z*!_>x#ArDVaX~P zl&rsJ4$92w@_jz_u!%B9{rb;V5S6O)_h0_6ZYQZ;zvL#C`JZt4!bZrrl*A%->dW`Jt$IFU3#!~vm;Aqu%i3EBWaWW2l3x3 zp=1q7Kh;Jh*dm|VLe*>auM)DPWv4#HF6SLPX;I~YWuQylZ{$+Loq&NeDP;hp-6pBF z0t1jt&S6ffpb`vDdYc8I7x<1dLlOybNLTH@^2||~R^!al+23x4pz?jf`maJfMUVIl z1Frt0I+gz1iYBiumMqqXALZd^WGNdGAHBy8V=w$X|30y1`VO{hBYO2HXE^ui;d`Ey z7L*}+YBxO9`U32Yuz8VGMMLPTDokreXzZyBYui9n$=@p{>LejYc%KbtiUg^+Rk zt^9fN1c*~#aYPX6cH$HC>}Lm&`_w<^$&`SBQ^vfKedt#ljI#Z6Ulpn6y%q#3h@rlG z#Kr~XSS9l$tVz63QA8`N~;})uVBDi!PgNpByOw80Vs^UbPx> zA=W+KRW@jRCG@gb&b=j`OfugddJu~F39ScvOvGNFn6P2*k1&F%qHdco?1#uWgKKX@ zXFwla=P-$b#SNDw^pDZ`KnIZ4?_9r=JIsQFr+2=4-@z4>j)n~*rqMU zePokU)Q2Ndg5Dnu%Cv+-I}ploBZci2mlvF@Sj4S~qRQAvDkh<`PuM9|I(^3Sj9)Nr zF$gR*)cBwC@1Bi7pCT(7;r!~SjSO_dD&+E*J7PxL5ni=A4z^|bcHuh1FI~rAXj5pL ziW+&4$@rOuKN}cBmy#KEF$$Pu)~DvrZ<~@Ulnqw}*GsYLzAZN=fE9C_?qCXI;Vw|g zNMcMY=AByvE+q42C*7jSviza}=D*Z;z<{Ee& z7EK%9tk6tpW75ja_^|FN^jIgkMg^3x_$X6{9qWj&oiz|+FrJY38GgwjDZi8Nw?6W2 z0up916uV9U&U2;SG~OlPm42??^5ZHPmo~?-xm;Cc2V=0_?X&og~~?c4hF?wQqRyjiSZz^NcmIUQ&u60d zStBl+ z4j&~5H2UysQKMU$xctR$2IyDjqDE#ajI<@2HxM9+oCS1<(!R(HCzTpHXq}bzer!z6 zO@hgo2Io0TnRkow3Z2hYVu?DYjbx>$0E-to7#{-LmY8wkx+efGV2|gy6cG{@UGJpB z{8|K;e8Yj?+XUT3l+=-%jGtHu;qC6CmiJMnCf2p>GJ&*3qx)%~q|E03J;2Jcg3_51 zmpSWEQ%b*EAHElbg&^KrUG53w;-(gp}?pt6=~922v|(^RkCPwZ}WUuOq2XUd&3T26A2l>lP}(ERSe! zT6`-JPj$(@OsjFJqMIZ=`)p~1!Zx-zmqiN3mZroOKEd+ZiZkJ2&_Tz%AvS?*j;kT> zb6lD&3;rRNW;7a3NIG8paG48dqFLR2rq$+spk6*N{4|+d_#$U&cMQM+d8dk1szr^~ zx4-Eha>rlJA827_qyBML=r{2IKA}O|XI;WjP>#=*zCAE&+Us=D)t|Vq(ZwSo2vbX-JDlH>V-S~y6Hx$2iSXg{;R3}iJmZ@#hcQGVv*j94e7jf zY7JOU*@dAkgHHL8YP7Yfx6LzdC)Hxe$Hnem2Z4f6<15$-K1P z*Dj97-NikBMLfZzYvhN`#$pki(SFbf$|0a-!P8;`NfzH)$VbAP3f`E@mD=mq0D^|; z&lE2-0!@4(nZ-zK1fVH2o6Izhns%d=+Stx^qR@8m7L>aZtEQts66vGu_DjjH1FpaneSIT#Z7a})fy_Bxn88o5;ERQ_)RK9J+ zudcy}yo$z-ji0Zi<4R_%t@cPii9sZhYw0H`sE|L8M{EV@07+{Er7M-Y!dIvTA!mnv zxL%G{QNH|LIe)#6d!onEU=*{`MiYMl{>~k^u4*+Ds2*lBFYuK}q-kp!+x*)p33-%1 zGJ4~YD8?~+LlxHv6R~Sun)bB~D8&8~#ps6{&tjXe^M4x?Q()E^+Rp-OEJ--{siJVN6&ODMIB9hwB)P0w-G{55`8->{ zH%j<7{TXJG=@o9s(I%xmF3%zUO!6=5szyg@S%;>09!Kp}cZTicj4!2{rgB8;NTjwJ z`?%ADe({fK1&aY|==x&sG_DA_SJS0~%a@SUCYUK%Fg=^>gNJgl5-cuZ`d#mLJ+Mcv zK6UO{UI?@tjdh3k2sp;ui-VBjs1m>7=hTXybnNAPsE)W}(^XV-$0EoNTY5(@ZdzfE zXK6!Ya|X8Jqh8;Ngu?*QjWfl`!-V~$j2{1X2s=Q@*IfYNM0b{=zLrP?kG3jQr}cUT zQl<8T64;%_xFftzR-_?KqAb#i3Rr{)lD98!0#X#_zrojQNZdDWlR)r(C;xsQ1)kx_ zMlZw6SDs$d>l4uC5e0wi#tU%}?>zq|=Wf)t4WnDP>_oH)Y}~QP57}UwqrS8c`IdIZ zwapQzK6BMAu7166bS(8n!5wiWFr#M8phFUlA=T}30t2Z9PlDd#oPC+9(fcI$(>xYL zWs;EY!ydCWK`DR+AQnyB)D{_q$X=L!#A5}iat5@f z1dL#2@~bHpzqP&J29GkEYEsnpo?%$azbquYx5=e1>Fp9?^y|{I18X~k+8+m>`fP+h z81fz6cC!rOCO>3YQ3B-4=?mxQ z&+&L++Vy|Ti%?GJIOm~CR{eu-6-CH5ev=dfu2Y zN&GL}Zdz@mOo)2!Q7aOZIkIe57=gns4VtA`-XM7;OslZ?<_|lR!Vc*@*w7K>UbkPi ze5gbs@G|&aWE-GJBTMZ02L~&vb@9qiSAD^6XL%etok41Ct-D|p^1&Qw9Q`4S@qXjX z=B^wkCFdy}1OfKceKyDC=aYE8!RN|w;QVIfUxBtoSx5`0;jqLV(n}e;?uk-aSzlISi zZaWVgK-zG6$31=>LhYyNau21?;J5tZS=Jr+PUfPhMh2noywJYZdMmqQnnHhzQBB1CD#t&P z`aze9?jC*e$@a^TdqF%7R(4=rVlePnu?E=NDMw*!v<9 zjTVzR5O(p$QOwM_@rda;?Q8Yr026%x*-pQTf0Cyg|N8Vk{^?GD19zZTJ!Wz%&am&F@=3QK9QV(|$t^BX%BFyr$gO5xxY%*YF*L9A0&vpW~54PX= zT1p0bB>DLi;Aw&JH%uFGV5Z6WrzFvk=YS#KPp@JjgD>`JKYg!ff{B|7fv&3(M%x|Q zM4-PvF|_Vz6Qa1hIEOik%Nj*+&+YkwV-@Ovi@a9@d+@fcDc5?m>vuwbO_Re$hfUQT zd*?{!NuM)Gy;6}E(0Gc7+SeKuA#+pws(TO{d&;exmfbi?7OlE!Y{WFXitb-5WZv%q zkUI&#Ak*$!`E}!TlBwhhHLXaZA^wa@OU1?L&EGYNx(&*o?y}bIq5>XAm$KVg5()(B z(y7h+VSY?nmo<|HkQ?Stq)p!)id}K@5>Nm3G`7qA)p&hAps3O%gS(&FK;&hvofkH( zhM^M&zP+RBH4-kxJ=35G^R-yxDOI(MTS(bvH&rzwEtWJUCY3j%$THH-TR_}TZHHM_ zi_!-g<@CmH86H1&LwYfGA1s&JCi*j|Fj&$t8Kg+Q!}oOofxt>*1}$%m4T@0|O>@8jXh6Jr4QZYZJ zmRx^QuDg~U?}F!yXus#Z+h+$9+%(xKpOC#;SJTJu2iJD=YrSq(^`SP1E;T27lJMZN zw~tTT9VXllj0D)=?h8%==p13D7^q}u#f8|H5Zw-e;P(g41ni!ZJ*)>-V#)7YEyll4 znOkR_Z-L2WZ12B1=<`ZXo9UQ(0H*|6_6BtriWFI$1p-0m(lDNwrG^^ZjZ3`1c+euB7sL(SbJ2z<+cV}XGW zeiNDlgsT$Zb2PfUjdCLnDE6eOP7T=@@Cg3)05Rf{G%qB-@hiy=6|SALNNqB; zVju~Xfjq|>l*{2c5O0TfqCt0TqVia)U>6%1^uRufR|w4gP8ytt5U7%HYRr6^s0QYy4#;Dia5tFp&g3Wc1=>Qu1=*C7Y*v*hefS_K6 zsdAU(5MNfP-fPhS?lU>kKc+C$2wJHF8ODd{ST#fXL>7-+>y5=5{dGBT?uflDCOI=1 z?PTK$8$U%QvtqAz99!*Dtt{B=YI85PizT0#OVp(XDiv4nl&@0ZI{eDei0iaRjqKqeX7gEuTU- z+)dXU?TM3g^*`fxNsgNY!)8C2Ubh*p!H{}nk3JwKanm!0aU(eEC;Sqw_P&EFcm97Q zU3oy$_4}{LsI<@jl19kFL6{Ep2)Mp*S*XRM-ySDs_2?zfeC#_iX(>4Pe$X2C|SFX zfMV(gNXNYbq-=WNHD)6&7nBM17F@@^cTFgF`3xp!L&&!>2O^qT9g$L7p*rz z{oYxkJPMOgy33MYMur%k)&10ltg7z90Sf1T|ON_^l_tqm!V z-;Ky&Un%q^=|S=49GGmHsiF2qc(`U>NclJIFx^d@f9Mlsn3Eg6@8s@Ldtk16=z~8y zjyX(L-3`6ogwWAWvhX@AL)xlg7Tptj>|y#MZekXW0p^;Nl@d1aeD>8zX8ePCxqdvU zyZIXo>(>@SrMxQNe|0y`NuDS#m6+jliyZR>jJt!g)y+HqCk%dW?!UsvNaJn^+plN+ zHO4M{oS}J3hP~5bOdS}*jt-<)(u^8HxpI#0K+J#ygeMe;b5fdDl@Xg5mF($gFd)OV zktw+hQtz_z)&wP!?=v6w_eakEs%#6TJr= zB7LWqrWAmqVttjei=^iw?@F(iOjw>cNxJvl!DD4lnBL}ILNC&fTw5F(1m(`OkF8Qq zh5(&Zu3RPVL4F#zwL(-5fPq!z8++>|_Nj!Ls@X+B3x^^x*^&c6b>*@7tU=tD1U<`% z)USR3)FCcoyM8sUT@mgL%Dv_Kuf*GPPd`sDbnpAXE-qb zPIY_PTZ*^2&)?!sNPjHZH=9}nY;k2D7B_C9jLIZl;_)aU4dGv%?z zWnB{?Qs{8&dwtT0uqZ;$`bv>08EbUW*#?9%&RI4pn>3AFP{A~toR--k)wnx4Q)<)A zz*^tq)-zxj23CcY&jcFs&Ltrxp%+v@bSCk65(F?gj<-7g#2vQz%;*+23=^`t&gqpL z;f1XKX|NVVpET1X)-4I49Pf>~>Ci04>d4w1V{R{}dHN+#tTj+OORV zRAOY;h4gydI@Ui{e`!x*W@Gfw!q*U0!7v(s@!T#o;!43^Bh}?r#J_LgOgT9PuP^I= zpFEXIiA|8qG7hQT?niFpJ$RLpPu7!JUi*Y=B`E7ED?7UlTQcfbxvRba6$y)UlU;H% zK2DjS>2QTIYm@QM;Nx`6Fcd9l8fZy*T%YjhV4BU_AoIR&Asen?8U-t zVDaKV{G0>)@M5jcCW{{c8DWnrK?ff&1eQt?n<{P_a9(LFB-A+%4R;aoFmd#5MlK`) ziU#XFZXP4;+4155ScKRV^M7*G5FmHmSX^8UPH9ujSoyG_EL$2(?w;}}+fKX&cA+8O z`Bn(Q3vGvj<}l!92Ge8yXMlFhi?qNV8mY`1PHT3?eT~~tZXY!l&ksrV+97fanCr(a zfTe0x&C0wiLfu5D*KtSPNaxk(gZH;bNScKNLd6 z6qJw&Rd^qvV8l14<0^tS?p(u=o>)8l2PI!4{Tj+>l z_vXu`0|YGRJ278o>Ctj_rO9%^FIet+mha~q$SUz^oI)+Al}(&|tlf54Pv{4DrKXzz zzn)cAseth3>(oBfT$7U;5|dqb6>!EOt+U*h3e%B)TNy`Yl35b5)XkJyBhJ0uQX`)@ z$w=B!UWEzzIk!9tB>w2=xR*bP`Drq2VeRw-BVGz1@66*vuQ$GfOMlA|gpGQtI6(a- zWl7@#I8YE6Y5Uee0EAi^vLKX#!Zjm~NX{Id2gs{Vd z-Mkc}R_Kxk`x;5!JKzKVfchA82S&Morm8}4W9D^So zrz3t0rX8JsAY(Z^=qH}NkuS+j`1Wa21Qq3JYLS(@Z(9l;H#=$cTXrCxQ@P`D95$d+ zlk^Gv;r9He-WylocR*k@`*Z=UZ;f-&S!jSpaH$)QgusFe6#maBsMp6u26H-*{GsM; zH}wlLn>S*(^~~uka{m5~9)3!y->SH!aqRxP2y%;3tRwMZQD3{yB7+nX4r(6t;=^ix zi>vqsjz=(y!Bb*Dn*+OlsW0S2SXb3rg#UuBmGW#r3#|sKYDGV8ORmK~$@|7SL-j8C zi%mGXESG+(lSlxXxPP~CmGFHIZT#l~1QgtlI?j%wh=>7D%6CJgJF<1Cq2b0;1_LQu zo}(kmCZhVJ#t3quM)qEQEk~T9neV!Rkf|~SiT)imcs!=LYTT?U;_j{#hA63sgUyyw zaWsr@Dr*kiSki`Sn4n(&dv%IIZLB`d?u)($jTi;xt1ZM9L*D>t zJxvg}01lWfh&h#wnh7*%TYXzRpxwY>w~>Z4uHw!EB){z8V+DDn4{}o8s0Ep-CAzfVs}>6;DG}+--4J6HLdM7#$z*D@hx=?j#SDU}EP?I_p6FoM?ou+^zrl|h`6GKPE!2*g;{SpDR z^jjXMQTLM8nv$UMgLgq!^ib4Th>C0kWwY`nrRz=TCp1jy$gL&4#i$aLJl&5^JJmEr zYXEark?*MUy(Po@M78ovp9O=!*|UbHD8mN&?g*{~bxPk}dW%FTV#S8wTvT^sT^VVL zDr}?q@#0XDNM%gYWfyS}&rw$l{R_^0AKhzYK_$A3jU!2A?PXYjBp$6N^l~JrkZg5` zCcI6Shi4Uma##_aH>RaQA!RqD)ojLE1*Niy0w8U9_M^?5q->NJbhi_0qc`CQyJZx- zi6rC)y>@0J7lci}ONINvdV{h}EaQb?6T(uwQ{?$>J!*0%HZ~11qgz^ zquLvdci}gv$KAs^ejh`-bG|DxnI1DaHl_OyTax=Xcf*2E1CPq^$vq{0lIi()?htP0CJZ0tu^0FoE1#V!uI5yd_9{&F&?aXed z!L-(WWycqofYTcn3&ZxT#qsW&wNP?}fY_tID59eP1oNmPKbXZx4NjW{cSsek-pWyl zOS)Um3t=dmkg1p-&(sjj8p93WVkK^8o+u+dKnx#dnvkhRqyo-Q;5AMHY4EZtsR!;Ky?U&J3Zt z)ing-jNjlQ#Sx|xinXelWr0!vlF>s8l9OnJE&srWm4Sn%@~d36u@P=3F#o1)F>F%@ zec4FovV78ST-F#(x+M@@M5mCxXTNmAgs}6tVw78FhCmi&d1$wb#`GdAm@=h{3> z84qGZU{JXW{f;lfo{@>S-7k<_>#b9On2Z^^F3M+i0JwJc8IJ9L=(Ni_P3DTaf+U(t zMe*Z^Qtzuv#jD3Q-cPe8lU%?;o+?F*f8WNDtGG$bO{htWgs*_o?ss!Hq6oU-KssXI zOkh*Si8ueBv;@XEUNQ6G&?B>eUW;hHqi^Xg*hSsvgX zK5+czt6d~qex2TnQ&%S~)y>2$NHONxpth;xGrxPY7`W}}n6*q3akgJft#G$S^(@un z=^QM(Sv`L(n|S4^I>*K&u&p?f<&@d%*jn8}Z(t~FwRm1NvT1d8G}j$!gz{1I%p4To z+*NyJM}h$Rnl7rk$z4g}N2QIOA1faa zB)2q$XcuI3YAPXrLsQgzod1A$o71LrCkGjM3A$Gwv`&J}cO#N;MkIdqH*Q;^si5rb zd3_~pD+qDyKqYj6>|%Sf?`DB6`PzG}4~bN$EBtgGu;cPX-UhF=@Iuayf>Y^Wu*rT| zRee$n**|fUVhbvyxzME9HkJlujhi$C?=HI-(1NVHLC`nK6C&2JW#qKIVBSHRX-P8+ zy5Z5}qk-1L5yTs+_i?P-fSo9~T`zRNTapD~;$96(f11*lzO880@0s=feLi8eEcT=n43W`ohs!$tB2%b3}BJH z2?1(P=}DsZ!*??55=ju)wt3UumoCDaq(yz*8IMO-YGSSOi=pT6Hsz_efIK{xO24Nf zVZL^I_wDwxaw+uuT}6i z2j6f`n~_-P|EjZ^N`t<}aBF4s#*y_Gzs%Er?DR7$3jQu7q(iF8P@+RvUweqK$z`kB zOP6suW+=?q4=nSAwqA^L)kGx}5~`K4hDYEiQ52j_EqDM`{z=+B2$>3_KeRg|!{(a} zxhn{A7sA?~VA+nUZljs)o;0Yz@f4?Nujehq(H|(aFhCo(OS)0>Qz<%JB?CR*Dxj*y zvse3D)I-GrpO#sgah+rwn)Xx;v?&y8yy_4ew5Htjj|O8uGl$(iKp3wDK`d$cY0!z$ z0oG=vxI`k2wj4r=kgZCR!t!K#=ZDuuL(#8>^^XiAtg7;dIF9J-XtdhlEkTi`#IuS$ zl1C8D!RE*}+i@iB_zh*w&BJZE-(=d~FCh3~y1EV~OV1jpIEsLpCN+Lo{tqhIr6E@- zkHln9+6Kxb5giW4X_7`2mPsnIPBN?0FL4S@;|*_evaDlEST7t$%Ltz}J)-u7gt%N~ z_NrAb$b^&{f8OFj8*+aY+-3wXY6scvJ$K3Nb7o(S1}<3}9M$ed2&RyH8=lV-odVi0 zGJRFW5Nxuqm(cRm4S*h|)2{#No?`(bC7ScWu)U+>6&1ZO<0Ua)45yPO4ODyvIo7=- z97s7{a)yE`@IuOZ;eSB`&bJru02ygM*Ss)i5w1bLf8QcPsr^qA>iL-A%~8Ge zXLev|IUnb|D}D@9%)Le;AsKyBlZ5F0#BDnKHIZx5c#gixc`sND#TY0|>M9xWDfnT^ zHrgHpDBZ@=ML$AmsC!i}gR~_xqF0O@H}FScFHAFI!ndW7C{Gx@>pE=?$#_Pz<$7>4v zQ|32imfFs&&hvS~%t^j;Z8tvFBfnkNW?~y3GZW??q6=aAC8c+J+VINfx4Va@%sMm9m-p8? z^Dw{-M_%Z3mwo8Tq9o)x)0=V;=xm93-#*le2-UAz_X+sN7}h;rQit{HT3x=vj#R^C z3Dc^B$j2#4{tEkmu>8gS9`bM$u+R?S8lFxXZDq?%hOVeocp+AWLl~)TwL)QQ1fi8c zrhL7x|8bDPF>1JvMjpH-+$Yi#4&P_VoPq?Rlp>uFwakg2x6G-P(U;(wt=oPR~{ zj_^-~npD{WELl!j23QT(KcYoKpj8AhraL>q2?+*}T5U#stW}2P?V%G&WUW^2{_jYXu^=t(mE&N#Bp<*il zR>^~r>^Rbe;3xgw{Da?YQ^>*v)_u;9@uoy3FABgY6o+2dpvJKiiYMRhi@h?6+;O+FO%>d&c_)cqa z5Y7im#S0j=A5*>aFfPpj&jaM;sCfb8(DtxRU;J=_oCmebg|k>_H)+p(7Isq8QW<@g z30G=Ze^?$^g1g|C5V43&Zk_FU%Df6!jBC2$-FF?wV6p5ErZ0X~_*18TV9IVp-qf6bR;Nf9YjF=88E**zn7q8* zOQ@Jv|{iVNg3tGYXSP9~DmI7ZE-rQ(9N=6aYHX?O1P@hCnl}aza6>cNHjV zn$hO-{;+A@JmqrIQtH0Sa72Itbl1YbUf=JqH&#CUD%D1!2ho>kFGguXq%)j#A&+Lh zo}HDq8xU3A8ar$4Tbw27{9(IAJMr#nd8+tiAkMq;grE+)&^?7QV*3JY=~!J&-a87a zc4ObG zENa!jhzWK^9USx^^q4i=twcsRMX-y03@8Qg=6!Eh;H|0@PisQ~L}Gw}?ciI4EA=yj z@+uH}6pqYg;N!zNcv4u6p>= zD?A*p-JCMRKAhxS#;YD|rcz7PMmno&MC8oxE!Y1FeZsuKWY!<&8!(8Z;1A1(9LVu421?9i5%LRYQ$N{@`_%0u z*D46etDLU-ZxqOS${nso2BHnDdClwHNCL|;na57Hz%%CQe+~K;4$G%BSvefu0+XCM z6xp^^2rv_rjJ|+;Q0$^Vq_r4PZIub%f8`p2W3QQP97l?C-c=22KSVn07a2@(Ay}iE z`JK6*aHRSs-||xd&$Mf8b)nq7iQu3)C&m3_wSFhctZda}Jfv*)NlF`T(+n#Z*P7M!t6gMaHy&TZu3oYE)j=q&gU#PhTZGCd`WPw)f-p@YX`j4aeQ1 ztS%mJv82Tv78JhF(oafyvkj+<*GPa<$ry**Xe1=$1eA0m(vKB+wqi0oG7AfrF#9G* z_sSTL=5>}L0=%^$CP9X~`69cHzE2BpNMG5g%td_6mC&5B;*dzNSKC?axeTNCoVtH? z82&={yH^t_{Klxyxs54-SeW9S+yM|M?NES&F=k%inwPgAItdA!C;hSm7v0J6Q{f|+JPDGGkMXsd55$W;gbgMczc^`JU z`)X3r8^~Q5=qy1R!hG+fj0#@(zQ-TbAxfuGie7TrCEoTb5nmHh z`8WSSJmw#{HtVP|96PY@X7-*k&@njUd?XzNxu(vq=egq~wLM6=`36^Z`(Z3 zYvBb_mSID{N3VKuvy$jv)C9vgm`h)!L?QO`3zil^tz1^gwRO+M#!wuh-4U$}vj4qq zOCF;5;(fy)2?=-6T=}rbNsOg~6}Sl{%)ueU_B5F0AkB=(vGyXzf89Sskm95F*~}>; zIOby7BF1ZSy%%spq3qpM{8*HC2o+GwHKiLSW#XdSagLd$zEI^5ITvBw>Hr+8@bjlC zT=RjOX`h$sAgvg|r0eZuhqkSOGXFVQh3*7Dy_+nVDf9jA>2j z=Xy*XJNYbkQ!tcqy?Z`AK`@U4VS|l`Xq~g%%ppX?oZ@&mGadHB>D%Z9gdGwoxj#~S z1Y888lXw6`4}JTy>vcw?GoTnQyt(BwSbXXh^GCZF5T4X|Dy)eOnCr>}U7So%lPPhGVL5VdDQLZcD1$C?H@ zB~ft9v8Ib9j|tMFpKcV))xvIzpO=)yiOBanep`r6t{r}(&zjLMI9qT%e`!1yrMvK}d4P`YHxBbPt zS)T|!{IaW@^ypkJQ3oMwkxuY=}0amgU>3S;~f+&}JibgSemicx(e6 z6cS99d8jDY-uNQ9+6~50c&z?nAs-DXfwY**wX!#{MU=2M0lfB&+N0hqq@ZP;Zv_20 z$M7$K+kG4F!mslNe?PI9xXUxDdz^3bh`#ihi2Fp7rPF44%(@OVIZ^Ujhrzsw^^hc3 zSHd(FpK!=X!ZD}Kp>>0tmBFdAyJZ7#Z)?ix)@q_*>R$Y8&E5|Dq~gacJ2QNg{^kq^ z0STO%6FQU@A!GgWYiOPG2}A#F%FT+;cjE~-5?aeyKb!zxLs^a+OguX{lQZ{xQU}jT zwkN4#$PPzRJAsp~0zshv3FnWq&W{;fg)H$ocDZ4Gd9@nAUZ&9|OBxE*hcY`s^63I% zPU=>f1FHSerIeUkLfk`A)AXG1wi*W4PwnJ<9;;t*F|ga8z*h|J{C&I&MHED$ ziFjwGLl(OH=6pM{Z!`h!PIJ0&BPi@$%il+CT8|Jgt+t#Wfr8O8+J3YeZ-Alrvi-0d z@nKQ_a9!hJ<2|GwkKV|aLy$H>**LU~AOb1SssE_C(vGdm6N6#P_M<84s=idT+& zR-uAYMr32c@$0epm*U6Kh1GDY`fq*u*5lYcVe%LY2~Q^c(aAHq44gJ$veJ%pD81QW zt{nCy&U3H0Js!L7K-Frct1{6RI+_})u190!V9c682lfu^+!l+ndRu_X9jW`5pp<^l zFsZFxM*5jG^P~1Ba$&reKAu?(DokKr>oK7KWK)L3t&$TZg3X{`<$isR48MqZFZYV* zfcrqzTT@3P?;#RL5B&2g8u_E)3Ol-T3RFcDs|L3N=EVR2-p5N22e2yhPAxeE6HvV~ z-UW{-Ew!@nEjvtJpzI5eCkS53e#o{LL#U8>JaN^@2dfD2E^2%IADl5ZQ-$J^H(Upe zwcPf9iCA5+CSAV8ZVeX1wrBnrtDv`bV=>B-B9lJG5hkFRAxufr{079{Kg_$#*L+-- z+{V~^-xbXbrUMnn_M$(;(cy|yd6lQyRf5b9$8Uh^R$IbqFUjM82Zfr?xP=Sl_ZKR{^tZ-Pr?sGj*{Nm)d4 zQhn_Y@l&_ONS#^w0d$*Sf~B=Nfshd-7)M6jAy*NgP$J8qZ%m0NSD3AOa?ZZ`m_c0~ zw@8|aUJ_GSYq|+RP``K2oSX635$CL$0_*MgLkV?@#(NHyRCRr5%70_)lMuI5+K7Em zdz%9ppZu0OC%*L5PQ*d>6YZ#a#5MAYi)Hf)NMgP+qD@)~ny&gh?HO@Yn(>U)NN_{E zl4DK?36iJ_AlxV`?{)l2Zh~ zL9oH5cQTSz@UN%3e?su7yQypPAC$7qQ=3?uZPo@Xg87;8F6R$Itr>0=^S7|~!BF=< zkc)7Zy}k`Q(XLWhHnl1xfn?^)53OI$e1^AF3Pg1TBsWbEy6Jt8s_G?uZ0tkrlcUy8 zbk=D7q`kaDgHTtX*S;DulVi&x3Ztzv^ zN>Ida3$6?P3+o5ck`@?eLjVmF*@h$gH{x5>+44B=sYY@WTuNTL48s!VwI#a$r0zmY zUCLX(pc~OU`kynFot)n5$hayN1%Jc+RlC@fHFF%IS-ysrZPQG&A3bTpg<4 zdHPYn*b!}v(09fl1VQ6YaNiTkSisl1@jgI7b$&ysH6Oq)cHP+_B5=PS*Lky?Mv79# zeS#qmgyE{w#k&m#96T#DUL!7Xdo#37!NW8w&VT&#By18e%+20-4llc~Si(%Yn--sc zk^)l^GgE0dHp+zQEqGRhaS<<+a}AV^4TP;y_Lzrgx`a|b{{YIm`Wc&POg)|ydZJc0 z%w(?La_!kT6biG&R~Zk81XPmmYzGaxK}=d(-8Z&H{m+vdSAENp#+x))4*oLJ%!`RP_p^N)lR-?wUQ-!hX4}4 zsq;Rm;dJL~rydR=#p3oB@75LY+SU(my28l`q8Tp~C>~=L)*-ZH2kE=d`cjxs@q3&ey^4FwJPw|Gg;E^DS-ddDV;3hy#iyMI?Uh#GCNWkNq`Mv+j8>11H%zhfwHy` zl0!A-9IEpEYJ3V!*sC6w(OqRFdd>axFGXzj2z)uYo{@97p!FcH=WcOlc8ZB3?Y|Cw41q zBC(T7*qIWCTq22nF2`Ap5J|hSZ-oJ;JJM`4T+byjU53rY@rqYM^be!cI)0Y zFh5`0jE0HP;jg;A-Fy^Al+%&sb0G@f>$d3(H9gKr3#Ie-!Y;JL80Ip47ZC)m(&f`= zGcxigf{uW=Rc8tIFNc)DWjvsWZI=D*ctVoe>s3giFtuhPt!rNtj6$$s&8ShBn0#@p>$cgSGn;(136U8HGY3LRkIUD})Zl4^-aU72wg9lNP5sS59EdOR3w^ZlD!%o8lQGQ` zhPz+T`Q6)IeE2AsbUN@WY{mOCQlhd5kXn{TC|gN9jylE%{A44qA{Tl?xu(AK$P@+E z?-Z{h$zg_TCyxfXwZ}gj@;D&jCWuNepTbw8Htcr~)2H&N7M#m0H==DT4!`mkG&}RL zqkG}}O=o;IXF(x9+_k-Y37DNwK4sS%Scvtey7VsG;(1yeWLR7nP1vp-e&Q&0253OF z`d=xfgaB=vJIC{>6~3mZJR|>91q$)z-&u=MYJ+>-tOth3Fgzb{!-O6OrLKVndZv(X z{)?yp>vDZ*_hP_7QXbQ!nik>{B(}~i09y3VJrBJgm6?F;>VpP=Hx1)lM!#smS!}G> z%v%R}b@dibed2B;?a_jQK?Ope5bE9z5e$)sFZW!Rj;O%AYO;OY01IXKDC`u_vIap+ zb=~n3Xos4_X5R(ilqPAab0f3F98-R!_~0=c({4Ha3dOTPmi}|Ar%Dj#g}Ue_Pt3YA zOcH#&Qi#vuGkX@HnJ?FLI>IcXOA}v1$krJh;D7yeVGHed4 za`NbpqukH$JGza_qP68 zr>8P}t@0yX4PN0;_-uXMg0Y_W#$e*3y5@(qeSt=-F`GI#9|PXgS9$*32%bHK+WxH- z7{YbqM6)rQAMR8~8LI2`VWA;J)pC+u_W-$O1=Rd@Dui0HIc+*L-oys&%#0+| zZ^@O2f5C%|?QL#uKa2BSR`nwKtsah>dn@ye}&KdD*RXxJW`VM>A7x{ zGEp!}H73iUtX5={*I3}#wf#PWp)kGCZ=5S)o84^Zbe)Y7F1<}4xqAz^i2LroE1Z^Dc%C91}(1O)cHe* zr&%5WTh?`xR=a5WW{Z~#mFwYWGm5mk89+ioJNLf$=W^p9bCQ5%8M9Wx~P3Ny#y%8-^$(607 z-=4z%A$Lp!Pfu@4m_*7|tVJ)^WJ3}GJ*Fn*Fm(WJj-RJYt|J#sXXtgXU7d+_|0bBZ zIjwVyP|F|pnv8p@l65YJhy?DFvZ)kc4En{+w3E(u>KfBGB!f`RG96R>3Qt-@OH@E2 zJH&9{M!t?1;0dPl=w`+lD^1&*1@FdpGD@ByNUe2A?|lv2HI4J|R5K}nB%8l>cuo37 z8W+1({QL~nHbs7q?66xKY~8-hG!FtQ>R%;TE@%UmrJNo%PGyg1O57G`MX-`wZrszH zp)!!~XxbSfqWRfsMEXdoSr|d0y4VkvPpa<8bjZQ!I z1X>(}=ZY>plnq%k_oTV{OioFq{J8OkIK>EDv84M<+r7xxqU#54R$T zdQL4oNU}=BWNy3>v>5%(sfCLW8XVCr`Q7bjU`qUF9)V)y|7Zjj%!LTHzK$nv3R^&? z&B(4vAvlHsap?ttO*VsPi&hNMmK!nO6xqHR<26@Txi@ASN!D zxq?(zN`lTMEC>NxtuDPRs{o4~V;|3ugZqU3m(rS-_%Oe!V|$87V5PC8?dd}TR+*o; zy95ZbaU?UYShgPcd%|`9F+@qF-6hDwRgBwY*rNomBHd|~O%8n?-ub| z>NTLbA*w}-Wo)gjwrX5f1sq3|N8BF>sR89tZ(|5JfiC)}o}G~3HYvMHm3J$gVW-*W zX1X|oa@FP+^&hf&M1j;tpcZTu2al}edVhx&t1C81qxiT?)q$H^`o!qQt<671=^}P- z^dfWV8KA?a(}my7{1Bgl_6YPE;tw5eHq7J-e4}x$$*nU!`Ypx6{LHXqe&}O{Q=Y!m zPk0PJq~6=HBjd;5QSgmiol_v|uZ~FGzj`bFB7B{F;~{dM+%|P9qH&769Slh>;V;E1 zHR3*H>veYMm_nh|>f?AD_ChK2H}LSIfviXOGvW{jdkEVNPFxBZkl?rCH#cx=tp23b zKEofH-s}sCvb8_q+r|{wC*xKSH1Xa zqX4*!v__vh=Q574Q$9!giAm{pik|iPX*_wAGCRnIovf~_8kOL*`M7ny>nWH+)jBQ94EP!?|GXT$sN=uE%)dnP!oiPrkz=Ll*f z<<9+N9Bg;_T}~%3lUCElNCSEGFsqJ)JRsz>jEG-}Y6phML_8bg7R_mOnDYl&oMV^i z%_s^l_PfMdO#_2su(l_TRGjz^d{yljE>U7!JBvGL3C_Cdv0ro<9GI5hGjmKSKYYM7 z4c$u`-?U|$R^vGo&TlC^GJExf-dY+4wH~LzMc{^ktk4>8*2n!*!yCpvtSxlO z&(!V4Z+WX6L`t{FdEXy&AI1)j>y!y zrzp(x_TuZiK7HV;2qb2FGb3Ez)}N9+$a=~tDKx#sTw;^3DQVIOwpGk4=k3BMbmNFe zLB4D<=Id;6T%bWp9rnRtNy?b0#dFI8m3$eTm&cq$pO&5EJr*!FRN@bg(kVeN2INu* zUaaR#Te1XOet5gTW5$@A;3ALC0dj6;s+Rn`6VKDodc~IrM)+x_@&o{SF-K3XAym`6 zvd#!1jFvmjEDTE_kSu5dD}Ueu6k@D1ee_@u3l=&%DTRE7wqn&jBS^7@pQ?Tu-Gl?< z`B$6(n_OIy?)eFEjS>Hon8S#6jc2b5{pS8Gpw;y04)NdP#s-@*^IZaIxNXWv zlR>~Mz%qF-)PP>0U{BEwGlBrx`qp?8h?U{jdn%rqfN_|{EB;3uM~z3E?sRJ_w3o5cbWe&+3)rY{n;A52OE>t~4!<>&Ck3>$3SmNLR@w zrI+p}+_>jro3_XA(IJ2;FT0m01h9|yyuBntS9RAH*0d&HU`ZzDu<)iDz>7^fb=r19 zEQvHMk{#Sf6a$P}T6zo@s~)}d@N6E8ZPLN}qRlH{vHhgt_Op{Q@HFiHOA*K8{bWIz z)Z@IhIgii~SZ=ls7sw%xd{##CLeAbyJK0*I`KNLtJt=?*^M3Vy3=iKPdRq^BlW#CJ zGRQFQaZ}9$sMVFU?2NJ*org89PAVOz$Gav6AD&9;7q)2&r>>oY-sjhvAKJQcLX$bo zLE>~{SR%cg^ZqB?(Czsn=A0t~C|c=upFMzIg+!8JS_1Kvr%MgD#WVovy$65qJ`5c9 zL6(bT2jV%!ZjZ`)WSPU{1q5lQ8Aman^WN*~ZHt zj?|V`@Yu6$puWX9ugeY^OFNexP8kKs!XvB5icF>pzgOq%fyj@2cxw8UWiZOL{evk| zd`7UD)!>ZoGv49t^?yD{Mpa#xn0(awB{c3h!$)h&aV{{0tnB3zBYw&JlbA+&SgVv9 z%4%e2U1q;LrgX*4L#a3J!;635SvjT#7AXOmep$tRufxw4x!dTh2`&IxNz125b>chuRaru%r`}kmD}oeLBIzX_q?dLr=i5Po(`K--%ZLG(gR)B8 zOkUnlb50COd>E6{@maGgIf938&EIW=lgT_*Y(sFnmQ7XT*eemBlxhQ)iEIf0&jpv?f zD3c6W*SFga1;R4x3}xm4f`@0!8s07h(~mbgF5@maSK>Jx-#&!qKhk~b-wHDVKacfo z-mzWiBegJoGpQM`qd0Q7o^Y|ad_SvM2+lzHkG_UZ9S=2I7LiXRF}gMFf00UQq=~Jc zFew*5B6BXD;8`5JdKa7jpjIk(lRS5{VcJ(%E=FULZ~}rG8U4ObI8d%kjPBR{F3$r4aR8Dm&#RL(BFX==$SZc(-C}LszKb1zbSF&bmx;X z3ysUpHqED_SguGC5LL;l-GP45D2=P~@8tG@Q`?nH9a(>vgWoVY8y0r}c%wLuSjrC) zuxNW;3ArC2UM?a-T`{)(2LCt21Ce{n7fX=)WhzaqQD^Ke(<4(;1%u@nc|mocmkabL zPRuJ2AZ8uyj31{pqC~ho|E9%vZMI}-!#-H-h_mCRdE4^>Pt><=AV(QWmgbNwayFQI z)H&y;oMPd9Dds;<-$q_gG^KMvG+b6mzz@1DE@U|2hSBrhJ4v6a@*|aZVgf0#-sdi1 z6Y?A-c@`TL-Gq|xUUX(aM>pzg{SJaJZm@d~veZ_?Y2Axv0&PFoMtPo#$S&qMKk`x_ zxm(OtNk2%jRW*=~AqX667GHH`W)z&2S?ky)g$S+>w=r!dXtIvgSut^Bc1MD3T*NCx z@#&O|JIX|BEjCo*;d|`{oU#18?eE;_#0U5pZu}GGfH0CSrTWH1kppdckGR zaWs;ELTo_S<%R7?%W$c>Mx2e?${wrqru&hZN(L8&f`<$rN%R{zc(QNBO#?&=0gUR* z>10MrEKP9Is|$SCZj1Jfo9M9lY4iW*f4319~s|iSWcqv#X8xF+wd(gsSi7#35?3u=r}U%SR%@Q2{^?$t81-0qBbFW z#4BYi&D55c7yU!9`jp*&1pj}q>GZoj@!V~S;QCoOLHu;ZRua`ThPBk|LjeUSS#-lM z(%*1_orlX@B;!zNvtI7|f@CDx5JRZ?7D*w*rr{Q%!*oAw)cN#7>_;ZE`P4;OrE{3?YQR#x)twkby~}x-&Xpp7hT= zS)|`L&%QU;sjx+in51vE{e)tqJNBGyD{FD#qwLtN5b8#CHk~T$v?c=KRxKMi2t)fE z=mvi@`>U&|OEP1=&nJ_EuVvLj74e8F-imdcZ^pgCOmEi6d&p1N-Qce+JBZ(!^*%ie zX3xkM$JE;3jLLk2dP<0Tw&qGqum($A*Hq_l{PdIU2DQJ}dG|{|E*A6tbl?BgJ>1If zl=rKyYjDF9EjM3F0mTIR%-2P$!Hw{eTXlG5k^zz$+cpoCut;9&95oYttq2mRK zhc>&v;>~DyZ&fm^q{698CJXcB)u5f|qG_Nj*!5(7uJD-EVk!K}=vo>nx4+9x zevF0JspYKYeU*4h4|zljk(7Wi)r&~dNWV8YV8MDxA1`;%Ni>7!&FfE7lwtl^=*hMJdVR;Tt9jPeP3ia43q<`RPCP~2M!p9Ic~jqYoP-DY7zIgARtB z&Jfw%7t9; zJVzu}Gg9&#jNiei(v^b9zH)4q1O12G1TW}ozV-T+0&tFczV63OS{yl-AaC)ird8U36z0otPa-MU7h!*RE|AN=k6Izo=#>4tWEw!2SUtY@;}r{2N?a zuH0G2tsWCJuB^V3gKBQ*F~uzeC%L9)QS-?-aoM8we*>el=Jp-#0ucY`rEX$FAKwh%jk*>L`pfb2C3$@HCZ#}ow+zGu_muLk|?UYTP#?=gQP%-XOt4@ zExsgSBo||#!@YYWhund3h45pq<+da1z*N<(e5@&wu)S><5~ec9D(4}vH~%i)1zK3Rqh@#VWhWqt_Cp%6c9!dhN*A-bw>`r1J#J((PGNcBgD-My1+g#CM_S_dB$&J zFAcNAOda>fz`D+qljBzHAz6H4MQ{-fT|tV{HUa6MH6~r8evJd6&h`(SM5^!JBEcm| z5!QQG{QM*;DP7s>nhAu`7#ww&r~H{DRUNyl%*6i6+|jS}i`bOu@BIR-n~0!|(njzH z8?9dPa^1j14q^3Z0>`H3-i;jWv?TX6Bjx>9lZlf=pvUO2#3{W?zv}plU>#dO-$A>; zis*e8W-IF#!`hUxUR!kP1(@dQB%7zjgcx7y;r3|!N?lWX^-483Td~yPfP0XKP%rq| z;W-~6z%8r5sw;370VRj1(9+gi5p=<-luT~n3YNX6bv#6*sbIleoOpA&8h(1gqxRqKM` zfo<9r?m33DrrXx{-Kl26wzJJ{1J~<{plAKu%sd=J^uf%p zb3{Z6L_2nV?_k~}lPG}5i;c$7w5VGh4Uh5$-v$Q-GCM1ihF&oBKy zE+t~qZTeY6X{fq#4MDy(NM85xR%t2dt8dy zxC}YIA=yiiF@(eo$65`~A^gU9;n_f@?Vuaoq9<199KUdD77=7w*fRN(bezkP_?}=# zBB6S>TC4uph%y(XWgN;e9q5SY*hnrRmfsc}P0lJHYs=3qA!rQ$}4}B*~H}QQc&v>2*!NaK6N8dE~@Al!#5T&TsBH zi9SgTwqw2UG?R`wJK+rW??669miFhy$f`-SJY*p1j~CjrNuRN7s@PU^i1^=oTz5qO z_apu3kBGqCb2cwYgih$P3~|yHY>G*L-s*GEZ_tvng|;#StK_EU#QcG)Mc>du4~!QwHa@ z(>z`LfRP9ZZi!UxAf{GztkSC#1ruq$^lL&F^kujIHx>Qbl7Y$yQc+H&U8nj{$wUp8 zA?x~iAXHHQyKf&u<{1m>uH3XHfz8BTi~qSHpMtDKHFFuc{Ve75%`Nco^4M$5d$2Ij z`$@`6Avmxs>`lrWZ&ZN=ii|=Kg|jSbL?lEtWkmmPI^r3hI7w@7#S+*wafXx0vrPv{ zTHO)h_Ev}i{cC^ZA6gwgmih^ob98eg*=$>%qqqVks;wCZGn`1Y<4ygB@Uz+UzEuxl z2Zzt~oEZSe$wQgOuj;#r&o$1~PM1vu^M-Z8DU!f0{0=`_cP**s>IR|s2`b$JK`$;C zlx~F=D=T2-RF-L^@N48t>h+}59lbH4{6QE}S~nW;vZw(jbi&w+SOw zcB1@O``h1n38MsLo7gPIyX!Nf=befqS`d>sY=1dSTKCeq6<}>%u22m_G);*29Qp z!8*{NY=PfwwmGGGp>aYPyTT>E3{~C0EL-EU5UBMXDzyuASE4t!Q93>)jc7Wn_JsyQ zyGY&|VGQ#E1x*pa7S@u9!)wHjUrj+I8tzKtR2lIrXmhp92!%mW<@Z~I*b2|rlMmT~ zw%So=<~G7?cj5dmAvFv_CerLfUWKHaXHv*#4)Am(s`Dewe~oEZz5Ir*+l?4G_1z89cA`1Ag{vT|2-aj5; zjH6#;@eU$&a+mVAyh1ic7SulUZwBy})3oOE8IQ@YRL7MNY z5O;xkI+ROFI%%9WcGUo{meHP4JKV9WbAInwlZ-z0+U!3VVkDM%1wnzs=%1p+o#=8a zM$o6-K-EFg6GCNmw5G9Hur4v4M4#q`T%K(U%|-j4vqcmBA6t){`Ys9qi#pq;%u9qs z*>pvli$sJoTz0~M$>*YI^EYGIowxBDnLncOwAP`oz?QI6-%WKP3W4mxm$jZ?G9^n{ zQA@A}yZ>ffC9_k%VGB3=K~EkCi$f!kuvH%G<{+3E&VRY!FFCC6Jw`vX@JEu}*%+PT zkcXu;`?jPd?cB=fXq%iD&FaM@C83PaoQ39{_VDHjn(!Zzs-wPC(DBFN z;u1#McF|1&yB~7wmHuD`Xlsp$neR=sEzn{LJv6TSEpCdxUFgDO0lC9FSHzT&8mz); zF!MfKAdVU9zdAWtsNv^N0ue;;rzG17YA$HY8v)c9w}t}@Ty2VT1VL?=1m*1@am}dB?Kotgm^DKkKmWjh<)C zfm^@4K&I|Lh#%<>9C=8JpdpRt;Qk#vUi^na;BqE*Z%DF)& zNci+|88wN_0|<^tby6Qv^vc|G;M)#Y2*@=}1b{jcYgFN4-LFla7n~aEU z#4Pgc9;~_-d*Rh@@WJx$M=!Uc_MtywR|AILXKfMrM@>4Uh*sV74^=e>6KK z#)@!jTIKUe%|&Tk&It`cs+P6IMcV_%U--t}`V^f<#@4#!5Gc+mT&@r!w;&m7%$Oud zB}@9_?6>6ho#uA7pwQHiqPP(<0GP08V~zRyhh5ZsqJsIC4l z65mg_QOW2g;EBQsn&Nx|1cTH?b>rl%IQ!O9X|MF~*g%jl{=}V4>&rNlUqgne_s^%* zgn_ZyE>`2=R)%%--L9#hipr8-%5)bBp)X`TY5AuCX}n!kaRSjXz~EM(8V~iH`{9XM zJg)md?~n8ZoE4A6RcXCdW6bY!U9?{)U!uVut_VuSc4dfq19M{_C3wT)pazL~4gW7= zN+q)Fc=|?Ogo`;cBiJgT^BZyia|f*2?h~Hq?r%y2hq0Z&SjV+pADSo7J2M?q zyQ2{R=r2{s*bEj_?}w?bGtOWM3T#KDd#JDuEz=GCka5PKn0C4+@EdIXWXC;s@%4ZS z@a&@_jVRZq##C#up}4LmPU9jE6$NTN2CWd11dqP*{1<0H)N4YI2R(a=K2hWP^+`Nm z+SglE=7J7G@Fz}Pm~ynWh%MZ4WETcu96QhbWCXMTl7AjzD)!uuDu(=~cWcqmLZTmB zw>K_m8CD|ZU_j|^6jBn0@6)S$G;#<&O1wXq0W!!MX=?sz29}{(g*S@Dq{0>+zmJ9< zbxH~EP7&T;O6v>&XD3%PgWBu;#3ZVm5_5924fkMRcEs0&jBm)X%`xI6XqF;YjHK0m zhr8du;;Y25Se9=eJX%F4y_~apkKZSa$f?>%+@0{*wEZcGozPD&4AbOz5g+2^*Df!l z;@mrZ61;lh3bcz+9(K!_$ZsHEykqq4}TC zCuEQ?uWqi*aC1?_#4;a3p#aj1296q0*?{^?@!MDbgZ^k1I$NVkF_2rb=^1_}E9)-f z3h9bT)|@SAL3&WPH&4*nggxxejTUYvz}Nc9_|o*>(FE`gZ>N3mf&=4DW8ESRb>lhK z+7hA=n?6qPl_?B;-BuWm&ohALRr4)Zy}~-Blw>r2Wq6DbD$x0X&$C=|`w#mmaIEZR zy2NxBUm#t6$-Dz1B1@6?{qP)e1*X=|7{+XhrRtWjwgZEy;J*=7F;NwfJUu({04{%P zLT?r;iqf|vcS-jhh_N$0DGNij_*18EN?(o5Xo+ic{#w*WyvhECAt4b{T=C2jrW%~_ zgXP~MA!MBzasAujAyCf#9XCpUz`0Jg_`qD&hD~=!6^Nt15@IjwoNuwqL=Xy#au!UR zL?^_!KW`3;(7KV*yXK5H(4-*ViCnEahqgmakOD)glD768E3EO9$Ht@TNr2a5==|Kl zAu_&cHeQ|IfJ7xUKY%Mfg_Gj3miJaF2h}w1{0bcoe2!Ew9g=;Zd~{Z|Gm!iSUZalZ zHlz`z+=+}b3Nh+_sdpQiWkId&^Pdq1omTi}qyK&s$(_FW^C1X?Fp`pP9bK<)0ODz? z*{9uu*n^rWEECdHV~IBM)KIoW2<*2gA_k_!WoABy5LWwfDolP(#VLqcaeM+HYms)H z>Yv8N=h@}oW8ui=8l#7QhxL^P?XwQ2Vxgt0X(_WbZ}E)KN2OZ{kNiO7_<$0^nWyz+ z4edE{78a$XpDS<03;2y}u!08ZrOF?sAyz1-B%Y{viWu9SWfA!|PTNP$`^;J& z@WO5nxZa6M26W7^DIi#3 z6QZhR{f2uG@*saB!CkcPA;lQ0Rx7FHI`V=^5&B=2<5NT0G6ZYGAR^#rT*vxOMrTF6 z;8cXPqpaXroWh!zJPqeCTGW&OKR$*eE7R#nU%@xo+#ZPaQt8VoWAE(4|I%z=Qfx>- zH8;x1Iu{1k#%rB%{u(A0UEpFW_a(@*GX{AubjMTle?AB51G?iw$uSWdQ%O1*P=Blk z=Ya7AyP2>9E0?6le}syoZG%2Be zw7_vyZ7lR6lZMWXqA0m9vVP1H$iFNt#@8t{b-F|3WxzF3xSU9DC-9nsQK#-yd!}(9 z&MY)6usC>1)YjoGxR8z8-SaHK={J0MjbQjPQ;2=OJjJ_^&AF7Wp|%EV-ez5A3- zkPb>7ehB^b86w%lkQ((V;6$LZm|tW@gw7jdsn^UA-0=3cHeOhZr!wl(Jnsqd-J;@F z3>y%9M6aM03+RXroxPU|b4c}DH=R!QNZ)&9&NmmxO?n#}xt4T~b2{e<`VbhA&iSQs zr=u`hj|O*d=c3B#Dk*%4we+izjPpt&+-l6-{pMXnSfhS@>n_a50O0&Ptt1R_flK@8 z)C;8kxrMu{ybJYSGRspfu&yDwZp)^8(I`NV9KQ2q&9`ahQAOhPGHL}N)jl@sQ!j>i z;O)FWNHy>{u!{V@$T(GqftM*N2UT>6oEmzw47`>A?K>WIhTzRW!~#LDY<-@~_MwL|bH-aUQ}1h}KTV>0YaP+2Dmorz*Rw8Eda0*(faMdQi> z*eMT>MKKXTCv#J%^b_{*{^DX`@w|6=mm~ytViUV*L}wedJt~J?P~Lku8T%JN*rDL>ryzeB+4Gj zH=W3)XzDBeTGV55PW&z_M>L4u{nrD~0%xA*vZc|lumX?$nevWh%MghqhIrC9kxw=k zHl=2a@fMrj)W0AT8PX%w@?ooPE=@V@F?dq=l;XWNb9;aq|NK-KT((* zeGrDbkS6&Yl~^Cc)#yKf>e6N&oRyu7SCj7*X?%$Sm4xFza*c3DWjUS*=_Ntp+)F8} z6(lN_Mx@w-tVeK({t`D6vj@jTy^!#r=Bz)S#KY&BDxA#}|3~-&vV=R`hsabBb-aD( zDRO{3%9@V=x}2hJ{yqNyKCj2Vvn4;0uuqGeW)t`q#jd-N2mv{;KMFRa5QR;rxhJ%T z2}P#qq~P5$eDQ{W5i=6*@3z$sfnT zP0~V7nHb989&2Qjiv94~kd;~G)&dkwf9+0Q2?0+OJQ@iI5Mhi~{#o+sGYj#Qf}_>% zK$^Ri9r}#fLFS6o|2NWzA*QRYFC+a43^uK=YF{IuP3;E!`E(3Jk2HH~?nD8n2!VboJdlp(p(KkgLSAle$*6~eGn zs5)kN#@tANHNp(CE>&lD1SOAxNNBb&sDMi8C}VGJ#2JUdd|{cpQvkWct;zK32?+40|0u(7F)4v>PkhZpBxYU7 zHzU2=K9ADOU$2nr3O8N)lty5Fn|PU95CvhV)v69eS)i^m@%tJ{EI7)ZjJ(^9enYUX zq86nLkc^k|BGCT~F{B*cHxnh#k^W>66E(4_pTZN+yvFQguig19&@=556~x(sxpfa+ zxd$Z!%mn5hszt5HFhg%rNRnwA`cfh`;>}sk4ywOQ#ygGX5FPW-eBN0Sf36tLq zdEqzg*SWur5Wna;Z}v|JubM=x=ItYN&(TaTM*13!mdWmP50bpP>?Tuo$$%= z7F=9=N@0o&1@TA+*R^vQ)pz zp5-HfZrP*^CKH>Y>eyG6IB+b@Fa7svc<+L3BMe1R1x(>{kpo6Kq0Lda=t?Da;Hqr& zW(*OSJIC^$mWB5Mxk_kI2RXGy^=@E5RDsoa z1?(Pk*d##4|3StP!5kJ{rMFU7Q@WigFk&8_gQI>CuUD2EZfB z0(!@EdfXPSS%<%IUOu>>;shxW%~V^F^h#OsRVF#hZyJ6W9wnl}MU<(-OGp>m**@C9 z-oiz9QR&%PNLr;K{*6J6*Zx<=r7X0`#yL?q*~2^20Yj{r%8eVvG*EY;S^h7*uhG_W z+hVi+n?uO4^q&dBFhZQ7GptS!lSzW1xFnW4@mc`iWta)TpIjpt%zKrH-(@b0{75F0 z2BTN{ogSMm$_Vnjfeln}CWJPT7j#H1QjGr}04Fh|0;L9UAvu|)0vDHEa$|&q#X)>O^|)#tSqc$^=g_CMl-JVHRZDTz z{4(N+IEeDT#2copx%|+5$`I~Kg6j?fnCRJZ#s-?Mhsv5WLxR~vDkR>&-U=ds>OUVE zhp8YKEZHADFA$$zX@A|~TQ9_&N`C{Lv{_~gXHYsZBAuevK41^38ny4iLY~v*RA~?40ZS6 zr{A@r1QZ+lf>r7=4MXw9?TUY})Ha~!1RJ}%@%gC-WGH3Ak@3I9f3Jq{-m4SR1{mqb zoFC*0+(?dAiCP=c-N;CMNNzDp68zgd^62GkahM$27DX+5y@^!pyamhBGP}pvWKzmz{^_GT0r6w#WDUfjOWEyZ8weLm8l@qGOjg#M=w0CJNX|1u z``a)F8TGP%%kz33BkYYDWa&LNt$c+U%B{TvESzGmRb8gEC#B*J3uO!;UW#apO! zRW4q{D(EUV`UyyDtKhX{zZyNqM2B|?Gz{m1?3;QN8J;#u=k1zKj4 zcY|}Z*)Z=`D_XUp_zV&LHOLc1e_dDvlj1Jg>m#;= zMRcN!K|HdB<^yR`#Ieq1Zv!STK>meb!E;{MF>G1z$Yo{&e64RbLzn(hGq0$zSz(iD zKc9;0Z*VBqRDm0`KW6@s8*G9=r-u5cmZSE|Ti zmVZD&1+2vs%WslN&1K7vsbJX#J%;}V7OLD(?@@-yH{uAU9rsLi!{M|M1erTy&q3pM z;w|aO=oSQ~lQr$!-a23X8Uz8<7RwVfvEX9`_ooa0rhufdALea3 zs08BSZWxo=EP~t zE0LHV>c3o7P!|lqfGRzaWkjsvms&1jMy)9t&?mfDJrULtewbLJQEQt2p5; zS`mS1z9TF$R>O@hzYQ`Yz$s}GZv=dTWrE!#-jEdsto?KB{#&_=A;8WZ^$zm(MN2i8 z*E_)(bR%{FTdlt|asddrRgqQbIa9z3p!HV~C6XURdcRx;%pcL-4~L^GsVosS4x3>j zn|(ULhJi-c48i*hstfR6HWfA3vHA+{-fmwr6peTjT;?8uQjR4%HM{vdYGw|p{TvA? z4S}+OJwN+k&|(`h^Vs~>cS{zyYT|`S*wg+s!+W1v3GtDHi-ORmlYe@$XrQ^ zjlBs$IJ8?)N@0$qH4+}uc_N3NqW&3dL2DEC?z!4100KztV;wYHtMpj z2dGe211R}&a0UK0H^!-!MQ zGJ-s>grS5YUM9)-Pe7zlZAAYN+AHTQ3)URc;4ItyW|^~TH;6lsIp5qADok9gqTmYwHH?a z$?o$E*)pO!q3L`eBLAT%HWr8ljhHrouj(N0y}uc*5+Shu4Z$8NLVL5e$AA?eZKd!> zVDLKVl^LN%;CM%Pk=5f+P&Yt2M4X9^wQc7jl8mHV-19$i?l#`ZZMO7D0lZ{hV!rv+ z?{RE8oTJ0!eI!s`AnP1Oby8XvDmel7VSO$7CozuVcFwn*<%EgcgVr<<$?CIWGk3SD z;BdrrhgLj5pJl6`3J)h)B^V8fGon2{>bHfpAKQ$>jOY<`$-FZ$AH+ahvk#A@;QB0W zX#^?juPAO+eMi_UqL+5Eu#`3{9iJ0Q)^Z2M{87XcRlBd4KSy;fHx-znbA+#@%t72# z1QhW`cjJkg82m(;E|}9y-mWyw!fgd%UbPgekL;45YzUx*OuOz!>`0Q={#rKa#Uu)j zv7U|vrM5LPB6}o`UMZjzloIRxw|@meoT`N*S3}a!GKidUWbyDRA{pg%)gNW+h``$M ze0h_A-KsC2e@TRz#n z8HIq5Qf>{m-3c`{*0;gK4bpy9d1y>jCaM1n4CUBd7#D^wyUitHS{ba__Hrk7uxAUA z_~UdP?C;<*FdFViDicW)1KdXtrDMD>m5nCQE6dG^aYg~cnxf|}#Ux~N6h3z<4kujz zpZwR(Xwt=sLe1%So(Cfuw=>aK_kdkbFO*4Y=BxYfRXt))7T}bNS4y48I7hj^*m5w~f=2zR{VqbyNxNKpTZW3u;YAg~F&@@|4XLy-WQEU8y)RipyBh6<%|}s8 zpSZF0r*lmh71J)(tdKO3Z+5gRnDF=%1v)zICv_VZJ$#oL8MDf64*D-c4lcIC4`+j- z3P=gd2~10-b)UX7p8s1RwrHq`FMm>+>W4^BxejA6PGv>N|FX08WRQ)2#C$yN4{qy2^xhRS| zdaf6FsWM5i=1pD;n9{DsK6#y_W_EKOg&PNdX~Ccu5dHQq^T@Hh-TK?yJ{-D>tYAYG z>3v`TbQF_vb+Y9)zkC~rL*@LXLEeo~Ic>Yd?G^fnM5gGf1x~wm=SB7r!8K95XnfdB zcRle2BK1JZJI#TwB{;kS{TJP0xJ$Q>T$!T?3siTJe>MSY+5N{m z&h0U%K+DRM(R*~v$k42u*0qA&O#9Kdw7by@rN5Sysqtcz#aXASrHn4Tud~Omj4T1T zZuGbLL#tbM!>t%M)>w!p7nCpS(9+U>5~on!}cZHXaDT ztRzz?9Xd=_no5s58r(W@7wTu+@d`l!K1Huul2A#y6Z(a+!IR`FzvhQd8vg;?By0ZA z8tngtfcHXVtzg2#v<01#!ybFvY;eChcDIFfW;Y{#Kt^A&6{i0)j*gxj!szP*2l(r|? zb@~Cw_4;Lxsr(!Axd5poYQX@ zZbG;bI)NNR3@RfT&30JyJOg7R92_ zXCjyLmkz)-s4e&}a|u={D=91$5_JVr?@jDcp)Z#UIPd@jj{OFjHWm{iAF4yH|C(=c zDYqp4e8v)HFseRgbq}0YZv1bp7Npy&y!|8R8EI+DW%ukZ5>FrcXV-1(G)ffeptg}O zU{!*5Xn!KJO`VJn6YpGs#iUr>9J?Ne_tFH%C!fKugr1tO(Bc|6v#L4eQcPpH25UW6 zM^vH)>R53`p}{;a5@^B zdu@7AzC*oDV&zX7MpC|O^dvMa>Sw%kP6iB9oari6Am*X@HYG+TZTF3g}S`zv4J=LO2cQc1O4mI;L920QxVoCdt zFi`>o zLDLlCS(R)50FFXmDm_07+pMse7|a-(P|y`fQGUab;mtjl4cGH=+ya{j?7!z(R%{KL zMbK02nuc*wEB2!j(QEW_(JoQ1sBVMdh+UPkJ4A%b%G7+Q{E!Ajn@+F1Xt8S$4yBMa zvWPp@vFVMzN$7!KQ`%}Er#!QOtFPd*oKjfy7T%y+_@eu+& z=6%Bm(f&*ss#afF_W)i?Z|juX^AYj6=}o6uTCl?RBs6*nW`v@=Ky%Iv*l7j+^{%3@ zGfh}EYql_QknH@zE2K z`gI$yf}so%|Mi<0@KUpdU5ek`#X7LL+wI!?X(4iN|BfFod;({nI?Ii^1W$*#0E1I9 zYWay-TpQ zIfg)=c5b58fC>WlxMS18XSmtXbf-mSLr9M(>sl#8ZKPd3Jt4FeL`35GBfI&yTaq@0 zB*eJ^!Z6V*?9P$dZtZkai8KOsf;*tSU2_UGNQ4xUP7>gX-PS}`7w?3{sB*3v#lL`8 zTeaYG`UE`}Ndldka8y!?MXlqRY5sNsK!fNSn9e?jy>I_UxCu={tAQqW0;C7Wi>s3W zKG`l-o=KXsy;lkiQ^-0nlonDlk#$}ya~JKY$J3J&dbx^xSp~y8acinU1C4 zoCGk*scjT-J5BrMo;85``nVJgf;w%cB{PFCqBpT=?B;X-;oJ^Q%`YX4A_d!$JM9|K zZg5T255Tn5@k-KekXSUp?VZfVZL#0);N&!>Wel;_8vMmE5fWr({3a~GJ>7iPDV zg~4Q|rf>ZV(s&3+_PR11hN?b5Yi1{-vDype$tBTra(D+LT5w4OooS8EeiLiU2jL^bKwQGydsOT>pK<#eR_+t}^aod~Dm!PR_t#mz7Z?tBB%sLuDuI zAA1$=E`cG}9ou4G!XG27mU0k%pc2fSR(lV2J>oTW7}-aT?!$WO8!eQaW+ZQO$iUWX zye;|1NPMK2GP22v@c16qH6{$A!L8D*XT}haLH|1s-dlo}<9<5uFOjE)h%bAO4v>NZ z+fD{rJ4KPtO{`;aUjWVwWFeW7a0Y2Zd$5l6n;s<)hO2nvU&axFNM}F!=+TFyr^ryY z+);|!tL_%_=@9v}VzY!hse}k4RCDp73{}t8zVQhaW2JGOm2E#r6B5s&n`m02AOVGQ0+|$^4W=XC)>u#SrTep4ti@%ZsMV*P-quJVxt-(4h9Ck7_CD z^$w-A2LuWbvxmw?(odemL<#UNwqql@R)XjGdr^8a6kL=L57P2_yc!WNY=iw{F>3f{ z8Ezv1BSa)ZUv}4YY%QA(Nr&mG_BaEcD|5? zBh|gMxc)X7ZAh{|p{fc)mtFs$NG3Y7l623l+<8hrc{Ne_ix^N`3S8J7s5 z%R&S}5?_vBHKcV@V%WHB)RCmHX-kN^q3lp2CUB!Ppy4q)R`yvzMbA(-S%V8KNIGs&^5-g$9r6vlLg2Q(RY#_VFBVD z^=9Xe2Hc>zf>;fclnNXSMr#sAfd{uFstL=gKg})Ez!tP%WMdR74-5C?$9RySi?O{Fww$+B z4tA77e~%kCv>&TvP&eBj0BfROO#zb|VTL=;*-jImLJTG7FwXHN4UsPuFD|5GJ2k=} z55F|j_6_H#Blz-u zrZ&-shaV!6vexy;vSnanM4|pL%JmrGE9g56SJkSnQXNi%l{Gxp{)XrREw348IoL|E zj9c@I0Y{XR-E6tIi%g0q6@=yv=GS5{fY{*azx9yy+kG#R?R!EeWfn`k@caJ@j zF#g$!s?J@f;zv~IK)6u$@-LOVZdJI3P)tUR6-@==~sRF52VGk7Q@-a zum)g#)Vl_9h~!qtW0BgA06P~3?)CaKUPNIltY?!)0YYS&;)J73 zJ>Jz&cDLrel(4tABu=R!na~`HDjk_n{jmu)$YEGc$w#}e6P5|5#C;g&gP@euf+%M~ zU;X?;YozVe5f=pzl|FRki}GJtSa>vf{`iYzY-VVYNK~txH&|`2IXHoqPMbhYpB5TK)iHpz2gqovmcRX{MxoGEebk+>TAfRPf83JSkwEOAEz={f4AYN}cY zRbaX3A^$1Su&nRQPWK@F&3z)xeyfe6P>@_F2d1u*RwDLpKm?-xKh^fO6~H!Dhy81g zxq+?hQ?y^@J6PI~y~%=}B68YJTfD8ob&7gpzv;#k+;&;(I}4a?NNV!qBHts%FyBmj z?revQqutlb9VmB(PIx;NC^zRB$@A?1>v;L%4*T;Nuiyo_bEl7ptAS+lOQVE8hdy|J z7rR}Z{uiuhR>A`73In?8PEvs!&BQHct~mi1W!F)~L;Yw}YqDCOi`rls{C~~hqD0K= zb3L2-7<2_MFs!ROycjuPNn7yiiJfC$IglX)wP~k)-Ajr!{IRa?Yzs!tu7BB$f=3(&hAFo({x9 ziN3D7)g3f4RP|2CMH|pJY`FHfKXLxg(%mJmh@PRi_!qbSe{i{?_*%UNk)I<~y*W%+ zjF?I~LvwTlz#CeVY@i>_1Z`zl^X=U*t2$fFz3D=@ewo%qq{c&3I+KJdqO9g1Q-AgR zKNPNp%!<^2fHx(SF-H*1JFqijudIht^W8pjjW~i%dZtea8{vGspW|#U?$pj=zf>F2 z9nd5*c)YOzHj_iiq|JoBwMv9RlA=gitR5hFk~%Af$BN&pXt$~eWZ#jMa`k)IWM`$w zDPuV?RmGqID^Ea7*{>{6s>$$kz+*C5;@xJ(e+E8K*yzqCH4b%;JyW7s0kw-E#?E=s z1P*%A-0!j!bM)PkhxVf|a`Ci?=76}d+5AOa7ET!m0(A4_!LES_$#kbK?`JvS?k*lN zZWR=hoSmxOK)`(m?dP6=1;(sEafTb_gqzrz7V;Jng13CQ>5ocm!9ZyVMS#+*nr{|5 zbre>c{VMUN)Um#*Q1!-wU_-8imT1XD=6bzMO6;1#quNBLh99f?CHiW(A1uR?6*96l zMVhr8lX^9rNq5^yGJWXB)U`}H@zB6(zU$o=C4zM z6jUz`|C@|6Ioto3st5su*SBe}wHut*cOw@+fB&iY}fKxxHaYp_b73Nj}WnnpE~TsV5VKd@rYO9$d2@1 zFcRL6Tc(}_?a=&4QRQ;zuQapl8FWSkg9JS>=oC(?a77t*vFK z+lXD4DkNj&8tQ#2jYNUEAEA8ow~#hcbq z2VW=MJpxBv;3`02=Bzl#?pHHJxyt_pgmJ-HT>nCjGiumpbuf7)wyLqH&vIZ4fTMRh z<0d2Txyi!0P#Oe`A-6;vK%xZwH_T5P6A%;nJDG7voGNkTNBrZDezG&G_O>|<;bCMW zjS9t=a{sk8a9%c|JWDoshUL${Ne}`w+~!5UTSpY#Bo{~46RLI_(VYuF-oJ9N zT&?`4T?J=o0OVNA@lV$NM#lL?A^cQFWETGQhZ@jas4_XerHeGo=k_@!i*X(z86AZL z%G2qs9RzeJz%40*5B$LsnLN>EPWA(08Y)Nh&+sMu*e4V%giKc9VWFEw%9nMqmlDIk zGgUOSc6hiH;~k74c~Wo^Nhz2^F-5v-$i7A4nX%w-lgZ0%8x`0&zVJnwF%?^>USpMd z6;=cML2Ai6s8VgGoF?f=zr%9ccghr;VaC&l-2!|o>UeWQ7x*=AQcAs++JkRF`{sp8 zLNx<}+$Q$aNqb;dNBT?O5divb*$GB96Gg-V7zEPo&i~8<_LWg z6-P?hfpg|7=oTHe32KK#z+{AbNrmx zORc>oS{aKoz)jBzBjDS}tJ2zwW4PoT?~-Dl(f%5`{%GG-Q5!g(I`-|^h6|jka^@Bq zkarAKM@%DNn`BibBhnNBntp=jb5e*?jkaGJ#2LuoyxeOIxuz}jZ=2A`Gc*KrDYVFG zs3rGVC&AbPHoqWstpn~6=V>7W-AYhY)cX;zp=x8U2jGeCtaqbQIZXT*yIGeMD@Dd(T zA9s&7#!il^66|7Ldjg-YoS&cQt;l`uukbF_XD>iypf+apK}5;YDu3M}4q@4dJm22} zYvw+^ICEFsF4>{97DnnnP@N-~_@niym<2UTF0I(b0bbEuI8?kp4xmbfjSVw^u+V9l zl?R(m?qk=AGDW{7?I0-&!@Z?vCBYr4`ss_eBJ^Sm2OpSE$R)jI%m)fZktcGQzkcu$ zt!9D1)HK1lgYg8?B>$P_`pAc&pWw&Zq|hNzl;1Q2*Z&`45aUIz=Ei@39KQ29`f@Vn zs90B$T_q~QBiLAbwzlO4rW?EK9`xhZ=8q)OAB=%)3Wgf9c74Eqd1JFd^%7?%>LF!o z^;}XU7ml2K0fy&R-FIVJiRQ0l6@LL4=v3HUg3CIW|4cZ7ghKFK$jLu=2+@M{df_gx z@m!k!bNjfebEUvcbokZ3$0SVrkw^Ab70=uD8$VNR!6iNH(Y~fC9$aTZi|)@6Aixei zBH80-`!K99o*C(|Jsp8;;5*scZ`Xo6MlNRXL6dQG<1OJ8_nwE|0_g&y|NH5EPbfc! zwj8KJpkewqB}+C373^78>XF6@zwDI~qNT*xQ)$sNv{3MKO&;{Pj{{tG&(g*G#%~ZY zv0|MHKv+{3+8A(6Tfv~%1QzVA#lf|i>=zes60o}RWi4ZTpTe^v6N5oNsIRSTtG|mi zZkwj}QLKK}CYiV~x_Su;ab@@3bdty#9L&$J9=7`fiUh-fPbR#CjLX+GzyF2WMAq(S zjGvV-a}Bl~HgzVSL?wl;O?R5o4;XN$B`Y*727i~;A`4%{ zqNk?P-f6-4ajRd@QwUWZfeG!l@(F2a#U-^aUhuG?_^Z%8OZI`6i948SIE$xRMbVGQ z2z{M2tUZHd%>(=mdK3OKFdmvVG!W*CYPxPvvJAb1&K=n$@tDce`1P4QLV1{L_Tqu2 z<0zT*R$OaKnA>9S8w{|EanoyUJe&OF5;z zq7Fnv<<(PVYsx?(>DAi%)(dCXaDMo}IX^H@SQ)CfI3obk<_JQ#1y@l+Zs>^EQ$Yr- z(gH)g?f3W9&FoO-=D};3pFa4nt%VUGm_~QmhvG!qpOx(G4iFS8dDixLAc1E%&ELeR zi~NHF4hhN4D9}LABAQJQNfpWmY_NK;BazgvMZInjGz4$V`>D^bM z)nW^Tq$T5-wetvgc3wvene~=i>NXsliE6-O4bO2&7rvX=w7e~Np`g|MjQ-ylMPMJ_ z_3Ug;3tD42C%zrn;k+1Qnyk3DDp(by;~Huf4?$4GX6fJH2a?d2j5M!bYC*-*(@(?xVOhPHGv|W}}v#t@t z)ZuUZsyqql-^ltf+iRa8n_h17ZP7gJ*`e!;0jW$h+30>#h`Ug4dt{3O5+8NEce1h_ zwXjw1-iDpevp{I`rCS_Wy&8(!ErhZ%33B+b0Cvo-evoKUMvbY<&|zbQ=TS!qsY82w z;B91xyX_Ulbw2RbY^4T`FoPL)*+x$UIx`;82sjw|Opn*zfJYp- z&ZD(~vPeSzq26Z+ju?$OyN#P_9>t6?5j?=ZR#9D5=k(ZozVk-Eg@i7pf5lI4gclwo za)_E1k?cX4fc9H@#-$t}H6mt8v+IU;ascc-6!|lO{L*O8NL^a;G&AhlPJES6*Qu;O z>fVz*7DN^;1qNSgYSA64AIsZ$kR`6eanE~N7yJW8&;o*-z6s@lBgQ81arQH`>Z9KD zSN#rHYqcoqHx5q?5v{@#xn>wrZJAIL7P?df4^@<`-o^6A|7cfwF|qyKGhwMi;G=bw zlB~c$Q$QE;x7z@9)fQSW-selqD4|cjN!aLD+-@ewLVfMW5z@NKEqTMUa)8N@d!)_x z*+6RPimNkxV8jZ;q?tg|$b+WbeE%O_SRFR3D4YDy+i#}+2?yjK8t>ud^n(t&x1)arDrujQ}L*Dk3e^m zkI8nA#87YH2vRi0g^3q=sPDP#9R4=9gh&tc#@;=O_2P5HuZN#K1fPkF+U*S!YpRRY zR{l$xE{VlCdD&R=m;R#;=ysx3X!Y+7z+Ahlrv(hW_*-*I+BTvZT}>w6)%E@5jTjfn zvGuKbH31^8u2HRSigpA~xs0JYe`= z=gFS~mga(~bpGKTMF8iytfVXGiooh~RH55|1vp0KpL<3ZKmMdC< zxz0epc1KDa0_b?-22tAEE?CHv=|est_hDmcyL@L?VdkrFlfG=%b|4ValuQ2X3CdGB zcNYANjQKH@ZHNdPAgGXg)*knR5%kvZXmyuOqPmyfik)@uVhI3!m!og|^Z;(Xd7117 z2ejbg12l(tNI70v&Jogkj^`!j9f;;w92_6jpCs=N$oke@ca%9pm~mj$LvHj0?XQDv zczM-5(=~?fSLXOu(ETmsf#H z)karLy5~%0KUDi?%zrT2Zwv~ZC4|}c5R`_xS8+w{g+2)~GUo}=jAC-W4lk;vsFOh) z;~LzvFAh|mhi|u!hK?219YX{72o;hjcfGO#FxuNaYnphBQbbPGX6GTF?3T<5TO4+l zS9X&>x^4eF#0WJ5%^rK);KLkladOGPth8@Jr1qa1P-6fvi+I5Ruo}BeM3JCR-cr^m+d-A@C6kMDl(ENyuop? zi>L6x$;R}Fbs-fEq8bYIm+qAhQN}1?{i96B=Hv<*=b#KP9P$M{GJ4ATy_!=Z|AX^1 zZ1mDrq+C*LoRNhx=>TidV=$TuJ%YR=yN|#xa%6M;ob@mZ2csj}{!v^O#S4F-pWe9v z2!)2D7)g;54AG5*F}0LzOU_NSa{R`%Pl-|@j~CC^q*nZg?4$h~qxgeQ)G1!so-+;= zlacDD82^@k^gqkv)qk!@sMz(QjU_o^9kLcBAfLD5P4$68&F)oRnqRK|Cq^t${Km$0 zA~~)YV#K5Y(8^o=`(dU3 zDmUYdq2dFywWC&~VJvy#%jo86H@@N3x)AI}%WR_tRGkf?-qK@0Rs60y@LSpw_^iO< z+{{qg@7`n<#-{!OOsxuc*P%$xXzyB-T zi6$;S@EPU`16F9y{>BL)3VFq;=h0H}Fvu=_g}R>el?Wm|S0-%~^yT3-Lk^5wezXs_ zOQQQ|uO^P$!lX6xCXTB2zzNM44M>@zs2b&$v+c29mdgd+1h+Lnw?0}qo%E%~580)` zvV`u-W(|-)TmF28=`WZ(5^J9y`mcToGWVn1fQo~G!8h#Swf=-HJDf(d$n=CIgL2O- zX7;En08P%4U%(xZ=DU@4%3cKuG`IS#Tc>p47KN_hsEgnm#3Lbm2d8uoz9;iTEBN*aMY+}Nsx~`kMf>4~Y z3wz(*hwIRv(S zpK`M6%aA5`EvDp5x=OTSB8_v>fGCBu&Kc)h_=4w7I=s#W`hoKy$$e{V3>sYgkkZ}f zk((}=ayYy?ZWo6+sr_^vXzpC)i7xMU{f$hmGMaPqK8jh!qml!9{xC)--w5A3AhbH1 zH(+d)yROx%?A*Ku-qy23tptNw=D2T2iP3j$`xpHgk?}YqN+GS-w#jL>1U|tH2@9Zz^*jjY=(_9)cq$r-S=JJR#n^cXMcL*c_9zHHrb*W7%D5x zYE3|LWj0Ru>koSg_v-`Kg%ey6K$qw4P$4feM7;|D_2EIkuOe?}pj%O3Fe=N9<7CIa Lsr%&biJ$yGK}Cmv literal 0 HcmV?d00001 diff --git a/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy b/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy new file mode 100644 index 00000000000000..f286a2becaec49 --- /dev/null +++ b/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// when compaction for one rowsets with multiple segments, the delete bitmap can be deleted +suite("test_mow_compact_multi_segments", "nonConcurrent") { + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + def tableName = "test_mow_compact_multi_segments" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + def backendId_to_params = [string: [:]] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) + + def reset_be_param = { paramName -> + // for eache be node, reset paramName to default + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + def original_value = backendId_to_params.get(id).get(paramName) + def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, original_value)) + assertTrue(out.contains("OK")) + } + } + + def get_be_param = { paramName -> + // for eache be node, get param value by default + def paramValue = "" + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + // get the config value from be + def (code, out, err) = curl("GET", String.format("http://%s:%s/api/show_config?conf_item=%s", beIp, bePort, paramName)) + assertTrue(code == 0) + assertTrue(out.contains(paramName)) + // parsing + def resultList = parseJson(out)[0] + assertTrue(resultList.size() == 4) + // get original value + paramValue = resultList[2] + backendId_to_params.get(id, [:]).put(paramName, paramValue) + } + } + + def getTabletStatus = { tablet, rowsetIndex, lastRowsetSegmentNum, enableAssert = false -> + String compactionUrl = tablet["CompactionStatus"] + def (code, out, err) = curl("GET", compactionUrl) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + assertTrue(tabletJson.rowsets.size() >= rowsetIndex) + def rowset = tabletJson.rowsets.get(rowsetIndex - 1) + logger.info("rowset: ${rowset}") + int start_index = rowset.indexOf("]") + int end_index = rowset.indexOf("DATA") + def segmentNumStr = rowset.substring(start_index + 1, end_index).trim() + logger.info("segmentNumStr: ${segmentNumStr}") + if (enableAssert) { + assertEquals(lastRowsetSegmentNum, Integer.parseInt(segmentNumStr)) + } else { + return lastRowsetSegmentNum == Integer.parseInt(segmentNumStr); + } + } + + def getLocalDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_local?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + def waitForCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def running = true + do { + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + // batch_size is 4164 in csv_reader.cpp + // _batch_size is 8192 in vtablet_writer.cpp + onFinish { + GetDebugPoint().clearDebugPointsForAllBEs() + reset_be_param("doris_scanner_row_bytes") + reset_be_param("tablet_rowset_stale_sweep_time_sec") + } + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset") + GetDebugPoint().enableDebugPointForAllBEs("Tablet.delete_expired_stale_rowset.start_delete_unused_rowset") + get_be_param("doris_scanner_row_bytes") + set_be_param("doris_scanner_row_bytes", "1") + get_be_param("tablet_rowset_stale_sweep_time_sec") + set_be_param("tablet_rowset_stale_sweep_time_sec", "0") + + tableName = "test_compact_multi_segments_" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` int(11) NULL, + `v3` int(11) NULL, + `v4` int(11) NULL + ) unique KEY(`k1`, `k2`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + def tablets = sql_return_maparray """ show tablets from ${tableName}; """ + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + def backend_id = tablet.BackendId + + // load 1 + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'compress_type', 'GZ' + file 'test_schema_change_add_key_column.csv.gz' + time 10000 // limit inflight 10s + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(8192, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + def rowCount1 = sql """ select count() from ${tableName}; """ + logger.info("rowCount1: ${rowCount1}") + // check generate 3 segments + getTabletStatus(tablet, 2, 3) + + // trigger compaction + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", + [tablet_id: "${tablet.TabletId}", start_version: 2, end_version: 2]) + def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + logger.info("compact json: " + compactJson) + // check generate 1 segments + for (int i = 0; i < 20; i++) { + if (getTabletStatus(tablet, 2, 1, false)) { + break + } + sleep(100) + } + getTabletStatus(tablet, 2, 1) + sql """ select * from ${tableName} limit 1; """ + + // load 2 + streamLoad { + table "${tableName}" + set 'column_separator', ',' + set 'compress_type', 'GZ' + file 'test_schema_change_add_key_column1.csv.gz' + time 10000 // limit inflight 10s + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20480, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + def rowCount2 = sql """ select count() from ${tableName}; """ + logger.info("rowCount2: ${rowCount2}") + // check generate 3 segments + getTabletStatus(tablet, 3, 6) + def local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local delete bitmap 1: " + local_dm) + + // trigger compaction for load 2 + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", + [tablet_id: "${tablet.TabletId}", start_version: 3, end_version: 3]) + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + compactJson = parseJson(out.trim()) + logger.info("compact json: " + compactJson) + waitForCompaction(tablet) + // check generate 1 segments + for (int i = 0; i < 20; i++) { + if (getTabletStatus(tablet, 3, 1, false)) { + break + } + sleep(100) + } + getTabletStatus(tablet, 3, 1) + + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.vacuum_stale_rowsets") // cloud + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_rowset") // local + local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local delete bitmap 2: " + local_dm) + assertEquals(1, local_dm["delete_bitmap_count"]) +} diff --git a/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy b/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy new file mode 100644 index 00000000000000..53bc4f10e51e36 --- /dev/null +++ b/regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy @@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.atomic.AtomicBoolean +import org.apache.doris.regression.suite.ClusterOptions +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_mow_delete_unused_rowset_dm_docker", "docker") { + logger.info("test_mow_delete_unused_rowset_dm_docker") + def options = new ClusterOptions() + options.cloudMode = false + options.setFeNum(1) + options.setBeNum(1) + options.enableDebugPoints() + options.feConfigs.add("enable_workload_group=false") + // beConfigs + options.beConfigs.add('compaction_promotion_version_count=5') + options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') + options.beConfigs.add('enable_mow_verbose_log=true') + options.beConfigs.add('enable_java_support=false') + + def testTable = "test_mow_delete_unused_rowset_dm_docker" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + + def triggerCompaction = { tablet -> + def compact_type = "cumulative" + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + if (compact_type == "cumulative") { + def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } else if (compact_type == "full") { + def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) + assertEquals(code_2, 0) + return out_2 + } else { + assertFalse(True) + } + } + + def getTabletStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def waitForCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def running = true + do { + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + def getLocalDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + boolean running = true + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_local?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + docker(options) { + sql """ DROP TABLE IF EXISTS ${testTable} """ + sql """ + create table ${testTable} (`k` int NOT NULL, `v` int NOT NULL) + UNIQUE KEY(`k`) + DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset") + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_rowset") + + // 1. write some data + sql """ INSERT INTO ${testTable} VALUES (1,98); """ + sql """ INSERT INTO ${testTable} VALUES (1,99),(2,99); """ + sql """ INSERT INTO ${testTable} VALUES (3,99); """ + sql """ INSERT INTO ${testTable} VALUES (4,99); """ + sql """ INSERT INTO ${testTable} VALUES (5,99); """ + sql "sync" + order_qt_sql1 """ select * from ${testTable}; """ + + // 2. trigger compaction to generate base rowset + getTabletStatus(tablet) + assertTrue(triggerCompaction(tablet).contains("Success")) + waitForCompaction(tablet) + def tablet_status = getTabletStatus(tablet) + assertEquals(2, tablet_status["rowsets"].size()) + + // 3. wait for no delete bitmap and no stale rowsets + def local_dm = getLocalDeleteBitmapStatus(tablet) + assertEquals(0, local_dm["delete_bitmap_count"]) + tablet_status = getTabletStatus(tablet) + assertEquals(0, tablet_status["stale_rowsets"].size()) + + // 3. write some data + sql """ INSERT INTO ${testTable} VALUES (1,100); """ + sql """ INSERT INTO ${testTable} VALUES (1,101),(2,100); """ + sql """ INSERT INTO ${testTable} VALUES (3,100); """ + sql """ INSERT INTO ${testTable} VALUES (4,100); """ + sql """ INSERT INTO ${testTable} VALUES (5,100); """ + sql """ sync """ + order_qt_sql2 "select * from ${testTable}" + tablet_status = getTabletStatus(tablet) + assertEquals(7, tablet_status["rowsets"].size()) + + // 4. trigger compaction + GetDebugPoint().enableDebugPointForAllBEs("StorageEngine::start_delete_unused_rowset.block") + assertTrue(triggerCompaction(tablet).contains("Success")) + waitForCompaction(tablet) + tablet_status = getTabletStatus(tablet) + assertEquals(3, tablet_status["rowsets"].size()) + + // 5. block delete unused rowset, there are delete bitmaps; wait for no stale rowsets + GetDebugPoint().disableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_rowset") + local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local_dm 1: " + local_dm) + assertEquals(6, local_dm["delete_bitmap_count"]) + tablet_status = getTabletStatus(tablet) + assertEquals(0, tablet_status["stale_rowsets"].size()) + + // 6. restart be. check delete bitmap count + cluster.restartBackends() + tablet_status = getTabletStatus(tablet) + logger.info("tablet status after restart: " + tablet_status) + for (int i = 0; i < 300; i++) { + local_dm = getLocalDeleteBitmapStatus(tablet) + if (local_dm["delete_bitmap_count"] == 5) { + break + } + sleep(20) + } + local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local_dm 2: " + local_dm) + assertEquals(5, local_dm["delete_bitmap_count"]) + order_qt_sql3 """ select * from ${testTable}; """ + + // 7. restart be to check to the deleted delete bitmap is stored to local storage + cluster.restartBackends() + tablet_status = getTabletStatus(tablet) + logger.info("tablet status after restart2: " + tablet_status) + for (int i = 0; i < 300; i++) { + local_dm = getLocalDeleteBitmapStatus(tablet) + if (local_dm["delete_bitmap_count"] == 5) { + break + } + sleep(20) + } + local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local_dm 3: " + local_dm) + assertEquals(5, local_dm["delete_bitmap_count"]) + } +} diff --git a/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy b/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy new file mode 100644 index 00000000000000..b91a19784e6eb6 --- /dev/null +++ b/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy @@ -0,0 +1,249 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.atomic.AtomicBoolean; +import org.codehaus.groovy.runtime.IOGroovyMethods + +// when move rowsets from stale to unused, the delete bitmap are not deleted +// when delete unused rowsets, the delete bitmap are deleted +suite("test_mow_stale_rowset_delete_bitmap", "nonConcurrent") { + def testTable = "test_mow_stale_rowset_delete_bitmap" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + def backendId_to_params = [string: [:]] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + + def set_be_param = { paramName, paramValue -> + // for eache be node, set paramName=paramValue + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue)) + assertTrue(out.contains("OK")) + } + } + + def reset_be_param = { paramName -> + // for eache be node, reset paramName to default + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + def original_value = backendId_to_params.get(id).get(paramName) + def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, original_value)) + assertTrue(out.contains("OK")) + } + } + + def get_be_param = { paramName -> + // for eache be node, get param value by default + def paramValue = "" + for (String id in backendId_to_backendIP.keySet()) { + def beIp = backendId_to_backendIP.get(id) + def bePort = backendId_to_backendHttpPort.get(id) + // get the config value from be + def (code, out, err) = curl("GET", String.format("http://%s:%s/api/show_config?conf_item=%s", beIp, bePort, paramName)) + assertTrue(code == 0) + assertTrue(out.contains(paramName)) + // parsing + def resultList = parseJson(out)[0] + assertTrue(resultList.size() == 4) + // get original value + paramValue = resultList[2] + backendId_to_params.get(id, [:]).put(paramName, paramValue) + } + } + + def triggerCompaction = { tablet -> + def compact_type = "cumulative" + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + if (compact_type == "cumulative") { + def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } else if (compact_type == "full") { + def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) + assertEquals(code_2, 0) + return out_2 + } else { + assertFalse(True) + } + } + + def getTabletStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def waitForCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def running = true + do { + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + def getLocalDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + boolean running = true + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_local?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + AtomicBoolean query_result = new AtomicBoolean(true) + def query = { + logger.info("query start") + def results = sql_return_maparray """ select * from ${testTable}; """ + logger.info("query result: " + results) + Set keys = new HashSet<>() + for (final def result in results) { + if (keys.contains(result.k)) { + logger.info("find duplicate key: " + result.k) + query_result.set(false) + break + } + keys.add(result.k) + } + logger.info("query finish. query_result: " + query_result.get()) + } + + sql """ DROP TABLE IF EXISTS ${testTable} """ + sql """ + create table ${testTable} (`k` int NOT NULL, `v` int NOT NULL) + UNIQUE KEY(`k`) + DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + + def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + + try { + GetDebugPoint().clearDebugPointsForAllBEs() + get_be_param("tablet_rowset_stale_sweep_time_sec") + set_be_param("tablet_rowset_stale_sweep_time_sec", "0") + + // write some data + sql """ INSERT INTO ${testTable} VALUES (1,99); """ + sql """ INSERT INTO ${testTable} VALUES (2,99); """ + sql """ INSERT INTO ${testTable} VALUES (3,99); """ + sql """ INSERT INTO ${testTable} VALUES (4,99),(5,0); """ + sql """ INSERT INTO ${testTable} VALUES (5,99); """ + sql "sync" + getTabletStatus(tablet) + getLocalDeleteBitmapStatus(tablet) + + // trigger and block one query + GetDebugPoint().enableDebugPointForAllBEs("NewOlapScanner::_init_tablet_reader_params.block") + Thread query_thread = new Thread(() -> query()) + query_thread.start() + sleep(100) + + // trigger compaction + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset") + GetDebugPoint().enableDebugPointForAllBEs("Tablet.delete_expired_stale_rowset.start_delete_unused_rowset") + assertTrue(triggerCompaction(tablet).contains("Success")) + waitForCompaction(tablet) + // wait for stale rowsets are deleted + for (int i = 0; i < 10; i++) { + def tablet_status = getTabletStatus(tablet) + if (tablet_status["rowsets"].size() <= 2 && tablet_status["stale_rowsets"].size() == 0) { + break + } + sleep(200) + } + getLocalDeleteBitmapStatus(tablet) + + // unblock the query + GetDebugPoint().disableDebugPointForAllBEs("NewOlapScanner::_init_tablet_reader_params.block") + query_thread.join() + assertTrue(query_result.get(), "found duplicated keys") + + // wait for delete bitmap of unused rowsets are deleted + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.vacuum_stale_rowsets") // cloud + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_rowset") // local + for (int i = 0; i < 20; i++) { + def local_delete_bitmap_status = getLocalDeleteBitmapStatus(tablet) + if (local_delete_bitmap_status["delete_bitmap_count"] == 0) { + break + } + sleep(100) + } + } finally { + reset_be_param("tablet_rowset_stale_sweep_time_sec") + GetDebugPoint().clearDebugPointsForAllBEs() + } +} From 483440afa96dbd872c492b3ef64269ef8d922472 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 17:10:18 +0800 Subject: [PATCH 023/572] branch-3.0: [improve](auto partition) random choose dummy partition location to enhance fault tolerance #51732 (#51885) Cherry-picked from #51732 Co-authored-by: hui lai --- .../org/apache/doris/planner/OlapTableSink.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index 0f4f1f9f17d4b1..3f5af10d4b17f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -617,23 +617,22 @@ public List createDummyLocation(OlapTable table) throws } for (int i = 0; i < table.getIndexNumber(); i++) { // only one fake tablet here + Long[] nodes = aliveBe.toArray(new Long[0]); + Random random = new SecureRandom(); + int nodeIndex = random.nextInt(nodes.length); if (singleReplicaLoad) { - Long[] nodes = aliveBe.toArray(new Long[0]); List slaveBe = aliveBe; - - Random random = new SecureRandom(); - int masterNode = random.nextInt(nodes.length); locationParam.addToTablets(new TTabletLocation(fakeTabletId, - Arrays.asList(nodes[masterNode]))); + Arrays.asList(nodes[nodeIndex]))); - slaveBe.remove(masterNode); + slaveBe.remove(nodeIndex); slaveLocationParam.addToTablets(new TTabletLocation(fakeTabletId, slaveBe)); } else { locationParam.addToTablets(new TTabletLocation(fakeTabletId, - Arrays.asList(aliveBe.get(0)))); // just one fake location is enough + Arrays.asList(nodes[nodeIndex]))); // just one fake location is enough - LOG.info("created dummy location tablet_id={}, be_id={}", fakeTabletId, aliveBe.get(0)); + LOG.info("created dummy location tablet_id={}, be_id={}", fakeTabletId, nodes[nodeIndex]); } } From f440b741fb5b8de80263d16251c333834806f913 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Thu, 19 Jun 2025 18:11:27 +0800 Subject: [PATCH 024/572] branch-3.0: [fix](jdbc) make sure init the jdbc client before using it (#51540) This may cause some statistic collection task fail: ``` 2025-06-05 15:02:32,277 WARN (Statistics Job Appender|182) [ExternalCatalog.buildDbForInit():902] Failed to check db DORIS_TEST exist in remote system, ignore it. java.lang.NullPointerException: Cannot invoke "org.apache.doris.datasource.jdbc.client.JdbcClient.getDatabaseNameList()" because "this.jdbcClient" is null at org.apache.doris.datasource.jdbc.JdbcExternalCatalog.listDatabaseNames(JdbcExternalCatalog.java:267) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalCatalog.getFilteredDatabaseNames(ExternalCatalog.java:476) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.datasource.ExternalCatalog.lambda$makeSureInitialized$0(ExternalCatalog.java:318) ~[doris-fe.jar:1.2-SNAPSHOT] at com.github.benmanes.caffeine.cache.LocalLoadingCache.lambda$newMappingFunction$2(LocalLoadingCache.java:145) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.LocalCache.lambda$statsAware$0(LocalCache.java:139) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2406) ~[hive-catalog-shade-2.1.4.jar:2.1.4] at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1916) ~[?:?] ``` Only for branch-3.0. Fix master in #51471 --- .../org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index a8561a6ff8860b..5094207490ce5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -157,6 +157,7 @@ protected Map processCompatibleProperties(Map pr } public String getDatabaseTypeName() { + makeSureInitialized(); return jdbcClient.getDbType(); } From ec7314fd89a203321cf4db204ddd9c7d4523be8a Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Thu, 19 Jun 2025 18:12:39 +0800 Subject: [PATCH 025/572] [feat](desc) add comment column in desc statement (#51047) (#51916) bp #51047 --- .../apache/doris/analysis/DescribeStmt.java | 5 + .../common/proc/IndexSchemaProcNode.java | 49 ++++-- .../common/proc/RemoteIndexSchemaProcDir.java | 2 +- .../proc/RemoteIndexSchemaProcNode.java | 2 +- .../org/apache/doris/qe/SessionVariable.java | 9 + .../test_external_and_internal_describe.out | 83 +++++++++ ...test_external_and_internal_describe.groovy | 163 ++++++++++++++++++ 7 files changed, 300 insertions(+), 13 deletions(-) create mode 100644 regression-test/data/external_table_p0/test_external_and_internal_describe.out create mode 100644 regression-test/suites/external_table_p0/test_external_and_internal_describe.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index f5f7cdaf7490b4..b56dc09a42f447 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -99,6 +99,7 @@ public class DescribeStmt extends ShowStmt implements NotFallbackInParser { private boolean isAllTables; private boolean isOlapTable = false; + private boolean showComment = false; TableValuedFunctionRef tableValuedFunctionRef; boolean isTableValuedFunction; @@ -352,6 +353,7 @@ public List> getResultRows() throws AnalysisException { if (isTableValuedFunction) { return totalRows; } + showComment = ConnectContext.get().getSessionVariable().showColumnCommentInDescribe; Preconditions.checkNotNull(node); List> rows = node.fetchResult().getRows(); List> res = new ArrayList<>(); @@ -378,6 +380,9 @@ public ShowResultSetMetaData getMetaData() { for (String col : IndexSchemaProcNode.TITLE_NAMES) { builder.addColumn(new Column(col, ScalarType.createVarchar(30))); } + if (showComment) { + builder.addColumn(new Column(IndexSchemaProcNode.COMMENT_COLUMN_TITLE, ScalarType.createStringType())); + } return builder.build(); } else { if (isOlapTable) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 7ffff0c449ac76..31373ac781c454 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -20,13 +20,13 @@ import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; +import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; import java.util.List; import java.util.Set; @@ -39,6 +39,7 @@ public class IndexSchemaProcNode implements ProcNodeInterface { .add("Field").add("Type").add("Null").add("Key") .add("Default").add("Extra") .build(); + public static final String COMMENT_COLUMN_TITLE = "Comment"; private final List schema; private final Set bfColumns; @@ -48,10 +49,21 @@ public IndexSchemaProcNode(List schema, Set bfColumns) { this.bfColumns = bfColumns; } - public static ProcResult createResult(List schema, Set bfColumns) throws AnalysisException { + public static ProcResult createResult(List schema, Set bfColumns, List additionalColNames) { Preconditions.checkNotNull(schema); BaseProcResult result = new BaseProcResult(); - result.setNames(TITLE_NAMES); + List names = Lists.newArrayList(TITLE_NAMES); + for (String additionalColName : additionalColNames) { + switch (additionalColName.toLowerCase()) { + case "comment": + names.add(COMMENT_COLUMN_TITLE); + break; + default: + Preconditions.checkState(false, "Unknown additional column name: " + additionalColName); + break; + } + } + result.setNames(names); for (Column column : schema) { // Extra string (aggregation and bloom filter) @@ -69,14 +81,26 @@ public static ProcResult createResult(List schema, Set bfColumns extras.add("STORED GENERATED"); } String extraStr = StringUtils.join(extras, ","); + String comment = column.getComment(); - List rowList = Arrays.asList(column.getDisplayName(), - column.getOriginType().hideVersionForVersionColumn(true), - column.isAllowNull() ? "Yes" : "No", - ((Boolean) column.isKey()).toString(), - column.getDefaultValue() == null - ? FeConstants.null_string : column.getDefaultValue(), - extraStr); + List rowList = Lists.newArrayList(column.getDisplayName(), + column.getOriginType().hideVersionForVersionColumn(true), + column.isAllowNull() ? "Yes" : "No", + ((Boolean) column.isKey()).toString(), + column.getDefaultValue() == null + ? FeConstants.null_string : column.getDefaultValue(), + extraStr, comment); + + for (String additionalColName : additionalColNames) { + switch (additionalColName.toLowerCase()) { + case "comment": + rowList.add(column.getComment()); + break; + default: + Preconditions.checkState(false, "Unknown additional column name: " + additionalColName); + break; + } + } result.addRow(rowList); } return result; @@ -84,6 +108,9 @@ public static ProcResult createResult(List schema, Set bfColumns @Override public ProcResult fetchResult() throws AnalysisException { - return createResult(this.schema, this.bfColumns); + boolean showCommentInDescribe = ConnectContext.get() == null ? false + : ConnectContext.get().getSessionVariable().showColumnCommentInDescribe; + return createResult(this.schema, this.bfColumns, + showCommentInDescribe ? Lists.newArrayList(COMMENT_COLUMN_TITLE) : Lists.newArrayList()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcDir.java index f2531b7ec15fb5..05a029ccd04fc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcDir.java @@ -70,7 +70,7 @@ public ProcResult fetchResult() throws AnalysisException { } List remoteSchema = new FetchRemoteTabletSchemaUtil(tablets).fetch(); this.schema.addAll(remoteSchema); - return IndexSchemaProcNode.createResult(this.schema, this.bfColumns); + return IndexSchemaProcNode.createResult(this.schema, this.bfColumns, Lists.newArrayList()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcNode.java index cdb1bbc133e356..997f51556a5416 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/RemoteIndexSchemaProcNode.java @@ -73,6 +73,6 @@ public ProcResult fetchResult() throws AnalysisException { } List remoteSchema = new FetchRemoteTabletSchemaUtil(tablets).fetch(); this.schema.addAll(remoteSchema); - return IndexSchemaProcNode.createResult(this.schema, this.bfColumns); + return IndexSchemaProcNode.createResult(this.schema, this.bfColumns, Lists.newArrayList()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 78ea69113e6f09..ad75acdb9e6389 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -717,6 +717,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_SCHEMA_SCAN_FROM_MASTER_FE = "enable_schema_scan_from_master_fe"; + public static final String SHOW_COLUMN_COMMENT_IN_DESCRIBE = "show_column_comment_in_describe"; + public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; @@ -2454,6 +2456,13 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) { }) public boolean enableSchemaScanFromMasterFe = true; + @VariableMgr.VarAttr(name = SHOW_COLUMN_COMMENT_IN_DESCRIBE, needForward = true, + description = { + "是否在 DESCRIBE TABLE 语句中显示列注释", + "whether to show column comments in DESCRIBE TABLE statement" + }) + public boolean showColumnCommentInDescribe = false; + @VariableMgr.VarAttr(name = SQL_CONVERTOR_CONFIG, needForward = true, description = { "SQL 转换器的相关配置，使用 Json 格式。以 {} 为根元素。", diff --git a/regression-test/data/external_table_p0/test_external_and_internal_describe.out b/regression-test/data/external_table_p0/test_external_and_internal_describe.out new file mode 100644 index 00000000000000..4ee6d9cf4cb404 --- /dev/null +++ b/regression-test/data/external_table_p0/test_external_and_internal_describe.out @@ -0,0 +1,83 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc01 -- +user_id text Yes true \N +item_id text Yes true \N +category_id text Yes true \N +behavior_type text Yes true \N +user_age tinyint Yes true \N +gender text Yes true \N +province text Yes true \N +city text Yes true \N +behavior_time datetime(6) Yes true \N +behavior_date date Yes true \N +dt varchar(65533) Yes true \N + +-- !desc02 -- +user_id text Yes true \N Unique identifier for the user +item_id text Yes true \N Unique identifier for the product +category_id text Yes true \N Category ID of the product +behavior_type text Yes true \N 行为类型，包含pv(浏览)、buy(购买)、cart(加购)、fav(收藏) +user_age tinyint Yes true \N Age group of the user (1=18-25, 2=26-30, 3=31-35, 4=36-40, 5=40+) +gender text Yes true \N User gender (0=female, 1=male, null=unknown) +province text Yes true \N Province where the user is located +city text Yes true \N City where the user is located +behavior_time datetime(6) Yes true \N Timestamp when the behavior occurred +behavior_date date Yes true \N Date when the behavior occurred +dt varchar(65533) Yes true \N Partition field in yyyy-MM-dd format + +-- !proc_sql01 -- +user_id text Yes true \N +item_id text Yes true \N +category_id text Yes true \N +behavior_type text Yes true \N +user_age tinyint Yes true \N +gender text Yes true \N +province text Yes true \N +city text Yes true \N +behavior_time datetime(6) Yes true \N +behavior_date date Yes true \N +dt varchar(65533) Yes true \N + +-- !proc_sql02 -- +user_id text Yes true \N Unique identifier for the user +item_id text Yes true \N Unique identifier for the product +category_id text Yes true \N Category ID of the product +behavior_type text Yes true \N 行为类型，包含pv(浏览)、buy(购买)、cart(加购)、fav(收藏) +user_age tinyint Yes true \N Age group of the user (1=18-25, 2=26-30, 3=31-35, 4=36-40, 5=40+) +gender text Yes true \N User gender (0=female, 1=male, null=unknown) +province text Yes true \N Province where the user is located +city text Yes true \N City where the user is located +behavior_time datetime(6) Yes true \N Timestamp when the behavior occurred +behavior_date date Yes true \N Date when the behavior occurred +dt varchar(65533) Yes true \N Partition field in yyyy-MM-dd format + +-- !desc01 -- +k1 int Yes true \N +k2 text Yes false \N NONE +k3 text Yes false \N NONE + +-- !desc02 -- +k1 int Yes true \N first column +k2 text Yes false \N NONE +k3 text Yes false \N NONE 中文column + +-- !proc_sql01 -- +k1 int Yes true \N +k2 text Yes false \N NONE +k3 text Yes false \N NONE + +-- !proc_db_sql01 -- +k1 int Yes true \N +k2 text Yes false \N NONE +k3 text Yes false \N NONE + +-- !proc_sql02 -- +k1 int Yes true \N first column +k2 text Yes false \N NONE +k3 text Yes false \N NONE 中文column + +-- !proc_db_sql02 -- +k1 int Yes true \N first column +k2 text Yes false \N NONE +k3 text Yes false \N NONE 中文column + diff --git a/regression-test/suites/external_table_p0/test_external_and_internal_describe.groovy b/regression-test/suites/external_table_p0/test_external_and_internal_describe.groovy new file mode 100644 index 00000000000000..fde81af64d712b --- /dev/null +++ b/regression-test/suites/external_table_p0/test_external_and_internal_describe.groovy @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_and_internal_describe", "p0,external,hive,external_docker,external_docker_hive") { + String catalog_name = "test_test_external_describe" + + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + for (String hivePrefix : ["hive3"]) { + setHivePrefix(hivePrefix) + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + + // 1. test default catalog + sql """drop catalog if exists ${catalog_name};""" + sql """ + create catalog ${catalog_name} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}' + ); + """ + sql """switch ${catalog_name}""" + hive_docker """drop database if exists test_external_describe CASCADE""" + hive_docker """create database test_external_describe""" + hive_docker """ + CREATE TABLE IF NOT EXISTS test_external_describe.user_behavior_log ( + user_id STRING COMMENT 'Unique identifier for the user', + item_id STRING COMMENT 'Unique identifier for the product', + category_id STRING COMMENT 'Category ID of the product', + behavior_type STRING COMMENT '行为类型，包含pv(浏览)、buy(购买)、cart(加购)、fav(收藏)', + user_age TINYINT COMMENT 'Age group of the user (1=18-25, 2=26-30, 3=31-35, 4=36-40, 5=40+)', + gender STRING COMMENT 'User gender (0=female, 1=male, null=unknown)', + province STRING COMMENT 'Province where the user is located', + city STRING COMMENT 'City where the user is located', + behavior_time TIMESTAMP COMMENT 'Timestamp when the behavior occurred', + behavior_date DATE COMMENT 'Date when the behavior occurred' + ) + COMMENT 'Log table for user behaviors' + PARTITIONED BY (dt STRING COMMENT 'Partition field in yyyy-MM-dd format') + STORED AS PARQUET + """ + // no comment + sql """set show_column_comment_in_describe = false""" + qt_desc01 """desc test_external_describe.user_behavior_log""" + // set show comment + sql """set show_column_comment_in_describe = true""" + qt_desc02 """desc test_external_describe.user_behavior_log""" + + sql """unset variable show_column_comment_in_describe;""" + + // test show proc + def show_proc_string = """/catalogs/""" + List> res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("test_test_external_describe")) { + show_proc_string = """${show_proc_string}${res[i][0]}/""" + } + } + // show proc "/catalogs/1747727318719/" + res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("test_external_describe")) { + show_proc_string = """${show_proc_string}${res[i][0]}/""" + } + } + // show proc "/catalogs/1747727318719/2272230635936012419/" + res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("user_behavior_log")) { + show_proc_string = """${show_proc_string}${res[i][0]}/index_schema/""" + } + } + // show proc "/catalogs/1747727318719/2272230635936012419/4443123596601666371/index_schema" + res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("user_behavior_log")) { + show_proc_string = """${show_proc_string}${res[i][0]}""" + } + } + sql """set show_column_comment_in_describe = false""" + qt_proc_sql01 """show proc '${show_proc_string}'""" + sql """set show_column_comment_in_describe = true""" + qt_proc_sql02 """show proc '${show_proc_string}'""" + + sql """unset variable show_column_comment_in_describe;""" + // sql """drop table test_external_describe.user_behavior_log""" + } + + // desc internal table + sql "switch internal" + sql "drop database if exists test_external_and_internal_describe_db" + sql "create database test_external_and_internal_describe_db"; + sql "use test_external_and_internal_describe_db"; + sql """ + CREATE TABLE test_external_and_internal_describe_tbl ( + k1 int COMMENT "first column", + k2 string COMMENT "", + k3 string COMMENT "中文column" + ) DISTRIBUTED BY HASH(k1) BUCKETS 1 PROPERTIES("replication_num" = "1"); + """ + + // no comment + sql """set show_column_comment_in_describe = false""" + qt_desc01 """desc test_external_and_internal_describe_tbl""" + // set show comment + sql """set show_column_comment_in_describe = true""" + qt_desc02 """desc test_external_and_internal_describe_tbl""" + + // test show proc for internal + def show_proc_string = """/catalogs/0/""" + def show_proc_db_string = """/dbs/""" + List> res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("test_external_and_internal_describe_db")) { + show_proc_string = """${show_proc_string}${res[i][0]}/""" + show_proc_db_string = """${show_proc_db_string}${res[i][0]}/""" + } + } + // show proc "/catalogs/0/1747727318719/" + res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("test_external_and_internal_describe_tbl")) { + show_proc_string = """${show_proc_string}${res[i][0]}/index_schema/""" + show_proc_db_string = """${show_proc_db_string}${res[i][0]}/index_schema/""" + } + } + // show proc "/catalogs/0/1747727318719/2272230635936012419/4443123596601666371/index_schema" + res = sql """show proc '${show_proc_string}'""" + for (int i = 0; i < res.size(); i++) { + if (res[i][1].equals("test_external_and_internal_describe_tbl")) { + show_proc_string = """${show_proc_string}${res[i][0]}""" + show_proc_db_string = """${show_proc_db_string}${res[i][0]}""" + } + } + sql """set show_column_comment_in_describe = false""" + qt_proc_sql01 """show proc '${show_proc_string}'""" + qt_proc_db_sql01 """show proc '${show_proc_db_string}'""" + sql """set show_column_comment_in_describe = true""" + qt_proc_sql02 """show proc '${show_proc_string}'""" + qt_proc_db_sql02 """show proc '${show_proc_db_string}'""" + + + sql """unset variable show_column_comment_in_describe;""" + } +} + From abc190ae1c3a7b022a0d79d67136dfcbfddee106 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 19 Jun 2025 20:04:55 +0800 Subject: [PATCH 026/572] branch-3.0: [Fix](compile) Fix arm compile failure caused by undefined symbol #51715 (#51926) Cherry-picked from #51715 Co-authored-by: zclllyybb --- be/src/vec/common/arithmetic_overflow.h | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/be/src/vec/common/arithmetic_overflow.h b/be/src/vec/common/arithmetic_overflow.h index e030957c600aa2..1a86af01b05d14 100644 --- a/be/src/vec/common/arithmetic_overflow.h +++ b/be/src/vec/common/arithmetic_overflow.h @@ -115,9 +115,49 @@ inline bool mul_overflow(long long x, long long y, long long& res) { return __builtin_smulll_overflow(x, y, &res); } +// from __muloXi4 in llvm's compiler-rt +static inline __int128 int128_overflow_mul(__int128 a, __int128 b, int* overflow) { + const int N = (int)(sizeof(__int128) * CHAR_BIT); + const auto MIN = (__int128)((__uint128_t)1 << (N - 1)); + const __int128 MAX = ~MIN; + *overflow = 0; + __int128 result = (__uint128_t)a * b; + if (a == MIN) { + if (b != 0 && b != 1) { + *overflow = 1; + } + return result; + } + if (b == MIN) { + if (a != 0 && a != 1) { + *overflow = 1; + } + return result; + } + __int128 sa = a >> (N - 1); + __int128 abs_a = (a ^ sa) - sa; + __int128 sb = b >> (N - 1); + __int128 abs_b = (b ^ sb) - sb; + if (abs_a < 2 || abs_b < 2) { + return result; + } + if (sa == sb) { + if (abs_a > MAX / abs_b) { + *overflow = 1; + } + } else { + if (abs_a > MIN / -abs_b) { + *overflow = 1; + } + } + return result; +} + template <> inline bool mul_overflow(__int128 x, __int128 y, __int128& res) { - return __builtin_mul_overflow(x, y, &res); + int overflow = 0; + res = int128_overflow_mul(x, y, &overflow); + return overflow != 0; } template <> From 6f6335a25277feacef1d82ba8b8b7cc7784c3cf0 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 19 Jun 2025 21:11:40 +0800 Subject: [PATCH 027/572] [ci](perf) adjust exsits meta and data path (#51912) --- .../pipeline/nonConcurrent/conf/be.conf | 90 +++++++++++ .../pipeline/nonConcurrent/conf/fe.conf | 94 ++++++++++++ .../nonConcurrent/conf/regression-conf.groovy | 143 ++++++++++++++++++ .../pipeline/performance/deploy.sh | 4 - .../pipeline/performance/prepare.sh | 4 - .../pipeline/performance/run-clickbench.sh | 4 - .../pipeline/performance/run-load.sh | 4 - 7 files changed, 327 insertions(+), 16 deletions(-) create mode 100644 regression-test/pipeline/nonConcurrent/conf/be.conf create mode 100644 regression-test/pipeline/nonConcurrent/conf/fe.conf create mode 100644 regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy diff --git a/regression-test/pipeline/nonConcurrent/conf/be.conf b/regression-test/pipeline/nonConcurrent/conf/be.conf new file mode 100644 index 00000000000000..961756b2e7118e --- /dev/null +++ b/regression-test/pipeline/nonConcurrent/conf/be.conf @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +CUR_DATE=`date +%Y%m%d-%H%M%S` + +PPROF_TMPDIR="$DORIS_HOME/log/" + +# For jdk 8 +JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" + +# For jdk 17, this JAVA_OPTS will be used as default JVM options +JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DORIS_HOME/log/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" + +# Set your own JAVA_HOME +# JAVA_HOME=/path/to/jdk/ + +# https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile +# https://jemalloc.net/jemalloc.3.html +JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:5000,dirty_decay_ms:5000,oversize_threshold:0,prof:true,prof_active:false,lg_prof_interval:-1" +JEMALLOC_PROF_PRFIX="jemalloc_heap_profile_" + +# INFO, WARNING, ERROR, FATAL +sys_log_level = INFO +sys_log_verbose_modules=query_context,runtime_query_statistics_mgr +be_port = 9161 +webserver_port = 8141 +heartbeat_service_port = 9151 +brpc_port = 8161 +arrow_flight_sql_port = 8181 + +path_gc_check_interval_second=1 +max_garbage_sweep_interval=180 + +log_buffer_level = -1 + +enable_stream_load_record = true +stream_load_record_batch_size = 500 +storage_root_path=/mnt/ssd01/cluster_storage/doris.SSD/P0/cluster1;/mnt/ssd01/cluster_storage/doris.SSD + +priority_networks=172.19.0.0/24 +enable_fuzzy_mode=true +max_depth_of_expr_tree=200 +enable_feature_binlog=true +max_sys_mem_available_low_water_mark_bytes=69206016 +user_files_secure_path=/ +enable_debug_points=true +# debug scanner context dead loop +enable_debug_log_timeout_secs=0 +enable_missing_rows_correctness_check=true + +flush_thread_num_per_store = 24 +high_priority_flush_thread_num_per_store = 24 + +trino_connector_plugin_dir=/tmp/trino_connector/connectors + +enable_jvm_monitor = true + +enable_be_proc_monitor = true +be_proc_monitor_interval_ms = 30000 +webserver_num_workers = 128 +pipeline_task_leakage_detect_period_sec=1 +crash_in_memory_tracker_inaccurate = true +#enable_table_size_correctness_check=true +enable_brpc_connection_check=true +enable_write_index_searcher_cache=true + +# enable download small files in batch, see apache/doris#45061 for details +enable_batch_download = true + +remove_unused_remote_files_interval_sec=60 +cold_data_compaction_interval_sec=60 +large_cumu_compaction_task_min_thread_num=3 + +# So feature has bug, so by default is false, only open it in pipeline to observe +enable_parquet_page_index=true + diff --git a/regression-test/pipeline/nonConcurrent/conf/fe.conf b/regression-test/pipeline/nonConcurrent/conf/fe.conf new file mode 100644 index 00000000000000..85d7a87815541b --- /dev/null +++ b/regression-test/pipeline/nonConcurrent/conf/fe.conf @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +##################################################################### +## The uppercase properties are read and exported by bin/start_fe.sh. +## To see all Frontend configurations, +## see fe/src/org/apache/doris/common/Config.java +##################################################################### + +CUR_DATE=`date +%Y%m%d-%H%M%S` + +# the output dir of stderr and stdout +LOG_DIR = ${DORIS_HOME}/log + +# For jdk 8 +JAVA_OPTS="-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx4096m -XX:+HeapDumpOnOutOfMemoryError -XX:+UnlockExperimentalVMOptions -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintClassHistogramAfterFullGC -Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Dlog4j2.formatMsgNoLookups=true -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" + +# For jdk 17, this JAVA_OPTS will be used as default JVM options +JAVA_OPTS_FOR_JDK_17="-Djavax.security.auth.useSubjectCredsOnly=false -Xmx8192m -Xms8192m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$DORIS_HOME/log/ -Xlog:gc*,classhisto*=trace:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED" + +sys_log_level = INFO +sys_log_mode = NORMAL +sys_log_verbose_modules = org.apache.doris.master.MasterImpl,org.apache.doris.common.profile,org.apache.doris.qe.QeProcessorImpl,org.apache.doris.load.ExportTaskExecutor,org.apache.doris.planner.OlapScanNode +arrow_flight_sql_port = 8081 +catalog_trash_expire_second=1 +#enable ssl for test +enable_ssl = true + +enable_outfile_to_local = true +tablet_create_timeout_second=100 +remote_fragment_exec_timeout_ms=120000 +fuzzy_test_type=p0 +use_fuzzy_session_variable=true + +enable_feature_binlog=true + +enable_debug_points=true + +# enable mtmv +enable_mtmv = true + +dynamic_partition_check_interval_seconds=3 + +desired_max_waiting_jobs=200 + +# make checkpoint more frequent +edit_log_roll_num = 1000 + +# make job/label clean more frequent +history_job_keep_max_second = 300 +streaming_label_keep_max_second = 300 +label_keep_max_second = 300 + +# job test configurations +#allows the creation of jobs with an interval of second +enable_job_schedule_second_for_test = true +mtmv_task_queue_size = 4096 + +enable_workload_group = true +publish_topic_info_interval_ms = 1000 +workload_sched_policy_interval_ms = 1000 + +disable_decimalv2 = false +disable_datev1 = false + +master_sync_policy = WRITE_NO_SYNC +replica_sync_policy = WRITE_NO_SYNC + +enable_advance_next_id = true +# enable deadlock detection +enable_deadlock_detection = true +max_lock_hold_threshold_seconds = 10 + +force_olap_table_replication_allocation=tag.location.default:1 + +# profile related +max_query_profile_num = 2000 +max_spilled_profile_num = 2000 + +check_table_lock_leaky=true diff --git a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy new file mode 100644 index 00000000000000..c1c1ad91865aed --- /dev/null +++ b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* ******* Do not commit this file unless you know what you are doing ******* */ + +// **Note**: default db will be create if not exist +defaultDb = "regression_test" + +jdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +targetJdbcUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round" +jdbcUser = "root" +jdbcPassword = "" + +ccrDownstreamUrl = "jdbc:mysql://172.19.0.2:9131/?useLocalSessionState=true&allowLoadLocalInfile=true" +ccrDownstreamUser = "root" +ccrDownstreamPassword = "" +ccrDownstreamFeThriftAddress = "127.0.0.1:9020" + +feSourceThriftAddress = "127.0.0.1:9020" +feTargetThriftAddress = "127.0.0.1:9020" +feSyncerUser = "root" +feSyncerPassword = "" + +feHttpAddress = "172.19.0.2:8131" +feHttpUser = "root" +feHttpPassword = "" + +// set DORIS_HOME by system properties +// e.g. java -DDORIS_HOME=./ +suitePath = "${DORIS_HOME}/regression-test/suites" +dataPath = "${DORIS_HOME}/regression-test/data" +pluginPath = "${DORIS_HOME}/regression-test/plugins" +realDataPath = "${DORIS_HOME}/regression-test/realdata" +trinoPluginsPath = "/tmp/trino_connector" +// sf1DataPath can be url like "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com" or local path like "/data" +//sf1DataPath = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com" + +// will test /.groovy +// empty group will test all group +testGroups = "nonConcurrent" +// empty suite will test all suite +testSuites = "" +// empty directories will test all directories +testDirectories = "" + +// this groups will not be executed +excludeGroups = "p1,p2" +// this suites will not be executed + +// this suites will not be executed +excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line + "test_write_inverted_index_exception_fault_injection," + // cause core dump + "zzz_the_end_sentinel_do_not_touch"// keep this line as the last line + +// this directories will not be executed +excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line + "variant_github_events_nonConcurrent_p2," + + "variant_github_events_new_p2," + + "hdfs_vault_p2," + + "nereids_p0/hbo," + + "zzz_the_end_sentinel_do_not_touch"// keep this line as the last line + +// for test csv with header +enableHdfs=false // set to true if hdfs is ready +hdfsFs = "hdfs://127.0.0.1:9000" +hdfsUser = "doris-test" +hdfsPasswd = "" +brokerName = "broker_name" + +// broker load test config +enableBrokerLoad=true + +// jdbc connector test config +// To enable jdbc test, you need first start mysql/pg container. +// See `docker/thirdparties/start-thirdparties-docker.sh` +enableJdbcTest=false +mysql_57_port=7111 +pg_14_port=7121 +mariadb_10_port=3326 +// hive catalog test config +// To enable jdbc test, you need first start hive container. +// See `docker/thirdparties/start-thirdparties-docker.sh` +enableHiveTest=false +enablePaimonTest=false + +// port of hive2 docker +hive2HmsPort=9083 +hive2HdfsPort=8020 +hive2ServerPort=10000 +hive2PgPort=5432 + +// port of hive3 docker +hive3HmsPort=9383 +hive3HdfsPort=8320 +hive3ServerPort=13000 +hive3PgPort=5732 + +// kafka test config +// to enable kafka test, you need firstly to start kafka container +// See `docker/thirdparties/start-thirdparties-docker.sh` +enableKafkaTest=true +kafka_port=19193 + +// iceberg test config +iceberg_rest_uri_port=18181 +iceberg_minio_port=19001 + +enableEsTest=false +es_6_port=19200 +es_7_port=29200 +es_8_port=39200 + +cacheDataPath = "/data/regression/" + +s3Source = "aliyun" +s3Endpoint = "oss-cn-hongkong-internal.aliyuncs.com" + +//arrow flight sql test config +extArrowFlightSqlHost = "127.0.0.1" +extArrowFlightSqlPort = 8081 +extArrowFlightSqlUser = "root" +extArrowFlightSqlPassword= "" + +max_failure_num=50 + +externalEnvIp="127.0.0.1" + +// trino-connector catalog test config +enableTrinoConnectorTest = false diff --git a/regression-test/pipeline/performance/deploy.sh b/regression-test/pipeline/performance/deploy.sh index 36870058641fab..9f29fd8d256385 100644 --- a/regression-test/pipeline/performance/deploy.sh +++ b/regression-test/pipeline/performance/deploy.sh @@ -68,10 +68,6 @@ exit_flag=0 cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/fe_custom.conf "${DORIS_HOME}"/fe/conf/ cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/be_custom.conf "${DORIS_HOME}"/be/conf/ target_branch="$(echo "${target_branch}" | sed 's| ||g;s|\.||g;s|-||g')" # remove space、dot、hyphen from branch name - if [[ "${target_branch}" == "branch30" ]]; then - # branch-3.0 also use master data - target_branch="master" - fi sed -i "s|^meta_dir=/data/doris-meta-\${branch_name}|meta_dir=/data/doris-meta-${target_branch}${meta_changed_suffix:-}|g" "${DORIS_HOME}"/fe/conf/fe_custom.conf sed -i "s|^storage_root_path=/data/doris-storage-\${branch_name}|storage_root_path=/data/doris-storage-${target_branch}${meta_changed_suffix:-}|g" "${DORIS_HOME}"/be/conf/be_custom.conf diff --git a/regression-test/pipeline/performance/prepare.sh b/regression-test/pipeline/performance/prepare.sh index 635b4510eba4fb..1f578b2c6793e7 100644 --- a/regression-test/pipeline/performance/prepare.sh +++ b/regression-test/pipeline/performance/prepare.sh @@ -98,10 +98,6 @@ if _get_pr_changed_files "${pr_num_from_trigger}"; then # if PR changed the doris meta file, the next PR deployment on the same mechine which built this PR will fail. # make a copy of the meta file for the meta changed PR. target_branch="$(echo "${target_branch}" | sed 's| ||g;s|\.||g;s|-||g')" # remove space、dot、hyphen from branch name - if [[ "${target_branch}" == "branch30" ]]; then - # branch-3.0 also use master data - target_branch="master" - fi meta_changed_suffix="_2" rsync -a --delete "/data/doris-meta-${target_branch}/" "/data/doris-meta-${target_branch}${meta_changed_suffix}" rsync -a --delete "/data/doris-storage-${target_branch}/" "/data/doris-storage-${target_branch}${meta_changed_suffix}" diff --git a/regression-test/pipeline/performance/run-clickbench.sh b/regression-test/pipeline/performance/run-clickbench.sh index 98f8e34ffd6e95..6ed29a464f9e57 100644 --- a/regression-test/pipeline/performance/run-clickbench.sh +++ b/regression-test/pipeline/performance/run-clickbench.sh @@ -91,10 +91,6 @@ exit_flag=0 cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/clickbench/conf/fe_custom.conf "${DORIS_HOME}"/fe/conf/ cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/clickbench/conf/be_custom.conf "${DORIS_HOME}"/be/conf/ target_branch="$(echo "${target_branch}" | sed 's| ||g;s|\.||g;s|-||g')" # remove space、dot、hyphen from branch name - if [[ "${target_branch}" == "branch30" ]]; then - # branch-3.0 also use master data - target_branch="master" - fi sed -i "s|^meta_dir=/data/doris-meta-\${branch_name}|meta_dir=/data/doris-meta-${target_branch}|g" "${DORIS_HOME}"/fe/conf/fe_custom.conf sed -i "s|^storage_root_path=/data/doris-storage-\${branch_name}|storage_root_path=/data/doris-storage-${target_branch}|g" "${DORIS_HOME}"/be/conf/be_custom.conf if ! restart_doris; then echo "ERROR: Restart doris failed" && exit 1; fi diff --git a/regression-test/pipeline/performance/run-load.sh b/regression-test/pipeline/performance/run-load.sh index 46b89f09d76154..bd67102f2030a1 100644 --- a/regression-test/pipeline/performance/run-load.sh +++ b/regression-test/pipeline/performance/run-load.sh @@ -649,10 +649,6 @@ exit_flag=0 cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/fe_custom.conf "${DORIS_HOME}"/fe/conf/ cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/be_custom.conf "${DORIS_HOME}"/be/conf/ target_branch="$(echo "${target_branch}" | sed 's| ||g;s|\.||g;s|-||g')" # remove space、dot、hyphen from branch name - if [[ "${target_branch}" == "branch30" ]]; then - # branch-3.0 also use master data - target_branch="master" - fi sed -i "s|^meta_dir=/data/doris-meta-\${branch_name}|meta_dir=/data/doris-meta-${target_branch}|g" "${DORIS_HOME}"/fe/conf/fe_custom.conf sed -i "s|^storage_root_path=/data/doris-storage-\${branch_name}|storage_root_path=/data/doris-storage-${target_branch}|g" "${DORIS_HOME}"/be/conf/be_custom.conf if ! restart_doris; then echo "ERROR: Restart doris failed" && exit 1; fi From 459632e0f4e324e267c76c5b2fadbacc795587f7 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Fri, 20 Jun 2025 09:26:45 +0800 Subject: [PATCH 028/572] [feat](jdbc) support custom function rules in catalog properties (#51471) (#51918) bp #51471 --- .../apache/doris/catalog/JdbcResource.java | 1 + .../org/apache/doris/catalog/JdbcTable.java | 20 + .../doris/datasource/ExternalCatalog.java | 68 +-- .../doris/datasource/ExternalDatabase.java | 72 +-- .../datasource/ExternalFunctionRules.java | 287 ++++++++++++ .../datasource/jdbc/JdbcExternalCatalog.java | 11 + .../datasource/jdbc/JdbcExternalTable.java | 1 + .../jdbc/source/JdbcFunctionPushDownRule.java | 84 ++-- .../datasource/jdbc/source/JdbcScanNode.java | 10 +- .../ExternalFunctionPushDownRulesTest.java | 431 +++++++++++++++++ .../ExternalFunctionRewriteRulesTest.java | 443 ++++++++++++++++++ ...meComparedLowercaseMetaCacheFalseTest.java | 1 - .../jdbc/test_clickhouse_jdbc_catalog.groovy | 71 ++- .../jdbc/test_jdbc_query_mysql.groovy | 134 +++++- .../jdbc/test_mysql_jdbc_catalog.groovy | 76 ++- .../jdbc/test_oracle_jdbc_catalog.groovy | 72 ++- 16 files changed, 1644 insertions(+), 138 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index bbd3e6df802049..c3c8b4d49a6736 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -109,6 +109,7 @@ public class JdbcResource extends Resource { public static final String CHECK_SUM = "checksum"; public static final String CREATE_TIME = "create_time"; public static final String TEST_CONNECTION = "test_connection"; + public static final String FUNCTION_RULES = "function_rules"; private static final ImmutableList ALL_PROPERTIES = new ImmutableList.Builder().add( JDBC_URL, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index 6dce40a2684fbc..adf013576b3962 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -22,6 +22,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.DeepCopy; import org.apache.doris.common.io.Text; +import org.apache.doris.datasource.ExternalFunctionRules; import org.apache.doris.thrift.TJdbcTable; import org.apache.doris.thrift.TOdbcTableType; import org.apache.doris.thrift.TTableDescriptor; @@ -103,6 +104,8 @@ public class JdbcTable extends Table { private int connectionPoolMaxLifeTime; private boolean connectionPoolKeepAlive; + private ExternalFunctionRules functionRules; + static { Map tempMap = new CaseInsensitiveMap(); tempMap.put("mysql", TOdbcTableType.MYSQL); @@ -128,6 +131,8 @@ public JdbcTable(long id, String name, List schema, Map throws DdlException { super(id, name, TableType.JDBC, schema); validate(properties); + // check and set external function rules + checkAndSetExternalFunctionRules(properties); } public JdbcTable(long id, String name, List schema, TableType type) { @@ -412,6 +417,12 @@ private void validate(Map properties) throws DdlException { } } + private void checkAndSetExternalFunctionRules(Map properties) throws DdlException { + ExternalFunctionRules.check(properties.getOrDefault(JdbcResource.FUNCTION_RULES, "")); + this.functionRules = ExternalFunctionRules.create(jdbcTypeName, + properties.getOrDefault(JdbcResource.FUNCTION_RULES, "")); + } + /** * Formats the provided name (for example, a database, table, or schema name) according to the specified parameters. * @@ -509,4 +520,13 @@ public static String properNameWithRemoteName(TOdbcTableType tableType, String r public static String formatNameWithRemoteName(String remoteName, String wrapStart, String wrapEnd) { return wrapStart + remoteName + wrapEnd; } + + // This is used when converting JdbcExternalTable to JdbcTable. + public void setExternalFunctionRules(ExternalFunctionRules functionRules) { + this.functionRules = functionRules; + } + + public ExternalFunctionRules getExternalFunctionRules() { + return functionRules; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index a96221057ea01c..0bf975a75d4ddc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -166,6 +166,8 @@ public abstract class ExternalCatalog private volatile Configuration cachedConf = null; private byte[] confLock = new byte[0]; + private volatile boolean isInitializing = false; + public ExternalCatalog() { } @@ -289,38 +291,46 @@ public boolean tableExistInLocal(String dbName, String tblName) { * So you have to make sure the client of third system is initialized before any method was called. */ public final synchronized void makeSureInitialized() { - initLocalObjects(); - if (!initialized) { - if (useMetaCache.get()) { - if (metaCache == null) { - metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( - name, - OptionalLong.of(86400L), - OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_meta_object_cache_num, - ignored -> getFilteredDatabaseNames(), - localDbName -> Optional.ofNullable( - buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, - true)), - (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); - } - setLastUpdateTime(System.currentTimeMillis()); - } else { - if (!Env.getCurrentEnv().isMaster()) { - // Forward to master and wait the journal to replay. - int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); - MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); - try { - remoteExecutor.forward(id, -1); - } catch (Exception e) { - Util.logAndThrowRuntimeException(LOG, - String.format("failed to forward init catalog %s operation to master.", name), e); + if (isInitializing) { + return; + } + isInitializing = true; + try { + initLocalObjects(); + if (!initialized) { + if (useMetaCache.get()) { + if (metaCache == null) { + metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( + name, + OptionalLong.of(86400L), + OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), + Config.max_meta_object_cache_num, + ignored -> getFilteredDatabaseNames(), + localDbName -> Optional.ofNullable( + buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, + true)), + (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); + } + setLastUpdateTime(System.currentTimeMillis()); + } else { + if (!Env.getCurrentEnv().isMaster()) { + // Forward to master and wait the journal to replay. + int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); + MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); + try { + remoteExecutor.forward(id, -1); + } catch (Exception e) { + Util.logAndThrowRuntimeException(LOG, + String.format("failed to forward init catalog %s operation to master.", name), e); + } + return; } - return; + init(); } - init(); + initialized = true; } - initialized = true; + } finally { + isInitializing = false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 5d5348b1f0d7ad..ece9d7e265c32f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -101,6 +101,8 @@ public abstract class ExternalDatabase private MetaCache metaCache; + private volatile boolean isInitializing = false; + /** * Create external database. * @@ -154,39 +156,49 @@ public boolean isInitialized() { } public final synchronized void makeSureInitialized() { - extCatalog.makeSureInitialized(); - if (!initialized) { - if (extCatalog.getUseMetaCache().get()) { - if (metaCache == null) { - metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( - name, - OptionalLong.of(86400L), - OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_meta_object_cache_num, - ignored -> listTableNames(), - localTableName -> Optional.ofNullable( - buildTableForInit(null, localTableName, - Util.genIdByName(extCatalog.getName(), name, localTableName), extCatalog, - this, true)), - (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); - } - setLastUpdateTime(System.currentTimeMillis()); - } else { - if (!Env.getCurrentEnv().isMaster()) { - // Forward to master and wait the journal to replay. - int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); - MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); - try { - remoteExecutor.forward(extCatalog.getId(), id); - } catch (Exception e) { - Util.logAndThrowRuntimeException(LOG, - String.format("failed to forward init external db %s operation to master", name), e); + if (isInitializing) { + return; + } + isInitializing = true; + try { + extCatalog.makeSureInitialized(); + if (!initialized) { + if (extCatalog.getUseMetaCache().get()) { + if (metaCache == null) { + metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( + name, + OptionalLong.of(86400L), + OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), + Config.max_meta_object_cache_num, + ignored -> listTableNames(), + localTableName -> Optional.ofNullable( + buildTableForInit(null, localTableName, + Util.genIdByName(extCatalog.getName(), name, localTableName), + extCatalog, + this, true)), + (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); + } + setLastUpdateTime(System.currentTimeMillis()); + } else { + if (!Env.getCurrentEnv().isMaster()) { + // Forward to master and wait the journal to replay. + int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); + MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); + try { + remoteExecutor.forward(extCatalog.getId(), id); + } catch (Exception e) { + Util.logAndThrowRuntimeException(LOG, + String.format("failed to forward init external db %s operation to master", name), + e); + } + return; } - return; + init(); } - init(); + initialized = true; } - initialized = true; + } finally { + isInitializing = false; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java new file mode 100644 index 00000000000000..c88eae9a686a75 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.jdbc.source.JdbcFunctionPushDownRule; + +import com.google.common.base.Strings; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.gson.Gson; +import lombok.Data; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * External push down rules for functions. + * This class provides a way to define which functions can be pushed down to external data sources. + * It supports both supported and unsupported functions in a JSON format. + */ +public class ExternalFunctionRules { + private static final Logger LOG = LogManager.getLogger(ExternalFunctionRules.class); + + private FunctionPushDownRule functionPushDownRule; + private FunctionRewriteRules functionRewriteRules; + + public static ExternalFunctionRules create(String datasource, String jsonRules) { + ExternalFunctionRules rules = new ExternalFunctionRules(); + rules.functionPushDownRule = FunctionPushDownRule.create(datasource, jsonRules); + rules.functionRewriteRules = FunctionRewriteRules.create(datasource, jsonRules); + return rules; + } + + public static void check(String jsonRules) throws DdlException { + if (Strings.isNullOrEmpty(jsonRules)) { + return; + } + FunctionPushDownRule.check(jsonRules); + FunctionRewriteRules.check(jsonRules); + } + + public FunctionPushDownRule getFunctionPushDownRule() { + return functionPushDownRule; + } + + public FunctionRewriteRules getFunctionRewriteRule() { + return functionRewriteRules; + } + + /** + * FunctionPushDownRule is used to determine if a function can be pushed down + */ + public static class FunctionPushDownRule { + private final Set supportedFunctions = Sets.newHashSet(); + private final Set unsupportedFunctions = Sets.newHashSet(); + + public static FunctionPushDownRule create(String datasource, String jsonRules) { + FunctionPushDownRule funcRule = new FunctionPushDownRule(); + try { + // Add default push down rules + switch (datasource.toLowerCase()) { + case "mysql": + funcRule.unsupportedFunctions.addAll(JdbcFunctionPushDownRule.MYSQL_UNSUPPORTED_FUNCTIONS); + break; + case "clickhouse": + funcRule.supportedFunctions.addAll(JdbcFunctionPushDownRule.CLICKHOUSE_SUPPORTED_FUNCTIONS); + break; + case "oracle": + funcRule.supportedFunctions.addAll(JdbcFunctionPushDownRule.ORACLE_SUPPORTED_FUNCTIONS); + break; + default: + break; + } + if (!Strings.isNullOrEmpty(jsonRules)) { + // set custom rules + Gson gson = new Gson(); + PushDownRules rules = gson.fromJson(jsonRules, PushDownRules.class); + funcRule.setCustomRules(rules); + } + return funcRule; + } catch (Exception e) { + LOG.warn("should not happen", e); + return funcRule; + } + } + + public static void check(String jsonRules) throws DdlException { + try { + Gson gson = new Gson(); + PushDownRules rules = gson.fromJson(jsonRules, PushDownRules.class); + if (rules == null) { + throw new DdlException("Push down rules cannot be null"); + } + rules.check(); + } catch (Exception e) { + throw new DdlException("Failed to parse push down rules: " + jsonRules, e); + } + } + + private void setCustomRules(PushDownRules rules) { + if (rules != null && rules.getPushdown() != null) { + if (rules.getPushdown().getSupported() != null) { + rules.getPushdown().getSupported().stream() + .map(String::toLowerCase) + .forEach(supportedFunctions::add); + } + if (rules.getPushdown().getUnsupported() != null) { + rules.getPushdown().getUnsupported().stream() + .map(String::toLowerCase) + .forEach(unsupportedFunctions::add); + } + } + } + + /** + * Checks if the function can be pushed down. + * + * @param functionName the name of the function to check + * @return true if the function can be pushed down, false otherwise + */ + public boolean canPushDown(String functionName) { + if (supportedFunctions.isEmpty() && unsupportedFunctions.isEmpty()) { + return false; + } + + // If supportedFunctions is not empty, only functions in supportedFunctions can return true + if (!supportedFunctions.isEmpty()) { + return supportedFunctions.contains(functionName.toLowerCase()); + } + + // For functions contained in unsupportedFunctions, return false + if (unsupportedFunctions.contains(functionName.toLowerCase())) { + return false; + } + + // In other cases, return true + return true; + } + } + + /** + * FunctionRewriteRule is used to rewrite function names based on provided rules. + * It allows for mapping one function name to another. + */ + public static class FunctionRewriteRules { + private final Map rewriteMap = Maps.newHashMap(); + + public static FunctionRewriteRules create(String datasource, String jsonRules) { + FunctionRewriteRules rewriteRule = new FunctionRewriteRules(); + try { + // Add default rewrite rules + switch (datasource.toLowerCase()) { + case "mysql": + rewriteRule.rewriteMap.putAll(JdbcFunctionPushDownRule.REPLACE_MYSQL_FUNCTIONS); + break; + case "clickhouse": + rewriteRule.rewriteMap.putAll(JdbcFunctionPushDownRule.REPLACE_CLICKHOUSE_FUNCTIONS); + break; + case "oracle": + rewriteRule.rewriteMap.putAll(JdbcFunctionPushDownRule.REPLACE_ORACLE_FUNCTIONS); + break; + default: + break; + } + if (!Strings.isNullOrEmpty(jsonRules)) { + // set custom rules + Gson gson = new Gson(); + RewriteRules rules = gson.fromJson(jsonRules, RewriteRules.class); + rewriteRule.setCustomRules(rules); + } + return rewriteRule; + } catch (Exception e) { + LOG.warn("should not happen", e); + return rewriteRule; + } + } + + private void setCustomRules(RewriteRules rules) { + if (rules != null && rules.getRewrite() != null) { + this.rewriteMap.putAll(rules.getRewrite()); + } + } + + public String rewriteFunction(String origFuncName) { + return rewriteMap.getOrDefault(origFuncName, origFuncName); + } + + public static void check(String jsonRules) throws DdlException { + try { + Gson gson = new Gson(); + RewriteRules rules = gson.fromJson(jsonRules, RewriteRules.class); + if (rules == null) { + throw new DdlException("Rewrite rules cannot be null"); + } + rules.check(); + } catch (Exception e) { + throw new DdlException("Failed to parse rewrite rules: " + jsonRules, e); + } + } + } + + /** + * push down rules in json format. + * eg: + * { + * "pushdown": { + * "supported": ["function1", "function2"], + * "unsupported": ["function3", "function4"] + * } + * } + */ + @Data + public static class PushDownRules { + private PushDown pushdown; + + @Data + public static class PushDown { + private List supported; + private List unsupported; + } + + public void check() { + if (pushdown != null) { + if (pushdown.getSupported() != null) { + for (String func : pushdown.getSupported()) { + if (Strings.isNullOrEmpty(func)) { + throw new IllegalArgumentException("Supported function name cannot be empty"); + } + } + } + if (pushdown.getUnsupported() != null) { + for (String func : pushdown.getUnsupported()) { + if (Strings.isNullOrEmpty(func)) { + throw new IllegalArgumentException("Unsupported function name cannot be empty"); + } + } + } + } + } + } + + /** + * push down rules in json format. + * eg: + * { + * "rewrite": { + * "func1": "func2", + * "func3": "func4" + * } + * } + */ + @Data + public static class RewriteRules { + private Map rewrite; + + public void check() { + if (rewrite != null) { + for (Map.Entry entry : rewrite.entrySet()) { + String origFunc = entry.getKey(); + String newFunc = entry.getValue(); + if (Strings.isNullOrEmpty(origFunc) || Strings.isNullOrEmpty(newFunc)) { + throw new IllegalArgumentException("Function names in rewrite rules cannot be empty"); + } + } + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 5094207490ce5e..4899cb2ec1cef1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalFunctionRules; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; @@ -77,6 +78,7 @@ public class JdbcExternalCatalog extends ExternalCatalog { // or Gson will throw exception with HikariCP private transient JdbcClient jdbcClient; private IdentifierMapping identifierMapping; + private ExternalFunctionRules functionRules; public JdbcExternalCatalog(long catalogId, String name, String resource, Map props, String comment) @@ -107,6 +109,9 @@ public void checkProperties() throws DdlException { getExcludeDatabaseMap()); JdbcResource.checkConnectionPoolProperties(getConnectionPoolMinSize(), getConnectionPoolMaxSize(), getConnectionPoolMaxWaitTime(), getConnectionPoolMaxLifeTime()); + + // check function rules + ExternalFunctionRules.check(catalogProperty.getProperties().getOrDefault(JdbcResource.FUNCTION_RULES, "")); } @Override @@ -223,6 +228,8 @@ public boolean isTestConnection() { @Override protected void initLocalObjectsImpl() { jdbcClient = createJdbcClient(); + this.functionRules = ExternalFunctionRules.create(jdbcClient.getDbType(), + catalogProperty.getOrDefault(JdbcResource.FUNCTION_RULES, "")); } private JdbcClient createJdbcClient() { @@ -437,4 +444,8 @@ private JdbcTable getTestConnectionJdbcTable(JdbcClient testClient) throws DdlEx return testTable; } + + public ExternalFunctionRules getFunctionRules() { + return functionRules; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index b3ff728bb7d5ab..2bcd2277251bea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -187,6 +187,7 @@ private JdbcTable toJdbcTable() { remoteColumnNames.put(column.getName(), remoteColumnName); } jdbcTable.setRemoteColumnNames(remoteColumnNames); + jdbcTable.setExternalFunctionRules(jdbcCatalog.getFunctionRules()); return jdbcTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java index a765681c402b84..48aac798cf088a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java @@ -23,6 +23,8 @@ import org.apache.doris.analysis.TimestampArithmeticExpr; import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.Config; +import org.apache.doris.datasource.ExternalFunctionRules; +import org.apache.doris.datasource.ExternalFunctionRules.FunctionRewriteRules; import org.apache.doris.thrift.TOdbcTableType; import com.google.common.base.Preconditions; @@ -34,11 +36,10 @@ import java.util.List; import java.util.Map; import java.util.TreeSet; -import java.util.function.Predicate; public class JdbcFunctionPushDownRule { private static final Logger LOG = LogManager.getLogger(JdbcFunctionPushDownRule.class); - private static final TreeSet MYSQL_UNSUPPORTED_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + public static final TreeSet MYSQL_UNSUPPORTED_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); static { MYSQL_UNSUPPORTED_FUNCTIONS.add("date_trunc"); @@ -47,14 +48,14 @@ public class JdbcFunctionPushDownRule { MYSQL_UNSUPPORTED_FUNCTIONS.addAll(Arrays.asList(Config.jdbc_mysql_unsupported_pushdown_functions)); } - private static final TreeSet CLICKHOUSE_SUPPORTED_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + public static final TreeSet CLICKHOUSE_SUPPORTED_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); static { CLICKHOUSE_SUPPORTED_FUNCTIONS.add("from_unixtime"); CLICKHOUSE_SUPPORTED_FUNCTIONS.add("unix_timestamp"); } - private static final TreeSet ORACLE_SUPPORTED_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + public static final TreeSet ORACLE_SUPPORTED_FUNCTIONS = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); static { ORACLE_SUPPORTED_FUNCTIONS.add("nvl"); @@ -73,21 +74,21 @@ private static boolean isOracleFunctionUnsupported(String functionName) { return !ORACLE_SUPPORTED_FUNCTIONS.contains(functionName.toLowerCase()); } - private static final Map REPLACE_MYSQL_FUNCTIONS = Maps.newHashMap(); + public static final Map REPLACE_MYSQL_FUNCTIONS = Maps.newHashMap(); static { REPLACE_MYSQL_FUNCTIONS.put("nvl", "ifnull"); REPLACE_MYSQL_FUNCTIONS.put("to_date", "date"); } - private static final Map REPLACE_CLICKHOUSE_FUNCTIONS = Maps.newHashMap(); + public static final Map REPLACE_CLICKHOUSE_FUNCTIONS = Maps.newHashMap(); static { REPLACE_CLICKHOUSE_FUNCTIONS.put("from_unixtime", "FROM_UNIXTIME"); REPLACE_CLICKHOUSE_FUNCTIONS.put("unix_timestamp", "toUnixTimestamp"); } - private static final Map REPLACE_ORACLE_FUNCTIONS = Maps.newHashMap(); + public static final Map REPLACE_ORACLE_FUNCTIONS = Maps.newHashMap(); static { REPLACE_ORACLE_FUNCTIONS.put("ifnull", "nvl"); @@ -105,77 +106,54 @@ private static boolean isReplaceOracleFunctions(String functionName) { return REPLACE_ORACLE_FUNCTIONS.containsKey(functionName.toLowerCase()); } - public static Expr processFunctions(TOdbcTableType tableType, Expr expr, List errors) { - if (tableType == null || expr == null) { + public static Expr processFunctions(TOdbcTableType tableType, Expr expr, List errors, + ExternalFunctionRules functionRules) { + if (tableType == null || expr == null || functionRules == null) { return expr; } - Predicate checkFunction; - Predicate replaceFunction; - - if (TOdbcTableType.MYSQL.equals(tableType)) { - replaceFunction = JdbcFunctionPushDownRule::isReplaceMysqlFunctions; - checkFunction = JdbcFunctionPushDownRule::isMySQLFunctionUnsupported; - } else if (TOdbcTableType.CLICKHOUSE.equals(tableType)) { - replaceFunction = JdbcFunctionPushDownRule::isReplaceClickHouseFunctions; - checkFunction = JdbcFunctionPushDownRule::isClickHouseFunctionUnsupported; - } else if (TOdbcTableType.ORACLE.equals(tableType)) { - replaceFunction = JdbcFunctionPushDownRule::isReplaceOracleFunctions; - checkFunction = JdbcFunctionPushDownRule::isOracleFunctionUnsupported; - } else { - return expr; - } - - return processFunctionsRecursively(expr, checkFunction, replaceFunction, errors, tableType); + return processFunctionsRecursively(expr, functionRules, errors, tableType); } - private static Expr processFunctionsRecursively(Expr expr, Predicate checkFunction, - Predicate replaceFunction, List errors, TOdbcTableType tableType) { + private static Expr processFunctionsRecursively(Expr expr, ExternalFunctionRules functionRules, List errors, + TOdbcTableType tableType) { if (expr instanceof FunctionCallExpr) { FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; String func = functionCallExpr.getFnName().getFunction(); Preconditions.checkArgument(!func.isEmpty(), "function can not be empty"); - if (checkFunction.test(func)) { - String errMsg = "Unsupported function: " + func + " in expr: " + expr.toExternalSql( - TableType.JDBC_EXTERNAL_TABLE, null) - + " in JDBC Table Type: " + tableType; - LOG.warn(errMsg); - errors.add(errMsg); + // 1. check can push down + if (!functionRules.getFunctionPushDownRule().canPushDown(func)) { + if (LOG.isDebugEnabled()) { + String errMsg = "Unsupported function: " + func + " in expr: " + expr.toExternalSql( + TableType.JDBC_EXTERNAL_TABLE, null) + + " in JDBC Table Type: " + tableType; + LOG.debug(errMsg); + } + errors.add("has error"); } - replaceFunctionNameIfNecessary(func, replaceFunction, functionCallExpr, tableType); - + // 2. replace function + replaceFunctionNameIfNecessary(func, functionRules.getFunctionRewriteRule(), functionCallExpr); expr = replaceGenericFunctionExpr(functionCallExpr, func); } List children = expr.getChildren(); for (int i = 0; i < children.size(); i++) { Expr child = children.get(i); - Expr newChild = processFunctionsRecursively(child, checkFunction, replaceFunction, errors, tableType); + Expr newChild = processFunctionsRecursively(child, functionRules, errors, tableType); expr.setChild(i, newChild); } return expr; } - private static void replaceFunctionNameIfNecessary(String func, Predicate replaceFunction, - FunctionCallExpr functionCallExpr, TOdbcTableType tableType) { - if (replaceFunction.test(func)) { - String newFunc; - if (TOdbcTableType.MYSQL.equals(tableType)) { - newFunc = REPLACE_MYSQL_FUNCTIONS.get(func.toLowerCase()); - } else if (TOdbcTableType.CLICKHOUSE.equals(tableType)) { - newFunc = REPLACE_CLICKHOUSE_FUNCTIONS.get(func); - } else if (TOdbcTableType.ORACLE.equals(tableType)) { - newFunc = REPLACE_ORACLE_FUNCTIONS.get(func); - } else { - newFunc = null; - } - if (newFunc != null) { - functionCallExpr.setFnName(FunctionName.createBuiltinName(newFunc)); - } + private static void replaceFunctionNameIfNecessary(String func, FunctionRewriteRules rewriteRule, + FunctionCallExpr functionCallExpr) { + String newFuncName = rewriteRule.rewriteFunction(func); + if (!newFuncName.equals(func)) { + functionCallExpr.setFnName(FunctionName.createBuiltinName(newFuncName)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java index 31026e6b8770da..019cceca6a7229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java @@ -39,6 +39,7 @@ import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.ExternalFunctionRules; import org.apache.doris.datasource.ExternalScanNode; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.planner.PlanNodeId; @@ -131,7 +132,8 @@ private void createJdbcFilters() { ArrayList conjunctsList = Expr.cloneList(conjuncts, sMap); List errors = Lists.newArrayList(); - List pushDownConjuncts = collectConjunctsToPushDown(conjunctsList, errors); + List pushDownConjuncts = collectConjunctsToPushDown(conjunctsList, errors, + tbl.getExternalFunctionRules()); for (Expr individualConjunct : pushDownConjuncts) { String filter = conjunctExprToString(jdbcType, individualConjunct, tbl); @@ -140,13 +142,15 @@ private void createJdbcFilters() { } } - private List collectConjunctsToPushDown(List conjunctsList, List errors) { + private List collectConjunctsToPushDown(List conjunctsList, List errors, + ExternalFunctionRules functionRules) { List pushDownConjuncts = new ArrayList<>(); for (Expr p : conjunctsList) { if (shouldPushDownConjunct(jdbcType, p)) { List individualConjuncts = p.getConjuncts(); for (Expr individualConjunct : individualConjuncts) { - Expr newp = JdbcFunctionPushDownRule.processFunctions(jdbcType, individualConjunct, errors); + Expr newp = JdbcFunctionPushDownRule.processFunctions(jdbcType, individualConjunct, errors, + functionRules); if (!errors.isEmpty()) { errors.clear(); continue; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java new file mode 100644 index 00000000000000..5f8591ed1d43b3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java @@ -0,0 +1,431 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.ExternalFunctionRules.FunctionPushDownRule; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ExternalFunctionPushDownRulesTest { + + @Test + public void testFunctionPushDownRuleCreateWithMysqlDataSource() { + // Test MySQL datasource with default rules + FunctionPushDownRule rule = FunctionPushDownRule.create("mysql", null); + + // MySQL has unsupported functions by default, supported functions is empty + Assertions.assertFalse(rule.canPushDown("date_trunc")); + Assertions.assertFalse(rule.canPushDown("money_format")); + Assertions.assertFalse(rule.canPushDown("negative")); + + // Test case insensitivity + Assertions.assertFalse(rule.canPushDown("DATE_TRUNC")); + Assertions.assertFalse(rule.canPushDown("Money_Format")); + + // Functions not in unsupported list should be allowed (since supportedFunctions is empty) + Assertions.assertTrue(rule.canPushDown("sum")); + Assertions.assertTrue(rule.canPushDown("count")); + } + + @Test + public void testFunctionPushDownRuleCreateWithClickHouseDataSource() { + // Test ClickHouse datasource with default rules + FunctionPushDownRule rule = FunctionPushDownRule.create("clickhouse", null); + + // ClickHouse has supported functions by default, so only supported functions return true + Assertions.assertTrue(rule.canPushDown("from_unixtime")); + Assertions.assertTrue(rule.canPushDown("unix_timestamp")); + + // Test case insensitivity + Assertions.assertTrue(rule.canPushDown("FROM_UNIXTIME")); + Assertions.assertTrue(rule.canPushDown("Unix_Timestamp")); + + // Functions not in supported list should be denied (since supportedFunctions is not empty) + Assertions.assertFalse(rule.canPushDown("unknown_function")); + Assertions.assertFalse(rule.canPushDown("custom_func")); + Assertions.assertFalse(rule.canPushDown("sum")); + Assertions.assertFalse(rule.canPushDown("count")); + } + + @Test + public void testFunctionPushDownRuleCreateWithOracleDataSource() { + // Test Oracle datasource with default rules + FunctionPushDownRule rule = FunctionPushDownRule.create("oracle", null); + + // Oracle has supported functions by default, so only supported functions return true + Assertions.assertTrue(rule.canPushDown("nvl")); + Assertions.assertTrue(rule.canPushDown("ifnull")); + + // Test case insensitivity + Assertions.assertTrue(rule.canPushDown("NVL")); + Assertions.assertTrue(rule.canPushDown("IfNull")); + + // Functions not in supported list should be denied (since supportedFunctions is not empty) + Assertions.assertFalse(rule.canPushDown("unknown_function")); + Assertions.assertFalse(rule.canPushDown("custom_func")); + Assertions.assertFalse(rule.canPushDown("sum")); + Assertions.assertFalse(rule.canPushDown("count")); + } + + @Test + public void testFunctionPushDownRuleCreateWithUnknownDataSource() { + // Test unknown datasource should have no default rules + FunctionPushDownRule rule = FunctionPushDownRule.create("unknown", null); + + // With no rules, all functions should be denied + Assertions.assertFalse(rule.canPushDown("any_function")); + Assertions.assertFalse(rule.canPushDown("sum")); + Assertions.assertFalse(rule.canPushDown("count")); + } + + @Test + public void testFunctionPushDownRuleCreateWithValidCustomRules() { + // Test custom rules with supported functions + String jsonRules = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"custom_func1\", \"Custom_Func2\"],\n" + + " \"unsupported\": [\"blocked_func1\", \"Blocked_Func2\"]\n" + + " }\n" + + "}"; + + FunctionPushDownRule rule = FunctionPushDownRule.create("mysql", jsonRules); + + // Since supportedFunctions is not empty, only supported functions return true + Assertions.assertTrue(rule.canPushDown("custom_func1")); + Assertions.assertTrue(rule.canPushDown("custom_func2")); // case insensitive + Assertions.assertTrue(rule.canPushDown("CUSTOM_FUNC1")); + + // Functions not in supported list should be denied (even if not in unsupported list) + Assertions.assertFalse(rule.canPushDown("other_function")); + Assertions.assertFalse(rule.canPushDown("sum")); + Assertions.assertFalse(rule.canPushDown("count")); + + // Functions in unsupported list should still be denied + Assertions.assertFalse(rule.canPushDown("blocked_func1")); + Assertions.assertFalse(rule.canPushDown("blocked_func2")); // case insensitive + Assertions.assertFalse(rule.canPushDown("BLOCKED_FUNC1")); + + // Default MySQL unsupported functions should be denied + Assertions.assertFalse(rule.canPushDown("date_trunc")); + } + + @Test + public void testFunctionPushDownRuleCreateWithOnlySupportedCustomRules() { + // Test custom rules with only supported functions + String jsonRules = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"allowed_func1\", \"allowed_func2\"]\n" + + " }\n" + + "}"; + + FunctionPushDownRule rule = FunctionPushDownRule.create("unknown", jsonRules); + + // Since supportedFunctions is not empty, only supported functions return true + Assertions.assertTrue(rule.canPushDown("allowed_func1")); + Assertions.assertTrue(rule.canPushDown("allowed_func2")); + + // Other functions should be denied (since supportedFunctions is not empty) + Assertions.assertFalse(rule.canPushDown("other_function")); + Assertions.assertFalse(rule.canPushDown("sum")); + Assertions.assertFalse(rule.canPushDown("count")); + } + + @Test + public void testFunctionPushDownRuleCreateWithOnlyUnsupportedCustomRules() { + // Test custom rules with only unsupported functions + String jsonRules = "{\n" + + " \"pushdown\": {\n" + + " \"unsupported\": [\"blocked_func1\", \"blocked_func2\"]\n" + + " }\n" + + "}"; + + FunctionPushDownRule rule = FunctionPushDownRule.create("unknown", jsonRules); + + // Custom unsupported functions should be denied + Assertions.assertFalse(rule.canPushDown("blocked_func1")); + Assertions.assertFalse(rule.canPushDown("blocked_func2")); + + // Other functions should be allowed (default behavior) + Assertions.assertTrue(rule.canPushDown("other_function")); + } + + @Test + public void testFunctionPushDownRuleCreateWithEmptyCustomRules() { + // Test empty custom rules + String jsonRules = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [],\n" + + " \"unsupported\": []\n" + + " }\n" + + "}"; + + FunctionPushDownRule rule = FunctionPushDownRule.create("unknown", jsonRules); + + // With empty rules, all functions should be denied + Assertions.assertFalse(rule.canPushDown("any_function")); + Assertions.assertFalse(rule.canPushDown("sum")); + } + + @Test + public void testFunctionPushDownRuleCreateWithNullCustomRules() { + // Test null pushdown section + String jsonRules = "{\n" + + " \"pushdown\": null\n" + + "}"; + + FunctionPushDownRule rule = FunctionPushDownRule.create("unknown", jsonRules); + + // With null rules, all functions should be denied + Assertions.assertFalse(rule.canPushDown("any_function")); + } + + @Test + public void testFunctionPushDownRuleCreateWithInvalidJson() { + // Test invalid JSON should not throw exception but return default rule + String invalidJson = "{ invalid json }"; + + FunctionPushDownRule rule = FunctionPushDownRule.create("unknown", invalidJson); + + // Should return a rule with no functions configured + Assertions.assertFalse(rule.canPushDown("any_function")); + } + + @Test + public void testFunctionPushDownRuleCreateWithEmptyJsonRules() { + // Test empty string rules + FunctionPushDownRule rule = FunctionPushDownRule.create("mysql", ""); + + // Should only have default MySQL rules + Assertions.assertFalse(rule.canPushDown("date_trunc")); + Assertions.assertTrue(rule.canPushDown("other_function")); + } + + @Test + public void testFunctionPushDownRuleCreateCaseInsensitiveDataSource() { + // Test case insensitivity for datasource names + FunctionPushDownRule mysqlRule = FunctionPushDownRule.create("MYSQL", null); + FunctionPushDownRule clickhouseRule = FunctionPushDownRule.create("ClickHouse", null); + FunctionPushDownRule oracleRule = FunctionPushDownRule.create("Oracle", null); + + // Should apply correct default rules regardless of case + Assertions.assertFalse(mysqlRule.canPushDown("date_trunc")); + Assertions.assertTrue(clickhouseRule.canPushDown("from_unixtime")); + Assertions.assertTrue(oracleRule.canPushDown("nvl")); + } + + @Test + public void testFunctionPushDownRuleCanPushDownLogic() { + // Test the canPushDown logic with different scenarios + + // 1. Both supported and unsupported are empty -> return false + FunctionPushDownRule emptyRule = FunctionPushDownRule.create("unknown", null); + Assertions.assertFalse(emptyRule.canPushDown("any_function")); + + // 2. Function in supported list -> return true (only when supportedFunctions is not empty) + String supportedJson = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"func1\"]\n" + + " }\n" + + "}"; + FunctionPushDownRule supportedRule = FunctionPushDownRule.create("unknown", supportedJson); + Assertions.assertTrue(supportedRule.canPushDown("func1")); + + // 3. Function not in supported list when supportedFunctions is not empty -> return false + Assertions.assertFalse(supportedRule.canPushDown("other_func")); + + // 4. Function in unsupported list -> return false + String unsupportedJson = "{\n" + + " \"pushdown\": {\n" + + " \"unsupported\": [\"func1\"]\n" + + " }\n" + + "}"; + FunctionPushDownRule unsupportedRule = FunctionPushDownRule.create("unknown", unsupportedJson); + Assertions.assertFalse(unsupportedRule.canPushDown("func1")); + + // 5. Function not in unsupported list when supportedFunctions is empty -> return true + Assertions.assertTrue(unsupportedRule.canPushDown("other_func")); + + // 6. Priority test: when supportedFunctions is not empty, only supported functions return true + String bothJson = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"func1\"],\n" + + " \"unsupported\": [\"func2\"]\n" + + " }\n" + + "}"; + FunctionPushDownRule bothRule = FunctionPushDownRule.create("unknown", bothJson); + Assertions.assertTrue(bothRule.canPushDown("func1")); // in supported list + Assertions.assertFalse(bothRule.canPushDown("func2")); // not in supported list (even though in unsupported) + Assertions.assertFalse(bothRule.canPushDown("func3")); // not in supported list + } + + @Test + public void testFunctionPushDownRuleCheck() throws DdlException { + // Test valid JSON rules + String validJson = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"func1\", \"func2\"],\n" + + " \"unsupported\": [\"func3\", \"func4\"]\n" + + " }\n" + + "}"; + + // Should not throw exception + Assertions.assertDoesNotThrow(() -> { + FunctionPushDownRule.check(validJson); + }); + } + + @Test + public void testFunctionPushDownRuleCheckWithInvalidJson() { + // Test invalid JSON rules + String invalidJson = "{ invalid json }"; + + // Should throw DdlException + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionPushDownRule.check(invalidJson); + }); + + Assertions.assertTrue(exception.getMessage().contains("Failed to parse push down rules")); + } + + @Test + public void testFunctionPushDownRuleCheckWithEmptyJson() throws DdlException { + // Test empty JSON + String emptyJson = "{}"; + + // Should not throw exception + Assertions.assertDoesNotThrow(() -> { + FunctionPushDownRule.check(emptyJson); + }); + } + + @Test + public void testFunctionPushDownRuleCheckWithNullJson() throws DdlException { + // Test null JSON + String nullJson = null; + + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionPushDownRule.check(nullJson); + }); + Assertions.assertTrue(exception.getMessage().contains("Failed to parse push down rules")); + } + + @Test + public void testFunctionPushDownRuleCheckWithMalformedJson() { + // Test various malformed JSON strings + String[] malformedJsons = { + "{ \"pushdown\": }", // Missing value + "{ \"pushdown\": { \"supported\": } }", // Missing array + "{ \"pushdown\" { \"supported\": [] } }", // Missing colon + "{ \"pushdown\": { \"supported\": [\"func1\",] } }", // Trailing comma + "{ \"pushdown\": { \"supported\": [\"func1\" \"func2\"] } }" // Missing comma + }; + + for (String malformedJson : malformedJsons) { + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionPushDownRule.check(malformedJson); + }); + + Assertions.assertTrue(exception.getMessage().contains("Failed to parse push down rules")); + } + } + + @Test + public void testFunctionPushDownRuleWithComplexCustomRules() { + // Test complex custom rules that override and extend default rules + String complexJson = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"date_trunc\", \"custom_func\"],\n" + + " \"unsupported\": [\"from_unixtime\", \"another_func\"]\n" + + " }\n" + + "}"; + + // Test with MySQL (has default unsupported functions) + FunctionPushDownRule mysqlRule = FunctionPushDownRule.create("mysql", complexJson); + + // Since supportedFunctions is not empty, only supported functions return true + Assertions.assertTrue(mysqlRule.canPushDown("date_trunc")); // in supported list + Assertions.assertTrue(mysqlRule.canPushDown("custom_func")); // in supported list + + // Functions not in supported list should be denied + Assertions.assertFalse(mysqlRule.canPushDown("from_unixtime")); // not in supported list + Assertions.assertFalse(mysqlRule.canPushDown("another_func")); // not in supported list + Assertions.assertFalse(mysqlRule.canPushDown("money_format")); // not in supported list + Assertions.assertFalse(mysqlRule.canPushDown("sum")); // not in supported list + Assertions.assertFalse(mysqlRule.canPushDown("count")); // not in supported list + } + + @Test + public void testFunctionPushDownRuleNewLogicCases() { + // Additional test cases for the new logic + + // Test case 1: Only unsupported functions defined (supportedFunctions is empty) + String onlyUnsupportedJson = "{\n" + + " \"pushdown\": {\n" + + " \"unsupported\": [\"blocked_func\"]\n" + + " }\n" + + "}"; + FunctionPushDownRule onlyUnsupportedRule = FunctionPushDownRule.create("unknown", onlyUnsupportedJson); + + // Functions in unsupported list should be denied + Assertions.assertFalse(onlyUnsupportedRule.canPushDown("blocked_func")); + + // Other functions should be allowed (since supportedFunctions is empty) + Assertions.assertTrue(onlyUnsupportedRule.canPushDown("allowed_func")); + Assertions.assertTrue(onlyUnsupportedRule.canPushDown("sum")); + + // Test case 2: Both supported and unsupported functions defined + String bothListsJson = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"func1\", \"func2\"],\n" + + " \"unsupported\": [\"func3\", \"func4\"]\n" + + " }\n" + + "}"; + FunctionPushDownRule bothListsRule = FunctionPushDownRule.create("unknown", bothListsJson); + + // Only supported functions return true + Assertions.assertTrue(bothListsRule.canPushDown("func1")); + Assertions.assertTrue(bothListsRule.canPushDown("func2")); + + // Functions in unsupported list should be denied (not in supported list) + Assertions.assertFalse(bothListsRule.canPushDown("func3")); + Assertions.assertFalse(bothListsRule.canPushDown("func4")); + + // Other functions should be denied (not in supported list) + Assertions.assertFalse(bothListsRule.canPushDown("func5")); + Assertions.assertFalse(bothListsRule.canPushDown("other_func")); + + // Test case 3: MySQL with custom supported functions + String mysqlSupportedJson = "{\n" + + " \"pushdown\": {\n" + + " \"supported\": [\"date_trunc\", \"money_format\"]\n" + + " }\n" + + "}"; + FunctionPushDownRule mysqlSupportedRule = FunctionPushDownRule.create("mysql", mysqlSupportedJson); + + // Only supported functions return true (overrides default MySQL unsupported functions) + Assertions.assertTrue(mysqlSupportedRule.canPushDown("date_trunc")); + Assertions.assertTrue(mysqlSupportedRule.canPushDown("money_format")); + + // Other functions should be denied + Assertions.assertFalse(mysqlSupportedRule.canPushDown("negative")); + Assertions.assertFalse(mysqlSupportedRule.canPushDown("sum")); + Assertions.assertFalse(mysqlSupportedRule.canPushDown("count")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java new file mode 100644 index 00000000000000..3bca159575ab2b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java @@ -0,0 +1,443 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.ExternalFunctionRules.FunctionRewriteRules; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class ExternalFunctionRewriteRulesTest { + + @Test + public void testFunctionRewriteRuleCreateWithMysqlDataSource() { + // Test MySQL datasource with default rewrite rules + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", null); + + // MySQL has default rewrite rules + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + + // Test case sensitivity - original function names should be used as-is + Assertions.assertEquals("NVL", rule.rewriteFunction("NVL")); + Assertions.assertEquals("To_Date", rule.rewriteFunction("To_Date")); + + // Functions not in rewrite map should return original name + Assertions.assertEquals("sum", rule.rewriteFunction("sum")); + Assertions.assertEquals("count", rule.rewriteFunction("count")); + Assertions.assertEquals("unknown_func", rule.rewriteFunction("unknown_func")); + } + + @Test + public void testFunctionRewriteRuleCreateWithClickHouseDataSource() { + // Test ClickHouse datasource with default rewrite rules + FunctionRewriteRules rule = FunctionRewriteRules.create("clickhouse", null); + + // ClickHouse has default rewrite rules + Assertions.assertEquals("FROM_UNIXTIME", rule.rewriteFunction("from_unixtime")); + Assertions.assertEquals("toUnixTimestamp", rule.rewriteFunction("unix_timestamp")); + + // Test case sensitivity + Assertions.assertEquals("FROM_UNIXTIME", rule.rewriteFunction("FROM_UNIXTIME")); + Assertions.assertEquals("Unix_Timestamp", rule.rewriteFunction("Unix_Timestamp")); + + // Functions not in rewrite map should return original name + Assertions.assertEquals("sum", rule.rewriteFunction("sum")); + Assertions.assertEquals("count", rule.rewriteFunction("count")); + Assertions.assertEquals("unknown_func", rule.rewriteFunction("unknown_func")); + } + + @Test + public void testFunctionRewriteRuleCreateWithOracleDataSource() { + // Test Oracle datasource with default rewrite rules + FunctionRewriteRules rule = FunctionRewriteRules.create("oracle", null); + + // Oracle has default rewrite rules + Assertions.assertEquals("nvl", rule.rewriteFunction("ifnull")); + + // Test case sensitivity + Assertions.assertEquals("IFNULL", rule.rewriteFunction("IFNULL")); + Assertions.assertEquals("IfNull", rule.rewriteFunction("IfNull")); + + // Functions not in rewrite map should return original name + Assertions.assertEquals("sum", rule.rewriteFunction("sum")); + Assertions.assertEquals("count", rule.rewriteFunction("count")); + Assertions.assertEquals("unknown_func", rule.rewriteFunction("unknown_func")); + } + + @Test + public void testFunctionRewriteRuleCreateWithUnknownDataSource() { + // Test unknown datasource should have no default rewrite rules + FunctionRewriteRules rule = FunctionRewriteRules.create("unknown", null); + + // All functions should return original name (no rewrite rules) + Assertions.assertEquals("any_function", rule.rewriteFunction("any_function")); + Assertions.assertEquals("sum", rule.rewriteFunction("sum")); + Assertions.assertEquals("count", rule.rewriteFunction("count")); + Assertions.assertEquals("nvl", rule.rewriteFunction("nvl")); + Assertions.assertEquals("ifnull", rule.rewriteFunction("ifnull")); + } + + @Test + public void testFunctionRewriteRuleCreateWithValidCustomRules() { + // Test custom rewrite rules + String jsonRules = "{\n" + + " \"rewrite\": {\n" + + " \"old_func1\": \"new_func1\",\n" + + " \"Old_Func2\": \"New_Func2\",\n" + + " \"CUSTOM_FUNC\": \"custom_replacement\"\n" + + " }\n" + + "}"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", jsonRules); + + // Custom rewrite rules should work + Assertions.assertEquals("new_func1", rule.rewriteFunction("old_func1")); + Assertions.assertEquals("New_Func2", rule.rewriteFunction("Old_Func2")); + Assertions.assertEquals("custom_replacement", rule.rewriteFunction("CUSTOM_FUNC")); + + // Default MySQL rewrite rules should still work + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + + // Functions not in any rewrite map should return original name + Assertions.assertEquals("other_function", rule.rewriteFunction("other_function")); + Assertions.assertEquals("sum", rule.rewriteFunction("sum")); + } + + @Test + public void testFunctionRewriteRuleCreateWithEmptyCustomRules() { + // Test empty custom rewrite rules + String jsonRules = "{\n" + + " \"rewrite\": {}\n" + + "}"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", jsonRules); + + // Default MySQL rewrite rules should still work + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + + // Other functions should return original name + Assertions.assertEquals("other_function", rule.rewriteFunction("other_function")); + } + + @Test + public void testFunctionRewriteRuleCreateWithNullCustomRules() { + // Test null rewrite section + String jsonRules = "{\n" + + " \"rewrite\": null\n" + + "}"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", jsonRules); + + // Default MySQL rewrite rules should still work + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + + // Other functions should return original name + Assertions.assertEquals("other_function", rule.rewriteFunction("other_function")); + } + + @Test + public void testFunctionRewriteRuleCreateWithNullRewriteMap() { + // Test null rewrite map - this test is no longer relevant with the new format + // Since rewrite is now directly the map, we just test with null rewrite + String jsonRules = "{\n" + + " \"rewrite\": null\n" + + "}"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", jsonRules); + + // Default MySQL rewrite rules should still work + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + + // Other functions should return original name + Assertions.assertEquals("other_function", rule.rewriteFunction("other_function")); + } + + @Test + public void testFunctionRewriteRuleCreateWithInvalidJson() { + // Test invalid JSON should not throw exception but return default rule + String invalidJson = "{ invalid json }"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", invalidJson); + + // Should still have default MySQL rewrite rules + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + + // Other functions should return original name + Assertions.assertEquals("other_function", rule.rewriteFunction("other_function")); + } + + @Test + public void testFunctionRewriteRuleCreateWithEmptyJsonRules() { + // Test empty string rules + FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", ""); + + // Should only have default MySQL rewrite rules + Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl")); + Assertions.assertEquals("date", rule.rewriteFunction("to_date")); + Assertions.assertEquals("other_function", rule.rewriteFunction("other_function")); + } + + @Test + public void testFunctionRewriteRuleCreateCaseInsensitiveDataSource() { + // Test case insensitivity for datasource names + FunctionRewriteRules mysqlRule = FunctionRewriteRules.create("MYSQL", null); + FunctionRewriteRules clickhouseRule = FunctionRewriteRules.create("ClickHouse", null); + FunctionRewriteRules oracleRule = FunctionRewriteRules.create("Oracle", null); + + // Should apply correct default rules regardless of case + Assertions.assertEquals("ifnull", mysqlRule.rewriteFunction("nvl")); + Assertions.assertEquals("FROM_UNIXTIME", clickhouseRule.rewriteFunction("from_unixtime")); + Assertions.assertEquals("nvl", oracleRule.rewriteFunction("ifnull")); + } + + @Test + public void testFunctionRewriteRuleRewriteFunction() { + // Test the rewriteFunction logic with different scenarios + + // Test with custom rewrite rules + String jsonRules = "{\n" + + " \"rewrite\": {\n" + + " \"func1\": \"replacement1\",\n" + + " \"func2\": \"replacement2\"\n" + + " }\n" + + "}"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("unknown", jsonRules); + + // Functions in rewrite map should be replaced + Assertions.assertEquals("replacement1", rule.rewriteFunction("func1")); + Assertions.assertEquals("replacement2", rule.rewriteFunction("func2")); + + // Functions not in rewrite map should return original name + Assertions.assertEquals("func3", rule.rewriteFunction("func3")); + Assertions.assertEquals("unknown_func", rule.rewriteFunction("unknown_func")); + + // Test with null function name + Assertions.assertEquals(null, rule.rewriteFunction(null)); + + // Test with empty function name + Assertions.assertEquals("", rule.rewriteFunction("")); + } + + @Test + public void testFunctionRewriteRuleCheck() throws DdlException { + // Test valid JSON rewrite rules + String validJson = "{\n" + + " \"rewrite\": {\n" + + " \"func1\": \"replacement1\",\n" + + " \"func2\": \"replacement2\"\n" + + " }\n" + + "}"; + + // Should not throw exception + Assertions.assertDoesNotThrow(() -> { + FunctionRewriteRules.check(validJson); + }); + } + + @Test + public void testFunctionRewriteRuleCheckWithInvalidJson() { + // Test invalid JSON rules + String invalidJson = "{ invalid json }"; + + // Should throw DdlException + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionRewriteRules.check(invalidJson); + }); + + Assertions.assertTrue(exception.getMessage().contains("Failed to parse rewrite rules")); + } + + @Test + public void testFunctionRewriteRuleCheckWithEmptyJson() throws DdlException { + // Test empty JSON + String emptyJson = "{}"; + + // Should not throw exception + Assertions.assertDoesNotThrow(() -> { + FunctionRewriteRules.check(emptyJson); + }); + } + + @Test + public void testFunctionRewriteRuleCheckWithNullJson() { + // Test null JSON + String nullJson = null; + + // Should throw DdlException due to new null check + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionRewriteRules.check(nullJson); + }); + Assertions.assertTrue(exception.getMessage().contains("Failed to parse rewrite rules")); + } + + @Test + public void testFunctionRewriteRuleCheckWithMalformedJson() { + // Test various malformed JSON strings + String[] malformedJsons = { + "{ \"rewrite\": }", // Missing value + "{ \"rewrite\": { } }", // Missing object - this is actually valid now + "{ \"rewrite\" { } }", // Missing colon + "{ \"rewrite\": {\"func1\": \"replacement1\",} }", // Trailing comma + "{ \"rewrite\": {\"func1\" \"replacement1\"} }" // Missing colon + }; + + for (String malformedJson : malformedJsons) { + // Skip the second case as it's now valid + if (malformedJson.equals("{ \"rewrite\": { } }")) { + continue; + } + + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionRewriteRules.check(malformedJson); + }); + + Assertions.assertTrue(exception.getMessage().contains("Failed to parse rewrite rules")); + } + } + + @Test + public void testFunctionRewriteRuleCheckWithEmptyFunctionNames() { + // Test empty function names in rewrite rules should throw exception + String emptyKeyJson = "{\n" + + " \"rewrite\": {\n" + + " \"\": \"replacement1\",\n" + + " \"func2\": \"replacement2\"\n" + + " }\n" + + "}"; + + DdlException exception1 = Assertions.assertThrows(DdlException.class, () -> { + FunctionRewriteRules.check(emptyKeyJson); + }); + Assertions.assertTrue(exception1.getMessage().contains("Failed to parse rewrite rules")); + + String emptyValueJson = "{\n" + + " \"rewrite\": {\n" + + " \"func1\": \"\",\n" + + " \"func2\": \"replacement2\"\n" + + " }\n" + + "}"; + + DdlException exception2 = Assertions.assertThrows(DdlException.class, () -> { + FunctionRewriteRules.check(emptyValueJson); + }); + Assertions.assertTrue(exception2.getMessage().contains("Failed to parse rewrite rules")); + } + + @Test + public void testFunctionRewriteRuleCheckWithNullFunctionNames() { + // Test null function names in rewrite rules should throw exception + // Note: JSON parsing will not allow null keys, but null values are possible + String nullValueJson = "{\n" + + " \"rewrite\": {\n" + + " \"func1\": null,\n" + + " \"func2\": \"replacement2\"\n" + + " }\n" + + "}"; + + DdlException exception = Assertions.assertThrows(DdlException.class, () -> { + FunctionRewriteRules.check(nullValueJson); + }); + Assertions.assertTrue(exception.getMessage().contains("Failed to parse rewrite rules")); + } + + @Test + public void testFunctionRewriteRuleWithComplexCustomRules() { + // Test complex custom rewrite rules that override and extend default rules + String complexJson = "{\n" + + " \"rewrite\": {\n" + + " \"nvl\": \"custom_nvl\",\n" + + " \"custom_func1\": \"transformed_func1\",\n" + + " \"old_function\": \"new_function\"\n" + + " }\n" + + "}"; + + // Test with MySQL (has default rewrite rules) + FunctionRewriteRules mysqlRule = FunctionRewriteRules.create("mysql", complexJson); + + // Custom rewrite rules should override default rules + Assertions.assertEquals("custom_nvl", mysqlRule.rewriteFunction("nvl")); // overridden + Assertions.assertEquals("transformed_func1", mysqlRule.rewriteFunction("custom_func1")); // custom + Assertions.assertEquals("new_function", mysqlRule.rewriteFunction("old_function")); // custom + + // Default MySQL rewrite rules that are not overridden should still work + Assertions.assertEquals("date", mysqlRule.rewriteFunction("to_date")); // default + + // Functions not in any rewrite map should return original name + Assertions.assertEquals("sum", mysqlRule.rewriteFunction("sum")); + Assertions.assertEquals("count", mysqlRule.rewriteFunction("count")); + } + + @Test + public void testFunctionRewriteRuleCreateWithMultipleDataSources() { + // Test that different datasources have different default rules + FunctionRewriteRules mysqlRule = FunctionRewriteRules.create("mysql", null); + FunctionRewriteRules clickhouseRule = FunctionRewriteRules.create("clickhouse", null); + FunctionRewriteRules oracleRule = FunctionRewriteRules.create("oracle", null); + + // Same function should be rewritten differently for different datasources + Assertions.assertEquals("ifnull", mysqlRule.rewriteFunction("nvl")); // MySQL: nvl -> ifnull + Assertions.assertEquals("nvl", clickhouseRule.rewriteFunction("nvl")); // ClickHouse: no rewrite + Assertions.assertEquals("nvl", oracleRule.rewriteFunction("nvl")); // Oracle: no rewrite + + Assertions.assertEquals("ifnull", mysqlRule.rewriteFunction("ifnull")); // MySQL: no rewrite + Assertions.assertEquals("ifnull", clickhouseRule.rewriteFunction("ifnull")); // ClickHouse: no rewrite + Assertions.assertEquals("nvl", oracleRule.rewriteFunction("ifnull")); // Oracle: ifnull -> nvl + + Assertions.assertEquals("FROM_UNIXTIME", + clickhouseRule.rewriteFunction("from_unixtime")); // ClickHouse specific + Assertions.assertEquals("from_unixtime", mysqlRule.rewriteFunction("from_unixtime")); // No rewrite in MySQL + Assertions.assertEquals("from_unixtime", oracleRule.rewriteFunction("from_unixtime")); // No rewrite in Oracle + } + + @Test + public void testFunctionRewriteRuleRewriteFunctionEdgeCases() { + // Test edge cases for rewriteFunction method + String jsonRules = "{\n" + + " \"rewrite\": {\n" + + " \"normal_func\": \"replaced_func\",\n" + + " \"UPPER_CASE\": \"lower_case\",\n" + + " \"Mixed_Case\": \"another_case\"\n" + + " }\n" + + "}"; + + FunctionRewriteRules rule = FunctionRewriteRules.create("unknown", jsonRules); + + // Test exact matches + Assertions.assertEquals("replaced_func", rule.rewriteFunction("normal_func")); + Assertions.assertEquals("lower_case", rule.rewriteFunction("UPPER_CASE")); + Assertions.assertEquals("another_case", rule.rewriteFunction("Mixed_Case")); + + // Test case sensitivity - should not match different cases + Assertions.assertEquals("Normal_Func", rule.rewriteFunction("Normal_Func")); // different case + Assertions.assertEquals("upper_case", rule.rewriteFunction("upper_case")); // different case + Assertions.assertEquals("mixed_case", rule.rewriteFunction("mixed_case")); // different case + + // Test special characters + Assertions.assertEquals("func_with_underscore", rule.rewriteFunction("func_with_underscore")); + Assertions.assertEquals("func123", rule.rewriteFunction("func123")); + Assertions.assertEquals("func-with-dash", rule.rewriteFunction("func-with-dash")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java index e470cebbeec631..4f0c6742240573 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java @@ -76,7 +76,6 @@ protected void runAfterAll() throws Exception { env.getCatalogMgr().dropCatalog(stmt); } - @Test public void testGlobalVariable() { Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); diff --git a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy index 3e625596d99c6a..9d4295fc6337d9 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy @@ -143,6 +143,75 @@ suite("test_clickhouse_jdbc_catalog", "p0,external,clickhouse,external_docker,ex order_qt_clickhouse_7_schema_tvf """ select * from query('catalog' = 'clickhouse_7_schema', 'query' = 'select * from doris_test.type;') order by 1; """ order_qt_clickhouse_7_schema_tvf_arr """ select * from query('catalog' = 'clickhouse_7_schema', 'query' = 'select * from doris_test.arr;') order by 1; """ - sql """ drop catalog if exists clickhouse_7_schema """ + // test function rules + // test push down + sql """ drop catalog if exists clickhouse_7_catalog """ + // test invalid config + test { + sql """ create catalog if not exists clickhouse_7_catalog properties( + "type"="jdbc", + "user"="default", + "password"="123456", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test?databaseTerm=schema", + "driver_url" = "${driver_url_7}", + "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver", + "function_rules" = '{"pushdown" : {"supported" : [null]}}' + );""" + + exception """Failed to parse push down rules: {"pushdown" : {"supported" : [null]}}""" + } + + sql """ create catalog if not exists clickhouse_7_catalog properties( + "type"="jdbc", + "user"="default", + "password"="123456", + "jdbc_url" = "jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test?databaseTerm=schema", + "driver_url" = "${driver_url_7}", + "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver", + "function_rules" = '{"pushdown" : {"supported": ["abs"]}}' + );""" + sql "use clickhouse_7_catalog.doris_test" + explain { + sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4) > 0") + contains """SELECT "k4" FROM "doris_test"."type" WHERE ((abs("k4") > 0)) AND ((toUnixTimestamp("k4") > 0))""" + contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND (unix_timestamp(k4[#3]) > 0))""" + } + sql """alter catalog clickhouse_7_catalog set properties("function_rules" = '');""" + explain { + sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4) > 0") + contains """QUERY: SELECT "k4" FROM "doris_test"."type" WHERE ((toUnixTimestamp("k4") > 0))""" + contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND (unix_timestamp(k4[#3]) > 0))""" + } + + sql """alter catalog clickhouse_7_catalog set properties("function_rules" = '{"pushdown" : {"supported": ["abs"]}}')""" + explain { + sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4) > 0") + contains """SELECT "k4" FROM "doris_test"."type" WHERE ((abs("k4") > 0)) AND ((toUnixTimestamp("k4") > 0))""" + contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND (unix_timestamp(k4[#3]) > 0))""" + } + + // test rewrite + sql """alter catalog clickhouse_7_catalog set properties("function_rules" = '{"pushdown" : {"supported": ["abs"]}, "rewrite" : {"unix_timestamp" : "rewrite_func"}}')""" + explain { + sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4) > 0") + contains """QUERY: SELECT "k4" FROM "doris_test"."type" WHERE ((abs("k4") > 0)) AND ((rewrite_func("k4") > 0))""" + contains """((abs(CAST(k4[#3] AS double)) > 0) AND (unix_timestamp(k4[#3]) > 0))""" + } + + // reset function rules + sql """alter catalog clickhouse_7_catalog set properties("function_rules" = '');""" + explain { + sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4) > 0") + contains """QUERY: SELECT "k4" FROM "doris_test"."type" WHERE ((toUnixTimestamp("k4") > 0))""" + contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND (unix_timestamp(k4[#3]) > 0))""" + } + + // test invalid config + test { + sql """alter catalog clickhouse_7_catalog set properties("function_rules" = 'invalid_json')""" + exception """Failed to parse push down rules: invalid_json""" + } + + // sql """ drop catalog if exists clickhouse_7_schema """ } } diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy index 4a32e2e206a42c..16c9982631115e 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy @@ -941,6 +941,121 @@ suite("test_jdbc_query_mysql", "p0,external,mysql,external_docker,external_docke order_qt_sql111 """ SELECT rank() OVER () FROM (SELECT k8 FROM $jdbcMysql57Table1 LIMIT 10) as t LIMIT 3 """ order_qt_sql112 """ SELECT k7, count(DISTINCT k8) FROM $jdbcMysql57Table1 WHERE k8 > 110 GROUP BY GROUPING SETS ((), (k7)) """ + // test function rules + sql """ drop table if exists jdbc_table_function_rule """ + test { + sql """ + CREATE EXTERNAL TABLE `jdbc_table_function_rule` ( + `products_id` int(11) NOT NULL, + `orders_id` int(11) NOT NULL, + `sales_add_time` datetime NOT NULL, + `sales_update_time` datetime NOT NULL, + `finance_admin` int(11) NOT NULL + ) ENGINE=JDBC + COMMENT "JDBC Mysql 外部表" + PROPERTIES ( + "resource" = "$jdbcResourceMysql57", + "table" = "ex_tb4", + "table_type"="mysql", + "function_rules" = '{"pushdown" : {"supported" : [null]}}' + ); + """ + + exception """Failed to parse push down rules: {"pushdown" : {"supported" : [null]}}""" + } + + sql """ + CREATE EXTERNAL TABLE `jdbc_table_function_rule` ( + `products_id` int(11) NOT NULL, + `orders_id` int(11) NOT NULL, + `sales_add_time` datetime NOT NULL, + `sales_update_time` datetime NOT NULL, + `finance_admin` int(11) NOT NULL + ) ENGINE=JDBC + COMMENT "JDBC Mysql 外部表" + PROPERTIES ( + "resource" = "$jdbcResourceMysql57", + "table" = "ex_tb4", + "table_type"="mysql", + "function_rules" = '{"pushdown" : {"supported" : ["date_trunc"]}}' + ); + """ + explain { + sql """select products_id from jdbc_table_function_rule where abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `products_id`, `sales_add_time` FROM `ex_tb4` WHERE (date_trunc(`sales_add_time`, 'month') = '2013-10-01 00:00:00')""" + contains """PREDICATES: ((abs(products_id[#0]) > 0) AND (date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))""" + } + + sql """drop table jdbc_table_function_rule""" + sql """ + CREATE EXTERNAL TABLE `jdbc_table_function_rule` ( + `products_id` int(11) NOT NULL, + `orders_id` int(11) NOT NULL, + `sales_add_time` datetime NOT NULL, + `sales_update_time` datetime NOT NULL, + `finance_admin` int(11) NOT NULL + ) ENGINE=JDBC + COMMENT "JDBC Mysql 外部表" + PROPERTIES ( + "resource" = "$jdbcResourceMysql57", + "table" = "ex_tb4", + "table_type"="mysql", + "function_rules" = '' + ); + """ + explain { + sql """select products_id from jdbc_table_function_rule where abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `products_id`, `sales_add_time` FROM `ex_tb4` WHERE ((abs(`products_id`) > 0))""" + contains """PREDICATES: ((abs(products_id[#0]) > 0) AND (date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))""" + } + + sql """drop table jdbc_table_function_rule""" + sql """ + CREATE EXTERNAL TABLE `jdbc_table_function_rule` ( + `products_id` int(11) NOT NULL, + `orders_id` int(11) NOT NULL, + `sales_add_time` datetime NOT NULL, + `sales_update_time` datetime NOT NULL, + `finance_admin` int(11) NOT NULL + ) ENGINE=JDBC + COMMENT "JDBC Mysql 外部表" + PROPERTIES ( + "resource" = "$jdbcResourceMysql57", + "table" = "ex_tb4", + "table_type"="mysql", + "function_rules" = '{"pushdown" : {"supported": ["date_trunc"], "unsupported" : ["abs"]}}' + ); + """ + explain { + sql """select products_id from jdbc_table_function_rule where abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `products_id`, `sales_add_time` FROM `ex_tb4` WHERE (date_trunc(`sales_add_time`, 'month') = '2013-10-01 00:00:00')""" + contains """PREDICATES: ((abs(products_id[#0]) > 0) AND (date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))""" + } + + // test rewrite + sql """drop table jdbc_table_function_rule""" + sql """ + CREATE EXTERNAL TABLE `jdbc_table_function_rule` ( + `products_id` int(11) NOT NULL, + `orders_id` int(11) NOT NULL, + `sales_add_time` datetime NOT NULL, + `sales_update_time` datetime NOT NULL, + `finance_admin` int(11) NOT NULL + ) ENGINE=JDBC + COMMENT "JDBC Mysql 外部表" + PROPERTIES ( + "resource" = "$jdbcResourceMysql57", + "table" = "ex_tb4", + "table_type"="mysql", + "function_rules" = '{"pushdown" : {"supported": ["to_date"], "unsupported" : ["abs"]}, "rewrite" : {"to_date" : "date2"}}' + ); + """ + explain { + sql """select products_id from jdbc_table_function_rule where to_date(sales_add_time) = "2013-10-01" and abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `products_id`, `sales_add_time` FROM `ex_tb4` WHERE (date2(`sales_add_time`) = '2013-10-01')""" + contains """PREDICATES: (((to_date(sales_add_time[#2]) = '2013-10-01') AND (abs(products_id[#0]) > 0)) AND (date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))""" + } + // TODO: check this, maybe caused by datasource in JDBC // test alter resource sql """alter resource $jdbcResourceMysql57 properties("password" = "1234567")""" @@ -950,25 +1065,6 @@ suite("test_jdbc_query_mysql", "p0,external,mysql,external_docker,external_docke } sql """alter resource $jdbcResourceMysql57 properties("password" = "123456")""" -// // test for type check -// sql """ drop table if exists ${exMysqlTypeTable} """ -// sql """ -// CREATE EXTERNAL TABLE ${exMysqlTypeTable} ( -// `id` bigint NOT NULL, -// `count_value` varchar(100) NULL -// ) ENGINE=JDBC -// COMMENT "JDBC Mysql 外部表" -// PROPERTIES ( -// "resource" = "$jdbcResourceMysql57", -// "table" = "ex_tb2", -// "table_type"="mysql" -// ); -// """ -// -// test { -// sql """select * from ${exMysqlTypeTable} order by id""" -// exception "Fail to convert jdbc type of java.lang.Integer to doris type BIGINT on column: id" -// } } } diff --git a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy index 46f2bb371e993e..2e06703e11552e 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy @@ -72,6 +72,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc String dt_null = "dt_null"; String test_zd = "test_zd" + sql """switch internal""" try_sql("DROP USER ${user}") sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" @@ -96,7 +97,7 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc "driver_class" = "${driver_class}" );""" - sql """use ${internal_db_name}""" + sql """use internal.${internal_db_name}""" sql """ drop table if exists ${internal_db_name}.${inDorisTable} """ sql """ CREATE TABLE ${internal_db_name}.${inDorisTable} ( @@ -653,6 +654,79 @@ suite("test_mysql_jdbc_catalog", "p0,external,mysql,external_docker,external_doc // so need to test both. sql """drop catalog if exists mysql_conjuncts;""" sql """set enable_nereids_planner=true""" + + + // test function rules + // test push down + sql """ drop catalog if exists mysql_function_rules""" + // test invalid config + test { + sql """create catalog if not exists mysql_function_rules properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false&zeroDateTimeBehavior=convertToNull", + "driver_url" = "${driver_url}", + "driver_class" = "${driver_class}", + "metadata_refresh_interval_sec" = "5", + "function_rules" = '{"pushdown" : {"supported" : [null]}}' + );""" + + exception """Failed to parse push down rules: {"pushdown" : {"supported" : [null]}}""" + } + + sql """create catalog if not exists mysql_function_rules properties( + "type"="jdbc", + "user"="root", + "password"="123456", + "jdbc_url" = "jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false&zeroDateTimeBehavior=convertToNull", + "driver_url" = "${driver_url}", + "driver_class" = "${driver_class}", + "metadata_refresh_interval_sec" = "5", + "function_rules" = '{"pushdown" : {"supported" : ["date_trunc"]}}' + );""" + + sql "use mysql_function_rules.doris_test" + explain { + sql """select tinyint_u from all_types where abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `tinyint_u`, `datetime` FROM `doris_test`.`all_types` WHERE (date_trunc(`datetime`, 'month') = '2013-10-01 00:00:00')""" + contains """PREDICATES: ((abs(tinyint_u[#0]) > 0) AND (date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))""" + } + sql """alter catalog mysql_function_rules set properties("function_rules" = '');""" + explain { + sql """select tinyint_u from all_types where abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `tinyint_u`, `datetime` FROM `doris_test`.`all_types` WHERE ((abs(`tinyint_u`) > 0))""" + contains """PREDICATES: ((abs(tinyint_u[#0]) > 0) AND (date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))""" + } + + sql """alter catalog mysql_function_rules set properties("function_rules" = '{"pushdown" : {"supported": ["date_trunc"], "unsupported" : ["abs"]}}')""" + explain { + sql """select tinyint_u from all_types where abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `tinyint_u`, `datetime` FROM `doris_test`.`all_types` WHERE (date_trunc(`datetime`, 'month') = '2013-10-01 00:00:00')""" + contains """PREDICATES: ((abs(tinyint_u[#0]) > 0) AND (date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))""" + } + + // test rewrite + sql """alter catalog mysql_function_rules set properties("function_rules" = '{"pushdown" : {"supported": ["to_date"], "unsupported" : ["abs"]}, "rewrite" : {"to_date" : "date2"}}');""" + explain { + sql """select tinyint_u from all_types where to_date(`datetime`) = "2013-10-01" and abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `tinyint_u`, `datetime` FROM `doris_test`.`all_types` WHERE (date2(`datetime`) = '2013-10-01')""" + contains """PREDICATES: (((to_date(datetime[#17]) = '2013-10-01') AND (abs(tinyint_u[#0]) > 0)) AND (date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))""" + } + + // reset function rules + sql """alter catalog mysql_function_rules set properties("function_rules" = '');""" + explain { + sql """select tinyint_u from all_types where to_date(`datetime`) = "2013-10-01" and abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";""" + contains """QUERY: SELECT `tinyint_u`, `datetime` FROM `doris_test`.`all_types` WHERE (date(`datetime`) = '2013-10-01') AND ((abs(`tinyint_u`) > 0))""" + contains """PREDICATES: (((to_date(datetime[#17]) = '2013-10-01') AND (abs(tinyint_u[#0]) > 0)) AND (date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))""" + } + + // test invalid config + test { + sql """alter catalog mysql_function_rules set properties("function_rules" = 'invalid_json')""" + exception """Failed to parse push down rules: invalid_json""" + } } } diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy index 4dd2607d48414f..818de72bca74b8 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy @@ -387,7 +387,77 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d order_qt_null_operator9 """ SELECT * FROM STUDENT WHERE (id IS NOT NULL AND NULL); """ order_qt_null_operator10 """ SELECT * FROM STUDENT WHERE (name IS NULL OR age IS NOT NULL); """ - sql """ drop catalog if exists oracle_null_operator; """ + // test function rules + // test push down + sql """ drop catalog if exists oracle_function_rules""" + // test invalid config + test { + sql """create catalog if not exists oracle_function_rules properties( + "type"="jdbc", + "user"="doris_test", + "password"="123456", + "jdbc_url" = "jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}", + "driver_url" = "${driver_url}", + "driver_class" = "oracle.jdbc.driver.OracleDriver", + "function_rules" = '{"pushdown" : {"supported" : [null]}}' + );""" + + exception """Failed to parse push down rules: {"pushdown" : {"supported" : [null]}}""" + } + + sql """create catalog if not exists oracle_function_rules properties( + "type"="jdbc", + "user"="doris_test", + "password"="123456", + "jdbc_url" = "jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}", + "driver_url" = "${driver_url}", + "driver_class" = "oracle.jdbc.driver.OracleDriver", + "function_rules" = '{"pushdown" : {"supported" : ["abs"]}}' + );""" + + sql "use oracle_function_rules.DORIS_TEST" + explain { + sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3) = 3;""" + contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE ((abs("ID") > 0)) AND ((nvl("ID", 3) = 3))""" + contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3) = 3))""" + } + sql """alter catalog oracle_function_rules set properties("function_rules" = '');""" + explain { + sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3) = 3;""" + contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE ((nvl("ID", 3) = 3))""" + contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3) = 3))""" + } + + sql """alter catalog oracle_function_rules set properties("function_rules" = '{"pushdown" : {"supported": ["abs"], "unsupported" : []}}')""" + explain { + sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3) = 3;""" + contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE ((abs("ID") > 0)) AND ((nvl("ID", 3) = 3))""" + contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3) = 3))""" + } + + // test rewrite + sql """alter catalog oracle_function_rules set properties("function_rules" = '{"pushdown" : {"supported": ["abs"]}, "rewrite" : {"abs" : "abs2"}}');""" + explain { + sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3) = 3;""" + contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE ((abs2("ID") > 0)) AND ((nvl("ID", 3) = 3))""" + contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3) = 3))""" + } + + // reset function rules + sql """alter catalog oracle_function_rules set properties("function_rules" = '');""" + explain { + sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3) = 3;""" + contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE ((nvl("ID", 3) = 3))""" + contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3) = 3))""" + } + + // test invalid config + test { + sql """alter catalog oracle_function_rules set properties("function_rules" = 'invalid_json')""" + exception """Failed to parse push down rules: invalid_json""" + } + + // sql """ drop catalog if exists oracle_null_operator; """ } } From 0ec6d154fe969b5285dd2551ee895c9ca64b4226 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:50:46 +0800 Subject: [PATCH 029/572] branch-3.0: [test](mv) Fix mv regression test is not chosen stable by CBO #50690 (#51943) Cherry-picked from #50690 Co-authored-by: seawinde --- .../test_dup_mv_plus/test_dup_mv_plus.out | 46 +++++++++++++------ .../testAggQueryOnAggMV10.out | 18 +++++--- .../testAggQueryOnAggMV11.out | 7 ++- .../testAggQueryOnAggMV2.out | 11 +++-- .../testAggQueryOnAggMV3.out | 6 ++- .../testAggQuqeryOnAggMV5.out | 4 ++ .../testAggQuqeryOnAggMV7.out | 4 ++ .../testAggregateMVCalcAggFunctionQuery.out | 4 ++ .../testBitmapUnionInQuery.out | 3 ++ .../testCountDistinctToBitmap.out | 3 ++ .../testIncorrectMVRewriteInSubquery.out | 3 ++ .../testIncorrectRewriteCountDistinct.out | 3 ++ .../testJoinOnLeftProjectToJoin.out | 4 +- .../mv_p0/ut/testNDVToHll/testNDVToHll.out | 3 ++ .../testOrderByQueryOnProjectView.out | 8 ++++ .../testProjectionMV1/testProjectionMV1.out | 14 ++++-- .../testProjectionMV2/testProjectionMV2.out | 7 +++ .../testProjectionMV3/testProjectionMV3.out | 7 +++ .../testProjectionMV4/testProjectionMV4.out | 5 ++ .../testSingleMVMultiUsage.out | 4 ++ .../mv_p0/ut/testSubQuery/testSubQuery.out | 8 ++++ .../testUnionDistinct/testUnionDistinct.out | 4 ++ .../test_dup_mv_plus/test_dup_mv_plus.groovy | 6 ++- .../testAggQueryOnAggMV10.groovy | 6 ++- .../testAggQueryOnAggMV11.groovy | 6 ++- .../testAggQueryOnAggMV2.groovy | 5 +- .../testAggQueryOnAggMV3.groovy | 6 ++- .../testAggQuqeryOnAggMV5.groovy | 6 ++- .../testAggQuqeryOnAggMV7.groovy | 7 ++- ...stAggTableCountDistinctInBitmapType.groovy | 5 +- ...testAggregateMVCalcAggFunctionQuery.groovy | 6 ++- .../testBitmapUnionInQuery.groovy | 5 +- .../testCountDistinctToBitmap.groovy | 5 +- .../testIncorrectMVRewriteInSubquery.groovy | 5 +- .../testIncorrectRewriteCountDistinct.groovy | 5 +- .../testJoinOnLeftProjectToJoin.groovy | 10 +++- .../mv_p0/ut/testNDVToHll/testNDVToHll.groovy | 5 +- .../testOrderByQueryOnProjectView.groovy | 6 ++- .../testProjectionMV1.groovy | 5 +- .../testProjectionMV2.groovy | 5 +- .../testProjectionMV3.groovy | 5 +- .../testProjectionMV4.groovy | 5 +- .../testSingleMVMultiUsage.groovy | 6 ++- .../mv_p0/ut/testSubQuery/testSubQuery.groovy | 6 ++- .../testUnionDistinct.groovy | 6 ++- .../usercase_union_rewrite.groovy | 2 +- 46 files changed, 251 insertions(+), 59 deletions(-) diff --git a/regression-test/data/mv_p0/test_dup_mv_plus/test_dup_mv_plus.out b/regression-test/data/mv_p0/test_dup_mv_plus/test_dup_mv_plus.out index 0f9a3544165959..91228895b36375 100644 --- a/regression-test/data/mv_p0/test_dup_mv_plus/test_dup_mv_plus.out +++ b/regression-test/data/mv_p0/test_dup_mv_plus/test_dup_mv_plus.out @@ -1,55 +1,75 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- -4 -4 -4 d +-4 -4 -4 d +1 1 1 a 1 1 1 a 2 2 2 b +2 2 2 b +3 -3 \N c 3 -3 \N c -- !select_mv -- -4 -3 +-4 -3 1 2 +1 2 +2 3 2 3 3 -2 +3 -2 -- !select_mv_sub -- -3 +-3 2 +2 +3 3 -2 +-2 -- !select_mv_sub_add -- -4 +-4 1 +1 +2 2 -3 - --- !select_group_mv -- -3 -2 -3 --2 -- !select_group_mv -- +-6 +4 +6 -4 -3 -1 + +-- !select_group_mv -- +-8 +6 2 +4 -- !select_group_mv_add -- --4 -3 -1 +-8 +6 2 +4 -- !select_group_mv_not -- --3 --4 -1 +-6 +-8 2 +4 -- !select_mv -- -4 -3 +-4 -3 +3 -2 3 -2 1 2 +1 2 +2 3 2 3 diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.out index 83bfded26fcd9b..c14d0402be30ef 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.out @@ -2,15 +2,19 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- -\N \N 8 -1 \N 3 -1 1 3 -2 \N 3 -2 2 3 -3 \N 4 -3 3 4 +\N \N 15 +1 \N 5 +1 1 5 +2 \N 5 +2 2 5 +3 \N 7 +3 3 7 diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.out index d0b1871ed2f217..e12b89b07a386e 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.out @@ -2,11 +2,14 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 3 -- !select_mv -- -1 4 -2 2 +1 6 +2 6 3 2 diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.out index 94559632cba57e..ae6b7ab10b83fa 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.out @@ -1,14 +1,17 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_emps_mv -- -2 9 -3 3 +2 18 +3 6 -- !select_star -- 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 7 2 2020-01-02 2 b 2 7 2 2020-01-03 3 c 3 3 3 +2020-01-03 3 c 3 3 3 -- !select_mv -- -2 9 -3 3 +2 18 +3 6 diff --git a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.out b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.out index 0ae3aec19b4214..165729c3dbc834 100644 --- a/regression-test/data/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.out +++ b/regression-test/data/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.out @@ -1,12 +1,16 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 2020-01-03 3 c 3 3 10 +2020-01-03 3 c 3 3 10 +2020-01-04 4 d 21 4 4 2020-01-04 4 d 21 4 4 -- !select_mv -- -4 4 +4 8 -- !select_mv -- diff --git a/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.out b/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.out index b10e432dadb571..6d14c178c4bd4c 100644 --- a/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.out +++ b/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.out @@ -2,7 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.out b/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.out index b10e432dadb571..6d14c178c4bd4c 100644 --- a/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.out +++ b/regression-test/data/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.out @@ -2,7 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/data/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.out b/regression-test/data/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.out index b10e432dadb571..6d14c178c4bd4c 100644 --- a/regression-test/data/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.out +++ b/regression-test/data/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.out @@ -2,7 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out b/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out index 75a47cb33192a5..d8a22797e50bf9 100644 --- a/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out +++ b/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out @@ -1,8 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 +2020-01-01 1 a 1 +2020-01-01 1 a 2 2020-01-01 1 a 2 2020-01-02 2 b 2 +2020-01-02 2 b 2 -- !select_mv -- 1 2 diff --git a/regression-test/data/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.out b/regression-test/data/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.out index e5696983e12d59..75885fdb163de8 100644 --- a/regression-test/data/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.out +++ b/regression-test/data/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.out @@ -1,8 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 +2020-01-01 1 a 1 +2020-01-01 1 a 2 2020-01-01 1 a 2 2020-01-02 2 b 2 +2020-01-02 2 b 2 -- !select_mv -- 1 2 diff --git a/regression-test/data/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.out b/regression-test/data/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.out index ab95a5b36faee4..ba6d1c2674a85a 100644 --- a/regression-test/data/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.out +++ b/regression-test/data/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.out @@ -1,8 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 +2020-01-01 1 a 1 +2020-01-01 1 a 2 2020-01-01 1 a 2 2020-01-02 2 b 2 +2020-01-02 2 b 2 -- !select_mv -- 1 \N diff --git a/regression-test/data/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.out b/regression-test/data/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.out index dd06cd3927c59d..5f491967b17384 100644 --- a/regression-test/data/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.out +++ b/regression-test/data/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.out @@ -1,8 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 +2020-01-01 1 a 1 +2020-01-01 1 a 2 2020-01-01 1 a 2 2020-01-02 2 b 2 +2020-01-02 2 b 2 -- !select_mv -- a 2 diff --git a/regression-test/data/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.out b/regression-test/data/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.out index cf5140bb02272b..838e50a149cef3 100644 --- a/regression-test/data/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.out +++ b/regression-test/data/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.out @@ -1,5 +1,5 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_mv -- -2 9 2 2 -3 3 3 3 +2 18 2 2 +3 6 3 3 diff --git a/regression-test/data/mv_p0/ut/testNDVToHll/testNDVToHll.out b/regression-test/data/mv_p0/ut/testNDVToHll/testNDVToHll.out index 7afb3700d9e87d..dc95ffa41f4d51 100644 --- a/regression-test/data/mv_p0/ut/testNDVToHll/testNDVToHll.out +++ b/regression-test/data/mv_p0/ut/testNDVToHll/testNDVToHll.out @@ -1,8 +1,11 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_star -- 2020-01-01 1 a 1 +2020-01-01 1 a 1 +2020-01-01 1 a 2 2020-01-01 1 a 2 2020-01-02 2 b 2 +2020-01-02 2 b 2 -- !select_mv -- 1 2 diff --git a/regression-test/data/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.out b/regression-test/data/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.out index beefefba56c876..ae4b592b5dc0be 100644 --- a/regression-test/data/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.out +++ b/regression-test/data/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.out @@ -2,12 +2,20 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- 1 1 +1 +1 2 +2 +3 3 diff --git a/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out b/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out index 523176bb0a5fdd..adbcee9429dfb3 100644 --- a/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out +++ b/regression-test/data/mv_p0/ut/testProjectionMV1/testProjectionMV1.out @@ -2,18 +2,24 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 -- !select_mv -- 1 1 1 1 +1 1 +1 1 +2 2 2 2 -- !select_mv -- -1 2 -2 2 +1 4 +2 4 -- !select_mv -- -1 2 -2 2 +1 4 +2 4 diff --git a/regression-test/data/mv_p0/ut/testProjectionMV2/testProjectionMV2.out b/regression-test/data/mv_p0/ut/testProjectionMV2/testProjectionMV2.out index 934d3208d48fc8..2a4293882a6db1 100644 --- a/regression-test/data/mv_p0/ut/testProjectionMV2/testProjectionMV2.out +++ b/regression-test/data/mv_p0/ut/testProjectionMV2/testProjectionMV2.out @@ -2,13 +2,20 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 -- !select_mv -- 2 2 +2 +2 -- !select_base -- a a +a +a diff --git a/regression-test/data/mv_p0/ut/testProjectionMV3/testProjectionMV3.out b/regression-test/data/mv_p0/ut/testProjectionMV3/testProjectionMV3.out index 9537036f4128dc..c8c96b2438332a 100644 --- a/regression-test/data/mv_p0/ut/testProjectionMV3/testProjectionMV3.out +++ b/regression-test/data/mv_p0/ut/testProjectionMV3/testProjectionMV3.out @@ -2,13 +2,20 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 -- !select_mv -- 2 a 2 a +2 a +2 a -- !select_mv2 -- a a +a +a diff --git a/regression-test/data/mv_p0/ut/testProjectionMV4/testProjectionMV4.out b/regression-test/data/mv_p0/ut/testProjectionMV4/testProjectionMV4.out index 45a962ed59c94d..52a5842d59da3e 100644 --- a/regression-test/data/mv_p0/ut/testProjectionMV4/testProjectionMV4.out +++ b/regression-test/data/mv_p0/ut/testProjectionMV4/testProjectionMV4.out @@ -2,11 +2,16 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-02 2 b 2 2 2 2020-01-02 2 b 2 2 2 -- !select_mv -- b +b -- !select_base -- 2 +2 diff --git a/regression-test/data/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.out b/regression-test/data/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.out index b10e432dadb571..6d14c178c4bd4c 100644 --- a/regression-test/data/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.out +++ b/regression-test/data/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.out @@ -2,7 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/data/mv_p0/ut/testSubQuery/testSubQuery.out b/regression-test/data/mv_p0/ut/testSubQuery/testSubQuery.out index a30c9136e426c3..a0a66b8ef87358 100644 --- a/regression-test/data/mv_p0/ut/testSubQuery/testSubQuery.out +++ b/regression-test/data/mv_p0/ut/testSubQuery/testSubQuery.out @@ -2,12 +2,20 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- 1 1 1 1 1 1 +1 1 1 +1 1 1 2 2 2 +2 2 2 +3 3 3 3 3 3 diff --git a/regression-test/data/mv_p0/ut/testUnionDistinct/testUnionDistinct.out b/regression-test/data/mv_p0/ut/testUnionDistinct/testUnionDistinct.out index cd7fbba9aa266d..e2c8e5e603db3e 100644 --- a/regression-test/data/mv_p0/ut/testUnionDistinct/testUnionDistinct.out +++ b/regression-test/data/mv_p0/ut/testUnionDistinct/testUnionDistinct.out @@ -2,7 +2,11 @@ -- !select_star -- 2020-01-01 1 a 1 1 1 2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 +2020-01-01 1 a 1 1 1 2020-01-02 2 b 2 2 2 +2020-01-02 2 b 2 2 2 +2020-01-03 3 c 3 3 3 2020-01-03 3 c 3 3 3 -- !select_mv -- diff --git a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy index 86f06b665df2bb..213d8853db2b4c 100644 --- a/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy +++ b/regression-test/suites/mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy @@ -33,15 +33,19 @@ suite ("test_dup_mv_plus") { """ sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 1,1,1,'a';" + sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 2,2,2,'b';" sql "insert into d_table select 3,-3,null,'c';" + sql "insert into d_table select 3,-3,null,'c';" createMV ("create materialized view k12p as select k1,k2+1 from d_table;") + sql "insert into d_table select -4,-4,-4,'d';" sql "insert into d_table select -4,-4,-4,'d';" sql "analyze table d_table with sync;" - sql """alter table d_table modify column k4 set stats ('row_count'='3');""" + sql """alter table d_table modify column k4 set stats ('row_count'='8');""" sql """set enable_stats=false;""" qt_select_star "select * from d_table order by k1;" diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy index 8747f7641a91ca..9ee40c2b80a891 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV10/testAggQueryOnAggMV10.groovy @@ -33,15 +33,19 @@ suite ("testAggQueryOnAggMV10") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy index dcd4f05f76cd9a..029117ff0eed19 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV11/testAggQueryOnAggMV11.groovy @@ -33,15 +33,19 @@ suite ("testAggQueryOnAggMV11") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, count(salary) from emps group by deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy index f2aadc63e17b9c..a8c66710604201 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV2/testAggQueryOnAggMV2.groovy @@ -31,8 +31,11 @@ suite ("testAggQueryOnAggMV2") { sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" sql """insert into emps values("2020-01-02",2,"b",2,7,2);""" + sql """insert into emps values("2020-01-02",2,"b",2,7,2);""" explain { sql("select deptno, sum(salary) from emps group by deptno order by deptno;") @@ -43,7 +46,7 @@ suite ("testAggQueryOnAggMV2") { createMV("create materialized view emps_mv as select deptno, sum(salary) from emps group by deptno ;") sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='3');""" + sql """alter table emps modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy index bf46b5b0af7101..ab0e806a8ebdb9 100644 --- a/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQueryOnAggMV3/testAggQueryOnAggMV3.groovy @@ -33,15 +33,19 @@ suite ("testAggQueryOnAggMV3") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,10);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,10);""" + sql """insert into emps values("2020-01-04",4,"d",21,4,4);""" sql """insert into emps values("2020-01-04",4,"d",21,4,4);""" createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;") sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy index 1e2b01dbc61abe..e57df1332aeb5f 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV5/testAggQuqeryOnAggMV5.groovy @@ -33,15 +33,19 @@ suite ("testAggQuqeryOnAggMV5") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """analyze table emps with sync;""" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy index 82e2c878956ec7..bf2ed58a376881 100644 --- a/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy +++ b/regression-test/suites/mv_p0/ut/testAggQuqeryOnAggMV7/testAggQuqeryOnAggMV7.groovy @@ -33,16 +33,19 @@ suite ("testAggQuqeryOnAggMV7") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, commission, sum(salary) from emps group by deptno, commission;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """analyze table emps with sync;""" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy b/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy index 2af8c74566e98e..e68a207a3247ce 100644 --- a/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy +++ b/regression-test/suites/mv_p0/ut/testAggTableCountDistinctInBitmapType/testAggTableCountDistinctInBitmapType.groovy @@ -25,12 +25,15 @@ suite ("testAggTableCountDistinctInBitmapType") { """ sql """insert into test_tb values(1,to_bitmap(1));""" + sql """insert into test_tb values(1,to_bitmap(1));""" + sql """insert into test_tb values(2,to_bitmap(2));""" sql """insert into test_tb values(2,to_bitmap(2));""" sql """insert into test_tb values(3,to_bitmap(3));""" + sql """insert into test_tb values(3,to_bitmap(3));""" sql "analyze table test_tb with sync;" - sql """alter table test_tb modify column k1 set stats ('row_count'='3');""" + sql """alter table test_tb modify column k1 set stats ('row_count'='6');""" sql """set enable_stats=false;""" qt_select_star "select * from test_tb order by 1;" diff --git a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy index 9fdee877cd69e1..90308b50badd4b 100644 --- a/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testAggregateMVCalcAggFunctionQuery/testAggregateMVCalcAggFunctionQuery.groovy @@ -32,16 +32,20 @@ suite ("testAggregateMVCalcAggFunctionQuery") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, empid, sum(salary) from emps group by empid, deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy index ab6632ca48765c..ca427baad9464f 100644 --- a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy @@ -30,14 +30,17 @@ suite ("testBitmapUnionInQuery") { """ sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-02",2,"b",2);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags group by user_id;") + sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql "analyze table user_tags with sync;" - sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy index 4e1de1eedee3e1..ed0b296ff0d01c 100644 --- a/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy +++ b/regression-test/suites/mv_p0/ut/testCountDistinctToBitmap/testCountDistinctToBitmap.groovy @@ -30,14 +30,17 @@ suite ("testCountDistinctToBitmap") { """ sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-02",2,"b",2);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags group by user_id;") + sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql "analyze table user_tags with sync;" - sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy index 4de20757ef8fc8..07466312cfc0a3 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectMVRewriteInSubquery/testIncorrectMVRewriteInSubquery.groovy @@ -29,14 +29,17 @@ suite ("testIncorrectMVRewriteInSubquery") { """ sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-02",2,"b",2);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags group by user_id;") + sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql "analyze table user_tags with sync;" - sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy index dfd212a7fad631..5f600dfc3a98a8 100644 --- a/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy +++ b/regression-test/suites/mv_p0/ut/testIncorrectRewriteCountDistinct/testIncorrectRewriteCountDistinct.groovy @@ -29,14 +29,17 @@ suite ("testIncorrectRewriteCountDistinct") { """ sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-02",2,"b",2);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" createMV("create materialized view user_tags_mv as select user_id, bitmap_union(to_bitmap(tag_id)) from user_tags group by user_id;") + sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql "analyze table user_tags with sync;" - sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy index 6f6097438421d7..f4b786c04aaed8 100644 --- a/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy +++ b/regression-test/suites/mv_p0/ut/testJoinOnLeftProjectToJoin/testJoinOnLeftProjectToJoin.groovy @@ -30,8 +30,11 @@ suite ("testJoinOnLeftProjectToJoin") { """ sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" sql """insert into emps values("2020-01-02",2,"b",2,7,2);""" + sql """insert into emps values("2020-01-02",2,"b",2,7,2);""" sql """ DROP TABLE IF EXISTS depts; """ sql """ @@ -43,17 +46,20 @@ suite ("testJoinOnLeftProjectToJoin") { partition by range (time_col) (partition p1 values less than MAXVALUE) distributed by hash(time_col) buckets 3 properties('replication_num' = '1'); """ - sql """alter table depts modify column time_col set stats ('row_count'='3');""" + sql """alter table depts modify column time_col set stats ('row_count'='6');""" sql """insert into depts values("2020-01-02",2,"b",2);""" + sql """insert into depts values("2020-01-02",2,"b",2);""" + sql """insert into depts values("2020-01-03",3,"c",3);""" sql """insert into depts values("2020-01-03",3,"c",3);""" sql """insert into depts values("2020-01-02",2,"b",1);""" + sql """insert into depts values("2020-01-02",2,"b",1);""" createMV("create materialized view emps_mv as select deptno, sum(salary), sum(commission) from emps group by deptno;") createMV("create materialized view depts_mv as select deptno, max(cost) from depts group by deptno;") sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='3');""" + sql """alter table emps modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" diff --git a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy index 769d483544ca0b..c02f4eb02eb1e9 100644 --- a/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy +++ b/regression-test/suites/mv_p0/ut/testNDVToHll/testNDVToHll.groovy @@ -31,14 +31,17 @@ suite ("testNDVToHll") { """ sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-01",1,"a",1);""" + sql """insert into user_tags values("2020-01-02",2,"b",2);""" sql """insert into user_tags values("2020-01-02",2,"b",2);""" createMV("create materialized view user_tags_mv as select user_id, hll_union(hll_hash(tag_id)) from user_tags group by user_id;") + sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql """insert into user_tags values("2020-01-01",1,"a",2);""" sql "analyze table user_tags with sync;" - sql """alter table user_tags modify column time_col set stats ('row_count'='3');""" + sql """alter table user_tags modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from user_tags order by time_col;", "user_tags_mv") diff --git a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy index b5c11b3fbdbafe..5d78d91c95dfa3 100644 --- a/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy +++ b/regression-test/suites/mv_p0/ut/testOrderByQueryOnProjectView/testOrderByQueryOnProjectView.groovy @@ -32,15 +32,19 @@ suite ("testOrderByQueryOnProjectView") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, empid from emps;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy index dcc8245a45f922..b78d251da6dfa1 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV1/testProjectionMV1.groovy @@ -32,6 +32,8 @@ suite ("testProjectionMV1") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" test { @@ -41,10 +43,11 @@ suite ("testProjectionMV1") { createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='3');""" + sql """alter table emps modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy index 8d65df35835086..3af9fd26cc3b15 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV2/testProjectionMV2.groovy @@ -32,14 +32,17 @@ suite ("testProjectionMV2") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" createMV("create materialized view emps_mv as select deptno, empid from emps order by deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='3');""" + sql """alter table emps modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy index 5c24070d9cc935..8082eaa1352d8d 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV3/testProjectionMV3.groovy @@ -32,16 +32,19 @@ suite ("testProjectionMV3") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" def result = "null" createMV("create materialized view emps_mv as select deptno, empid, name from emps order by deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """analyze table emps with sync;""" - sql """alter table emps modify column time_col set stats ('row_count'='3');""" + sql """alter table emps modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy index 370f21b3d73ff5..68c0505d83f2bd 100644 --- a/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy +++ b/regression-test/suites/mv_p0/ut/testProjectionMV4/testProjectionMV4.groovy @@ -32,16 +32,19 @@ suite ("testProjectionMV4") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" def result = "null" createMV("create materialized view emps_mv as select name, deptno, salary from emps;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='3');""" + sql """alter table emps modify column time_col set stats ('row_count'='6');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy index 5da7c15b186767..a2076a9a0b8b44 100644 --- a/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy +++ b/regression-test/suites/mv_p0/ut/testSingleMVMultiUsage/testSingleMVMultiUsage.groovy @@ -32,15 +32,19 @@ suite ("testSingleMVMultiUsage") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, empid, salary from emps order by deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy index 58100561862f83..6e40eec885c373 100644 --- a/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy +++ b/regression-test/suites/mv_p0/ut/testSubQuery/testSubQuery.groovy @@ -35,15 +35,19 @@ suite ("testSubQuery") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select deptno, empid from emps;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") diff --git a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy index c6243a3f923616..6b2ee26d221a0a 100644 --- a/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy +++ b/regression-test/suites/mv_p0/ut/testUnionDistinct/testUnionDistinct.groovy @@ -32,15 +32,20 @@ suite ("testUnionDistinct") { """ sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" + sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-02",2,"b",2,2,2);""" sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" + sql """insert into emps values("2020-01-03",3,"c",3,3,3);""" createMV("create materialized view emps_mv as select empid, deptno from emps order by empid, deptno;") + sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql """insert into emps values("2020-01-01",1,"a",1,1,1);""" sql "analyze table emps with sync;" + sql """alter table emps modify column time_col set stats ('row_count'='8');""" sql """set enable_stats=false;""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") @@ -53,7 +58,6 @@ suite ("testUnionDistinct") { } qt_select_mv "select * from (select empid, deptno from emps where empid >1 union select empid, deptno from emps where empid <0) t order by 1;" sql """set enable_stats=true;""" - sql """alter table emps modify column time_col set stats ('row_count'='4');""" mv_rewrite_fail("select * from emps order by empid;", "emps_mv") explain { diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy index a90b7a98044e00..559a72a9208066 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy @@ -106,7 +106,7 @@ suite ("usercase_union_rewrite") { sql """analyze table orders_user with sync;""" sql """analyze table lineitem_user with sync;""" - sql """alter table orders_user modify column o_comment set stats ('row_count'='7');""" + sql """alter table orders_user modify column o_comment set stats ('row_count'='21');""" sql """alter table lineitem_user modify column l_comment set stats ('row_count'='3');""" def create_mv_orders = { mv_name, mv_sql -> From 3f733ceb7915e7389933999de68f68fb815c7a62 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:51:28 +0800 Subject: [PATCH 030/572] =?UTF-8?q?branch-3.0:=20[improvement](cloud)=20en?= =?UTF-8?q?able=20s3=20load=20from=20volcano=20engine=20object=20storag?= =?UTF-8?q?=E2=80=A6=20#51794=20(#51909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #51794 Co-authored-by: HonestManXin --- .../doris/cloud/storage/RemoteBase.java | 2 + .../apache/doris/cloud/storage/TosRemote.java | 42 +++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/storage/TosRemote.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java index a1bbf656404a5c..cad8dcb6f02c8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/RemoteBase.java @@ -153,6 +153,8 @@ public static RemoteBase newInstance(ObjectInfo obj) throws Exception { return new BosRemote(obj); case AZURE: return new AzureRemote(obj); + case TOS: + return new TosRemote(obj); default: throw new Exception("current not support obj : " + obj.toString()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/TosRemote.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/TosRemote.java new file mode 100644 index 00000000000000..4e1969455c2559 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/TosRemote.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.storage; + +import org.apache.doris.common.DdlException; + +import org.apache.commons.lang3.tuple.Triple; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +public class TosRemote extends DefaultRemote { + private static final Logger LOG = LogManager.getLogger(TosRemote.class); + + public TosRemote(ObjectInfo obj) { + super(obj); + } + + public String getPresignedUrl(String fileName) { + throw new UnsupportedOperationException("not unsupported for tos yet"); + } + + @Override + public Triple getStsToken() throws DdlException { + throw new DdlException("Get sts token for tos is unsupported"); + } +} From d3148dced233b92be71e13c644136fe68c4817d8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:52:32 +0800 Subject: [PATCH 031/572] branch-3.0: [enhancement](cloud) monitor evict size of file cache active gc #51197 (#51793) Cherry-picked from #51197 Signed-off-by: zhengyu Co-authored-by: zhengyu --- be/src/io/cache/block_file_cache.cpp | 6 ++++++ be/src/io/cache/block_file_cache.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 39e1a4ae28a9bc..72bbd708506734 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -87,6 +87,10 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path, _cache_base_path.c_str(), "file_cache_ttl_cache_evict_size"); _total_evict_size_metrics = std::make_shared>( _cache_base_path.c_str(), "file_cache_total_evict_size"); + _gc_evict_bytes_metrics = std::make_shared>(_cache_base_path.c_str(), + "file_cache_gc_evict_bytes"); + _gc_evict_count_metrics = std::make_shared>(_cache_base_path.c_str(), + "file_cache_gc_evict_count"); _evict_by_time_metrics_matrix[FileCacheType::DISPOSABLE][FileCacheType::NORMAL] = std::make_shared>(_cache_base_path.c_str(), @@ -1146,6 +1150,8 @@ void BlockFileCache::remove_if_cached_async(const UInt128Wrapper& file_key) { std::vector to_remove; if (iter != _files.end()) { for (auto& [_, cell] : iter->second) { + *_gc_evict_bytes_metrics << cell.size(); + *_gc_evict_count_metrics << 1; if (cell.releasable()) { to_remove.push_back(&cell); } else { diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index c12b03cc998792..7c046cc162783a 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -546,6 +546,8 @@ class BlockFileCache { std::shared_ptr> _cur_disposable_queue_cache_size_metrics; std::array>, 4> _queue_evict_size_metrics; std::shared_ptr> _total_evict_size_metrics; + std::shared_ptr> _gc_evict_bytes_metrics; + std::shared_ptr> _gc_evict_count_metrics; std::shared_ptr> _evict_by_time_metrics_matrix[4][4]; std::shared_ptr> _evict_by_size_metrics_matrix[4][4]; std::shared_ptr> _evict_by_self_lru_metrics_matrix[4]; From 75d29d8e104f1f88819cf32e5b2a31eb2efec40c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:54:22 +0800 Subject: [PATCH 032/572] branch-3.0: [fix](trash) Fix shadow variable causing garbage scheduling time errors #51647 (#51687) Cherry-picked from #51647 Co-authored-by: deardeng --- be/src/olap/olap_server.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index df201c7d699112..fc601903a54cc9 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -388,7 +388,7 @@ void StorageEngine::_garbage_sweeper_thread_callback() { // when usage = 0.88, ratio is approximately 0.0057. double ratio = (1.1 * (pi / 2 - std::atan(usage * 100 / 5 - 14)) - 0.28) / pi; ratio = ratio > 0 ? ratio : 0; - auto curr_interval = uint32_t(max_interval * ratio); + curr_interval = uint32_t(max_interval * ratio); curr_interval = std::max(curr_interval, min_interval); curr_interval = std::min(curr_interval, max_interval); @@ -403,6 +403,8 @@ void StorageEngine::_garbage_sweeper_thread_callback() { << "see previous message for detail. err code=" << res; // do nothing. continue next loop. } + LOG(INFO) << "trash thread check usage=" << usage << " ratio=" << ratio + << " curr_interval=" << curr_interval; } while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(curr_interval))); } From a7dddf011833d4c539060d75321c4c0456a7cc74 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:55:05 +0800 Subject: [PATCH 033/572] branch-3.0: [fix](auto bucket)Set the estimated partition size to 5G in non cloud #51258 (#51682) Cherry-picked from #51258 Co-authored-by: deardeng --- .../src/main/java/org/apache/doris/common/Config.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 6f2d8d06de3011..c046e515eaa3bf 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2648,12 +2648,12 @@ public class Config extends ConfigBase { public static int arrow_flight_max_connections = 4096; @ConfField(mutable = true, masterOnly = true, description = { - "Auto Buckets中按照partition size去估算bucket数，存算一体partition size 1G估算一个bucket，" - + "但存算分离下partition size 10G估算一个bucket。 若配置小于0，会在在代码中会自适应存算一体模式默认1G，在存算分离默认10G", + "Auto Buckets中按照partition size去估算bucket数，存算一体partition size 5G估算一个bucket，" + + "但存算分离下partition size 10G估算一个bucket。 若配置小于0，会在在代码中会自适应存算一体模式默认5G，在存算分离默认10G", "In Auto Buckets, the number of buckets is estimated based on the partition size. " - + "For storage and computing integration, a partition size of 1G is estimated as one bucket." + + "For storage and computing integration, a partition size of 5G is estimated as one bucket." + " but for cloud, a partition size of 10G is estimated as one bucket. " - + "If the configuration is less than 0, the code will have an adaptive non-cloud mode with a default of 1G," + + "If the configuration is less than 0, the code will have an adaptive non-cloud mode with a default of 5G," + " and in cloud mode with a default of 10G." }) public static int autobucket_partition_size_per_bucket_gb = -1; From 542cc933588722d3e8de7d9bab0ade3111b90e15 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:56:26 +0800 Subject: [PATCH 034/572] branch-3.0: [chore](cloud) Add cloud meta mgr timeout counter #51836 (#51882) Cherry-picked from #51836 Co-authored-by: Gavin Chou --- be/src/cloud/cloud_meta_mgr.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 2971ff33f8e75e..e4054bdd885fd7 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -134,8 +134,11 @@ Status bthread_fork_join(const std::vector>& tasks, int namespace { constexpr int kBrpcRetryTimes = 3; -bvar::LatencyRecorder _get_rowset_latency("doris_CloudMetaMgr", "get_rowset"); +bvar::LatencyRecorder _get_rowset_latency("doris_cloud_meta_mgr_get_rowset"); bvar::LatencyRecorder g_cloud_commit_txn_resp_redirect_latency("cloud_table_stats_report_latency"); +bvar::Adder g_cloud_meta_mgr_rpc_timeout_count("cloud_meta_mgr_rpc_timeout_count"); +bvar::Window> g_cloud_ms_rpc_timeout_count_window( + "cloud_meta_mgr_rpc_timeout_qps", &g_cloud_meta_mgr_rpc_timeout_count, 30); class MetaServiceProxy { public: @@ -404,6 +407,10 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res, error_msg = res->status().msg(); } + if (error_code == brpc::ERPCTIMEDOUT) { + g_cloud_meta_mgr_rpc_timeout_count << 1; + } + ++retry_times; if (retry_times > config::meta_service_rpc_retry_times || (retry_times > config::meta_service_rpc_timeout_retry_times && From b48e0c0d51e34e108e9f0d9dbf51df9eeaaef8a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:58:12 +0800 Subject: [PATCH 035/572] branch-3.0: [fix](alter) alter partition without storage_policy property will also cancel storage_policy #51662 (#51779) Cherry-picked from #51662 Co-authored-by: camby <104178625@qq.com> --- .../src/main/java/org/apache/doris/alter/Alter.java | 3 ++- .../apache/doris/common/util/PropertyAnalyzer.java | 7 +++++++ .../test_show_storage_policy_using.groovy | 13 ++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index ca6268c5b03dab..37b1f177d45887 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -852,7 +852,8 @@ public void modifyPartitionsProperty(Database db, // check currentStoragePolicy resource exist. Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(currentStoragePolicy); partitionInfo.setStoragePolicy(partition.getId(), currentStoragePolicy); - } else { + } else if (PropertyAnalyzer.hasStoragePolicy(properties)) { + // only set "storage_policy" = "", means cancel storage policy // if current partition is already in remote storage if (partition.getRemoteDataSize() > 0) { throw new AnalysisException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 2460119f60a13b..1c12d2b106bace 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -1194,6 +1194,13 @@ public static String analyzeStoragePolicy(Map properties) { return storagePolicy; } + public static boolean hasStoragePolicy(Map properties) { + if (properties != null && properties.containsKey(PROPERTIES_STORAGE_POLICY)) { + return true; + } + return false; + } + public static String analyzeStorageVaultName(Map properties) { String storageVaultName = null; if (properties != null && properties.containsKey(PROPERTIES_STORAGE_VAULT_NAME)) { diff --git a/regression-test/suites/cold_heat_separation_p2/test_show_storage_policy_using.groovy b/regression-test/suites/cold_heat_separation_p2/test_show_storage_policy_using.groovy index a6277e663d77ba..b3ea5c7c8de6d5 100644 --- a/regression-test/suites/cold_heat_separation_p2/test_show_storage_policy_using.groovy +++ b/regression-test/suites/cold_heat_separation_p2/test_show_storage_policy_using.groovy @@ -156,6 +156,17 @@ suite("test_show_storage_policy_using") { """ assertTrue(show_result.size() >= 4) + // alter other property, will not cancel storage_policy + sql """ ALTER STORAGE POLICY ${policy_name} PROPERTIES("cooldown_ttl" = "1"); """ + sql """ + ALTER TABLE partition_with_multiple_storage_policy MODIFY PARTITION (`p201701`) SET ("replication_num"="1") + """ + show_result = sql """ + show storage policy using for ${policy_name} + """ + assertEquals(show_result.size(), 2) + sql """ ALTER STORAGE POLICY ${policy_name} PROPERTIES("cooldown_ttl" = "300"); """ + // test cancel a partition's storage policy sql """ ALTER TABLE partition_with_multiple_storage_policy MODIFY PARTITION (`p201701`) SET ("storage_policy"="") @@ -187,4 +198,4 @@ suite("test_show_storage_policy_using") { sql """ DROP TABLE IF EXISTS table_no_storage_policy_1 """ sql """ DROP TABLE IF EXISTS partition_with_multiple_storage_policy """ sql """ DROP TABLE IF EXISTS table_with_storage_policy_2""" -} \ No newline at end of file +} From 2251d1f89a338a151d9261817b87e57f2e79daed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:58:58 +0800 Subject: [PATCH 036/572] branch-3.0: [fix](hudi catalog) Fix the Kerberos authentication error when querying hudi table #51713 (#51902) Cherry-picked from #51713 Co-authored-by: heguanhui --- .../doris/datasource/hudi/source/HudiScanNode.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index a6156924e27967..8e33dfb24692a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -383,7 +383,14 @@ public List getSplits(int numBackends) throws UserException { partitionInit = true; } List splits = Collections.synchronizedList(new ArrayList<>()); - getPartitionsSplits(prunedPartitions, splits); + try { + hmsTable.getCatalog().getPreExecutionAuthenticator().execute(() -> { + getPartitionsSplits(prunedPartitions, splits); + return null; + }); + } catch (Exception e) { + throw new UserException(ExceptionUtils.getRootCauseMessage(e), e); + } return splits; } From c4884ad079c4330ec067f8d0ce88aa344b86461a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Jun 2025 09:59:53 +0800 Subject: [PATCH 037/572] branch-3.0: [opt](storage) Add log and metric when aws/azure sdk do retry operation #51485 (#51686) Cherry-picked from #51485 Co-authored-by: Lei Zhang --- be/src/common/config.cpp | 13 ++++++- be/src/common/config.h | 7 ++++ be/src/util/s3_util.cpp | 36 +++++++++++++++++- .../io/fs/azure_obj_storage_client_test.cpp | 18 +++++---- cloud/src/common/config.h | 12 +++++- cloud/src/recycler/s3_accessor.cpp | 31 ++++++++++++++- common/cpp/aws_logger.h | 2 +- common/cpp/obj_retry_strategy.cpp | 38 ++++++++++++------- common/cpp/obj_retry_strategy.h | 8 ++-- conf/be.conf | 9 ++++- 10 files changed, 141 insertions(+), 33 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 02cd267cc2456d..3df7058c12d1ce 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -841,7 +841,18 @@ DEFINE_mInt32(zone_map_row_num_threshold, "20"); // Info = 4, // Debug = 5, // Trace = 6 -DEFINE_Int32(aws_log_level, "2"); +DEFINE_Int32(aws_log_level, "3"); +DEFINE_Validator(aws_log_level, + [](const int config) -> bool { return config >= 0 && config <= 6; }); + +// azure sdk log level +// Verbose = 1, +// Informational = 2, +// Warning = 3, +// Error = 4 +DEFINE_Int32(azure_log_level, "3"); +DEFINE_Validator(azure_log_level, + [](const int config) -> bool { return config >= 1 && config <= 4; }); // the buffer size when read data from remote storage like s3 DEFINE_mInt32(remote_storage_read_buffer_mb, "16"); diff --git a/be/src/common/config.h b/be/src/common/config.h index b60e10fe49d609..0fa93d0f8f8ef1 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -887,6 +887,13 @@ DECLARE_mInt32(zone_map_row_num_threshold); // Trace = 6 DECLARE_Int32(aws_log_level); +// azure sdk log level +// Verbose = 1, +// Informational = 2, +// Warning = 3, +// Error = 4 +DECLARE_Int32(azure_log_level); + // the buffer size when read data from remote storage like s3 DECLARE_mInt32(remote_storage_read_buffer_mb); diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index ce8d9c2d6c2635..f650558333794d 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -32,6 +32,7 @@ #include #ifdef USE_AZURE +#include #include #endif #include @@ -150,6 +151,33 @@ S3ClientFactory::S3ClientFactory() { config::s3_put_token_per_second, config::s3_put_bucket_tokens, config::s3_put_token_limit, metric_func_factory(put_rate_limit_ns, put_rate_limit_exceed_req_num))}; + +#ifdef USE_AZURE + auto azureLogLevel = + static_cast(config::azure_log_level); + Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel); + Azure::Core::Diagnostics::Logger::SetListener( + [&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) { + switch (level) { + case Azure::Core::Diagnostics::Logger::Level::Verbose: + LOG(INFO) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Informational: + LOG(INFO) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Warning: + LOG(WARNING) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Error: + LOG(ERROR) << message; + break; + default: + LOG(WARNING) << "Unknown level: " << static_cast(level) + << ", message: " << message; + break; + } + }); +#endif } S3ClientFactory::~S3ClientFactory() { @@ -204,7 +232,13 @@ std::shared_ptr S3ClientFactory::_create_azure_client( } } - auto containerClient = std::make_shared(uri, cred); + Azure::Storage::Blobs::BlobClientOptions options; + options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests); + options.Retry.MaxRetries = config::max_s3_client_retry; + options.PerRetryPolicies.emplace_back(std::make_unique()); + + auto containerClient = std::make_shared( + uri, cred, std::move(options)); LOG_INFO("create one azure client with {}", s3_conf.to_string()); return std::make_shared(std::move(containerClient)); #else diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 1297e7c75f77a1..4ef1770128c17b 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -21,6 +21,7 @@ #include "io/fs/file_system.h" #include "io/fs/obj_storage_client.h" +#include "util/s3_util.h" #ifdef USE_AZURE #include @@ -49,13 +50,16 @@ class AzureObjStorageClientTest : public testing::Test { std::string accountKey = std::getenv("AZURE_ACCOUNT_KEY"); std::string containerName = std::getenv("AZURE_CONTAINER_NAME"); - auto cred = std::make_shared(accountName, - accountKey); - const std::string uri = - fmt::format("https://{}.blob.core.windows.net/{}", accountName, containerName); - auto containerClient = std::make_shared(uri, cred); - AzureObjStorageClientTest::obj_storage_client = - std::make_shared(std::move(containerClient)); + // Initialize Azure SDK + [[maybe_unused]] auto& s3ClientFactory = S3ClientFactory::instance(); + + AzureObjStorageClientTest::obj_storage_client = S3ClientFactory::instance().create( + {.endpoint = fmt::format("https://{}.blob.core.windows.net", accountName), + .region = "dummy-region", + .ak = accountName, + .sk = accountKey, + .bucket = containerName, + .provider = io::ObjStorageType::AZURE}); } void SetUp() override { diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 14047123b4cbb3..23bb4f74b6a4e3 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -285,7 +285,17 @@ CONF_Strings(recycler_storage_vault_white_list, ""); // Info = 4, // Debug = 5, // Trace = 6 -CONF_Int32(aws_log_level, "2"); +CONF_Int32(aws_log_level, "3"); +CONF_Validator(aws_log_level, [](const int config) -> bool { return config >= 0 && config <= 6; }); + +// azure sdk log level +// Verbose = 1, +// Informational = 2, +// Warning = 3, +// Error = 4 +CONF_Int32(azure_log_level, "3"); +CONF_Validator(azure_log_level, + [](const int config) -> bool { return config >= 1 && config <= 4; }); // ca_cert_file is in this path by default, Normally no modification is required // ca cert default path is different from different OS diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index ada9a4a1e8cc36..3c36f5a01bb248 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -29,6 +29,7 @@ #include #ifdef USE_AZURE +#include #include #include #endif @@ -120,6 +121,33 @@ class S3Environment { return std::make_shared(logLevel); }; Aws::InitAPI(aws_options_); + +#ifdef USE_AZURE + auto azureLogLevel = + static_cast(config::azure_log_level); + Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel); + Azure::Core::Diagnostics::Logger::SetListener( + [&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) { + switch (level) { + case Azure::Core::Diagnostics::Logger::Level::Verbose: + LOG(INFO) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Informational: + LOG(INFO) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Warning: + LOG(WARNING) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Error: + LOG(ERROR) << message; + break; + default: + LOG(WARNING) << "Unknown level: " << static_cast(level) + << ", message: " << message; + break; + } + }); +#endif } ~S3Environment() { Aws::ShutdownAPI(aws_options_); } @@ -308,8 +336,7 @@ int S3Accessor::init() { // Within the RetryPolicy, the nextPolicy is called multiple times inside a loop. // All policies in the PerRetryPolicies are downstream of the RetryPolicy. // Therefore, you only need to add a policy to check if the response code is 429 and if the retry count meets the condition, it can record the retry count. - options.PerRetryPolicies.emplace_back( - std::make_unique(config::max_s3_client_retry)); + options.PerRetryPolicies.emplace_back(std::make_unique()); auto container_client = std::make_shared( uri_, cred, std::move(options)); // uri format for debug: ${scheme}://${ak}.blob.core.windows.net/${bucket}/${prefix} diff --git a/common/cpp/aws_logger.h b/common/cpp/aws_logger.h index ca607cab0566f7..85734d13e1411c 100644 --- a/common/cpp/aws_logger.h +++ b/common/cpp/aws_logger.h @@ -19,7 +19,7 @@ #include #include -#include // IWYU pragma: export +#include class DorisAWSLogger final : public Aws::Utils::Logging::LogSystemInterface { public: diff --git a/common/cpp/obj_retry_strategy.cpp b/common/cpp/obj_retry_strategy.cpp index 0226d1af28f038..6da4c23980a9b7 100644 --- a/common/cpp/obj_retry_strategy.cpp +++ b/common/cpp/obj_retry_strategy.cpp @@ -19,10 +19,11 @@ #include #include +#include namespace doris { -bvar::Adder s3_too_many_request_retry_cnt("s3_too_many_request_retry_cnt"); +bvar::Adder object_request_retry_count("object_request_retry_count"); S3CustomRetryStrategy::S3CustomRetryStrategy(int maxRetries) : DefaultRetryStrategy(maxRetries) {} @@ -35,33 +36,42 @@ bool S3CustomRetryStrategy::ShouldRetry(const Aws::Client::AWSError AzureRetryRecordPolicy::Send( Azure::Core::Http::Request& request, Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, Azure::Core::Context const& context) const { - auto resp = nextPolicy.Send(request, context); - if (retry_cnt != 0 && - resp->GetStatusCode() == Azure::Core::Http::HttpStatusCode::TooManyRequests) { - retry_cnt--; - s3_too_many_request_retry_cnt << 1; + // https://learn.microsoft.com/en-us/azure/developer/cpp/sdk/fundamentals/http-pipelines-and-retries + + std::unique_ptr response = nextPolicy.Send(request, context); + int32_t retry_count = + Azure::Core::Http::Policies::_internal::RetryPolicy::GetRetryCount(context); + + if (static_cast(response->GetStatusCode()) > 299 || + static_cast(response->GetStatusCode()) < 200) { + if (retry_count > 0) { + object_request_retry_count << 1; + } + + // If the response is not successful, we log the retry attempt and status code. + LOG(INFO) << "azure retry retry_count: " << retry_count + << ", status code: " << static_cast(response->GetStatusCode()) + << ", reason: " << response->GetReasonPhrase(); } - return resp; + + return response; } std::unique_ptr AzureRetryRecordPolicy::Clone() const { - auto ret = std::make_unique(*this); - ret->retry_cnt = 0; - return ret; + return std::make_unique(*this); } #endif } // namespace doris \ No newline at end of file diff --git a/common/cpp/obj_retry_strategy.h b/common/cpp/obj_retry_strategy.h index b081ca91a22c36..dd98f871716fe7 100644 --- a/common/cpp/obj_retry_strategy.h +++ b/common/cpp/obj_retry_strategy.h @@ -37,16 +37,14 @@ class S3CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy { #ifdef USE_AZURE class AzureRetryRecordPolicy final : public Azure::Core::Http::Policies::HttpPolicy { public: - AzureRetryRecordPolicy(int retry_cnt); - ~AzureRetryRecordPolicy() override; + AzureRetryRecordPolicy() = default; + ~AzureRetryRecordPolicy() override = default; + std::unique_ptr Clone() const override; std::unique_ptr Send( Azure::Core::Http::Request& request, Azure::Core::Http::Policies::NextHttpPolicy nextPolicy, Azure::Core::Context const& context) const override; - -private: - mutable int retry_cnt; }; #endif } // namespace doris \ No newline at end of file diff --git a/conf/be.conf b/conf/be.conf index 961b41b51fdfb3..28207c16962810 100644 --- a/conf/be.conf +++ b/conf/be.conf @@ -88,6 +88,13 @@ sys_log_level = INFO # Debug = 5, # Trace = 6 # Default to turn off aws sdk log, because aws sdk errors that need to be cared will be output through Doris logs -aws_log_level=2 +aws_log_level = 3 + +# azure sdk log level +# Verbose = 1, +# Informational = 2, +# Warning = 3, +# Error = 4 +azure_log_level = 3 ## If you are not running in aws cloud, you can disable EC2 metadata AWS_EC2_METADATA_DISABLED=true From cec412a853d040c3973f495bbcdbb9a56c1134b3 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Fri, 20 Jun 2025 10:01:19 +0800 Subject: [PATCH 038/572] branch-3.0: [opt](cache) Reset initial capacity of all caches after Cgroup memory limit changes (#51698) ### What problem does this PR solve? pick #51216 --- be/src/olap/lru_cache.cpp | 4 ++ be/src/runtime/memory/cache_manager.cpp | 7 +++ be/src/runtime/memory/cache_manager.h | 2 + be/src/runtime/memory/cache_policy.h | 1 + be/src/runtime/memory/lru_cache_policy.h | 25 +++++++++-- be/src/util/mem_info.cpp | 8 +++- be/test/olap/lru_cache_test.cpp | 57 ++++++++++++++++++++++++ 7 files changed, 100 insertions(+), 4 deletions(-) diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp index 8b004d294af577..9bb21ef717deab 100644 --- a/be/src/olap/lru_cache.cpp +++ b/be/src/olap/lru_cache.cpp @@ -181,6 +181,10 @@ PrunedInfo LRUCache::set_capacity(size_t capacity) { LRUHandle* last_ref_list = nullptr; { std::lock_guard l(_mutex); + if (capacity > _capacity) { + _capacity = capacity; + return {0, 0}; + } _capacity = capacity; _evict_from_lru(0, &last_ref_list); } diff --git a/be/src/runtime/memory/cache_manager.cpp b/be/src/runtime/memory/cache_manager.cpp index ec57ffba50d318..f823d23df2b9ca 100644 --- a/be/src/runtime/memory/cache_manager.cpp +++ b/be/src/runtime/memory/cache_manager.cpp @@ -81,4 +81,11 @@ int64_t CacheManager::for_each_cache_refresh_capacity(double adjust_weighted, return freed_size; } +void CacheManager::for_each_cache_reset_initial_capacity(double adjust_weighted) { + std::lock_guard l(_caches_lock); + for (const auto& pair : _caches) { + pair.second->reset_initial_capacity(adjust_weighted); + } +} + } // namespace doris diff --git a/be/src/runtime/memory/cache_manager.h b/be/src/runtime/memory/cache_manager.h index a2a089b929dbdf..04e611c5a87093 100644 --- a/be/src/runtime/memory/cache_manager.h +++ b/be/src/runtime/memory/cache_manager.h @@ -84,6 +84,8 @@ class CacheManager { int64_t for_each_cache_refresh_capacity(double adjust_weighted, RuntimeProfile* profile = nullptr); + void for_each_cache_reset_initial_capacity(double adjust_weighted); + private: std::mutex _caches_lock; std::unordered_map _caches; diff --git a/be/src/runtime/memory/cache_policy.h b/be/src/runtime/memory/cache_policy.h index b7f7b65ee1631b..5c5c3bfddb52f4 100644 --- a/be/src/runtime/memory/cache_policy.h +++ b/be/src/runtime/memory/cache_policy.h @@ -156,6 +156,7 @@ class CachePolicy { CacheType type() { return _type; } size_t initial_capacity() const { return _initial_capacity; } + virtual int64_t reset_initial_capacity(double adjust_weighted) = 0; bool enable_prune() const { return _enable_prune; } RuntimeProfile* profile() { return _profile.get(); } diff --git a/be/src/runtime/memory/lru_cache_policy.h b/be/src/runtime/memory/lru_cache_policy.h index 7bd2ca486acdf4..83c7f46585ab65 100644 --- a/be/src/runtime/memory/lru_cache_policy.h +++ b/be/src/runtime/memory/lru_cache_policy.h @@ -240,9 +240,9 @@ class LRUCachePolicy : public CachePolicy { } } - int64_t adjust_capacity_weighted(double adjust_weighted) override { - std::lock_guard l(_lock); - auto capacity = static_cast(_initial_capacity * adjust_weighted); + int64_t adjust_capacity_weighted_unlocked(double adjust_weighted) { + auto capacity = + static_cast(static_cast(_initial_capacity) * adjust_weighted); COUNTER_SET(_freed_entrys_counter, (int64_t)0); COUNTER_SET(_freed_memory_counter, (int64_t)0); COUNTER_SET(_cost_timer, (int64_t)0); @@ -271,6 +271,25 @@ class LRUCachePolicy : public CachePolicy { return _freed_entrys_counter->value(); } + int64_t adjust_capacity_weighted(double adjust_weighted) override { + std::lock_guard l(_lock); + return adjust_capacity_weighted_unlocked(adjust_weighted); + } + + int64_t reset_initial_capacity(double adjust_weighted) override { + DCHECK(adjust_weighted != 0.0); // otherwise initial_capacity will always to be 0. + std::lock_guard l(_lock); + int64_t prune_num = adjust_capacity_weighted_unlocked(adjust_weighted); + size_t old_capacity = _initial_capacity; + _initial_capacity = + static_cast(static_cast(_initial_capacity) * adjust_weighted); + LOG(INFO) << fmt::format( + "[MemoryGC] {} reset initial capacity, new capacity {}, old capacity {}, prune num " + "{}", + type_string(_type), _initial_capacity, old_capacity, prune_num); + return prune_num; + }; + protected: void _init_mem_tracker(const std::string& type_name) { if (std::find(CachePolicy::MetadataCache.begin(), CachePolicy::MetadataCache.end(), diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index fe9cf84b2aed54..f0efffd7c925ae 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -20,7 +20,7 @@ #include "mem_info.h" -#include "gutil/strings/split.h" +#include "runtime/memory/cache_manager.h" #ifdef __APPLE__ #include @@ -254,6 +254,12 @@ void MemInfo::refresh_proc_meminfo() { // 2. if physical_mem changed, refresh mem limit and gc size. if (physical_mem > 0 && _s_physical_mem.load(std::memory_order_relaxed) != physical_mem) { + if (_s_physical_mem != std::numeric_limits::max()) { + // After MemInfo is initialized, if physical memory changed, reset initial capacity of all caches. + CacheManager::instance()->for_each_cache_reset_initial_capacity( + physical_mem / (_s_physical_mem * 1.0)); + } + _s_physical_mem.store(physical_mem); bool is_percent = true; diff --git a/be/test/olap/lru_cache_test.cpp b/be/test/olap/lru_cache_test.cpp index 1acc38f2b9e084..8c260d69755f43 100644 --- a/be/test/olap/lru_cache_test.cpp +++ b/be/test/olap/lru_cache_test.cpp @@ -704,4 +704,61 @@ TEST_F(CacheTest, SetCapacity) { ASSERT_EQ(0, cache()->get_usage()); } +TEST_F(CacheTest, ResetInitialCapacity) { + init_number_cache(); + for (int i = 0; i < kCacheSize; i++) { + Insert(i, 1000 + i, 1); + EXPECT_EQ(1000 + i, Lookup(i)); + } + ASSERT_EQ(kCacheSize, cache()->get_capacity()); + ASSERT_EQ(kCacheSize, cache()->get_usage()); + + int64_t prune_num = cache()->adjust_capacity_weighted(0.5); + ASSERT_EQ(prune_num, kCacheSize / 2); + ASSERT_EQ(kCacheSize / 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); + + prune_num = cache()->adjust_capacity_weighted(2); + ASSERT_EQ(prune_num, 0); + ASSERT_EQ(kCacheSize * 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); + + prune_num = cache()->reset_initial_capacity(0.5); + ASSERT_EQ(prune_num, 0); + ASSERT_EQ(kCacheSize / 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); + + prune_num = cache()->adjust_capacity_weighted(2); + ASSERT_EQ(prune_num, 0); + ASSERT_EQ(kCacheSize, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); + + prune_num = cache()->adjust_capacity_weighted(1); + ASSERT_EQ(prune_num, 0); + ASSERT_EQ(kCacheSize / 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); + + prune_num = cache()->reset_initial_capacity(4); + ASSERT_EQ(prune_num, 0); + ASSERT_EQ(kCacheSize * 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); + + for (int i = kCacheSize; i < kCacheSize * 2; i++) { + Insert(i, 1000 + i, 1); + EXPECT_EQ(1000 + i, Lookup(i)); + } + ASSERT_EQ(kCacheSize * 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize + kCacheSize / 2, cache()->get_usage()); + + prune_num = cache()->adjust_capacity_weighted(0.5); + ASSERT_EQ(prune_num, kCacheSize / 2); + ASSERT_EQ(kCacheSize, cache()->get_capacity()); + ASSERT_EQ(kCacheSize, cache()->get_usage()); + + prune_num = cache()->reset_initial_capacity(0.25); + ASSERT_EQ(prune_num, kCacheSize / 2); + ASSERT_EQ(kCacheSize / 2, cache()->get_capacity()); + ASSERT_EQ(kCacheSize / 2, cache()->get_usage()); +} + } // namespace doris From e757432bfd4710503bef8200b0fb25548d9ac0f9 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 20 Jun 2025 12:06:49 +0800 Subject: [PATCH 039/572] [bugfix](memleak) fix memory leak for tabletschema and result cache (#51931) (#51952) pick #51786 --- be/src/olap/tablet_schema.cpp | 21 +++++---------------- be/src/olap/tablet_schema.h | 3 --- be/src/runtime/cache/result_cache.h | 6 +++++- be/src/runtime/exec_env_init.cpp | 1 + 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 43a2a034722845..3f268e5edbe070 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -871,9 +871,7 @@ void TabletIndex::to_schema_pb(TabletIndexPB* index) const { TabletSchema::TabletSchema() = default; -TabletSchema::~TabletSchema() { - clear_column_cache_handlers(); -} +TabletSchema::~TabletSchema() = default; int64_t TabletSchema::get_metadata_size() const { return sizeof(TabletSchema) + _vl_field_mem_size; @@ -968,14 +966,6 @@ void TabletSchema::clear_columns() { _num_null_columns = 0; _num_key_columns = 0; _cols.clear(); - clear_column_cache_handlers(); -} - -void TabletSchema::clear_column_cache_handlers() { - for (auto* cache_handle : _column_cache_handlers) { - TabletColumnObjectPool::instance()->release(cache_handle); - } - _column_cache_handlers.clear(); } void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extracted_columns, @@ -990,7 +980,6 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac _field_name_to_index.clear(); _field_id_to_index.clear(); _cluster_key_idxes.clear(); - clear_column_cache_handlers(); for (const auto& i : schema.cluster_key_idxes()) { _cluster_key_idxes.push_back(i); } @@ -1000,7 +989,10 @@ void TabletSchema::init_from_pb(const TabletSchemaPB& schema, bool ignore_extrac auto pair = TabletColumnObjectPool::instance()->insert( deterministic_string_serialize(column_pb)); column = pair.second; - _column_cache_handlers.push_back(pair.first); + // Release the handle quickly, because we use shared ptr to manage column. + // It often core during tablet schema copy to another schema because handle's + // reference count should be managed mannually. + TabletColumnObjectPool::instance()->release(pair.first); } else { column = std::make_shared(); column->init_from_pb(column_pb); @@ -1089,8 +1081,6 @@ void TabletSchema::shawdow_copy_without_columns(const TabletSchema& tablet_schem _num_null_columns = 0; _num_key_columns = 0; _cols.clear(); - // notice : do not ref columns - _column_cache_handlers.clear(); } void TabletSchema::update_index_info_from(const TabletSchema& tablet_schema) { @@ -1153,7 +1143,6 @@ void TabletSchema::build_current_tablet_schema(int64_t index_id, int32_t version _sequence_col_idx = -1; _version_col_idx = -1; _cluster_key_idxes.clear(); - clear_column_cache_handlers(); for (const auto& i : ori_tablet_schema._cluster_key_idxes) { _cluster_key_idxes.push_back(i); } diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index ce114bf80f5cea..7e5cc1216081eb 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -544,13 +544,10 @@ class TabletSchema : public MetadataAdder { friend bool operator!=(const TabletSchema& a, const TabletSchema& b); TabletSchema(const TabletSchema&) = default; - void clear_column_cache_handlers(); - KeysType _keys_type = DUP_KEYS; SortType _sort_type = SortType::LEXICAL; size_t _sort_col_num = 0; std::vector _cols; - std::vector _column_cache_handlers; std::vector _indexes; std::unordered_map _field_name_to_index; diff --git a/be/src/runtime/cache/result_cache.h b/be/src/runtime/cache/result_cache.h index 7473a42d918b8d..1af42223acc013 100644 --- a/be/src/runtime/cache/result_cache.h +++ b/be/src/runtime/cache/result_cache.h @@ -79,7 +79,11 @@ class ResultCache { _partition_count = 0; } - virtual ~ResultCache() {} + virtual ~ResultCache() { + _node_list.clear(); + _node_map.clear(); + } + void update(const PUpdateCacheRequest* request, PCacheResponse* response); void fetch(const PFetchCacheRequest* request, PFetchCacheResult* result); bool contains(const UniqueId& sql_key); diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index a0bed40953a53a..ee0ad2c93f5db7 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -722,6 +722,7 @@ void ExecEnv::destroy() { // Free resource after threads are stopped. // Some threads are still running, like threads created by _new_load_stream_mgr ... SAFE_DELETE(_tablet_schema_cache); + SAFE_DELETE(_tablet_column_object_pool); // _scanner_scheduler must be desotried before _storage_page_cache SAFE_DELETE(_scanner_scheduler); From 7df2b14a12ad1d4f1220b64bf5c48861cb5c15c3 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 20 Jun 2025 16:27:59 +0800 Subject: [PATCH 040/572] [cherry-pick](branch-30) support regexp_replace function with ignore flag options (#50245) (#51917) cherry-pick from master (#50245) --- be/src/exprs/string_functions.cpp | 16 +- be/src/exprs/string_functions.h | 3 +- be/src/vec/functions/function_regexp.cpp | 176 +++++++++++++++--- .../functions/scalar/RegexpReplace.java | 45 ++++- .../functions/scalar/RegexpReplaceOne.java | 45 ++++- 5 files changed, 247 insertions(+), 38 deletions(-) diff --git a/be/src/exprs/string_functions.cpp b/be/src/exprs/string_functions.cpp index ce5738dba9509e..e21f9e365ed98f 100644 --- a/be/src/exprs/string_functions.cpp +++ b/be/src/exprs/string_functions.cpp @@ -25,6 +25,8 @@ #include +#include "util/string_util.h" + // NOTE: be careful not to use string::append. It is not performant. namespace doris { @@ -61,7 +63,7 @@ bool StringFunctions::set_re2_options(const StringRef& match_parameter, std::str // The caller owns the returned regex. Returns nullptr if the pattern could not be compiled. bool StringFunctions::compile_regex(const StringRef& pattern, std::string* error_str, const StringRef& match_parameter, - std::unique_ptr& re) { + const StringRef& options_value, std::unique_ptr& re) { re2::StringPiece pattern_sp(pattern.data, pattern.size); re2::RE2::Options options; // Disable error logging in case e.g. every row causes an error @@ -70,6 +72,18 @@ bool StringFunctions::compile_regex(const StringRef& pattern, std::string* error // Return the leftmost longest match (rather than the first match). // options.set_longest_match(true); options.set_dot_nl(true); + + if ((options_value.data != nullptr) && (options_value.size > 0)) { + auto options_split = split(options_value.to_string(), ","); + for (const auto& option : options_split) { + if (iequal("ignore_invalid_escape", option)) { + options.set_ignore_replace_escape(true); + } else { + // "none" do nothing, and could add more options for future extensibility. + } + } + } + if (match_parameter.size > 0 && !StringFunctions::set_re2_options(match_parameter, error_str, &options)) { return false; diff --git a/be/src/exprs/string_functions.h b/be/src/exprs/string_functions.h index 8ed917b82d25a1..ab7079c846e8d5 100644 --- a/be/src/exprs/string_functions.h +++ b/be/src/exprs/string_functions.h @@ -36,6 +36,7 @@ class StringFunctions { // The caller owns the returned regex. Returns nullptr if the pattern could not be compiled. static bool compile_regex(const StringRef& pattern, std::string* error_str, - const StringRef& match_parameter, std::unique_ptr& re); + const StringRef& match_parameter, const StringRef& options_value, + std::unique_ptr& re); }; } // namespace doris diff --git a/be/src/vec/functions/function_regexp.cpp b/be/src/vec/functions/function_regexp.cpp index f03ae176c207af..77ffd6286ae61d 100644 --- a/be/src/vec/functions/function_regexp.cpp +++ b/be/src/vec/functions/function_regexp.cpp @@ -20,7 +20,6 @@ #include #include -#include #include #include #include @@ -51,12 +50,124 @@ namespace doris::vectorized { +struct ThreeParamTypes { + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared(), + std::make_shared()}; + } +}; + +struct FourParamTypes { + static DataTypes get_variadic_argument_types() { + return {std::make_shared(), std::make_shared(), + std::make_shared(), std::make_shared()}; + } +}; + +// template FunctionRegexpFunctionality is used for regexp_replace/regexp_replace_one +template +class FunctionRegexpReplace : public IFunction { +public: + static constexpr auto name = Impl::name; + + static FunctionPtr create() { return std::make_shared(); } + + String get_name() const override { return name; } + + size_t get_number_of_arguments() const override { + return get_variadic_argument_types_impl().size(); + } + + bool is_variadic() const override { return true; } + + DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { + return make_nullable(std::make_shared()); + } + + DataTypes get_variadic_argument_types_impl() const override { + return ParamTypes::get_variadic_argument_types(); + } + + Status open(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { + if (scope == FunctionContext::THREAD_LOCAL) { + if (context->is_col_constant(1)) { + DCHECK(!context->get_function_state(scope)); + const auto pattern_col = context->get_constant_col(1)->column_ptr; + const auto& pattern = pattern_col->get_data_at(0); + if (pattern.size == 0) { + return Status::OK(); + } + + std::string error_str; + std::unique_ptr scoped_re; + StringRef options_value; + if (context->get_num_args() == 4) { + DCHECK(context->is_col_constant(3)); + const auto options_col = context->get_constant_col(3)->column_ptr; + options_value = options_col->get_data_at(0); + } + + bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), + options_value, scoped_re); + if (!st) { + context->set_error(error_str.c_str()); + return Status::InvalidArgument(error_str); + } + std::shared_ptr re(scoped_re.release()); + context->set_function_state(scope, re); + } + } + return Status::OK(); + } + + Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + size_t result, size_t input_rows_count) const override { + size_t argument_size = arguments.size(); + + auto result_null_map = ColumnUInt8::create(input_rows_count, 0); + auto result_data_column = ColumnString::create(); + auto& result_data = result_data_column->get_chars(); + auto& result_offset = result_data_column->get_offsets(); + result_offset.resize(input_rows_count); + + bool col_const[3]; + ColumnPtr argument_columns[3]; + for (int i = 0; i < 3; ++i) { + col_const[i] = is_column_const(*block.get_by_position(arguments[i]).column); + } + argument_columns[0] = col_const[0] ? static_cast( + *block.get_by_position(arguments[0]).column) + .convert_to_full_column() + : block.get_by_position(arguments[0]).column; + + default_preprocess_parameter_columns(argument_columns, col_const, {1, 2}, block, arguments); + + StringRef options_value; + if (col_const[1] && col_const[2]) { + Impl::execute_impl_const_args(context, argument_columns, options_value, + input_rows_count, result_data, result_offset, + result_null_map->get_data()); + } else { + // the options have check in FE, so is always const, and get idx of 0 + if (argument_size == 4) { + options_value = block.get_by_position(arguments[3]).column->get_data_at(0); + } + Impl::execute_impl(context, argument_columns, options_value, input_rows_count, + result_data, result_offset, result_null_map->get_data()); + } + + block.get_by_position(result).column = + ColumnNullable::create(std::move(result_data_column), std::move(result_null_map)); + return Status::OK(); + } +}; + struct RegexpReplaceImpl { static constexpr auto name = "regexp_replace"; - // 3 args static void execute_impl(FunctionContext* context, ColumnPtr argument_columns[], - size_t input_rows_count, ColumnString::Chars& result_data, - ColumnString::Offsets& result_offset, NullMap& null_map) { + const StringRef& options_value, size_t input_rows_count, + ColumnString::Chars& result_data, ColumnString::Offsets& result_offset, + NullMap& null_map) { const auto* str_col = check_and_get_column(argument_columns[0].get()); const auto* pattern_col = check_and_get_column(argument_columns[1].get()); const auto* replace_col = check_and_get_column(argument_columns[2].get()); @@ -66,12 +177,13 @@ struct RegexpReplaceImpl { StringOP::push_null_string(i, result_data, result_offset, null_map); continue; } - _execute_inner_loop(context, str_col, pattern_col, replace_col, result_data, - result_offset, null_map, i); + _execute_inner_loop(context, str_col, pattern_col, replace_col, options_value, + result_data, result_offset, null_map, i); } } static void execute_impl_const_args(FunctionContext* context, ColumnPtr argument_columns[], - size_t input_rows_count, ColumnString::Chars& result_data, + const StringRef& options_value, size_t input_rows_count, + ColumnString::Chars& result_data, ColumnString::Offsets& result_offset, NullMap& null_map) { const auto* str_col = check_and_get_column(argument_columns[0].get()); const auto* pattern_col = check_and_get_column(argument_columns[1].get()); @@ -82,14 +194,14 @@ struct RegexpReplaceImpl { StringOP::push_null_string(i, result_data, result_offset, null_map); continue; } - _execute_inner_loop(context, str_col, pattern_col, replace_col, result_data, - result_offset, null_map, i); + _execute_inner_loop(context, str_col, pattern_col, replace_col, options_value, + result_data, result_offset, null_map, i); } } template static void _execute_inner_loop(FunctionContext* context, const ColumnString* str_col, const ColumnString* pattern_col, - const ColumnString* replace_col, + const ColumnString* replace_col, const StringRef& options_value, ColumnString::Chars& result_data, ColumnString::Offsets& result_offset, NullMap& null_map, const size_t index_now) { @@ -99,7 +211,8 @@ struct RegexpReplaceImpl { if (re == nullptr) { std::string error_str; const auto& pattern = pattern_col->get_data_at(index_check_const(index_now, Const)); - bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), scoped_re); + bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), + options_value, scoped_re); if (!st) { context->add_warning(error_str.c_str()); StringOP::push_null_string(index_now, result_data, result_offset, null_map); @@ -121,8 +234,9 @@ struct RegexpReplaceOneImpl { static constexpr auto name = "regexp_replace_one"; static void execute_impl(FunctionContext* context, ColumnPtr argument_columns[], - size_t input_rows_count, ColumnString::Chars& result_data, - ColumnString::Offsets& result_offset, NullMap& null_map) { + const StringRef& options_value, size_t input_rows_count, + ColumnString::Chars& result_data, ColumnString::Offsets& result_offset, + NullMap& null_map) { const auto* str_col = check_and_get_column(argument_columns[0].get()); const auto* pattern_col = check_and_get_column(argument_columns[1].get()); const auto* replace_col = check_and_get_column(argument_columns[2].get()); @@ -132,13 +246,14 @@ struct RegexpReplaceOneImpl { StringOP::push_null_string(i, result_data, result_offset, null_map); continue; } - _execute_inner_loop(context, str_col, pattern_col, replace_col, result_data, - result_offset, null_map, i); + _execute_inner_loop(context, str_col, pattern_col, replace_col, options_value, + result_data, result_offset, null_map, i); } } static void execute_impl_const_args(FunctionContext* context, ColumnPtr argument_columns[], - size_t input_rows_count, ColumnString::Chars& result_data, + const StringRef& options_value, size_t input_rows_count, + ColumnString::Chars& result_data, ColumnString::Offsets& result_offset, NullMap& null_map) { const auto* str_col = check_and_get_column(argument_columns[0].get()); const auto* pattern_col = check_and_get_column(argument_columns[1].get()); @@ -149,14 +264,14 @@ struct RegexpReplaceOneImpl { StringOP::push_null_string(i, result_data, result_offset, null_map); continue; } - _execute_inner_loop(context, str_col, pattern_col, replace_col, result_data, - result_offset, null_map, i); + _execute_inner_loop(context, str_col, pattern_col, replace_col, options_value, + result_data, result_offset, null_map, i); } } template static void _execute_inner_loop(FunctionContext* context, const ColumnString* str_col, const ColumnString* pattern_col, - const ColumnString* replace_col, + const ColumnString* replace_col, const StringRef& options_value, ColumnString::Chars& result_data, ColumnString::Offsets& result_offset, NullMap& null_map, const size_t index_now) { @@ -166,7 +281,8 @@ struct RegexpReplaceOneImpl { if (re == nullptr) { std::string error_str; const auto& pattern = pattern_col->get_data_at(index_check_const(index_now, Const)); - bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), scoped_re); + bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), + options_value, scoped_re); if (!st) { context->add_warning(error_str.c_str()); StringOP::push_null_string(index_now, result_data, result_offset, null_map); @@ -250,7 +366,8 @@ struct RegexpExtractImpl { if (re == nullptr) { std::string error_str; const auto& pattern = pattern_col->get_data_at(index_check_const(index_now, Const)); - bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), scoped_re); + bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), StringRef(), + scoped_re); if (!st) { context->add_warning(error_str.c_str()); StringOP::push_null_string(index_now, result_data, result_offset, null_map); @@ -328,7 +445,8 @@ struct RegexpExtractAllImpl { if (re == nullptr) { std::string error_str; const auto& pattern = pattern_col->get_data_at(index_check_const(index_now, Const)); - bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), scoped_re); + bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), StringRef(), + scoped_re); if (!st) { context->add_warning(error_str.c_str()); StringOP::push_null_string(index_now, result_data, result_offset, null_map); @@ -417,8 +535,8 @@ class FunctionRegexpFunctionality : public IFunction { std::string error_str; std::unique_ptr scoped_re; - bool st = - StringFunctions::compile_regex(pattern, &error_str, StringRef(), scoped_re); + bool st = StringFunctions::compile_regex(pattern, &error_str, StringRef(), + StringRef(), scoped_re); if (!st) { context->set_error(error_str.c_str()); return Status::InvalidArgument(error_str); @@ -484,17 +602,15 @@ class FunctionRegexpFunctionality : public IFunction { ColumnNullable::create(std::move(result_data_column), std::move(result_null_map)); return Status::OK(); } - - Status close(FunctionContext* context, FunctionContext::FunctionStateScope scope) override { - return Status::OK(); - } }; void register_function_regexp_extract(SimpleFunctionFactory& factory) { - factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); + factory.register_function>(); factory.register_function>>(); factory.register_function>>(); - factory.register_function>(); factory.register_function>(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java index 8a12b8d7205df7..46c7285bd8bf66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplace.java @@ -18,12 +18,15 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.VarcharType; @@ -42,7 +45,12 @@ public class RegexpReplace extends ScalarFunction FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(StringType.INSTANCE) - .args(StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE) + .args(StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE), + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, + VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(StringType.INSTANCE) + .args(StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE) ); /** @@ -52,13 +60,44 @@ public RegexpReplace(Expression arg0, Expression arg1, Expression arg2) { super("regexp_replace", arg0, arg1, arg2); } + /** + * constructor with 4 arguments. + */ + public RegexpReplace(Expression arg0, Expression arg1, Expression arg2, Expression arg3) { + super("regexp_replace", arg0, arg1, arg2, arg3); + } + /** * withChildren. */ @Override public RegexpReplace withChildren(List children) { - Preconditions.checkArgument(children.size() == 3); - return new RegexpReplace(children.get(0), children.get(1), children.get(2)); + Preconditions.checkArgument(children.size() == 3 || children.size() == 4, + "RegexpReplace should have 3 or 4 children, but got: " + children.size()); + if (children.size() == 3) { + return new RegexpReplace(children.get(0), children.get(1), children.get(2)); + } else { + return new RegexpReplace(children.get(0), children.get(1), children.get(2), children.get(3)); + } + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (children().size() == 3) { + return; + } + if (children().size() == 4) { + Expression value = child(3); + DataType type = value.getDataType(); + if (!type.isStringLikeType()) { + throw new AnalysisException( + "The fourth param of regexp_replace must be a string type: " + this.toSql()); + } + if (!(value instanceof Literal)) { + throw new AnalysisException( + "The fourth param of regexp_replace must be a constant value: " + this.toSql()); + } + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java index f31cf84cfa18e7..552d92d63a5c49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/RegexpReplaceOne.java @@ -18,12 +18,15 @@ package org.apache.doris.nereids.trees.expressions.functions.scalar; import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral; +import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.VarcharType; @@ -42,7 +45,12 @@ public class RegexpReplaceOne extends ScalarFunction FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT), FunctionSignature.ret(StringType.INSTANCE) - .args(StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE) + .args(StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE), + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, + VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(StringType.INSTANCE) + .args(StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE, StringType.INSTANCE) ); /** @@ -52,13 +60,44 @@ public RegexpReplaceOne(Expression arg0, Expression arg1, Expression arg2) { super("regexp_replace_one", arg0, arg1, arg2); } + /** + * constructor with 4 arguments. + */ + public RegexpReplaceOne(Expression arg0, Expression arg1, Expression arg2, Expression arg3) { + super("regexp_replace_one", arg0, arg1, arg2, arg3); + } + /** * withChildren. */ @Override public RegexpReplaceOne withChildren(List children) { - Preconditions.checkArgument(children.size() == 3); - return new RegexpReplaceOne(children.get(0), children.get(1), children.get(2)); + Preconditions.checkArgument(children.size() == 3 || children.size() == 4, + "RegexpReplaceOne should have 3 or 4 children"); + if (children.size() == 3) { + return new RegexpReplaceOne(children.get(0), children.get(1), children.get(2)); + } else { + return new RegexpReplaceOne(children.get(0), children.get(1), children.get(2), children.get(3)); + } + } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (children().size() == 3) { + return; + } + if (children().size() == 4) { + Expression value = child(3); + DataType type = value.getDataType(); + if (!type.isStringLikeType()) { + throw new AnalysisException( + "The fourth param of regexp_replace_one must be a string type: " + this.toSql()); + } + if (!(value instanceof Literal)) { + throw new AnalysisException( + "The fourth param of regexp_replace_one must be a constant value: " + this.toSql()); + } + } } @Override From 2d1de5558df505c0d80bfcf1474373e79397bf94 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Fri, 20 Jun 2025 17:32:26 +0800 Subject: [PATCH 041/572] [bugfix](memleak) fix memleak in arrow input stream (#51929) (#51992) (#52039) pick #51929 --- be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp | 2 +- be/src/vec/exec/format/arrow/arrow_pip_input_stream.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp b/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp index 7bc32c7ab4bb91..1c4462befcc53b 100644 --- a/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp +++ b/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp @@ -32,7 +32,7 @@ namespace doris::vectorized { ArrowPipInputStream::ArrowPipInputStream(io::FileReaderSPtr file_reader) - : _file_reader(file_reader), _pos(0), _begin(true), _read_buf(new uint8_t[4]) { + : _file_reader(file_reader), _pos(0), _begin(true) { set_mode(arrow::io::FileMode::READ); } diff --git a/be/src/vec/exec/format/arrow/arrow_pip_input_stream.h b/be/src/vec/exec/format/arrow/arrow_pip_input_stream.h index fef4cf10903b4c..61276ab190534a 100644 --- a/be/src/vec/exec/format/arrow/arrow_pip_input_stream.h +++ b/be/src/vec/exec/format/arrow/arrow_pip_input_stream.h @@ -59,7 +59,8 @@ class ArrowPipInputStream : public arrow::io::InputStream { io::FileReaderSPtr _file_reader; int64_t _pos; bool _begin; - uint8_t* _read_buf; + // The read buf is very small, so use stack memory directly. + uint8_t _read_buf[4]; }; } // namespace vectorized From dc679ae9670e8d9fc53e89df74e216332379c635 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Mon, 23 Jun 2025 21:18:35 +0800 Subject: [PATCH 042/572] branch-3.0: [Opt](cloud-mow) Add more delete bitmap verbose log (#51751) (#52121) pick https://github.com/apache/doris/pull/51751 --- be/src/cloud/cloud_meta_mgr.cpp | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index e4054bdd885fd7..70d4c9f4b6fd02 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -579,6 +579,76 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, return st; } tablet->tablet_meta()->delete_bitmap().merge(delete_bitmap); + if (config::enable_mow_verbose_log && !resp.rowset_meta().empty() && + delete_bitmap.cardinality() > 0) { + std::vector new_rowset_msgs; + std::vector old_rowset_msgs; + std::unordered_set new_rowset_ids; + int64_t new_max_version = resp.rowset_meta().rbegin()->end_version(); + for (const auto& rs : resp.rowset_meta()) { + RowsetId rowset_id; + rowset_id.init(rs.rowset_id_v2()); + new_rowset_ids.insert(rowset_id); + DeleteBitmap rowset_dbm(tablet_id); + delete_bitmap.subset( + {rowset_id, 0, 0}, + {rowset_id, std::numeric_limits::max(), + std::numeric_limits::max()}, + &rowset_dbm); + size_t cardinality = rowset_dbm.cardinality(); + size_t count = rowset_dbm.get_delete_bitmap_count(); + if (cardinality > 0) { + new_rowset_msgs.push_back(fmt::format( + "({}[{}-{}],{},{})", rs.rowset_id_v2(), rs.start_version(), + rs.end_version(), count, cardinality)); + } + } + + if (old_max_version > 0) { + std::vector old_rowsets; + RowsetIdUnorderedSet old_rowset_ids; + { + std::lock_guard rlock(tablet->get_header_lock()); + RETURN_IF_ERROR( + tablet->get_all_rs_id_unlocked(old_max_version, &old_rowset_ids)); + old_rowsets = tablet->get_rowset_by_ids(&old_rowset_ids); + } + for (const auto& rs : old_rowsets) { + if (!new_rowset_ids.contains(rs->rowset_id())) { + DeleteBitmap rowset_dbm(tablet_id); + delete_bitmap.subset( + {rs->rowset_id(), 0, 0}, + {rs->rowset_id(), + std::numeric_limits::max(), + std::numeric_limits::max()}, + &rowset_dbm); + size_t cardinality = rowset_dbm.cardinality(); + size_t count = rowset_dbm.get_delete_bitmap_count(); + if (cardinality > 0) { + old_rowset_msgs.push_back( + fmt::format("({}{},{},{})", rs->rowset_id().to_string(), + rs->version().to_string(), count, cardinality)); + } + } + } + } + + LOG_INFO("[verbose] sync tablet delete bitmap") + .tag("tablet_id", tablet->tablet_id()) + .tag("table_id", tablet->table_id()) + .tag("full_sync", full_sync) + .tag("old_max_version", old_max_version) + .tag("new_max_version", new_max_version) + .tag("cumu_compaction_cnt", resp.stats().cumulative_compaction_cnt()) + .tag("base_compaction_cnt", resp.stats().base_compaction_cnt()) + .tag("cumu_point", resp.stats().cumulative_point()) + .tag("rowset_num", resp.rowset_meta().size()) + .tag("delete_bitmap_cardinality", delete_bitmap.cardinality()) + .tag("old_rowsets(rowset,count,cardinality)", + fmt::format("[{}]", fmt::join(old_rowset_msgs, ", "))) + .tag("new_rowsets(rowset,count,cardinality)", + fmt::format("[{}]", fmt::join(new_rowset_msgs, ", "))); + } } DBUG_EXECUTE_IF("CloudMetaMgr::sync_tablet_rowsets.before.modify_tablet_meta", { auto target_tablet_id = dp->param("tablet_id", -1); From 3327cefa82976d8611735c8ba0428f49ecc4185b Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Mon, 23 Jun 2025 21:33:39 +0800 Subject: [PATCH 043/572] [fix](restore) Analyze idx meta in Olap Table after restore commit (#51907) reproduce: backup and restore Table or restart fe, then insert before: ``` desc tbl all *************************** 26. row *************************** IndexName: idx IndexKeysType: AGG_KEYS Field: mv_date_format(hours_add(happen_time, 1), '%Y-%m-%d %H:00:00') Type: varchar(65533) InternalType: varchar(65533) Null: Yes Key: true Default: NULL Extra: Visible: true DefineExpr: WhereClause: insert into wrong [ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = column has no source field, column=mva_SUM__CASE WHEN 1 IS NULL THEN 0 ELSE 1 END ``` now ``` desc tbl all *************************** 26. row *************************** IndexName: idx IndexKeysType: AGG_KEYS Field: mv_date_format(hours_add(happen_time, 1), '%Y-%m-%d %H:00:00') Type: varchar(65533) InternalType: varchar(65533) Null: Yes Key: true Default: NULL Extra: Visible: true DefineExpr: date_format(hours_add(`happen_time`, 1), '%Y-%m-%d %H:00:00') WhereClause: (`k1` > '2025-06-12 00:00:00') insert into ok ``` --- .../org/apache/doris/backup/RestoreJob.java | 3 + .../test_backup_restore_mv_write.groovy | 118 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 83569e10eabfaa..df5ea9b78f7d79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -2657,6 +2657,9 @@ private void setTableStateToNormalAndUpdateProperties(Database db, boolean commi if (committed && isBeingSynced) { olapTbl.setBeingSyncedProperties(); } + if (committed) { + olapTbl.analyze(db.getName()); + } } finally { tbl.writeUnlock(); } diff --git a/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy b/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy new file mode 100644 index 00000000000000..c9e9bddb6a1148 --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_mv_write", "backup_restore") { + String suiteName = "test_backup_restore_mv_write" + String dbName = context.dbName + String repoName = "${suiteName}_repo_" + UUID.randomUUID().toString().replace("-", "") + String snapshotName = "snapshot_${suiteName}" + String tableNamePrefix = "tbl_${suiteName}" + String viewName = "mv_${tableNamePrefix}" + + def syncer = getSyncer() + syncer.createS3Repository(repoName) + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + + sql "DROP TABLE IF EXISTS ${tableNamePrefix}" + sql "DROP materialized VIEW IF EXISTS ${viewName}" + + sql """ + CREATE TABLE `${tableNamePrefix}` ( + `k` int NOT NULL, + `k1` datetime NOT NULL, + `k2` datetime NOT NULL, + `vin` varchar(128) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k`, `k1`, `k2`, `vin`) + DISTRIBUTED BY HASH(`k`) BUCKETS 5 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ + CREATE materialized VIEW ${viewName} AS SELECT + DATE_FORMAT( date_add( k2, INTERVAL 1 HOUR ), '%Y-%m-%d %H:00:00' ) AS k2, + vin, + count( 1 ) AS val + FROM + ${tableNamePrefix} + WHERE + k1 > '2025-06-12 00:00:00' + GROUP BY + DATE_FORMAT( date_add( k2, INTERVAL 1 HOUR ), '%Y-%m-%d %H:00:00' ), + vin + """ + + sql """ + BACKUP SNAPSHOT ${snapshotName} + TO `${repoName}` + ON (`${tableNamePrefix}`) + PROPERTIES ("type" = "full") + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + def res0 = sql "desc ${tableNamePrefix} all" + + sql "DROP TABLE IF EXISTS ${tableNamePrefix}" + + sql """ + RESTORE SNAPSHOT ${snapshotName} + FROM `${repoName}` + ON ( `${tableNamePrefix}` ) + PROPERTIES + ( + "backup_timestamp"="${snapshot}", + "replication_num" = "1" + ); + """ + + syncer.waitAllRestoreFinish(dbName) + + def res = sql "desc ${tableNamePrefix} all" + + // DefineExpr and WhereClause should not be null after restore + // desc table all: + // IndexName: mv_t + // IndexKeysType: AGG_KEYS + // Field: mv_date_format(hours_add(k2, 1), '%Y-%m-%d %H:00:00') + // Type: varchar(65533) + // InternalType: varchar(65533) + // Null: Yes + // Key: true + // Default: NULL + // Extra: + // Visible: true + // DefineExpr: date_format(hours_add(`k2`, 1), '%Y-%m-%d %H:00:00') + // WhereClause: (`k1` > '2025-06-12 00:00:00') + + assertTrue(res0.toString() == res.toString()) + + sql "insert into ${tableNamePrefix} values (1, '2025-06-12 01:00:00', '2025-06-12 02:00:00', 'test_vin_0')" + sql "insert into ${tableNamePrefix} values (2, '2025-06-13 02:00:00', '2025-06-13 03:00:00', 'test_vin_1')" + sql "insert into ${tableNamePrefix} values (3, '2025-06-14 03:00:00', '2025-06-14 04:00:00', 'test_vin_2')" + + def select_res = sql "select * from ${tableNamePrefix}" + assertTrue(select_res.size() == 3) + + sql "DROP REPOSITORY `${repoName}`" +} From a1d5291c2a091984b57750b3c5e7ebbf6e85205d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:52:42 +0800 Subject: [PATCH 044/572] branch-3.0: [chore](storage vault) Make CacheHotspotManager exception msg more distinct #51861 (#52136) Cherry-picked from #51861 Co-authored-by: Lei Zhang --- .../java/org/apache/doris/cloud/CacheHotspotManager.java | 6 +++++- .../org/apache/doris/cloud/CacheHotspotManagerUtils.java | 4 ++++ .../org/apache/doris/common/util/PropertyAnalyzer.java | 2 +- .../suites/vault_p0/alter/test_alter_s3_vault.groovy | 5 ++++- .../concurent/test_create_vault_concurrently.groovy | 8 ++++---- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index a05518a6ee20a6..1e86fda15db9af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -124,12 +124,16 @@ public void runAfterCatalogReady() { jobDaemon.start(); startJobDaemon = true; } + if (!tableCreated) { try { CacheHotspotManagerUtils.execCreateCacheTable(); tableCreated = true; + this.intervalMs = Config.fetch_cluster_cache_hotspot_interval_ms; } catch (Exception e) { - LOG.warn("Create cache hot spot table failed", e); + // sleep 60s wait for syncing storage vault info from ms and retry + this.intervalMs = 60000; + LOG.warn("Create cache hot spot table failed, sleep 60s and retry", e); return; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java index ed2213e1208477..cd295d9707b512 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManagerUtils.java @@ -216,11 +216,15 @@ public static void execCreateCacheTable() throws Exception { Database db = Env.getCurrentInternalCatalog().getDbNullable(FeConstants.INTERNAL_DB_NAME); if (db == null) { LOG.warn("{} database doesn't exist", FeConstants.INTERNAL_DB_NAME); + throw new Exception( + String.format("Database %s doesn't exist", FeConstants.INTERNAL_DB_NAME)); } Table t = db.getTableNullable(FeConstants.INTERNAL_FILE_CACHE_HOTSPOT_TABLE_NAME); if (t == null) { LOG.warn("{} table doesn't exist", FeConstants.INTERNAL_FILE_CACHE_HOTSPOT_TABLE_NAME); + throw new Exception( + String.format("Table %s doesn't exist", FeConstants.INTERNAL_FILE_CACHE_HOTSPOT_TABLE_NAME)); } INTERNAL_TABLE_ID = String.valueOf(t.getId()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 1c12d2b106bace..b873cd8e7cf786 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -1236,7 +1236,7 @@ public static Pair analyzeStorageVault(Map prope } else { // continue to check default vault Pair info = Env.getCurrentEnv().getStorageVaultMgr().getDefaultStorageVault(); - if (info == null) { + if (info == null || Strings.isNullOrEmpty(info.first) || Strings.isNullOrEmpty(info.second)) { throw new AnalysisException("No default storage vault." + " You can use `SHOW STORAGE VAULT` to get all available vaults," + " and pick one set default vault with `SET AS DEFAULT STORAGE VAULT`"); diff --git a/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy b/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy index b9d6d1975cf8f6..f20bdafef57592 100644 --- a/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy +++ b/regression-test/suites/vault_p0/alter/test_alter_s3_vault.groovy @@ -203,8 +203,11 @@ suite("test_alter_s3_vault", "nonConcurrent") { for (int i = 0; i < vaultInfos.size(); i++) { logger.info("vault info: ${vaultInfos[i]}") if (vaultInfos[i][0].equals(s3VaultName)) { + // [s3_9f2c2f80a45c4fb48ad901f70675ffdf, 8, ctime: 1750217403 mtime: 1750217404 ak: "*******" sk: "xxxxxxx" bucket: "xxx" ...] def newProperties = vaultInfos[i][2] - assertTrue(properties.equals(newProperties), "Properties are not the same") + def oldAkSkStr = properties.substring(properties.indexOf("ak:") + 3, properties.indexOf("bucket:") - 1) + def newAkSkStr = newProperties.substring(newProperties.indexOf("ak:") + 3, newProperties.indexOf("bucket:") - 1) + assertTrue(oldAkSkStr.equals(newAkSkStr), "Ak and Sk string are not the same") } } diff --git a/regression-test/suites/vault_p0/concurent/test_create_vault_concurrently.groovy b/regression-test/suites/vault_p0/concurent/test_create_vault_concurrently.groovy index 985bf971e3a26b..86b01474c7d21c 100644 --- a/regression-test/suites/vault_p0/concurent/test_create_vault_concurrently.groovy +++ b/regression-test/suites/vault_p0/concurent/test_create_vault_concurrently.groovy @@ -35,7 +35,7 @@ suite("test_create_vault_concurrently", "nonConcurrent") { def future1 = thread("threadName1") { for (int i = 0; i < 100; i++) { - sql """ + try_sql """ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName} PROPERTIES ( "type"="S3", @@ -55,7 +55,7 @@ suite("test_create_vault_concurrently", "nonConcurrent") { def future2 = thread("threadName2") { for (int i = 0; i < 100; i++) { - sql """ + try_sql """ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName} PROPERTIES ( "type"="S3", @@ -75,7 +75,7 @@ suite("test_create_vault_concurrently", "nonConcurrent") { def future3 = thread("threadName3") { for (int i = 0; i < 100; i++) { - sql """ + try_sql """ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName} PROPERTIES ( "type"="S3", @@ -95,7 +95,7 @@ suite("test_create_vault_concurrently", "nonConcurrent") { def future4 = thread("threadName4") { for (int i = 0; i < 100; i++) { - sql """ + try_sql """ CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName} PROPERTIES ( "type"="S3", From 1a888e5d98c3e45de4d347af2d73c7fc04cecc40 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:54:21 +0800 Subject: [PATCH 045/572] branch-3.0: [chore](cloud) Implement idempotent injection framework for meta-service #51905 (#52102) Cherry-picked from #51905 Co-authored-by: Gavin Chou --- cloud/src/common/config.h | 8 +++ cloud/src/meta-service/meta_service.h | 52 ++++++++++++++++++- .../meta-service/meta_service_resource.cpp | 7 --- 3 files changed, 59 insertions(+), 8 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 23bb4f74b6a4e3..7dc63683688d03 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -35,6 +35,14 @@ CONF_Bool(use_mem_kv, "false"); CONF_Int32(meta_server_register_interval_ms, "20000"); CONF_Int32(meta_server_lease_ms, "60000"); +// for chaos testing +CONF_mBool(enable_idempotent_request_injection, "false"); +// idempotent_request_replay_delay_ms = idempotent_request_replay_delay_base_ms + random(-idempotent_request_replay_delay_range_ms, idempotent_request_replay_delay_range_ms) +CONF_mInt64(idempotent_request_replay_delay_base_ms, "10000"); +CONF_mInt64(idempotent_request_replay_delay_range_ms, "5000"); +// exclude some request that are meaningless to replay, comma separated list. e.g. GetTabletStatsRequest,GetVersionRequest +CONF_mString(idempotent_request_replay_exclusion, "GetTabletStatsRequest,GetVersionRequest"); + CONF_Int64(fdb_txn_timeout_ms, "10000"); CONF_Int64(brpc_max_body_size, "3147483648"); CONF_Int64(brpc_socket_max_unwritten_bytes, "1073741824"); diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 57f88d51dfe904..4be017edc9a386 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -46,6 +46,14 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, const std::string& instance_id, int64_t tablet_id, MetaServiceCode& code, std::string& msg, GetRowsetResponse* response); +// for wrapping stateful lambda to run in bthread +static void* run_bthread_work(void* arg) { + auto f = reinterpret_cast*>(arg); + (*f)(); + delete f; + return nullptr; +} + class MetaServiceImpl : public cloud::MetaService { public: MetaServiceImpl(std::shared_ptr txn_kv, std::shared_ptr resource_mgr, @@ -720,8 +728,12 @@ class MetaServiceProxy final : public MetaService { static_assert(std::is_base_of_v<::google::protobuf::Message, Response>); using namespace std::chrono; - brpc::ClosureGuard done_guard(done); + + // life span of this defer MUST be longer than `done` + std::unique_ptr> defer_injection( + (int*)(0x01), [&, this](int*) { idempotent_injection(method, req, resp); }); + if (!config::enable_txn_store_retry) { (impl_.get()->*method)(ctrl, req, resp, brpc::DoNothing()); if (DCHECK_IS_ON()) { @@ -791,6 +803,44 @@ class MetaServiceProxy final : public MetaService { } } + template + void idempotent_injection(MetaServiceMethod method, const Request* requ, + Response* resp) { + if (!config::enable_idempotent_request_injection) return; + + using namespace std::chrono; + auto s = system_clock::now(); + static std::mt19937_64 rng(duration_cast(s.time_since_epoch()).count()); + // clang-format off + // FIXME(gavin): make idempotent_request_replay_exclusion configurable via HTTP + static auto exclusion = []{ std::istringstream iss(config::idempotent_request_replay_exclusion); std::string e; std::unordered_set r; + while (std::getline(iss, e, ',')) { r.insert(e); } return r; + }(); + auto f = new std::function([s, req = *requ, res = *resp, method, this]() mutable { // copy and capture + auto dist = std::uniform_int_distribution(-config::idempotent_request_replay_delay_range_ms, + config::idempotent_request_replay_delay_range_ms); + int64_t sleep_ms = config::idempotent_request_replay_delay_base_ms + dist(rng); + LOG(INFO) << " request_name=" << req.GetDescriptor()->name() + << " response_name=" << res.GetDescriptor()->name() + << " queue_ts=" << duration_cast(s.time_since_epoch()).count() + << " now_ts=" << duration_cast(system_clock::now().time_since_epoch()).count() + << " idempotent_request_replay_delay_base_ms=" << config::idempotent_request_replay_delay_base_ms + << " idempotent_request_replay_delay_range_ms=" << config::idempotent_request_replay_delay_range_ms + << " idempotent_request_replay_delay_ms=" << sleep_ms + << " request=" << req.ShortDebugString(); + if (sleep_ms < 0 || exclusion.count(req.GetDescriptor()->name())) return; + brpc::Controller ctrl; + bthread_usleep(sleep_ms * 1000); + (impl_.get()->*method)(&ctrl, &req, &res, brpc::DoNothing()); + }); + // clang-format on + bthread_t bid; + if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { + LOG(WARNING) << "failed to bthread_start_background, run in current thread"; + run_bthread_work(f); + } + } + std::unique_ptr impl_; }; diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index bbd94b577b1bab..8c8b0646c94cdd 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -51,13 +51,6 @@ bool is_valid_storage_vault_name(const std::string& str) { namespace doris::cloud { -static void* run_bthread_work(void* arg) { - auto f = reinterpret_cast*>(arg); - (*f)(); - delete f; - return nullptr; -} - static std::string_view print_cluster_status(const ClusterStatus& status) { switch (status) { case ClusterStatus::UNKNOWN: From f0d6f778f99bb5446e68864666118781b9714ed8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:55:16 +0800 Subject: [PATCH 046/572] branch-3.0: [fix](meta) Do not copy it when input rowset meta dont have load id #52088 (#52096) Cherry-picked from #52088 Co-authored-by: Siyang Tang --- be/src/cloud/pb_convert.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp index 0d24192a758ae7..2883fb51d0f214 100644 --- a/be/src/cloud/pb_convert.cpp +++ b/be/src/cloud/pb_convert.cpp @@ -213,7 +213,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in, out->mutable_delete_predicate()->CopyFrom(in.delete_predicate()); } out->set_empty(in.empty()); - out->mutable_load_id()->CopyFrom(in.load_id()); + if (in.has_load_id()) { + out->mutable_load_id()->CopyFrom(in.load_id()); + } out->set_delete_flag(in.delete_flag()); out->set_creation_time(in.creation_time()); if (in.has_tablet_uid()) { @@ -268,7 +270,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in, out->mutable_delete_predicate()->Swap(in.mutable_delete_predicate()); } out->set_empty(in.empty()); - out->mutable_load_id()->CopyFrom(in.load_id()); + if (in.has_load_id()) { + out->mutable_load_id()->CopyFrom(in.load_id()); + } out->set_delete_flag(in.delete_flag()); out->set_creation_time(in.creation_time()); if (in.has_tablet_uid()) { From 15e71e6da3cd115cd4ae037a691548ac74bd0f3e Mon Sep 17 00:00:00 2001 From: lihangyu Date: Mon, 23 Jun 2025 21:57:38 +0800 Subject: [PATCH 047/572] Branch-3.0 [regression-test](point_query_p0) fix .out file (#52034) --- regression-test/data/point_query_p0/test_point_query.out | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/regression-test/data/point_query_p0/test_point_query.out b/regression-test/data/point_query_p0/test_point_query.out index 60aff86a589c0a..72dc8c310f137b 100644 --- a/regression-test/data/point_query_p0/test_point_query.out +++ b/regression-test/data/point_query_p0/test_point_query.out @@ -158,7 +158,7 @@ -- !sql -- -- !sql -- -10 20 aabc +10 20 aabc -- !sql -- -10 20 aabc update val @@ -203,4 +203,5 @@ user_guid feature sk feature_value 2021-01-01T00:00 4 -- !sql -- -v1 \ No newline at end of file +v1 + From 7eceebb7d7ec0e03d78eaa8be04100a966327118 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:58:57 +0800 Subject: [PATCH 048/572] branch-3.0: [Feature](recycler) Add recycler metrics for recycler layer #51409 (#51884) Cherry-picked from #51409 Co-authored-by: Uniqueyou --- cloud/src/common/bvars.cpp | 13 ++++ cloud/src/common/bvars.h | 107 ++++++++++++++++++++++++++++++++ cloud/src/main.cpp | 2 + cloud/src/recycler/recycler.cpp | 25 +++++++- 4 files changed, 144 insertions(+), 3 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index fe887760f7f32d..23e140d3c2d820 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -17,6 +17,10 @@ #include "common/bvars.h" +#include +#include +#include + #include #include @@ -98,6 +102,15 @@ BvarStatusWithTag g_bvar_recycler_recycle_partition_earlest_ts("recycle BvarStatusWithTag g_bvar_recycler_recycle_rowset_earlest_ts("recycler", "recycle_rowset_earlest_ts"); BvarStatusWithTag g_bvar_recycler_recycle_tmp_rowset_earlest_ts("recycler", "recycle_tmp_rowset_earlest_ts"); BvarStatusWithTag g_bvar_recycler_recycle_expired_txn_label_earlest_ts("recycler", "recycle_expired_txn_label_earlest_ts"); +bvar::Status g_bvar_recycler_task_max_concurrency("recycler_task_max_concurrency_num",0); +bvar::Adder g_bvar_recycler_task_concurrency; + +// recycler's mbvars +mBvarIntAdder g_bvar_recycler_instance_running("recycler_instance_running",{"instance_id"}); +mBvarLongStatus g_bvar_recycler_instance_last_recycle_duration("recycler_instance_last_recycle_duration_ms",{"instance_id"}); +mBvarLongStatus g_bvar_recycler_instance_next_time("recycler_instance_next_time_s",{"instance_id"}); +mBvarPairStatus g_bvar_recycler_instance_recycle_times("recycler_instance_recycle_times",{"instance_id"}); +mBvarLongStatus g_bvar_recycler_instance_recycle_last_success_times("recycler_instance_recycle_last_success_times",{"instance_id"}); // txn_kv's bvars bvar::LatencyRecorder g_bvar_txn_kv_get("txn_kv", "get"); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 7f616615394781..455d8caf45a1d4 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include #include @@ -27,6 +29,7 @@ #include #include #include +#include /** * Manage bvars that with similar names (identical prefix) @@ -97,6 +100,102 @@ template requires std::is_integral_v using BvarStatusWithTag = BvarWithTag, true>; +/** +@brief: A wrapper class for multidimensional bvar metrics. +This template class provides a convenient interface for managing multidimensional +bvar metrics. It supports various bvar types including Adder, IntRecorder, +LatencyRecorder, Maxer, and Status. +@param: BvarType The type of bvar metric to use (must be one of the supported types) +@output: Based on the bvar multidimensional counter implementation, +the metrics output format would typically follow this structure: +{metric_name}{dimension1="value1",dimension2="value2",...} value +@example: Basic usage with an Adder: +// Create a 2-dimensional counter with dimensions "region" and "service" +mBvarWrapper> request_counter("xxx_request_count", {"region", "service"}); +// Increment the counter for specific dimension values +request_counter.put({"east", "login"}, 1); +request_counter.put({"west", "search"}, 1); +request_counter.put({"east", "login"}, 1); // Now east/login has value 2 +// the output of above metrics: +xxx_request_count{region="east",service="login"} 2 +xxx_request_count{region="west",service="search"} 1 +@note: The dimensions provided in the constructor and the values provided to +put() and get() methods must match in count. Also, all supported bvar types +have different behaviors for how values are processed and retrieved. +*/ +template +class mBvarWrapper { +public: + mBvarWrapper(const std::string& metric_name, + const std::initializer_list& dim_names) + : counter_(metric_name, std::list(dim_names)) { + static_assert(is_valid_bvar_type::value, + "BvarType must be one of the supported bvar types (Adder, IntRecorder, " + "LatencyRecorder, Maxer, Status)"); + } + + template + void put(const std::initializer_list& dim_values, ValType value) { + BvarType* stats = counter_.get_stats(std::list(dim_values)); + if (stats) { + if constexpr (std::is_same_v> || + std::is_same_v> || + is_pair_status::value) { + stats->set_value(value); + } else { + *stats << value; + } + } + } + + auto get(const std::initializer_list& dim_values) { + BvarType* stats = counter_.get_stats(std::list(dim_values)); + if (stats) { + return stats->get_value(); + } + return std::declval(0); + } + +private: + template + struct is_valid_bvar_type : std::false_type {}; + template + struct is_pair_status : std::false_type {}; + template + struct is_valid_bvar_type> : std::true_type {}; + template <> + struct is_valid_bvar_type : std::true_type {}; + template + struct is_valid_bvar_type> : std::true_type {}; + template + struct is_valid_bvar_type> : std::true_type {}; + template + struct is_pair_status>> : std::true_type {}; + template <> + struct is_valid_bvar_type : std::true_type {}; + + bvar::MultiDimension counter_; +}; + +using mBvarIntAdder = mBvarWrapper>; +using mBvarDoubleAdder = mBvarWrapper>; +using mBvarIntRecorder = mBvarWrapper; +using mBvarLatencyRecorder = mBvarWrapper; +using mBvarIntMaxer = mBvarWrapper>; +using mBvarDoubleMaxer = mBvarWrapper>; +using mBvarLongStatus = mBvarWrapper>; +using mBvarDoubleStatus = mBvarWrapper>; + +namespace std { +template +inline std::ostream& operator<<(std::ostream& os, const std::pair& p) { + return os << "{" << p.first << "," << p.second << "}"; +} +} // namespace std + +template +using mBvarPairStatus = mBvarWrapper>>; + // meta-service's bvars extern BvarLatencyRecorderWithTag g_bvar_ms_begin_txn; extern BvarLatencyRecorderWithTag g_bvar_ms_precommit_txn; @@ -171,6 +270,14 @@ extern BvarStatusWithTag g_bvar_recycler_recycle_rowset_earlest_ts; extern BvarStatusWithTag g_bvar_recycler_recycle_tmp_rowset_earlest_ts; extern BvarStatusWithTag g_bvar_recycler_recycle_expired_txn_label_earlest_ts; +extern bvar::Status g_bvar_recycler_task_max_concurrency; +extern bvar::Adder g_bvar_recycler_task_concurrency; +extern mBvarIntAdder g_bvar_recycler_instance_running; +extern mBvarLongStatus g_bvar_recycler_instance_last_recycle_duration; +extern mBvarLongStatus g_bvar_recycler_instance_next_time; +extern mBvarPairStatus g_bvar_recycler_instance_recycle_times; +extern mBvarLongStatus g_bvar_recycler_instance_recycle_last_success_times; + // txn_kv's bvars extern bvar::LatencyRecorder g_bvar_txn_kv_get; extern bvar::LatencyRecorder g_bvar_txn_kv_range_get; diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 9115158743f20c..18cf98720e9cb0 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -236,6 +236,8 @@ int main(int argc, char** argv) { std::cout << "try to start meta_service, recycler" << std::endl; } + google::SetCommandLineOption("bvar_max_dump_multi_dimension_metric_number", "2000"); + brpc::Server server; brpc::FLAGS_max_body_size = config::brpc_max_body_size; brpc::FLAGS_socket_max_unwritten_bytes = config::brpc_socket_max_unwritten_bytes; diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index f9197a9101766c..78b6ec3cf6a2c7 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -17,8 +17,10 @@ #include "recycler/recycler.h" +#include #include #include +#include #include #include @@ -27,9 +29,11 @@ #include #include #include +#include #include #include #include +#include #include "common/stopwatch.h" #include "meta-service/meta_service.h" @@ -275,7 +279,12 @@ void Recycler::recycle_callback() { if (stopped()) return; LOG_INFO("begin to recycle instance").tag("instance_id", instance_id); auto ctime_ms = duration_cast(system_clock::now().time_since_epoch()).count(); + g_bvar_recycler_task_concurrency << 1; + g_bvar_recycler_instance_running.put({instance_id}, 1); + g_bvar_recycler_instance_recycle_times.put({instance_id}, std::make_pair(ctime_ms, -1)); ret = instance_recycler->do_recycle(); + g_bvar_recycler_task_concurrency << -1; + g_bvar_recycler_instance_running.put({instance_id}, -1); // If instance recycler has been aborted, don't finish this job if (!instance_recycler->stopped()) { finish_instance_recycle_job(txn_kv_.get(), recycle_job_key, instance_id, ip_port_, @@ -285,9 +294,18 @@ void Recycler::recycle_callback() { std::lock_guard lock(mtx_); recycling_instance_map_.erase(instance_id); } - auto elpased_ms = - duration_cast(system_clock::now().time_since_epoch()).count() - - ctime_ms; + auto now = duration_cast(system_clock::now().time_since_epoch()).count(); + auto elpased_ms = now - ctime_ms; + g_bvar_recycler_instance_recycle_times.put({instance_id}, std::make_pair(ctime_ms, now)); + g_bvar_recycler_instance_last_recycle_duration.put({instance_id}, elpased_ms); + g_bvar_recycler_instance_next_time.put({instance_id}, + now + config::recycle_interval_seconds * 1000); + LOG(INFO) << "recycle instance done, " + << "instance_id=" << instance_id << " ret=" << ret << " ctime_ms: " << ctime_ms + << " now: " << now; + + g_bvar_recycler_instance_recycle_last_success_times.put({instance_id}, now); + LOG_INFO("finish recycle instance") .tag("instance_id", instance_id) .tag("cost_ms", elpased_ms); @@ -344,6 +362,7 @@ void Recycler::check_recycle_tasks() { int Recycler::start(brpc::Server* server) { instance_filter_.reset(config::recycle_whitelist, config::recycle_blacklist); + g_bvar_recycler_task_max_concurrency.set_value(config::recycle_concurrency); if (config::enable_checker) { checker_ = std::make_unique(txn_kv_); From 61acb985d5edefcbf8a6d5248eb639938de7b4be Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 22:00:13 +0800 Subject: [PATCH 049/572] branch-3.0: [fix](group commit) add bvar for group commit block by memory counter #51851 (#52008) Cherry-picked from #51851 Co-authored-by: meiyi --- be/src/runtime/group_commit_mgr.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp index 70345acdfb76b6..553da51a96bf69 100644 --- a/be/src/runtime/group_commit_mgr.cpp +++ b/be/src/runtime/group_commit_mgr.cpp @@ -35,6 +35,8 @@ namespace doris { +bvar::Adder group_commit_block_by_memory_counter("group_commit_block_by_memory_counter"); + std::string LoadBlockQueue::_get_load_ids() { std::stringstream ss; ss << "["; @@ -80,6 +82,7 @@ Status LoadBlockQueue::add_block(RuntimeState* runtime_state, if (!runtime_state->is_cancelled() && status.ok() && _all_block_queues_bytes->load(std::memory_order_relaxed) >= config::group_commit_queue_mem_limit) { + group_commit_block_by_memory_counter << 1; DCHECK(_load_ids_to_write_dep.find(load_id) != _load_ids_to_write_dep.end()); _load_ids_to_write_dep[load_id]->block(); VLOG_DEBUG << "block add_block for load_id=" << load_id From c051c9d7faa06b0f3a09fae72caa114ee116302e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 22:01:01 +0800 Subject: [PATCH 050/572] branch-3.0: [chore](regression-test) move test_cold_data_compaction to `nonConcurrent` #50699 (#52066) Cherry-picked from #50699 Co-authored-by: yagagagaga --- .../cold_data_compaction.groovy | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy b/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy index bf9e33e7528759..c80e39b8a9d654 100644 --- a/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy +++ b/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy @@ -18,7 +18,7 @@ import com.amazonaws.services.s3.model.ListObjectsRequest import java.util.function.Supplier -suite("test_cold_data_compaction") { +suite("test_cold_data_compaction", "nonConcurrent") { def retryUntilTimeout = { int timeoutSecond, Supplier closure -> long start = System.currentTimeMillis() while (true) { @@ -87,7 +87,7 @@ suite("test_cold_data_compaction") { """ // wait until files upload to S3 - retryUntilTimeout(1800, { + retryUntilTimeout(3600, { def res = sql_return_maparray "show data from t_recycle_in_s3" String size = "" String remoteSize = "" @@ -112,9 +112,13 @@ suite("test_cold_data_compaction") { // trigger cold data compaction sql """alter table t_recycle_in_s3 set ("disable_auto_compaction" = "false")""" + def v = get_be_param("disable_auto_compaction").values().toArray()[0].toString() + if ("true" == v) { + set_be_param("disable_auto_compaction", "false") + } // wait until compaction finish - retryUntilTimeout(1800, { + retryUntilTimeout(3600, { def filesAfterCompaction = getS3Client().listObjects( new ListObjectsRequest().withBucketName(getS3BucketName()).withPrefix(s3Prefix+ "/data/${tabletId}")).getObjectSummaries() logger.info("t_recycle_in_s3's remote file number is ${filesAfterCompaction.size()}") @@ -122,8 +126,12 @@ suite("test_cold_data_compaction") { return filesAfterCompaction.size() == 2 }) + if ("true" == v) { + set_be_param("disable_auto_compaction", "true") + } + sql "drop table t_recycle_in_s3 force" - retryUntilTimeout(1800, { + retryUntilTimeout(3600, { def filesAfterDrop = getS3Client().listObjects( new ListObjectsRequest().withBucketName(getS3BucketName()).withPrefix(s3Prefix+ "/data/${tabletId}")).getObjectSummaries() logger.info("after drop t_recycle_in_s3, remote file number is ${filesAfterDrop.size()}") From 3310a8a5c3da61ec08b3d7a4ceac2163eb6b1425 Mon Sep 17 00:00:00 2001 From: camby <104178625@qq.com> Date: Mon, 23 Jun 2025 22:02:19 +0800 Subject: [PATCH 051/572] [opt](metrics) add metrics pipeline_task_queue_size (#51878) (#52139) ### What problem does this PR solve? pick https://github.com/apache/doris/pull/51878 to branch-3.0 --- be/src/pipeline/task_queue.cpp | 5 ++++- be/src/util/doris_metrics.cpp | 3 +++ be/src/util/doris_metrics.h | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/pipeline/task_queue.cpp b/be/src/pipeline/task_queue.cpp index f59707d872f297..b91b74dfdef9ba 100644 --- a/be/src/pipeline/task_queue.cpp +++ b/be/src/pipeline/task_queue.cpp @@ -53,6 +53,7 @@ void PriorityTaskQueue::close() { std::unique_lock lock(_work_size_mutex); _closed = true; _wait_task.notify_all(); + DorisMetrics::instance()->pipeline_task_queue_size->increment(-_total_task_size); } PipelineTask* PriorityTaskQueue::_try_take_unprotected(bool is_steal) { @@ -78,6 +79,7 @@ PipelineTask* PriorityTaskQueue::_try_take_unprotected(bool is_steal) { if (task) { task->update_queue_level(level); _total_task_size--; + DorisMetrics::instance()->pipeline_task_queue_size->increment(-1); } return task; } @@ -127,6 +129,7 @@ Status PriorityTaskQueue::push(PipelineTask* task) { _sub_queues[level].push_back(task); _total_task_size++; + DorisMetrics::instance()->pipeline_task_queue_size->increment(1); _wait_task.notify_one(); return Status::OK(); } @@ -217,4 +220,4 @@ void MultiCoreTaskQueue::update_statistics(PipelineTask* task, int64_t time_spen time_spent); } -} // namespace doris::pipeline \ No newline at end of file +} // namespace doris::pipeline diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index 8ec906cb630053..e7e700bfaf3b3c 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -227,6 +227,7 @@ DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(query_ctx_cnt, MetricUnit::NOUNIT); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(scanner_ctx_cnt, MetricUnit::NOUNIT); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(scanner_cnt, MetricUnit::NOUNIT); DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(scanner_task_cnt, MetricUnit::NOUNIT); +DEFINE_COUNTER_METRIC_PROTOTYPE_2ARG(pipeline_task_queue_size, MetricUnit::NOUNIT); const std::string DorisMetrics::_s_registry_name = "doris_be"; const std::string DorisMetrics::_s_hook_name = "doris_metrics"; @@ -373,6 +374,8 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_ctx_cnt); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_cnt); INT_COUNTER_METRIC_REGISTER(_server_metric_entity, scanner_task_cnt); + + INT_COUNTER_METRIC_REGISTER(_server_metric_entity, pipeline_task_queue_size); } void DorisMetrics::initialize(bool init_system_metrics, const std::set& disk_devices, diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index 5a4bef95a85862..d95eee6800e9f7 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -246,6 +246,7 @@ class DorisMetrics { IntCounter* scanner_ctx_cnt = nullptr; IntCounter* scanner_cnt = nullptr; IntCounter* scanner_task_cnt = nullptr; + IntCounter* pipeline_task_queue_size = nullptr; static DorisMetrics* instance() { static DorisMetrics instance; From 7dbe24412f5b56ebb0862560bfbb06719c852e1c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 22:04:26 +0800 Subject: [PATCH 052/572] branch-3.0: [fix](load) print real reason if fetching Kafka meta fail #51880 (#52058) Cherry-picked from #51880 Co-authored-by: hui lai --- .../org/apache/doris/datasource/kafka/KafkaUtil.java | 10 ++++++++-- .../routine_load/test_routine_load_error.groovy | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java index 3e78ba0d4a5265..a097f052aa9dea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/kafka/KafkaUtil.java @@ -231,6 +231,7 @@ private static InternalService.PProxyResult getInfoRequest(InternalService.PProx InternalService.PProxyResult result = null; Set failedBeIds = new HashSet<>(); TStatusCode code = null; + String errorMsg = null; try { while (retryTimes < 3) { @@ -257,7 +258,10 @@ private static InternalService.PProxyResult getInfoRequest(InternalService.PProx } if (backendIds.isEmpty()) { MetricRepo.COUNTER_ROUTINE_LOAD_GET_META_FAIL_COUNT.increase(1L); - throw new LoadException("Failed to get info. No alive backends"); + if (failedBeIds.isEmpty()) { + errorMsg = "no alive backends"; + } + throw new LoadException("failed to get info: " + errorMsg + ","); } Collections.shuffle(backendIds); Backend be = Env.getCurrentSystemInfo().getBackend(backendIds.get(0)); @@ -268,6 +272,7 @@ private static InternalService.PProxyResult getInfoRequest(InternalService.PProx future = BackendServiceProxy.getInstance().getInfo(address, request); result = future.get(Config.max_get_kafka_meta_timeout_second, TimeUnit.SECONDS); } catch (Exception e) { + errorMsg = e.getMessage(); LOG.warn("failed to get info request to " + address + " err " + e.getMessage()); failedBeIds.add(beId); retryTimes++; @@ -275,6 +280,7 @@ private static InternalService.PProxyResult getInfoRequest(InternalService.PProx } code = TStatusCode.findByValue(result.getStatus().getStatusCode()); if (code != TStatusCode.OK) { + errorMsg = result.getStatus().getErrorMsgsList().toString(); LOG.warn("failed to get info request to " + address + " err " + result.getStatus().getErrorMsgsList()); failedBeIds.add(beId); @@ -285,7 +291,7 @@ private static InternalService.PProxyResult getInfoRequest(InternalService.PProx } MetricRepo.COUNTER_ROUTINE_LOAD_GET_META_FAIL_COUNT.increase(1L); - throw new LoadException("Failed to get info"); + throw new LoadException("failed to get info: " + errorMsg + ","); } finally { // Ensure that not all BE added to the blacklist. // For single request: diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy index 825752941d5b91..844d4e5a1830e0 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy @@ -285,6 +285,7 @@ suite("test_routine_load_error","p0") { } log.info("reason of state changed: ${res[0][17].toString()}".toString()) assertTrue(res[0][17].toString().contains("may be Kafka properties set in job is error or no partition in this topic that should check Kafka")) + assertTrue(res[0][17].toString().contains("Unknown topic or partition")) break; } } finally { From 2f738c86760356f72422fc477e3d758b94b6f0f8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:12:07 +0800 Subject: [PATCH 053/572] branch-3.0: [fix](case) fix test_group_commit_schema_change #51950 (#52157) Cherry-picked from #51950 Co-authored-by: meiyi --- .../group_commit/test_group_commit_schema_change.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/insert_p0/group_commit/test_group_commit_schema_change.groovy b/regression-test/suites/insert_p0/group_commit/test_group_commit_schema_change.groovy index 06bbbebef5be54..99dedc620a691e 100644 --- a/regression-test/suites/insert_p0/group_commit/test_group_commit_schema_change.groovy +++ b/regression-test/suites/insert_p0/group_commit/test_group_commit_schema_change.groovy @@ -123,12 +123,12 @@ suite("test_group_commit_schema_change", "nonConcurrent") { def job_state = getJobState(tableName3) assertEquals("RUNNING", job_state) GetDebugPoint().disableDebugPointForAllFEs("FE.SchemaChangeJobV2.runRunning.block") - for (int i = 0; i < 10; i++) { + for (int i = 0; i < 30; i++) { job_state = getJobState(tableName3) if (job_state == "FINISHED") { break } - sleep(100) + sleep(500) } assertEquals("FINISHED", job_state) } From c78d91e5259c6833eda3255d27bc99a4494cd811 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:26:09 +0800 Subject: [PATCH 054/572] branch-3.0: [fix](broker) BE crashed because of empty broker #51274 (#51783) Cherry-picked from #51274 Co-authored-by: camby <104178625@qq.com> --- be/src/io/file_factory.cpp | 44 +++++++++++++++++-- .../doris/datasource/FileQueryScanNode.java | 24 +++++----- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index db0d1c2109bbec..899287dbef1c99 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -72,14 +72,39 @@ io::FileReaderOptions FileFactory::get_reader_options(RuntimeState* state, return opts; } +int32_t get_broker_index(const std::vector& brokers, const std::string& path) { + if (brokers.empty()) { + return -1; + } + + // firstly find local broker + const auto local_host = BackendOptions::get_localhost(); + for (int32_t i = 0; i < brokers.size(); ++i) { + if (brokers[i].hostname == local_host) { + return i; + } + } + + // secondly select broker by hash of file path + auto key = HashUtil::hash(path.data(), path.size(), 0); + return key % brokers.size(); +} + Result FileFactory::create_fs(const io::FSPropertiesRef& fs_properties, const io::FileDescription& file_description) { switch (fs_properties.type) { case TFileType::FILE_LOCAL: return io::global_local_filesystem(); - case TFileType::FILE_BROKER: - return io::BrokerFileSystem::create((*fs_properties.broker_addresses)[0], + case TFileType::FILE_BROKER: { + auto index = get_broker_index(*fs_properties.broker_addresses, file_description.path); + if (index < 0) { + return ResultError(Status::InternalError("empty broker_addresses")); + } + LOG_INFO("select broker: {} for file {}", (*fs_properties.broker_addresses)[index].hostname, + file_description.path); + return io::BrokerFileSystem::create((*fs_properties.broker_addresses)[index], *fs_properties.properties, io::FileSystem::TMP_FS_ID); + } case TFileType::FILE_S3: { S3URI s3_uri(file_description.path); RETURN_IF_ERROR_RESULT(s3_uri.parse()); @@ -129,7 +154,12 @@ Result FileFactory::create_file_writer( return file_writer; } case TFileType::FILE_BROKER: { - return io::BrokerFileWriter::create(env, broker_addresses[0], properties, path); + auto index = get_broker_index(broker_addresses, path); + if (index < 0) { + return ResultError(Status::InternalError("empty broker_addresses")); + } + LOG_INFO("select broker: {} for file {}", broker_addresses[index].hostname, path); + return io::BrokerFileWriter::create(env, broker_addresses[index], properties, path); } case TFileType::FILE_S3: { S3URI s3_uri(path); @@ -198,8 +228,14 @@ Result FileFactory::create_file_reader( }); } case TFileType::FILE_BROKER: { + auto index = get_broker_index(system_properties.broker_addresses, file_description.path); + if (index < 0) { + return ResultError(Status::InternalError("empty broker_addresses")); + } + LOG_INFO("select broker: {} for file {}", + system_properties.broker_addresses[index].hostname, file_description.path); // TODO(plat1ko): Create `FileReader` without FS - return io::BrokerFileSystem::create(system_properties.broker_addresses[0], + return io::BrokerFileSystem::create(system_properties.broker_addresses[index], system_properties.properties, io::FileSystem::TMP_FS_ID) .and_then([&](auto&& fs) -> Result { io::FileReaderSPtr file_reader; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index dedbedeafa3124..04adce89bb76db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -75,6 +75,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -467,21 +468,24 @@ private void setLocationPropertiesIfNecessary(Backend selectedBackend, TFileType params.setProperties(locationProperties); if (!params.isSetBrokerAddresses()) { - FsBroker broker; + List brokers; if (brokerName != null) { - broker = Env.getCurrentEnv().getBrokerMgr().getBroker(brokerName, selectedBackend.getHost()); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format( - "Set location for broker [%s], selected BE host: [%s] selected broker host: [%s]", - brokerName, selectedBackend.getHost(), broker.host)); - } + brokers = Env.getCurrentEnv().getBrokerMgr().getBrokers(brokerName); } else { - broker = Env.getCurrentEnv().getBrokerMgr().getAnyAliveBroker(); + brokers = Env.getCurrentEnv().getBrokerMgr().getAllBrokers(); + } + if (brokers == null || brokers.isEmpty()) { + throw new UserException("No alive broker."); + } + Collections.shuffle(brokers); + for (FsBroker broker : brokers) { + if (broker.isAlive) { + params.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port)); + } } - if (broker == null) { + if (params.getBrokerAddresses().isEmpty()) { throw new UserException("No alive broker."); } - params.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port)); } } } else if ((locationType == TFileType.FILE_S3 || locationType == TFileType.FILE_LOCAL) From 30ab8046765468d4a6400259bde7ecadcc6cd80f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:28:48 +0800 Subject: [PATCH 055/572] branch-3.0: [improve](cloud) make meta_service_endpoint configurable #51748 (#52153) Cherry-picked from #51748 Co-authored-by: HonestManXin --- be/src/agent/heartbeat_server.cpp | 31 +++++++++++++++---- be/src/cloud/cloud_meta_mgr.cpp | 5 +-- .../java/org/apache/doris/common/Config.java | 4 +-- .../org/apache/doris/common/ConfigBase.java | 16 ++++++++++ .../doris/cloud/rpc/MetaServiceClient.java | 8 ++--- .../org/apache/doris/system/HeartbeatMgr.java | 3 ++ 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 0cb6bea22207e2..498315f2baded2 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include "cloud/cloud_tablet_mgr.h" #include "cloud/config.h" @@ -260,16 +261,34 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) { << " " << st; } - if (master_info.meta_service_endpoint != config::meta_service_endpoint && - config::enable_meta_service_endpoint_consistency_check) { + if (master_info.meta_service_endpoint != config::meta_service_endpoint) { LOG(WARNING) << "Detected mismatch in meta_service_endpoint configuration between FE " "and BE. " << "FE meta_service_endpoint: " << master_info.meta_service_endpoint << ", BE meta_service_endpoint: " << config::meta_service_endpoint; - return Status::InvalidArgument( - "fe and be do not work in same mode or meta_service_endpoint mismatch," - "fe meta_service_endpoint: {}, be meta_service_endpoint: {}", - master_info.meta_service_endpoint, config::meta_service_endpoint); + std::vector old_endpoints = + doris::split(config::meta_service_endpoint, ","); + std::vector new_endpoints = + doris::split(master_info.meta_service_endpoint, ","); + auto has_intersection = false; + for (auto endpoint : new_endpoints) { + if (std::find(old_endpoints.begin(), old_endpoints.end(), endpoint) != + old_endpoints.end()) { + has_intersection = true; + } + } + if (has_intersection) { + auto st = config::set_config("meta_service_endpoint", + master_info.meta_service_endpoint, true); + LOG(INFO) << "change config meta_service_endpoint to " + << master_info.meta_service_endpoint << " " << st; + } + if (!has_intersection && config::enable_meta_service_endpoint_consistency_check) { + return Status::InvalidArgument( + "fe and be do not work in same mode or meta_service_endpoint mismatch," + "fe meta_service_endpoint: {}, be meta_service_endpoint: {}", + master_info.meta_service_endpoint, config::meta_service_endpoint); + } } } diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 70d4c9f4b6fd02..d1c6d4a5f2305b 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -187,15 +187,12 @@ class MetaServiceProxy { long deadline = now; // connection age only works without list endpoint. - if (!is_meta_service_endpoint_list() && - config::meta_service_connection_age_base_seconds > 0) { + if (config::meta_service_connection_age_base_seconds > 0) { std::default_random_engine rng(static_cast(now)); std::uniform_int_distribution<> uni( config::meta_service_connection_age_base_seconds, config::meta_service_connection_age_base_seconds * 2); deadline = now + duration_cast(seconds(uni(rng))).count(); - } else { - deadline = LONG_MAX; } // Last one WIN diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c046e515eaa3bf..0ff3300c6fab21 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3027,7 +3027,7 @@ public static boolean isNotCloudMode() { * If you want to access a group of meta services, separated the endpoints by comma, * like "host-1:port,host-2:port". */ - @ConfField + @ConfField(mutable = true, callback = CommaSeparatedIntersectConfHandler.class) public static String meta_service_endpoint = ""; @ConfField(mutable = true) @@ -3048,8 +3048,6 @@ public static int metaServiceRpcRetryTimes() { // A connection will expire after a random time during [base, 2*base), so that the FE // has a chance to connect to a new RS. Set zero to disable it. - // - // It only works if the meta_service_endpoint is not point to a group of meta services. @ConfField(mutable = true) public static int meta_service_connection_age_base_minutes = 5; diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java index 18ae1dc1c0171f..35b78e626ca573 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/ConfigBase.java @@ -20,6 +20,7 @@ import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -36,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -79,6 +81,20 @@ public void handle(Field field, String confVal) throws Exception { } } + static class CommaSeparatedIntersectConfHandler implements ConfHandler { + @Override + public void handle(Field field, String newVal) throws Exception { + String oldVal = String.valueOf(field.get(null)); + Set oldSets = Sets.newHashSet(oldVal.split(",")); + Set newSets = Sets.newHashSet(newVal.split(",")); + if (!oldSets.removeAll(newSets)) { + throw new ConfigException("Config '" + field.getName() + + "' must have intersection between the configs"); + } + setConfigField(field, newVal); + } + } + private static String confFile; private static String customConfFile; public static Class confClass; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java index 048d8ab93dfdb8..f17625a89eaaf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceClient.java @@ -48,7 +48,6 @@ public class MetaServiceClient { private final MetaServiceGrpc.MetaServiceBlockingStub blockingStub; private final ManagedChannel channel; private final long expiredAt; - private final boolean isMetaServiceEndpointList; private Random random = new Random(); static { @@ -64,10 +63,8 @@ public class MetaServiceClient { public MetaServiceClient(String address) { this.address = address; - isMetaServiceEndpointList = address.contains(","); - String target = address; - if (isMetaServiceEndpointList) { + if (address.contains(",")) { target = MetaServiceListResolverProvider.MS_LIST_SCHEME_PREFIX + address; } @@ -87,8 +84,7 @@ public MetaServiceClient(String address) { private long connectionAgeExpiredAt() { long connectionAgeBase = Config.meta_service_connection_age_base_minutes; - // Disable connection age if the endpoint is a list. - if (!isMetaServiceEndpointList && connectionAgeBase > 1) { + if (connectionAgeBase > 0) { long base = TimeUnit.MINUTES.toMillis(connectionAgeBase); long now = System.currentTimeMillis(); long rand = random.nextLong() % base; diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index 036efd05d79fa1..c839222af5e508 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -109,6 +109,9 @@ public void setMaster(int clusterId, String token, long epoch) { */ @Override protected void runAfterCatalogReady() { + if (Config.isCloudMode() && masterInfo.get() != null) { + masterInfo.get().setMetaServiceEndpoint(Config.meta_service_endpoint); + } // Get feInfos of previous iteration. List feInfos = Env.getCurrentEnv().getFrontendInfos(); List> hbResponses = Lists.newArrayList(); From 48b953235b2c7d8de7faaf3681d0c56d44c46176 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:29:24 +0800 Subject: [PATCH 056/572] branch-3.0: [chore](http) return NOT_FOUND if dir is not exists #51897 (#52155) Cherry-picked from #51897 Co-authored-by: walter --- be/src/http/utils.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/be/src/http/utils.cpp b/be/src/http/utils.cpp index ee7a78113e555a..7318e281b5395a 100644 --- a/be/src/http/utils.cpp +++ b/be/src/http/utils.cpp @@ -208,7 +208,13 @@ void do_dir_response(const std::string& dir_path, HttpRequest* req, bool is_acqu return; } - VLOG_DEBUG << "list dir: " << dir_path << ", file count: " << files.size(); + VLOG_DEBUG << "list dir: " << dir_path << ", exists: " << exists + << ", file count: " << files.size(); + + if (!exists) { + HttpChannel::send_error(req, HttpStatus::NOT_FOUND); + return; + } const std::string FILE_DELIMITER_IN_DIR_RESPONSE = "\n"; From a160dd234c051d52820a66f55576cc4d655f3f97 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 15:21:37 +0800 Subject: [PATCH 057/572] branch-3.0: [feat](hive) add catalog level schema cache property #50958 #51057 (#51011) Cherry-picked from #50958 #51057 --------- Co-authored-by: Mingyu Chen (Rayner) --- .../doris/datasource/ExternalCatalog.java | 29 +++- .../datasource/ExternalMetaCacheMgr.java | 35 +++-- .../doris/datasource/ExternalSchemaCache.java | 8 +- .../datasource/hive/HMSExternalCatalog.java | 5 - .../datasource/hive/HMSExternalTable.java | 11 +- .../datasource/hive/HiveMetaStoreCache.java | 9 +- .../hive/HiveMetaStoreClientHelper.java | 5 +- .../org/apache/doris/qe/ShowExecutor.java | 2 +- .../hive/test_hive_meta_cache.out | 69 ++++++++++ .../hive/test_hive_meta_cache.groovy | 127 ++++++++++++++++++ 10 files changed, 265 insertions(+), 35 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 0bf975a75d4ddc..0b391ca0435fb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -75,6 +75,7 @@ import lombok.Data; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.hadoop.conf.Configuration; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -116,6 +117,11 @@ public abstract class ExternalCatalog // https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0 public static final String OOS_ROOT_POLICY = "oss.root_policy"; + public static final String SCHEMA_CACHE_TTL_SECOND = "schema.cache.ttl-second"; + // -1 means cache with no ttl + public static final int CACHE_NO_TTL = -1; + // 0 means cache is disabled; >0 means cache with ttl; + public static final int CACHE_TTL_DISABLE_CACHE = 0; // Properties that should not be shown in the `show create catalog` result public static final Set HIDDEN_PROPERTIES = Sets.newHashSet( @@ -351,7 +357,7 @@ public void checkProperties() throws DdlException { Map properties = getCatalogProperty().getProperties(); if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) { try { - Integer metadataRefreshIntervalSec = Integer.valueOf( + int metadataRefreshIntervalSec = Integer.parseInt( properties.get(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)); if (metadataRefreshIntervalSec < 0) { throw new DdlException("Invalid properties: " + CatalogMgr.METADATA_REFRESH_INTERVAL_SEC); @@ -361,11 +367,13 @@ public void checkProperties() throws DdlException { } } - // if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE, "true").equals("false")) { - // LOG.warn("force to set use_meta_cache to true for catalog: {} when creating", name); - // getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true"); - // useMetaCache = Optional.of(true); - // } + // check schema.cache.ttl-second parameter + String schemaCacheTtlSecond = catalogProperty.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); + if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL) + < CACHE_TTL_DISABLE_CACHE) { + throw new DdlException( + "The parameter " + SCHEMA_CACHE_TTL_SECOND + " is wrong, value is " + schemaCacheTtlSecond); + } } /** @@ -1185,4 +1193,13 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hashCode(name); } + + @Override + public void notifyPropertiesUpdated(Map updatedProps) { + CatalogIf.super.notifyPropertiesUpdated(updatedProps); + String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); + if (java.util.Objects.nonNull(schemaCacheTtl)) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidSchemaCache(id); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index c2f50f929f8a83..b640aa08f7696c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -88,7 +88,7 @@ public class ExternalMetaCacheMgr { // catalog id -> HiveMetaStoreCache private final Map cacheMap = Maps.newConcurrentMap(); // catalog id -> table schema cache - private Map schemaCacheMap = Maps.newHashMap(); + private final Map schemaCacheMap = Maps.newHashMap(); // hudi partition manager private final HudiMetadataCacheMgr hudiMetadataCacheMgr; // all catalogs could share the same fsCache. @@ -221,8 +221,10 @@ public void removeCache(long catalogId) { if (cacheMap.remove(catalogId) != null) { LOG.info("remove hive metastore cache for catalog {}", catalogId); } - if (schemaCacheMap.remove(catalogId) != null) { - LOG.info("remove schema cache for catalog {}", catalogId); + synchronized (schemaCacheMap) { + if (schemaCacheMap.remove(catalogId) != null) { + LOG.info("remove schema cache for catalog {}", catalogId); + } } hudiMetadataCacheMgr.removeCache(catalogId); icebergMetadataCacheMgr.removeCache(catalogId); @@ -232,9 +234,11 @@ public void removeCache(long catalogId) { public void invalidateTableCache(long catalogId, String dbName, String tblName) { dbName = ClusterNamespace.getNameFromFullName(dbName); - ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId); - if (schemaCache != null) { - schemaCache.invalidateTableCache(dbName, tblName); + synchronized (schemaCacheMap) { + ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId); + if (schemaCache != null) { + schemaCache.invalidateTableCache(dbName, tblName); + } } HiveMetaStoreCache metaCache = cacheMap.get(catalogId); if (metaCache != null) { @@ -251,9 +255,11 @@ public void invalidateTableCache(long catalogId, String dbName, String tblName) public void invalidateDbCache(long catalogId, String dbName) { dbName = ClusterNamespace.getNameFromFullName(dbName); - ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId); - if (schemaCache != null) { - schemaCache.invalidateDbCache(dbName); + synchronized (schemaCacheMap) { + ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId); + if (schemaCache != null) { + schemaCache.invalidateDbCache(dbName); + } } HiveMetaStoreCache metaCache = cacheMap.get(catalogId); if (metaCache != null) { @@ -269,9 +275,8 @@ public void invalidateDbCache(long catalogId, String dbName) { } public void invalidateCatalogCache(long catalogId) { - ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId); - if (schemaCache != null) { - schemaCache.invalidateAll(); + synchronized (schemaCacheMap) { + schemaCacheMap.remove(catalogId); } HiveMetaStoreCache metaCache = cacheMap.get(catalogId); if (metaCache != null) { @@ -286,6 +291,12 @@ public void invalidateCatalogCache(long catalogId) { } } + public void invalidSchemaCache(long catalogId) { + synchronized (schemaCacheMap) { + schemaCacheMap.remove(catalogId); + } + } + public void addPartitionsCache(long catalogId, HMSExternalTable table, List partitionNames) { String dbName = ClusterNamespace.getNameFromFullName(table.getDbName()); HiveMetaStoreCache metaCache = cacheMap.get(catalogId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java index de3eeff75d97fa..0fb66c4e3f9e45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java @@ -28,6 +28,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.collect.ImmutableList; import lombok.Data; +import org.apache.commons.lang3.math.NumberUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -51,13 +52,16 @@ public ExternalSchemaCache(ExternalCatalog catalog, ExecutorService executor) { } private void init(ExecutorService executor) { + long schemaCacheTtlSecond = NumberUtils.toLong( + (catalog.getProperties().get(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND)), ExternalCatalog.CACHE_NO_TTL); CacheFactory schemaCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(schemaCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE + ? schemaCacheTtlSecond : 86400), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), Config.max_external_schema_cache_num, false, null); - schemaCache = schemaCacheFactory.buildCache(key -> loadSchema(key), null, executor); + schemaCache = schemaCacheFactory.buildCache(this::loadSchema, null, executor); } private void initMetrics() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 20d43263316cd1..2442eed0b9c36a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -81,11 +81,6 @@ public class HMSExternalCatalog extends ExternalCatalog { // from remoteTable object. public static final String GET_SCHEMA_FROM_TABLE = "get_schema_from_table"; - // -1 means cache with no ttl - public static final int CACHE_NO_TTL = -1; - // 0 means cache is disabled; >0 means cache with ttl; - public static final int CACHE_TTL_DISABLE_CACHE = 0; - private static final int FILE_SYSTEM_EXECUTOR_THREAD_NUM = 16; private ThreadPoolExecutor fileSystemExecutor; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 35627273f93604..b7353a59ddf5b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -579,7 +579,7 @@ private Optional getHiveSchema() { List schema = null; Map colDefaultValues = Maps.newHashMap(); if (getFromTable) { - schema = getSchemaFromRemoteTable(remoteTable); + schema = getSchemaFromRemoteTable(); } else { HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient(); schema = client.getSchema(dbName, name); @@ -597,10 +597,13 @@ private Optional getHiveSchema() { return Optional.of(new HMSSchemaCacheValue(columns, partitionColumns)); } - private static List getSchemaFromRemoteTable(Table table) { + private List getSchemaFromRemoteTable() { + // Here we should get a new remote table instead of using this.remoteTable + // Because we need to get the latest schema from HMS. + Table newTable = ((HMSExternalCatalog) catalog).getClient().getTable(dbName, name); List schema = Lists.newArrayList(); - schema.addAll(table.getSd().getCols()); - schema.addAll(table.getPartitionKeys()); + schema.addAll(newTable.getSd().getCols()); + schema.addAll(newTable.getPartitionKeys()); return schema; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index 6ef18c9f545fce..de867bfc2b928c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -37,6 +37,7 @@ import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo; import org.apache.doris.datasource.hive.HiveUtil.ACIDFileFilter; @@ -139,10 +140,10 @@ public HiveMetaStoreCache(HMSExternalCatalog catalog, public void init() { long partitionCacheTtlSecond = NumberUtils.toLong( (catalog.getProperties().get(HMSExternalCatalog.PARTITION_CACHE_TTL_SECOND)), - HMSExternalCatalog.CACHE_NO_TTL); + ExternalCatalog.CACHE_NO_TTL); CacheFactory partitionValuesCacheFactory = new CacheFactory( - OptionalLong.of(partitionCacheTtlSecond >= HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE + OptionalLong.of(partitionCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE ? partitionCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_hive_partition_table_cache_num, @@ -181,10 +182,10 @@ private void setNewFileCache() { // if the file.meta.cache.ttl-second is equal or greater than 0, the cache expired will be set to that value int fileMetaCacheTtlSecond = NumberUtils.toInt( (catalog.getProperties().get(HMSExternalCatalog.FILE_META_CACHE_TTL_SECOND)), - HMSExternalCatalog.CACHE_NO_TTL); + ExternalCatalog.CACHE_NO_TTL); CacheFactory fileCacheFactory = new CacheFactory( - OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE + OptionalLong.of(fileMetaCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE ? fileMetaCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_external_file_cache_num, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java index aed35dc47adef2..eb63aa1e5410f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java @@ -712,7 +712,10 @@ public static Type hiveTypeToDorisType(String hiveType, int timeScale) { return Type.UNSUPPORTED; } - public static String showCreateTable(org.apache.hadoop.hive.metastore.api.Table remoteTable) { + public static String showCreateTable(HMSExternalTable hmsTable) { + // Always use the latest schema + HMSExternalCatalog catalog = (HMSExternalCatalog) hmsTable.getCatalog(); + Table remoteTable = catalog.getClient().getTable(hmsTable.getDbName(), hmsTable.getRemoteName()); StringBuilder output = new StringBuilder(); if (remoteTable.isSetViewOriginalText() || remoteTable.isSetViewExpandedText()) { output.append(String.format("CREATE VIEW `%s` AS ", remoteTable.getTableName())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 2087fd283c3078..4be871f7af8d93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1156,7 +1156,7 @@ private void handleShowCreateTable() throws AnalysisException { try { if (table.getType() == TableType.HMS_EXTERNAL_TABLE) { rows.add(Arrays.asList(table.getName(), - HiveMetaStoreClientHelper.showCreateTable(((HMSExternalTable) table).getRemoteTable()))); + HiveMetaStoreClientHelper.showCreateTable((HMSExternalTable) table))); resultSet = new ShowResultSet(showStmt.getMetaData(), rows); return; } diff --git a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out index 7ab9a456bdc69c..7e031ce471d189 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out +++ b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out @@ -77,3 +77,72 @@ 2 2.0 2024 3 2.0 2024 +-- !sql_3col -- +id int Yes true \N +amount double Yes true \N +year int Yes true \N + +-- !sql_3col -- +id int Yes true \N +amount double Yes true \N +year int Yes true \N + +-- !sql_4col -- +id int Yes true \N +amount double Yes true \N +k3 text Yes true \N +year int Yes true \N + +-- !sql_4col -- +id int Yes true \N +amount double Yes true \N +k3 text Yes true \N +year int Yes true \N + +-- !sql_5col -- +id int Yes true \N +amount double Yes true \N +k3 text Yes true \N +k4 text Yes true \N +year int Yes true \N + +-- !sql_5col -- +id int Yes true \N +amount double Yes true \N +k3 text Yes true \N +k4 text Yes true \N +year int Yes true \N + +-- !sql_6col -- +id int Yes true \N +amount double Yes true \N +k3 text Yes true \N +k4 text Yes true \N +k5 text Yes true \N +year int Yes true \N + +-- !sql_3col -- +id int Yes true \N +amount double Yes true \N +year int Yes true \N + +-- !sql_4col -- +id int Yes true \N +amount double Yes true \N +k1 text Yes true \N +year int Yes true \N + +-- !sql_5col -- +id int Yes true \N +amount double Yes true \N +k1 text Yes true \N +k2 text Yes true \N +year int Yes true \N + +-- !sql_5col -- +id int Yes true \N +amount double Yes true \N +k1 text Yes true \N +k2 text Yes true \N +year int Yes true \N + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy index 3b6655f6e39a0d..aa5ba31af17343 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy @@ -229,6 +229,133 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_ // select 5 rows order_qt_sql_5row """select * from test_hive_meta_cache_db.sales""" sql """drop table test_hive_meta_cache_db.sales""" + + // test schema cache + sql """drop catalog if exists ${catalog_name_no_cache};""" + // 1. create catalog with default property fisrt + sql """ + create catalog ${catalog_name_no_cache} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}' + ); + """ + sql """switch ${catalog_name_no_cache}""" + hive_docker """drop database if exists test_hive_meta_cache_db CASCADE""" + hive_docker """create database test_hive_meta_cache_db""" + hive_docker """ + CREATE TABLE test_hive_meta_cache_db.sales ( + id INT, + amount DOUBLE + ) + PARTITIONED BY (year INT) + STORED AS PARQUET; + """ + // desc table, 3 columns + qt_sql_3col "desc test_hive_meta_cache_db.sales"; + // add a new column in hive + hive_docker "alter table test_hive_meta_cache_db.sales add columns(k3 string)" + // desc table, still 3 columns + qt_sql_3col "desc test_hive_meta_cache_db.sales"; + // refresh and check + sql "refresh table test_hive_meta_cache_db.sales"; + // desc table, 4 columns + qt_sql_4col "desc test_hive_meta_cache_db.sales"; + + // create catalog without schema cache + sql """drop catalog if exists ${catalog_name_no_cache};""" + sql """ + create catalog ${catalog_name_no_cache} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'schema.cache.ttl-second' = '0' + ); + """ + sql """switch ${catalog_name_no_cache}""" + // desc table, 4 columns + qt_sql_4col "desc test_hive_meta_cache_db.sales"; + // add a new column in hive + hive_docker "alter table test_hive_meta_cache_db.sales add columns(k4 string)" + // desc table, 5 columns + qt_sql_5col "desc test_hive_meta_cache_db.sales"; + + // modify property + // alter wrong catalog property + test { + sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "-2")""" + exception "is wrong" + } + sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "0")""" + // desc table, 5 columns + qt_sql_5col "desc test_hive_meta_cache_db.sales"; + // add a new column in hive + hive_docker "alter table test_hive_meta_cache_db.sales add columns(k5 string)" + // desc table, 6 columns + qt_sql_6col "desc test_hive_meta_cache_db.sales"; + sql """drop table test_hive_meta_cache_db.sales""" + + // test schema cache with get_schema_from_table + sql """drop catalog if exists ${catalog_name_no_cache};""" + // 1. create catalog with schema cache off and get_schema_from_table + sql """ + create catalog ${catalog_name_no_cache} properties ( + 'type'='hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}', + 'schema.cache.ttl-second' = '0', + 'get_schema_from_table' = 'true' + ); + """ + sql """switch ${catalog_name_no_cache}""" + hive_docker """drop database if exists test_hive_meta_cache_db CASCADE""" + hive_docker """create database test_hive_meta_cache_db""" + hive_docker """ + CREATE TABLE test_hive_meta_cache_db.sales ( + id INT, + amount DOUBLE + ) + PARTITIONED BY (year INT) + STORED AS PARQUET; + """ + // desc table, 3 columns + qt_sql_3col "desc test_hive_meta_cache_db.sales"; + // show create table , 3 columns + def sql_sct01_3col = sql "show create table test_hive_meta_cache_db.sales" + println "${sql_sct01_3col}" + assertTrue(sql_sct01_3col[0][1].contains("CREATE TABLE `sales`(\n `id` int,\n `amount` double)\nPARTITIONED BY (\n `year` int)")); + + // add a new column in hive + hive_docker "alter table test_hive_meta_cache_db.sales add columns(k1 string)" + // desc table, 4 columns + qt_sql_4col "desc test_hive_meta_cache_db.sales"; + // show create table, 4 columns + def sql_sct01_4col = sql "show create table test_hive_meta_cache_db.sales" + println "${sql_sct01_4col}" + assertTrue(sql_sct01_4col[0][1].contains("CREATE TABLE `sales`(\n `id` int,\n `amount` double,\n `k1` string)\nPARTITIONED BY (\n `year` int)")); + + // open schema cache + sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "120")""" + // add a new column in hive + hive_docker "alter table test_hive_meta_cache_db.sales add columns(k2 string)" + // desc table, 5 columns + qt_sql_5col "desc test_hive_meta_cache_db.sales"; + // show create table, 5 columns + def sql_sct01_5col = sql "show create table test_hive_meta_cache_db.sales" + println "${sql_sct01_5col}" + assertTrue(sql_sct01_5col[0][1].contains("CREATE TABLE `sales`(\n `id` int,\n `amount` double,\n `k1` string,\n `k2` string)\nPARTITIONED BY (\n `year` int)")); + // add a new column in hive + hive_docker "alter table test_hive_meta_cache_db.sales add columns(k3 string)" + // desc table, still 5 columns + qt_sql_5col "desc test_hive_meta_cache_db.sales"; + // show create table always see latest schema, 6 columns + def sql_sct01_6col = sql "show create table test_hive_meta_cache_db.sales" + println "${sql_sct01_6col}" + assertTrue(sql_sct01_6col[0][1].contains("CREATE TABLE `sales`(\n `id` int,\n `amount` double,\n `k1` string,\n `k2` string,\n `k3` string)\nPARTITIONED BY (\n `year` int)")); + sql """drop table test_hive_meta_cache_db.sales""" } } } From 868cc911b2a7914ab7658996e5679bda6c841b88 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 24 Jun 2025 16:05:34 +0800 Subject: [PATCH 058/572] [fix](iceberg)Table operations are not supported for catalogs of the dlf type for 3.0 #50696 (#50956) bp: #50696 --- .../iceberg/IcebergDLFExternalCatalog.java | 33 +++++++++++++++++++ .../client/IcebergDLFExternalCatalogTest.java | 15 +++++++++ 2 files changed, 48 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java index d3f192754ab4db..048117bce53505 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergDLFExternalCatalog.java @@ -17,10 +17,18 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.analysis.CreateDbStmt; +import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.DropDbStmt; +import org.apache.doris.analysis.DropTableStmt; +import org.apache.doris.analysis.TruncateTableStmt; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.iceberg.dlf.DLFCatalog; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.HMSProperties; +import org.apache.doris.nereids.exceptions.NotSupportedException; import java.util.Map; @@ -45,4 +53,29 @@ protected void initCatalog() { dlfCatalog.initialize(catalogName, catalogProperties); catalog = dlfCatalog; } + + @Override + public void createDb(CreateDbStmt stmt) throws DdlException { + throw new NotSupportedException("iceberg catalog with dlf type not supports 'create database'"); + } + + @Override + public void dropDb(DropDbStmt stmt) throws DdlException { + throw new NotSupportedException("iceberg catalog with dlf type not supports 'drop database'"); + } + + @Override + public boolean createTable(CreateTableStmt stmt) throws UserException { + throw new NotSupportedException("iceberg catalog with dlf type not supports 'create table'"); + } + + @Override + public void dropTable(DropTableStmt stmt) throws DdlException { + throw new NotSupportedException("iceberg catalog with dlf type not supports 'drop table'"); + } + + @Override + public void truncateTable(TruncateTableStmt stmt) throws DdlException { + throw new NotSupportedException("iceberg catalog with dlf type not supports 'truncate table'"); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java index bbd39b7b71bfc0..b14f5a217b3ff5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/dlf/client/IcebergDLFExternalCatalogTest.java @@ -17,6 +17,9 @@ package org.apache.doris.datasource.iceberg.dlf.client; +import org.apache.doris.datasource.iceberg.IcebergDLFExternalCatalog; +import org.apache.doris.nereids.exceptions.NotSupportedException; + import org.apache.hadoop.conf.Configuration; import org.junit.Assert; import org.junit.Test; @@ -38,4 +41,16 @@ public void testDatabaseList() { Assert.assertNotSame(dlfClientPool1, dlfClientPool2); } + + @Test + public void testNotSupportOperation() { + HashMap props = new HashMap<>(); + IcebergDLFExternalCatalog catalog = new IcebergDLFExternalCatalog(1, "test", "test", props, "test"); + Assert.assertThrows(NotSupportedException.class, () -> catalog.createDb(null)); + Assert.assertThrows(NotSupportedException.class, () -> catalog.dropDb(null)); + Assert.assertThrows(NotSupportedException.class, () -> catalog.createTable(null)); + Assert.assertThrows(NotSupportedException.class, () -> catalog.dropTable(null)); + Assert.assertThrows(NotSupportedException.class, () -> catalog.dropTable(null)); + Assert.assertThrows(NotSupportedException.class, () -> catalog.truncateTable(null)); + } } From da4ddf853d311f25981a66be1b4caa123f37c1c4 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 24 Jun 2025 18:13:40 +0800 Subject: [PATCH 059/572] branch-3.0: [opt](catalog) throw explicit error when operation is not supported(#51589) (#52199) bp #51589 --- .../doris/datasource/ExternalCatalog.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 0b391ca0435fb6..23d6cb958d414d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -219,14 +219,13 @@ private Configuration buildConf() { } /** - * set some default properties when creating catalog + * Lists all database names in this catalog. * * @return list of database names in this catalog */ protected List listDatabaseNames() { if (metadataOps == null) { - throw new UnsupportedOperationException("Unsupported operation: " - + "listDatabaseNames from remote client when init catalog with " + logType.name()); + throw new UnsupportedOperationException("List databases is not supported for catalog: " + getName()); } else { return metadataOps.listDatabaseNames(); } @@ -1001,8 +1000,7 @@ public void addDatabaseForTest(ExternalDatabase db) { public void createDb(CreateDbStmt stmt) throws DdlException { makeSureInitialized(); if (metadataOps == null) { - LOG.warn("createDb not implemented"); - return; + throw new DdlException("Create database is not supported for catalog: " + getName()); } try { metadataOps.createDb(stmt); @@ -1016,8 +1014,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException { public void dropDb(DropDbStmt stmt) throws DdlException { makeSureInitialized(); if (metadataOps == null) { - LOG.warn("dropDb not implemented"); - return; + throw new DdlException("Drop database is not supported for catalog: " + getName()); } try { metadataOps.dropDb(stmt); @@ -1031,8 +1028,7 @@ public void dropDb(DropDbStmt stmt) throws DdlException { public boolean createTable(CreateTableStmt stmt) throws UserException { makeSureInitialized(); if (metadataOps == null) { - LOG.warn("createTable not implemented"); - return false; + throw new DdlException("Create table is not supported for catalog: " + getName()); } try { return metadataOps.createTable(stmt); @@ -1046,8 +1042,7 @@ public boolean createTable(CreateTableStmt stmt) throws UserException { public void dropTable(DropTableStmt stmt) throws DdlException { makeSureInitialized(); if (metadataOps == null) { - LOG.warn("dropTable not implemented"); - return; + throw new DdlException("Drop table is not supported for catalog: " + getName()); } try { metadataOps.dropTable(stmt); From 02704141d2d27f5707b18e2cb4cedd905df5434e Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 24 Jun 2025 18:14:21 +0800 Subject: [PATCH 060/572] [fix](http) support show backend config when BE enable http auth (#51887) (#52202) bp #51887 --- .../doris/nereids/trees/plans/commands/ShowConfigCommand.java | 4 +++- .../src/main/java/org/apache/doris/qe/ShowExecutor.java | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java index 77bb5ee83d3a2c..a53f8febe12020 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowConfigCommand.java @@ -129,6 +129,7 @@ private ShowResultSet handShowBackendConfig() throws AnalysisException { try { URL url = new URL(urlString); URLConnection urlConnection = url.openConnection(); + urlConnection.setRequestProperty("Auth-Token", Env.getCurrentEnv().getTokenManager().acquireToken()); InputStream inputStream = urlConnection.getInputStream(); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); while (reader.ready()) { @@ -152,7 +153,8 @@ private ShowResultSet handShowBackendConfig() throws AnalysisException { } } catch (Exception e) { throw new AnalysisException( - String.format("Can’t get backend config, backendId: %d, host: %s", beId, host)); + String.format("Can’t get backend config, backendId: %d, host: %s. error: %s", + beId, host, e.getMessage()), e); } } return new ShowResultSet(getMetaData(BE_TITLE_NAMES), results); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 4be871f7af8d93..179ad443ec28fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1655,6 +1655,7 @@ private void handleShowLoadWarningsFromURL(ShowLoadWarningsStmt showWarningsStmt List> rows = Lists.newArrayList(); try { URLConnection urlConnection = url.openConnection(); + urlConnection.setRequestProperty("Auth-Token", Env.getCurrentEnv().getTokenManager().acquireToken()); InputStream inputStream = urlConnection.getInputStream(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { int limit = 100; @@ -2466,6 +2467,7 @@ private void handShowBackendConfig(ShowConfigStmt stmt) throws AnalysisException try { URL url = new URL(urlString); URLConnection urlConnection = url.openConnection(); + urlConnection.setRequestProperty("Auth-Token", Env.getCurrentEnv().getTokenManager().acquireToken()); InputStream inputStream = urlConnection.getInputStream(); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); while (reader.ready()) { From cf68a6485a84b432481f2ee6aab64fdb395b1b53 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 24 Jun 2025 19:43:32 +0800 Subject: [PATCH 061/572] [opt](audit) use one line in audit log and origin statement in audit table (#52032) (#52205) bp #52032 --- .../org/apache/doris/plugin/AuditEvent.java | 116 ++++++++++-------- .../doris/plugin/audit/AuditLoader.java | 66 +++++----- .../doris/plugin/audit/AuditLogBuilder.java | 6 + .../doris/plugin/audit/AuditStreamLoader.java | 2 + .../org/apache/doris/qe/AuditLogHelper.java | 7 +- .../plugin/audit/AuditLogBuilderTest.java | 24 +--- 6 files changed, 119 insertions(+), 102 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java index 20c05d982f806e..7a2f48d4e6f7f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java @@ -42,77 +42,95 @@ public enum EventType { } @Retention(RetentionPolicy.RUNTIME) - public static @interface AuditField { + public @interface AuditField { String value() default ""; + + String colName() default ""; } public EventType type; - // all fields which is about to be audit should be annotated by "@AuditField" + // all fields which is about to be audited should be annotated by "@AuditField" // make them all "public" so that easy to visit. - @AuditField(value = "Timestamp") + + // uuid and time + @AuditField(value = "QueryId", colName = "query_id") + public String queryId = ""; + @AuditField(value = "Timestamp", colName = "time") public long timestamp = -1; - @AuditField(value = "Client") + + // cs info + @AuditField(value = "Client", colName = "client_ip") public String clientIp = ""; - @AuditField(value = "User") + @AuditField(value = "User", colName = "user") public String user = ""; - @AuditField(value = "Ctl") + @AuditField(value = "FeIp", colName = "frontend_ip") + public String feIp = ""; + + // default ctl and db + @AuditField(value = "Ctl", colName = "catalog") public String ctl = ""; - @AuditField(value = "Db") + @AuditField(value = "Db", colName = "db") public String db = ""; - @AuditField(value = "CommandType") - public String commandType = ""; - @AuditField(value = "State") + + // query state + @AuditField(value = "State", colName = "state") public String state = ""; - @AuditField(value = "ErrorCode") + @AuditField(value = "ErrorCode", colName = "error_code") public int errorCode = 0; - @AuditField(value = "ErrorMessage") + @AuditField(value = "ErrorMessage", colName = "error_message") public String errorMessage = ""; - @AuditField(value = "Time(ms)") + + // execution info + @AuditField(value = "Time(ms)", colName = "query_time") public long queryTime = -1; - @AuditField(value = "ScanBytes") + @AuditField(value = "CpuTimeMS", colName = "cpu_time_ms") + public long cpuTimeMs = -1; + @AuditField(value = "PeakMemoryBytes", colName = "peak_memory_bytes") + public long peakMemoryBytes = -1; + @AuditField(value = "ScanBytes", colName = "scan_bytes") public long scanBytes = -1; - @AuditField(value = "ScanRows") + @AuditField(value = "ScanRows", colName = "scan_rows") public long scanRows = -1; - @AuditField(value = "ReturnRows") + @AuditField(value = "ReturnRows", colName = "return_rows") public long returnRows = -1; - @AuditField(value = "StmtId") - public long stmtId = -1; - @AuditField(value = "QueryId") - public String queryId = ""; - @AuditField(value = "IsQuery") - public boolean isQuery = false; - @AuditField(value = "IsNereids") - public boolean isNereids = false; - @AuditField(value = "FeIp") - public String feIp = ""; - @AuditField(value = "StmtType") - public String stmtType = ""; - @AuditField(value = "Stmt") - public String stmt = ""; - @AuditField(value = "CpuTimeMS") - public long cpuTimeMs = -1; - @AuditField(value = "ShuffleSendBytes") - public long shuffleSendBytes = -1; - @AuditField(value = "ShuffleSendRows") + @AuditField(value = "ShuffleSendRows", colName = "shuffle_send_rows") public long shuffleSendRows = -1; - @AuditField(value = "SqlHash") + @AuditField(value = "ShuffleSendBytes", colName = "shuffle_send_bytes") + public long shuffleSendBytes = -1; + @AuditField(value = "ScanBytesFromLocalStorage", colName = "scan_bytes_from_local_storage") + public long scanBytesFromLocalStorage = -1; + @AuditField(value = "ScanBytesFromRemoteStorage", colName = "scan_bytes_from_remote_storage") + public long scanBytesFromRemoteStorage = -1; + + @AuditField(value = "FuzzyVariables") + public String fuzzyVariables = ""; + + // type and digest + @AuditField(value = "CommandType") + public String commandType = ""; + @AuditField(value = "StmtType", colName = "stmt_type") + public String stmtType = ""; + @AuditField(value = "StmtId", colName = "stmt_id") + public long stmtId = -1; + @AuditField(value = "SqlHash", colName = "sql_hash") public String sqlHash = ""; - @AuditField(value = "PeakMemoryBytes") - public long peakMemoryBytes = -1; - @AuditField(value = "SqlDigest") + @AuditField(value = "SqlDigest", colName = "sql_digest") public String sqlDigest = ""; - @AuditField(value = "ComputeGroupName") - public String cloudClusterName = ""; - @AuditField(value = "WorkloadGroup") + @AuditField(value = "IsQuery", colName = "is_query") + public boolean isQuery = false; + @AuditField(value = "IsNereids", colName = "is_nereids") + public boolean isNereids = false; + + // resource + @AuditField(value = "WorkloadGroup", colName = "workload_group") public String workloadGroup = ""; - // note: newly added fields should be always before fuzzyVariables - @AuditField(value = "FuzzyVariables") - public String fuzzyVariables = ""; - @AuditField(value = "ScanBytesFromLocalStorage") - public long scanBytesFromLocalStorage = -1; - @AuditField(value = "ScanBytesFromRemoteStorage") - public long scanBytesFromRemoteStorage = -1; + @AuditField(value = "ComputeGroupName", colName = "compute_group") + public String cloudClusterName = ""; + + // stmt should be last one + @AuditField(value = "Stmt", colName = "stmt") + public String stmt = ""; public long pushToAuditLogQueueTime; diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java index 722ab48669b0d3..c1047bec1b13ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java @@ -46,6 +46,14 @@ public class AuditLoader extends Plugin implements AuditPlugin { public static final String AUDIT_LOG_TABLE = "audit_log"; + // the "\\u001F" and "\\u001E" are used to separate columns and lines in audit log data + public static final String AUDIT_TABLE_COL_SEPARATOR = "\\u001F"; + public static final String AUDIT_TABLE_LINE_DELIMITER = "\\u001E"; + // the "\\x1F" and "\\x1E" are used to specified column and line delimiter in stream load request + // which is corresponding to the "\\u001F" and "\\u001E" in audit log data. + public static final String AUDIT_TABLE_COL_SEPARATOR_STR = "\\x1F"; + public static final String AUDIT_TABLE_LINE_DELIMITER_STR = "\\x1E"; + private StringBuilder auditLogBuffer = new StringBuilder(); private int auditLogNum = 0; private long lastLoadTimeAuditLog = 0; @@ -139,40 +147,40 @@ private void assembleAudit(AuditEvent event) { private void fillLogBuffer(AuditEvent event, StringBuilder logBuffer) { // should be same order as InternalSchema.AUDIT_SCHEMA - logBuffer.append(event.queryId).append("\t"); - logBuffer.append(TimeUtils.longToTimeStringWithms(event.timestamp)).append("\t"); - logBuffer.append(event.clientIp).append("\t"); - logBuffer.append(event.user).append("\t"); - logBuffer.append(event.ctl).append("\t"); - logBuffer.append(event.db).append("\t"); - logBuffer.append(event.state).append("\t"); - logBuffer.append(event.errorCode).append("\t"); - logBuffer.append(event.errorMessage).append("\t"); - logBuffer.append(event.queryTime).append("\t"); - logBuffer.append(event.scanBytes).append("\t"); - logBuffer.append(event.scanRows).append("\t"); - logBuffer.append(event.returnRows).append("\t"); - logBuffer.append(event.shuffleSendRows).append("\t"); - logBuffer.append(event.shuffleSendBytes).append("\t"); - logBuffer.append(event.scanBytesFromLocalStorage).append("\t"); - logBuffer.append(event.scanBytesFromRemoteStorage).append("\t"); - logBuffer.append(event.stmtId).append("\t"); - logBuffer.append(event.stmtType).append("\t"); - logBuffer.append(event.isQuery ? 1 : 0).append("\t"); - logBuffer.append(event.isNereids ? 1 : 0).append("\t"); - logBuffer.append(event.feIp).append("\t"); - logBuffer.append(event.cpuTimeMs).append("\t"); - logBuffer.append(event.sqlHash).append("\t"); - logBuffer.append(event.sqlDigest).append("\t"); - logBuffer.append(event.peakMemoryBytes).append("\t"); - logBuffer.append(event.workloadGroup).append("\t"); - logBuffer.append(event.cloudClusterName).append("\t"); + logBuffer.append(event.queryId).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(TimeUtils.longToTimeStringWithms(event.timestamp)).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.clientIp).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.user).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.ctl).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.db).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.state).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.errorCode).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.errorMessage).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.queryTime).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.scanBytes).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.scanRows).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.returnRows).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.shuffleSendRows).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.shuffleSendBytes).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.scanBytesFromLocalStorage).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.scanBytesFromRemoteStorage).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.stmtId).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.stmtType).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.isQuery ? 1 : 0).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.isNereids ? 1 : 0).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.feIp).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.cpuTimeMs).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.sqlHash).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.sqlDigest).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.peakMemoryBytes).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.workloadGroup).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.cloudClusterName).append("AUDIT_TABLE_COL_SEPARATOR"); // already trim the query in org.apache.doris.qe.AuditLogHelper#logAuditLog String stmt = event.stmt; if (LOG.isDebugEnabled()) { LOG.debug("receive audit event with stmt: {}", stmt); } - logBuffer.append(stmt).append("\n"); + logBuffer.append(stmt).append(AUDIT_TABLE_LINE_DELIMITER); } // public for external call. diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLogBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLogBuilder.java index 4208d5def2ebac..94d7973f29431a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLogBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLogBuilder.java @@ -121,6 +121,12 @@ private String getAuditLogString(AuditEvent event) throws IllegalAccessException } } + // replace new line characters with escaped characters to make sure the stmt in one line + if (af.value().equals("Stmt")) { + fieldValue = ((String) fieldValue).replace("\n", "\\n") + .replace("\r", "\\r"); + } + sb.append("|").append(af.value()).append("=").append(fieldValue); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java index 0b70e9591d509d..d2576937d9894d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java @@ -68,6 +68,8 @@ private HttpURLConnection getConnection(String urlStr, String label, String clus InternalSchema.AUDIT_SCHEMA.stream().map(c -> c.getName()).collect( Collectors.joining(","))); conn.addRequestProperty("redirect-policy", "random-be"); + conn.addRequestProperty("column_separator", AuditLoader.AUDIT_TABLE_COL_SEPARATOR_STR); + conn.addRequestProperty("line_delimiter", AuditLoader.AUDIT_TABLE_LINE_DELIMITER_STR); conn.setDoOutput(true); conn.setDoInput(true); return conn; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index f1470d444b91d4..ecfd08aaa710b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -101,9 +101,7 @@ public static String handleStmt(String origStmt, StatementBase parsedStmt) { int maxLen = GlobalVariable.auditPluginMaxSqlLength; origStmt = truncateByBytes(origStmt, maxLen, " ... /* truncated. audit_plugin_max_sql_length=" + maxLen + " */"); - return origStmt.replace("\n", "\\n") - .replace("\t", "\\t") - .replace("\r", "\\r"); + return origStmt; } private static Optional handleInsertStmt(String origStmt, StatementBase parsedStmt) { @@ -134,9 +132,6 @@ private static Optional handleInsertStmt(String origStmt, StatementBase Math.min(GlobalVariable.auditPluginMaxInsertStmtLength, GlobalVariable.auditPluginMaxSqlLength)); origStmt = truncateByBytes(origStmt, maxLen, " ... /* total " + rowCnt + " rows, truncated. audit_plugin_max_insert_stmt_length=" + maxLen + " */"); - origStmt = origStmt.replace("\n", "\\n") - .replace("\t", "\\t") - .replace("\r", "\\r"); return Optional.of(origStmt); } else { return Optional.empty(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/plugin/audit/AuditLogBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/plugin/audit/AuditLogBuilderTest.java index 8c678447c3a1eb..f3e71c248df06d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/plugin/audit/AuditLogBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/audit/AuditLogBuilderTest.java @@ -80,12 +80,9 @@ public void testHandleStmtTruncationForNonInsertStmt() { // 4. Test statement with newlines, tabs, carriage returns String stmtWithSpecialChars = "SELECT *\nFROM table1\tWHERE id = 1\r"; result = AuditLogHelper.handleStmt(stmtWithSpecialChars, nonInsertStmt); - Assert.assertTrue("Should escape newlines", result.contains("\\n")); - Assert.assertTrue("Should escape tabs", result.contains("\\t")); - Assert.assertTrue("Should escape carriage returns", result.contains("\\r")); - Assert.assertFalse("Should not contain actual newlines", result.contains("\n")); - Assert.assertFalse("Should not contain actual tabs", result.contains("\t")); - Assert.assertFalse("Should not contain actual carriage returns", result.contains("\r")); + Assert.assertTrue("Should contain actual newlines", result.contains("\n")); + Assert.assertTrue("Should contain actual tabs", result.contains("\t")); + Assert.assertTrue("Should contain actual carriage returns", result.contains("\r")); // 5. Test long statement with Chinese characters truncation String chineseStmt @@ -118,12 +115,6 @@ public void testHandleStmtTruncationForNonInsertStmt() { String emptyStmt = ""; result = AuditLogHelper.handleStmt(emptyStmt, nonInsertStmt); Assert.assertEquals("Empty string should remain empty", "", result); - - // 9. Test statement with only special characters - String specialCharsStmt = "\n\t\r\n\t\r"; - result = AuditLogHelper.handleStmt(specialCharsStmt, nonInsertStmt); - Assert.assertEquals("Should escape all special characters", "\\n\\t\\r\\n\\t\\r", result); - } finally { // Restore original values GlobalVariable.auditPluginMaxSqlLength = originalMaxSqlLength; @@ -172,12 +163,9 @@ public void testHandleStmtTruncationForInsertStmt() { result = AuditLogHelper.handleStmt(insertWithSpecialChars, insertStmt); // Verify special characters are properly escaped - Assert.assertTrue("Should escape newlines in INSERT", result.contains("\\n")); - Assert.assertTrue("Should escape tabs in INSERT", result.contains("\\t")); - Assert.assertTrue("Should escape carriage returns in INSERT", result.contains("\\r")); - Assert.assertFalse("Should not contain actual newlines", result.contains("\n")); - Assert.assertFalse("Should not contain actual tabs", result.contains("\t")); - Assert.assertFalse("Should not contain actual carriage returns", result.contains("\r")); + Assert.assertTrue("Should contain actual newlines", result.contains("\n")); + Assert.assertTrue("Should contain actual tabs", result.contains("\t")); + Assert.assertTrue("Should contain actual carriage returns", result.contains("\r")); // 4. Test comparison: same length statements, different handling for INSERT vs non-INSERT // Create a statement with length between 80-200 From 6130427126a8f98bde9cf60c1ce138cc18d6d3dc Mon Sep 17 00:00:00 2001 From: meiyi Date: Tue, 24 Jun 2025 20:02:39 +0800 Subject: [PATCH 062/572] branch-3.0: [fix](mow) fix update delete bitmap lock not removed if schema change for empty tablet (#51780) (#52166) pick https://github.com/apache/doris/pull/51780 --- cloud/src/meta-service/meta_service_job.cpp | 30 ++++++++++----------- cloud/test/meta_service_job_test.cpp | 22 +++++++++++++++ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 8d8da279b733e0..c71859decb743e 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -1188,6 +1188,21 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str new_tablet_meta.SerializeToString(&new_tablet_val); txn->put(new_tablet_key, new_tablet_val); + // process mow table, check lock + if (new_tablet_meta.enable_unique_key_merge_on_write()) { + bool success = check_and_remove_delete_bitmap_update_lock( + code, msg, ss, txn, instance_id, new_table_id, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, + schema_change.delete_bitmap_lock_initiator()); + if (!success) { + return; + } + + std::string pending_key = meta_pending_delete_bitmap_key({instance_id, new_tablet_id}); + txn->remove(pending_key); + LOG(INFO) << "xxx sc remove delete bitmap pending key, pending_key=" << hex(pending_key) + << " tablet_id=" << new_tablet_id << "job_id=" << schema_change.id(); + } + //========================================================================== // move rowsets [2-alter_version] to recycle //========================================================================== @@ -1317,21 +1332,6 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str return; } - // process mow table, check lock - if (new_tablet_meta.enable_unique_key_merge_on_write()) { - bool success = check_and_remove_delete_bitmap_update_lock( - code, msg, ss, txn, instance_id, new_table_id, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, - schema_change.delete_bitmap_lock_initiator()); - if (!success) { - return; - } - - std::string pending_key = meta_pending_delete_bitmap_key({instance_id, new_tablet_id}); - txn->remove(pending_key); - LOG(INFO) << "xxx sc remove delete bitmap pending key, pending_key=" << hex(pending_key) - << " tablet_id=" << new_tablet_id << "job_id=" << schema_change.id(); - } - for (size_t i = 0; i < schema_change.txn_ids().size(); ++i) { auto tmp_rowset_key = meta_rowset_tmp_key({instance_id, schema_change.txn_ids().at(i), new_tablet_id}); diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp index cf1b8cd7d581d8..521d01d101fbad 100644 --- a/cloud/test/meta_service_job_test.cpp +++ b/cloud/test/meta_service_job_test.cpp @@ -1881,6 +1881,28 @@ TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) { remove_delete_bitmap_lock(meta_service.get(), table_id); res.Clear(); } + + // alter version < 2 + { + int64_t new_tablet_id = 16; + ASSERT_NO_FATAL_FAILURE(create_tablet(meta_service.get(), table_id, index_id, partition_id, + new_tablet_id, true, true)); + StartTabletJobResponse sc_res; + ASSERT_NO_FATAL_FAILURE(start_schema_change_job(meta_service.get(), table_id, index_id, + partition_id, tablet_id, new_tablet_id, + "job2", "be1", sc_res)); + std::vector output_rowsets; + auto res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 12345); + ASSERT_EQ(res_code, MetaServiceCode::OK); + FinishTabletJobResponse res; + finish_schema_change_job(meta_service.get(), tablet_id, new_tablet_id, "job2", "be1", + output_rowsets, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 100, -1); + ASSERT_EQ(res_code, MetaServiceCode::OK); + remove_delete_bitmap_lock(meta_service.get(), table_id); + res.Clear(); + } } TEST(MetaServiceJobTest, ConcurrentCompactionTest) { From cb511c903b8b965ea67e0f1e621b649f970235d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 21:52:01 +0800 Subject: [PATCH 063/572] branch-3.0: [fix](be) Fix be aws-sdk-cpp memory leak #52185 (#52210) Cherry-picked from #52185 Co-authored-by: Lei Zhang --- be/src/runtime/exec_env_init.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index ee0ad2c93f5db7..fd6f61a294d8e2 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -349,6 +349,8 @@ Status ExecEnv::_init(const std::vector& store_paths, _runtime_query_statistics_mgr->start_report_thread(); _s_ready = true; + // Make aws-sdk-cpp InitAPI and ShutdownAPI called in the same thread + S3ClientFactory::instance(); return Status::OK(); } From 28f9aca4be5296ea613e5df60bac6fefdb882e22 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Jun 2025 21:58:22 +0800 Subject: [PATCH 064/572] branch-3.0: [fix](case) fix mow case #52012 (#52237) Cherry-picked from #52012 Co-authored-by: meiyi --- .../suites/compaction/test_mow_compact_multi_segments.groovy | 1 - .../suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy | 1 - 2 files changed, 2 deletions(-) diff --git a/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy b/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy index f286a2becaec49..e6595cdfb13e99 100644 --- a/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy +++ b/regression-test/suites/compaction/test_mow_compact_multi_segments.groovy @@ -154,7 +154,6 @@ suite("test_mow_compact_multi_segments", "nonConcurrent") { ); """ def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - assertEquals(1, tablets.size()) def tablet = tablets[0] String tablet_id = tablet.TabletId def backend_id = tablet.BackendId diff --git a/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy b/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy index b91a19784e6eb6..78a57f97bfd1a6 100644 --- a/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy @@ -188,7 +188,6 @@ suite("test_mow_stale_rowset_delete_bitmap", "nonConcurrent") { def tablets = sql_return_maparray """ show tablets from ${testTable}; """ logger.info("tablets: " + tablets) - assertEquals(1, tablets.size()) def tablet = tablets[0] try { From 4a8260eb8de436780058ae1ee388d7a4bc5b2b33 Mon Sep 17 00:00:00 2001 From: meiyi Date: Wed, 25 Jun 2025 09:38:11 +0800 Subject: [PATCH 065/572] branch-3.0: [improve](cloud-mow) batch get tablet stats when get_delete_bitmap_update_lock (#47281) (#52225) pick https://github.com/apache/doris/pull/47281 --- cloud/src/common/config.h | 3 + cloud/src/meta-service/meta_service.cpp | 203 ++++++++++++++++++------ cloud/src/meta-service/meta_service.h | 5 + cloud/test/meta_service_test.cpp | 87 +++++----- 4 files changed, 210 insertions(+), 88 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 7dc63683688d03..31607057a057f3 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -285,6 +285,9 @@ CONF_Bool(enable_loopback_address_for_ms, "false"); // Comma seprated list: recycler_storage_vault_white_list="aaa,bbb,ccc" CONF_Strings(recycler_storage_vault_white_list, ""); +// for get_delete_bitmap_update_lock +CONF_mBool(enable_batch_get_mow_tablet_stats_and_meta, "true"); + // aws sdk log level // Off = 0, // Fatal = 1, diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 9ecb08363cdc65..66dec09662f430 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -2574,9 +2574,19 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl return; } + if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key)) { + return; + }; +} + +bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std::string& msg, + const GetDeleteBitmapUpdateLockRequest* request, + GetDeleteBitmapUpdateLockResponse* response, + std::string& instance_id, + std::string& lock_key) { bool require_tablet_stats = request->has_require_compaction_stats() ? request->require_compaction_stats() : false; - if (!require_tablet_stats) return; + if (!require_tablet_stats) return true; // this request is from fe when it commits txn for MOW table, we send the compaction stats // along with the GetDeleteBitmapUpdateLockResponse which will be sent to BE later to let // BE eliminate unnecessary sync_rowsets() calls if possible @@ -2587,79 +2597,168 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl // these steps can be done in different fdb txns StopWatch read_stats_sw; - err = txn_kv_->create_txn(&txn); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = "failed to init txn"; - return; + return false; } - - for (const auto& tablet_idx : request->tablet_indexes()) { + auto table_id = request->table_id(); + std::stringstream ss; + if (!config::enable_batch_get_mow_tablet_stats_and_meta) { + for (const auto& tablet_idx : request->tablet_indexes()) { + // 1. get compaction cnts + TabletStatsPB tablet_stat; + std::string stats_key = + stats_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_idx.tablet_id()}); + std::string stats_val; + TxnErrorCode err = txn->get(stats_key, &stats_val); + TEST_SYNC_POINT_CALLBACK( + "get_delete_bitmap_update_lock.get_compaction_cnts_inject_error", &err); + if (err == TxnErrorCode::TXN_TOO_OLD) { + code = MetaServiceCode::OK; + err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to init txn when get tablet stats"; + msg = ss.str(); + return false; + } + err = txn->get(stats_key, &stats_val); + } + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err, + tablet_idx.tablet_id()); + return false; + } + if (!tablet_stat.ParseFromArray(stats_val.data(), stats_val.size())) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = fmt::format("marformed tablet stats value, key={}", hex(stats_key)); + return false; + } + response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt()); + response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt()); + response->add_cumulative_points(tablet_stat.cumulative_point()); + + // 2. get tablet states + std::string tablet_meta_key = + meta_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_idx.tablet_id()}); + std::string tablet_meta_val; + err = txn->get(tablet_meta_key, &tablet_meta_val); + if (err != TxnErrorCode::TXN_OK) { + ss << "failed to get tablet meta" + << (err == TxnErrorCode::TXN_KEY_NOT_FOUND ? " (not found)" : "") + << " instance_id=" << instance_id << " tablet_id=" << tablet_idx.tablet_id() + << " key=" << hex(tablet_meta_key) << " err=" << err; + msg = ss.str(); + code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TABLET_NOT_FOUND + : cast_as(err); + return false; + } + doris::TabletMetaCloudPB tablet_meta; + if (!tablet_meta.ParseFromString(tablet_meta_val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = "malformed tablet meta"; + return false; + } + response->add_tablet_states( + static_cast>(tablet_meta.tablet_state())); + } + } else { // 1. get compaction cnts - TabletStatsPB tablet_stat; - std::string stats_key = - stats_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), - tablet_idx.partition_id(), tablet_idx.tablet_id()}); - std::string stats_val; - TxnErrorCode err = txn->get(stats_key, &stats_val); + std::vector stats_tablet_keys; + for (const auto& tablet_idx : request->tablet_indexes()) { + stats_tablet_keys.push_back( + stats_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_idx.tablet_id()})); + } + std::vector> stats_tablet_values; + err = txn->batch_get(&stats_tablet_values, stats_tablet_keys); TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_update_lock.get_compaction_cnts_inject_error", &err); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = fmt::format("failed to get tablet stats, err={} table_id={} lock_id={}", err, + table_id, request->lock_id()); + return false; + } + for (size_t i = 0; i < stats_tablet_keys.size(); i++) { + if (!stats_tablet_values[i].has_value()) { + code = cast_as(err); + msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err, + request->tablet_indexes(i).tablet_id()); + return false; + } + TabletStatsPB tablet_stat; + if (!tablet_stat.ParseFromString(stats_tablet_values[i].value())) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = fmt::format("marformed tablet stats value"); + return false; + } + response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt()); + response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt()); + response->add_cumulative_points(tablet_stat.cumulative_point()); + } + stats_tablet_keys.clear(); + stats_tablet_values.clear(); + DCHECK(request->tablet_indexes_size() == response->base_compaction_cnts_size()); + DCHECK(request->tablet_indexes_size() == response->cumulative_compaction_cnts_size()); + DCHECK(request->tablet_indexes_size() == response->cumulative_points_size()); + + // 2. get tablet states + std::vector tablet_meta_keys; + for (const auto& tablet_idx : request->tablet_indexes()) { + tablet_meta_keys.push_back( + meta_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_idx.tablet_id()})); + } + std::vector> tablet_meta_values; + err = txn->batch_get(&tablet_meta_values, tablet_meta_keys); if (err == TxnErrorCode::TXN_TOO_OLD) { code = MetaServiceCode::OK; err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); - ss << "failed to init txn when get tablet stats"; + ss << "failed to init txn when get tablet meta"; msg = ss.str(); - return; + return false; } - err = txn->get(stats_key, &stats_val); + err = txn->batch_get(&tablet_meta_values, tablet_meta_keys); } if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); - msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err, - tablet_idx.tablet_id()); - return; - } - if (!tablet_stat.ParseFromArray(stats_val.data(), stats_val.size())) { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - msg = fmt::format("marformed tablet stats value, key={}", hex(stats_key)); - return; - } - response->add_base_compaction_cnts(tablet_stat.base_compaction_cnt()); - response->add_cumulative_compaction_cnts(tablet_stat.cumulative_compaction_cnt()); - response->add_cumulative_points(tablet_stat.cumulative_point()); - - // 2. get tablet states - std::string tablet_meta_key = - meta_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), - tablet_idx.partition_id(), tablet_idx.tablet_id()}); - std::string tablet_meta_val; - err = txn->get(tablet_meta_key, &tablet_meta_val); - if (err != TxnErrorCode::TXN_OK) { - ss << "failed to get tablet meta" - << (err == TxnErrorCode::TXN_KEY_NOT_FOUND ? " (not found)" : "") - << " instance_id=" << instance_id << " tablet_id=" << tablet_idx.tablet_id() - << " key=" << hex(tablet_meta_key) << " err=" << err; - msg = ss.str(); - code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TABLET_NOT_FOUND - : cast_as(err); - return; + msg = fmt::format("failed to get tablet meta, err={} table_id={} lock_id={}", err, + table_id, request->lock_id()); + return false; } - doris::TabletMetaCloudPB tablet_meta; - if (!tablet_meta.ParseFromString(tablet_meta_val)) { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - msg = "malformed tablet meta"; - return; + for (size_t i = 0; i < tablet_meta_keys.size(); i++) { + if (!tablet_meta_values[i].has_value()) { + code = cast_as(err); + msg = fmt::format("failed to get tablet meta, err={} tablet_id={}", err, + request->tablet_indexes(i).tablet_id()); + return false; + } + doris::TabletMetaCloudPB tablet_meta; + if (!tablet_meta.ParseFromString(tablet_meta_values[i].value())) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = fmt::format("marformed tablet meta value"); + return false; + } + response->add_tablet_states( + static_cast>(tablet_meta.tablet_state())); } - response->add_tablet_states( - static_cast>(tablet_meta.tablet_state())); + DCHECK(request->tablet_indexes_size() == response->tablet_states_size()); } read_stats_sw.pause(); LOG(INFO) << fmt::format( - "tablet_idxes.size()={}, read tablet compaction cnts and tablet states cost={} ms", - request->tablet_indexes().size(), read_stats_sw.elapsed_us() / 1000); + "table_id={}, tablet_idxes.size()={}, read tablet compaction cnts and tablet states " + "cost={} ms", + table_id, request->tablet_indexes().size(), read_stats_sw.elapsed_us() / 1000); DeleteBitmapUpdateLockPB lock_info_tmp; if (!check_delete_bitmap_lock(code, msg, ss, txn, table_id, request->lock_id(), @@ -2669,7 +2768,9 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl << table_id << " request lock_id=" << request->lock_id() << " request initiator=" << request->initiator() << " code=" << code << " msg=" << msg; + return false; } + return true; } void MetaServiceImpl::remove_delete_bitmap_update_lock( diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index 4be017edc9a386..a4136ed39be26a 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -329,6 +329,11 @@ class MetaServiceImpl : public cloud::MetaService { const AlterInstanceRequest* request, std::function(InstanceInfoPB*)> action); + bool get_mow_tablet_stats_and_meta(MetaServiceCode& code, std::string& msg, + const GetDeleteBitmapUpdateLockRequest* request, + GetDeleteBitmapUpdateLockResponse* response, + std::string& instance_id, std::string& lock_key); + std::shared_ptr txn_kv_; std::shared_ptr resource_mgr_; std::shared_ptr rate_limiter_; diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index fbbfbff19fe802..d02ea7b5011257 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -4689,50 +4689,58 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockNoReadStats) { TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsNormal) { auto meta_service = get_meta_service(); - std::string instance_id = "test_get_delete_bitmap_update_lock_normal"; - [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [](int*) { - SyncPoint::get_instance()->disable_processing(); - SyncPoint::get_instance()->clear_all_call_backs(); - }); - sp->set_call_back("get_instance_id", [&](auto&& args) { - auto* ret = try_any_cast_ret(args); - ret->first = instance_id; - ret->second = true; - }); - sp->enable_processing(); + bool enable_batch_get_mow_tablet_stats_and_meta_vals[] = {false, true}; + for (bool val : enable_batch_get_mow_tablet_stats_and_meta_vals) { + config::enable_batch_get_mow_tablet_stats_and_meta = val; - int64_t db_id = 1000; - int64_t table_id = 2001; - int64_t index_id = 3001; - // [(partition_id, tablet_id)] - std::vector> tablet_idxes {{70001, 12345}, {80001, 3456}, {90001, 6789}}; + std::string instance_id = "test_get_delete_bitmap_update_lock_normal"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [](int*) { + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); + }); + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); - add_tablet_metas(meta_service.get(), instance_id, table_id, index_id, tablet_idxes); + int64_t db_id = 1000; + int64_t table_id = 2001; + int64_t index_id = 3001; + // [(partition_id, tablet_id)] + std::vector> tablet_idxes { + {70001, 12345}, {80001, 3456}, {90001, 6789}}; - GetDeleteBitmapUpdateLockResponse res; - get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id, tablet_idxes, - 5, 999999, -1, true); - ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + add_tablet_metas(meta_service.get(), instance_id, table_id, index_id, tablet_idxes); - ASSERT_EQ(res.base_compaction_cnts().size(), tablet_idxes.size()); - for (const auto& base_compaction_cnt : res.base_compaction_cnts()) { - ASSERT_EQ(base_compaction_cnt, 10); - } - ASSERT_EQ(res.cumulative_compaction_cnts().size(), tablet_idxes.size()); - for (const auto& cumu_compaction_cnt : res.cumulative_compaction_cnts()) { - ASSERT_EQ(cumu_compaction_cnt, 20); - } - ASSERT_EQ(res.cumulative_points().size(), tablet_idxes.size()); - for (const auto& cumulative_point : res.cumulative_points()) { - ASSERT_EQ(cumulative_point, 30); + GetDeleteBitmapUpdateLockResponse res; + get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id, + tablet_idxes, 5, 999999, -1, true); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + ASSERT_EQ(res.base_compaction_cnts().size(), tablet_idxes.size()); + for (const auto& base_compaction_cnt : res.base_compaction_cnts()) { + ASSERT_EQ(base_compaction_cnt, 10); + } + ASSERT_EQ(res.cumulative_compaction_cnts().size(), tablet_idxes.size()); + for (const auto& cumu_compaction_cnt : res.cumulative_compaction_cnts()) { + ASSERT_EQ(cumu_compaction_cnt, 20); + } + ASSERT_EQ(res.cumulative_points().size(), tablet_idxes.size()); + for (const auto& cumulative_point : res.cumulative_points()) { + ASSERT_EQ(cumulative_point, 30); + } } } TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) { auto meta_service = get_meta_service(); - { + bool enable_batch_get_mow_tablet_stats_and_meta_vals[] = {false, true}; + for (bool val : enable_batch_get_mow_tablet_stats_and_meta_vals) { + config::enable_batch_get_mow_tablet_stats_and_meta = val; // 2.1 abnormal path, lock has been expired and taken by another load/compaction during // the reading of tablet stats std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal1"; @@ -4773,7 +4781,9 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) { ASSERT_EQ(res.cumulative_points().size(), 0); } - { + for (bool val : enable_batch_get_mow_tablet_stats_and_meta_vals) { + config::enable_batch_get_mow_tablet_stats_and_meta = val; + // 2.2 abnormal path, lock has been taken by another load/compaction and been released during // the reading of tablet stats std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal2"; @@ -4815,7 +4825,9 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) { TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsError) { auto meta_service = get_meta_service(); - { + bool enable_batch_get_mow_tablet_stats_and_meta_vals[] = {false, true}; + for (bool val : enable_batch_get_mow_tablet_stats_and_meta_vals) { + config::enable_batch_get_mow_tablet_stats_and_meta = val; // 2.3 abnormal path, meeting error when reading tablets' stats std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal3"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); @@ -4853,7 +4865,8 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsError) { ASSERT_EQ(res.status().code(), MetaServiceCode::KV_TXN_GET_ERR); } - { + for (bool val : enable_batch_get_mow_tablet_stats_and_meta_vals) { + config::enable_batch_get_mow_tablet_stats_and_meta = val; // 2.4 abnormal path, meeting TXN_TOO_OLD error when reading tablets' stats, // this should not fail if lock is not expired std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal4"; From 2bc0c0eea4f0af228f99134450b8299643667225 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 09:43:09 +0800 Subject: [PATCH 066/572] branch-3.0: [enhancement](filecache) fix default capacity and add reset_capacity validation #51711 (#52152) Cherry-picked from #51711 Signed-off-by: zhengyu Co-authored-by: zhengyu --- be/src/io/cache/block_file_cache_factory.cpp | 62 +++++++++++++--- be/test/io/cache/block_file_cache_test.cpp | 77 ++++++++++++++++++++ 2 files changed, 128 insertions(+), 11 deletions(-) diff --git a/be/src/io/cache/block_file_cache_factory.cpp b/be/src/io/cache/block_file_cache_factory.cpp index 2d0d25735fe2fd..d43e3acea14daf 100644 --- a/be/src/io/cache/block_file_cache_factory.cpp +++ b/be/src/io/cache/block_file_cache_factory.cpp @@ -92,16 +92,13 @@ Status FileCacheFactory::create_file_cache(const std::string& cache_base_path, LOG_ERROR("").tag("file cache path", cache_base_path).tag("error", strerror(errno)); return Status::IOError("{} statfs error {}", cache_base_path, strerror(errno)); } - size_t disk_capacity = static_cast( - static_cast(stat.f_blocks) * static_cast(stat.f_bsize) * - (static_cast(config::file_cache_enter_disk_resource_limit_mode_percent) / - 100)); + size_t disk_capacity = static_cast(static_cast(stat.f_blocks) * + static_cast(stat.f_bsize)); if (file_cache_settings.capacity == 0 || disk_capacity < file_cache_settings.capacity) { LOG_INFO( - "The cache {} config size {} is larger than {}% disk size {} or zero, recalc " + "The cache {} config size {} is larger than disk size {} or zero, recalc " "it.", - cache_base_path, file_cache_settings.capacity, - config::file_cache_enter_disk_resource_limit_mode_percent, disk_capacity); + cache_base_path, file_cache_settings.capacity, disk_capacity); file_cache_settings = get_file_cache_settings(disk_capacity, file_cache_settings.max_query_cache_size); } @@ -174,16 +171,59 @@ std::vector FileCacheFactory::get_base_paths() { return paths; } +std::string validate_capacity(const std::string& path, int64_t new_capacity, + int64_t& valid_capacity) { + struct statfs stat; + if (statfs(path.c_str(), &stat) < 0) { + auto ret = fmt::format("reset capacity {} statfs error {}. ", path, strerror(errno)); + LOG_ERROR(ret); + valid_capacity = 0; // caller will handle the error + return ret; + } + size_t disk_capacity = static_cast(static_cast(stat.f_blocks) * + static_cast(stat.f_bsize)); + if (new_capacity == 0 || disk_capacity < new_capacity) { + auto ret = fmt::format( + "The cache {} config size {} is larger than disk size {} or zero, recalc " + "it to disk size. ", + path, new_capacity, disk_capacity); + valid_capacity = disk_capacity; + LOG_WARNING(ret); + return ret; + } + valid_capacity = new_capacity; + return ""; +} + std::string FileCacheFactory::reset_capacity(const std::string& path, int64_t new_capacity) { + std::stringstream ss; + size_t total_capacity = 0; if (path.empty()) { - std::stringstream ss; - for (auto& [_, cache] : _path_to_cache) { - ss << cache->reset_capacity(new_capacity); + for (auto& [p, cache] : _path_to_cache) { + int64_t valid_capacity = 0; + ss << validate_capacity(p, new_capacity, valid_capacity); + if (valid_capacity <= 0) { + return ss.str(); + } + ss << cache->reset_capacity(valid_capacity); + total_capacity += cache->capacity(); } + _capacity = total_capacity; return ss.str(); } else { if (auto iter = _path_to_cache.find(path); iter != _path_to_cache.end()) { - return iter->second->reset_capacity(new_capacity); + int64_t valid_capacity = 0; + ss << validate_capacity(path, new_capacity, valid_capacity); + if (valid_capacity <= 0) { + return ss.str(); + } + ss << iter->second->reset_capacity(valid_capacity); + + for (auto& [p, cache] : _path_to_cache) { + total_capacity += cache->capacity(); + } + _capacity = total_capacity; + return ss.str(); } } return "Unknown the cache path " + path; diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 0d6883e4c8b4d6..1408919fe1784a 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -7905,4 +7905,81 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader_normal_index) { FileCacheFactory::instance()->_capacity = 0; } +TEST_F(BlockFileCacheTest, test_reset_capacity) { + std::string cache_path2 = caches_dir / "cache2" / ""; + + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } + if (fs::exists(cache_path2)) { + fs::remove_all(cache_path2); + } + + io::FileCacheSettings settings; + settings.query_queue_size = 30; + settings.query_queue_elements = 5; + settings.index_queue_size = 30; + settings.index_queue_elements = 5; + settings.disposable_queue_size = 30; + settings.disposable_queue_elements = 5; + settings.capacity = 90; + settings.max_file_block_size = 30; + settings.max_query_cache_size = 30; + ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path, settings).ok()); + ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_path2, settings).ok()); + EXPECT_EQ(FileCacheFactory::instance()->get_cache_instance_size(), 2); + EXPECT_EQ(FileCacheFactory::instance()->get_capacity(), 180); + + // valid path + valid capacity + auto s = FileCacheFactory::instance()->reset_capacity(cache_base_path, 80); + LOG(INFO) << s; + EXPECT_EQ(FileCacheFactory::instance()->get_capacity(), 170); + + // empty path + valid capacity + s = FileCacheFactory::instance()->reset_capacity("", 70); + LOG(INFO) << s; + EXPECT_EQ(FileCacheFactory::instance()->get_capacity(), 140); + + // invalid path + valid capacity + s = FileCacheFactory::instance()->reset_capacity("/not/exist/haha", 70); + LOG(INFO) << s; + EXPECT_EQ(FileCacheFactory::instance()->get_capacity(), 140); + + // valid path + invalid capacity + s = FileCacheFactory::instance()->reset_capacity(cache_base_path, INT64_MAX); + LOG(INFO) << s; + EXPECT_LT(FileCacheFactory::instance()->get_capacity(), INT64_MAX); + EXPECT_GT(FileCacheFactory::instance()->get_capacity(), 70); + + // valid path + zero capacity + s = FileCacheFactory::instance()->reset_capacity(cache_base_path, 0); + LOG(INFO) << s; + EXPECT_LT(FileCacheFactory::instance()->get_capacity(), INT64_MAX); + EXPECT_GT(FileCacheFactory::instance()->get_capacity(), 70); + + // empty path + invalid capacity + s = FileCacheFactory::instance()->reset_capacity("", INT64_MAX); + LOG(INFO) << s; + EXPECT_LT(FileCacheFactory::instance()->get_capacity(), INT64_MAX); + EXPECT_GT(FileCacheFactory::instance()->get_capacity(), 70); + + // empty path + zero capacity + s = FileCacheFactory::instance()->reset_capacity("", 0); + LOG(INFO) << s; + EXPECT_LT(FileCacheFactory::instance()->get_capacity(), INT64_MAX); + EXPECT_GT(FileCacheFactory::instance()->get_capacity(), 70); + + FileCacheFactory::instance()->clear_file_caches(true); + std::this_thread::sleep_for(std::chrono::seconds(1)); + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } + if (fs::exists(cache_path2)) { + fs::remove_all(cache_path2); + } + FileCacheFactory::instance()->_caches.clear(); + FileCacheFactory::instance()->_path_to_cache.clear(); + FileCacheFactory::instance()->_capacity = 0; +} + } // namespace doris::io From caf00ea08643b6335f75ae7525caaf12ffb3bd53 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 09:45:18 +0800 Subject: [PATCH 067/572] branch-3.0: [fix](nereids) fix calc repeat property stack overflow #52159 (#52183) Cherry-picked from #52159 Co-authored-by: yujun --- .../ChildOutputPropertyDeriver.java | 5 +- .../nereids/trees/plans/algebra/Repeat.java | 4 +- .../test_repeat_no_stackflow.groovy | 429 ++++++++++++++++++ 3 files changed, 433 insertions(+), 5 deletions(-) create mode 100644 regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 7c78779b8b6f6f..66d30483b4e9db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -367,9 +367,8 @@ public PhysicalProperties visitPhysicalRepeat(PhysicalRepeat rep if (!groupingSets.isEmpty()) { Set intersectGroupingKeys = Utils.fastToImmutableSet(groupingSets.get(0)); for (int i = 1; i < groupingSets.size() && !intersectGroupingKeys.isEmpty(); i++) { - intersectGroupingKeys = Sets.intersection( - intersectGroupingKeys, Utils.fastToImmutableSet(groupingSets.get(i)) - ); + intersectGroupingKeys = Sets.intersection(intersectGroupingKeys, + Utils.fastToImmutableSet(groupingSets.get(i))).immutableCopy(); } List orderedShuffledColumns = distributionSpecHash.getOrderedShuffledColumns(); if (!intersectGroupingKeys.isEmpty() && intersectGroupingKeys.size() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java index 8925e597850818..06a1c7d47ff2a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Repeat.java @@ -111,8 +111,8 @@ default Set getCommonGroupingSetExpressions() { Iterator> iterator = groupingSets.iterator(); Set commonGroupingExpressions = Sets.newLinkedHashSet(iterator.next()); while (iterator.hasNext()) { - commonGroupingExpressions = - Sets.intersection(commonGroupingExpressions, Sets.newLinkedHashSet(iterator.next())); + commonGroupingExpressions = Sets.intersection(commonGroupingExpressions, + Sets.newLinkedHashSet(iterator.next())).immutableCopy(); if (commonGroupingExpressions.isEmpty()) { break; } diff --git a/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy b/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy new file mode 100644 index 00000000000000..2e0452371316b4 --- /dev/null +++ b/regression-test/suites/nereids_p0/test_repeat_no_stackflow.groovy @@ -0,0 +1,429 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_repeat_no_stackflow') { + multi_sql ''' + SET disable_nereids_rules="PRUNE_EMPTY_PARTITION"; + + DROP TABLE IF EXISTS test_repeat_no_stackflow_t1 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t2 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t3 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t4 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t5 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t6 FORCE; + + CREATE TABLE `test_repeat_no_stackflow_t1` ( + `id` int NOT NULL, + `agent_id` int NOT NULL, + `site_id` int NULL , + `tdate` varchar(255) NOT NULL , + `money` decimal(32,3) NOT NULL DEFAULT "0.000" , + `monies` decimal(13,3) NOT NULL DEFAULT "0.000" , + `account` decimal(13,3) NOT NULL DEFAULT "0.000" , + `reward_cost` decimal(13,3) NOT NULL DEFAULT "0.000" , + `finance_cost` decimal(13,3) NOT NULL DEFAULT "0.000" , + `ad_type` tinyint NOT NULL , + `agent_type` tinyint NULL , + `pay_type` tinyint NOT NULL , + `pay_date` varchar(255) NOT NULL, + `memo` text NULL, + `aorder` varchar(20) NULL, + `plat_id` tinyint NOT NULL DEFAULT "1", + `game_id` int NOT NULL DEFAULT "0", + `add_type` tinyint NOT NULL DEFAULT "0" , + `company_id` tinyint NOT NULL DEFAULT "0", + `nature` tinyint NOT NULL DEFAULT "0" , + `s_date` varchar(255) NOT NULL , + `e_date` varchar(255) NOT NULL , + `author` varchar(50) NULL , + `game_sign` varchar(64) NOT NULL DEFAULT "" , + `agent` varchar(64) NOT NULL DEFAULT "" , + `media` varchar(64) NOT NULL DEFAULT "" , + `company` varchar(16) NOT NULL DEFAULT "" , + `company_type` varchar(16) NOT NULL DEFAULT "self" , + `other` tinyint NOT NULL DEFAULT "0" , + `is_delete` tinyint NOT NULL DEFAULT "0" , + `created_at` varchar(255) NOT NULL, + `updated_at` varchar(255) NOT NULL +) ENGINE=OLAP +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 10 +PROPERTIES ( +"replication_num" = "1", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"enable_unique_key_merge_on_write" = "true", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"enable_mow_light_delete" = "false" +); + +CREATE TABLE `test_repeat_no_stackflow_t2` ( + `uid` varchar(255) NULL , + `game_sign` varchar(255) NULL , + `register_date` date NOT NULL , + `agent_id` varchar(255) NULL , + `site_id` varchar(255) NULL , + `game_id` varchar(255) NULL , + `guid` varchar(255) NOT NULL , + `openid` text NULL, + `mobile_phone` varchar(255) NULL , + `user_name` varchar(255) NULL , + `imei` text NULL , + `long_id` varchar(255) NULL , + `mtype` varchar(30) NULL , + `package` varchar(255) NULL , + `reg_time` varchar(255) NULL , + `logined` varchar(30) NULL , + `reg_type` varchar(30) NULL , + `os` varchar(20) NULL , + `game_aweme_id` varchar(255) NULL , + `match_type` varchar(30) NULL , + `network` varchar(255) NULL , + `idfv` text NULL , + `system` varchar(255) NULL , + `model` varchar(255) NULL , + `come_back_user` varchar(10) NULL , + `ip` varchar(255) NULL , + `ipv6` varchar(255) NULL , + `oaid` varchar(255) NULL , + `adinfo` text NULL , + `ad_device` text NULL , + `wx_platform` varchar(255) NULL , + `original_imei` varchar(255) NULL , + `total_reg_num` varchar(255) NULL , + `reg_hour` varchar(10) NOT NULL , + `reg_date_hour` varchar(128) NOT NULL +) ENGINE=OLAP +UNIQUE KEY(`uid`, `game_sign`, `register_date`) +DISTRIBUTED BY HASH(`uid`) BUCKETS 10 +PROPERTIES ( +"replication_num" = "1", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"enable_unique_key_merge_on_write" = "true", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"enable_mow_light_delete" = "false" +); + +CREATE TABLE `test_repeat_no_stackflow_t3` ( + `id` int NOT NULL, + `site_id` int NOT NULL DEFAULT "0" , + `site_name` varchar(255) NOT NULL DEFAULT "" , + `agent_id` int NOT NULL, + `plat` tinyint NOT NULL DEFAULT "0" , + `plat_id` tinyint NOT NULL DEFAULT "1" , + `chargeman` varchar(128) NOT NULL DEFAULT "" , + `site_url` varchar(128) NULL DEFAULT "" , + `https` tinyint NULL DEFAULT "0" , + `channel` varchar(60) NOT NULL DEFAULT "" , + `addtime` datetime NULL, + `edittime` datetime NULL, + `isdel` tinyint NOT NULL DEFAULT "0" , + `is_remote` tinyint NOT NULL DEFAULT "0" , + `convert_source_type` varchar(128) NOT NULL DEFAULT "" , + `convert_toolkit` varchar(128) NOT NULL DEFAULT "" , + `anchor_id` int NOT NULL DEFAULT "0" , + `anchor_account` varchar(128) NOT NULL DEFAULT "" , + `anchor_account_id` int NOT NULL DEFAULT "0" , + `create_user` varchar(128) NULL DEFAULT "" , + `update_user` varchar(128) NULL DEFAULT "" , + `action_track_type` int NULL DEFAULT "0" +) ENGINE=OLAP +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 10 +PROPERTIES ( +"replication_num" = "1", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"enable_unique_key_merge_on_write" = "true", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"enable_mow_light_delete" = "false" +); + +CREATE TABLE `test_repeat_no_stackflow_t4` ( + `id` varchar(255) NULL , + `agent_id` varchar(255) NULL , + `agent_name` varchar(255) NULL , + `group` varchar(255) NULL , + `media` varchar(255) NULL , + `agent` varchar(255) NULL , + `plat_id` varchar(10) NULL , + `reg_date` varchar(64) NULL , + `chargeman` varchar(128) NULL , + `agent_type` varchar(32) NULL , + `department_id` varchar(255) NULL , + `gr_agent_channel` varchar(64) NULL , + `channel_client` varchar(64) NULL , + `account_id` varchar(100) NULL , + `top` varchar(10) NULL , + `activity_at` varchar(64) NULL , + `updated_time` varchar(64) NULL , + `is_tw` varchar(10) NULL , + `system` varchar(32) NULL , + `subject` varchar(32) NULL , + `gr_agent_remarks` text NULL , + `agent_media_name` varchar(64) NULL , + `description` varchar(255) NULL , + `gr_agent_media_channel` varchar(32) NULL , + `callback` varchar(32) NULL , + `type` varchar(255) NULL , + `hot` varchar(10) NULL , + `default_media` varchar(3) NULL , + `agent_channel_name` varchar(32) NULL , + `gr_agent_channel_remarks` text NULL +) ENGINE=OLAP +UNIQUE KEY(`id`, `agent_id`) +DISTRIBUTED BY HASH(`agent_id`) BUCKETS 10 +PROPERTIES ( +"replication_num" = "1", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"enable_unique_key_merge_on_write" = "true", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"enable_mow_light_delete" = "false" +); + +CREATE TABLE `test_repeat_no_stackflow_t5` ( + `game_id` varchar(255) NULL , + `plat_id` varchar(20) NULL , + `game_app_name` varchar(255) NULL , + `game_sign` varchar(128) NULL , + `game_name` varchar(255) NULL , + `contract_name` varchar(255) NULL , + `company_qualification` varchar(255) NULL , + `system` varchar(64) NULL , + `sync` varchar(10) NULL , + `type` varchar(10) NULL , + `game_product_id` varchar(32) NULL , + `product_name` varchar(255) NULL , + `core_play` varchar(32) NULL , + `story_theme` varchar(32) NULL , + `art_style` varchar(32) NULL , + `contract_company` varchar(32) NULL , + `state` varchar(10) NULL , + `screen_type` varchar(10) NULL , + `float_position` varchar(20) NULL , + `is_open` varchar(10) NULL , + `is_sync` varchar(10) NULL , + `open_game` varchar(4) NULL , + `app_name` varchar(256) NULL , + `app_id` varchar(255) NULL , + `release_state` varchar(10) NULL , + `media_abbr` varchar(64) NULL , + `os` varchar(10) NULL , + `os_two` varchar(10) NULL , + `game_version` varchar(20) NULL , + `channel_show` varchar(10) NULL , + `autologin` varchar(10) NULL , + `relate_game` varchar(255) NULL , + `discount` varchar(32) NULL , + `disable_register` varchar(4) NULL , + `disable_unrelated_login` varchar(4) NULL , + `disable_related_back` varchar(4) NULL , + `disable_back` varchar(4) NULL , + `accept_related_game` varchar(20) NULL , + `disable_pay` varchar(4) NULL , + `tw_plat_id` varchar(4) NULL , + `tw_os` varchar(4) NULL , + `server_sign` varchar(32) NULL , + `objective_id` varchar(32) NULL , + `company_main` varchar(64) NULL , + `business_purpose` varchar(4) NULL , + `agent_sign` varchar(60) NULL , + `icon` varchar(128) NULL , + `package_name_id` varchar(30) NULL , + `server_group_id` varchar(40) NULL , + `client_type` varchar(10) NULL , + `platform` varchar(10) NULL +) ENGINE=OLAP +UNIQUE KEY(`game_id`, `plat_id`) +DISTRIBUTED BY HASH(`game_id`) BUCKETS 10 +PROPERTIES ( +"replication_num" = "1", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"enable_unique_key_merge_on_write" = "true", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"enable_mow_light_delete" = "false" +); + +CREATE TABLE `test_repeat_no_stackflow_t6` ( + `id` int NOT NULL , + `abbr` varchar(64) NOT NULL , + `name` varchar(64) NOT NULL , + `desc` varchar(255) NULL , + `created_at` datetime NULL DEFAULT CURRENT_TIMESTAMP , + `created_user` varchar(64) NULL , + `update_at` datetime NULL DEFAULT CURRENT_TIMESTAMP , + `update_user` varchar(64) NULL , + `hot` tinyint NULL , + `usage_status` tinyint NULL DEFAULT "1" +) ENGINE=OLAP +UNIQUE KEY(`id`) +DISTRIBUTED BY HASH(`id`) BUCKETS 10 +PROPERTIES ( +"replication_num" = "1", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"enable_unique_key_merge_on_write" = "true", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"enable_mow_light_delete" = "false" +); + ''' + + // the logical repeat had a long set intersection chain, + // need convert set intersection's result SetView to Set, + // otherwise it will cause stackoverflow + sql """ explain + SELECT + agent_dims.game_id, + cj.`system`, + cj.game_sign, + cj.os, + cj.os_two, + ci.account_id, + ci.media, + ci.gr_agent_media_channel, + cj.game_product_id, + ck.site_name, + ci.chargeman, + agent_dims.agent_id, + agent_dims.site_id, + ck.anchor_id, + cl.abbr, + SUM(agent_dims.total_money) AS total_cost, + SUM(user_guid_reg_real.guid) AS total_reg_users, + CASE + WHEN SUM(user_guid_reg_real.guid) = 0 OR SUM(user_guid_reg_real.guid) IS NULL THEN 0 + ELSE ROUND(SUM(agent_dims.total_money) / NULLIF(SUM(user_guid_reg_real.guid), 0), 2) + END AS reg_cost +FROM ( + SELECT + `site_id`, + `game_id`, + `agent_id`, + `game_sign`, + tdate, + COUNT(1), + SUM(`money`) AS total_money + FROM test_repeat_no_stackflow_t1 + WHERE tdate BETWEEN '2025-05-01' AND '2025-05-01' + GROUP BY + `site_id`, + `game_id`, + `agent_id`, + `game_sign`, + tdate +) AS agent_dims +LEFT JOIN ( + SELECT + `site_id`, + `game_id`, + `agent_id`, + `game_sign`, + register_date, + COUNT(DISTINCT guid) AS guid + FROM test_repeat_no_stackflow_t2 + WHERE `register_date` BETWEEN '2025-05-01' AND '2025-05-01' + GROUP BY + `site_id`, + `game_id`, + `agent_id`, + `game_sign`, + register_date +) AS user_guid_reg_real + ON agent_dims.game_id = user_guid_reg_real.game_id + AND agent_dims.game_sign = user_guid_reg_real.game_sign + AND agent_dims.agent_id = user_guid_reg_real.agent_id + AND agent_dims.site_id = user_guid_reg_real.site_id + AND agent_dims.tdate = user_guid_reg_real.register_date +LEFT JOIN `test_repeat_no_stackflow_t3` ck + ON agent_dims.site_id = ck.site_id +LEFT JOIN `test_repeat_no_stackflow_t4` ci + ON agent_dims.agent_id = ci.agent_id +LEFT JOIN `test_repeat_no_stackflow_t5` cj + ON agent_dims.game_id = cj.game_id +LEFT JOIN `test_repeat_no_stackflow_t6` cl + ON ci.agent = cl.abbr +WHERE + agent_dims.game_id = '3706' + AND agent_dims.site_id = '5100814' +GROUP BY + CUBE( + agent_dims.game_id, + cj.`system`, + cj.game_sign, + cj.os, + cj.os_two, + ci.account_id, + ci.media, + ci.gr_agent_media_channel, + cj.game_product_id, + ck.site_name, + ci.chargeman, + agent_dims.agent_id, + agent_dims.site_id, + ck.anchor_id, + cl.abbr + ); + """ + + multi_sql ''' + DROP TABLE IF EXISTS test_repeat_no_stackflow_t1 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t2 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t3 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t4 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t5 FORCE; + DROP TABLE IF EXISTS test_repeat_no_stackflow_t6 FORCE; + ''' +} From c8aaebb49be0ace547ea92b95365ad0cd40ffbe6 Mon Sep 17 00:00:00 2001 From: koarz Date: Wed, 25 Jun 2025 09:46:08 +0800 Subject: [PATCH 068/572] branch-3.0: [enhance](cloud)add log for compaction read time (#52168) https://github.com/apache/doris/pull/51355 --- be/src/cloud/cloud_base_compaction.cpp | 2 ++ be/src/cloud/cloud_cumulative_compaction.cpp | 2 ++ be/src/cloud/cloud_full_compaction.cpp | 2 ++ be/src/cloud/cloud_tablet.cpp | 15 +++++++++++++++ be/src/cloud/cloud_tablet.h | 4 ++++ be/src/olap/compaction.cpp | 6 ++++++ be/src/olap/merger.cpp | 6 ++++++ be/src/olap/merger.h | 2 ++ 8 files changed, 39 insertions(+) diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp index 2b9fd1c2e56558..963ce1fa587308 100644 --- a/be/src/cloud/cloud_base_compaction.cpp +++ b/be/src/cloud/cloud_base_compaction.cpp @@ -301,6 +301,8 @@ Status CloudBaseCompaction::execute_compact() { .tag("output_rowset_data_size", _output_rowset->data_disk_size()) .tag("output_rowset_index_size", _output_rowset->index_disk_size()) .tag("output_rowset_total_size", _output_rowset->total_disk_size()) + .tag("local_read_time_us", _stats.cloud_local_read_time) + .tag("remote_read_time_us", _stats.cloud_remote_read_time) .tag("local_read_bytes", _local_read_bytes_total) .tag("remote_read_bytes", _remote_read_bytes_total); diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index 8546ac04fcf92e..39954097324a99 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -224,6 +224,8 @@ Status CloudCumulativeCompaction::execute_compact() { .tag("cumulative_point", cloud_tablet()->cumulative_layer_point()) .tag("num_rowsets", cloud_tablet()->fetch_add_approximate_num_rowsets(0)) .tag("cumu_num_rowsets", cloud_tablet()->fetch_add_approximate_cumu_num_rowsets(0)) + .tag("local_read_time_us", _stats.cloud_local_read_time) + .tag("remote_read_time_us", _stats.cloud_remote_read_time) .tag("local_read_bytes", _local_read_bytes_total) .tag("remote_read_bytes", _remote_read_bytes_total); diff --git a/be/src/cloud/cloud_full_compaction.cpp b/be/src/cloud/cloud_full_compaction.cpp index f04b7d1d4a7e32..6bfab2ec69808d 100644 --- a/be/src/cloud/cloud_full_compaction.cpp +++ b/be/src/cloud/cloud_full_compaction.cpp @@ -183,6 +183,8 @@ Status CloudFullCompaction::execute_compact() { .tag("output_rowset_data_size", _output_rowset->data_disk_size()) .tag("output_rowset_index_size", _output_rowset->index_disk_size()) .tag("output_rowset_total_size", _output_rowset->total_disk_size()) + .tag("local_read_time_us", _stats.cloud_local_read_time) + .tag("remote_read_time_us", _stats.cloud_remote_read_time) .tag("local_read_bytes", _local_read_bytes_total) .tag("remote_read_bytes", _remote_read_bytes_total); diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index b4cf9d2e31c153..a26ce0dd2c7172 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -730,6 +730,21 @@ void CloudTablet::get_compaction_status(std::string* json_result) { static_cast(_last_full_compaction_status.length()), root.GetAllocator()); root.AddMember("last full status", full_compaction_status_value, root.GetAllocator()); + rapidjson::Value exec_compaction_time; + std::string num_str {std::to_string(exec_compaction_time_us.load())}; + exec_compaction_time.SetString(num_str.c_str(), static_cast(num_str.length()), + root.GetAllocator()); + root.AddMember("exec compaction time us", exec_compaction_time, root.GetAllocator()); + rapidjson::Value local_read_time; + num_str = std::to_string(local_read_time_us.load()); + local_read_time.SetString(num_str.c_str(), static_cast(num_str.length()), + root.GetAllocator()); + root.AddMember("compaction local read time us", local_read_time, root.GetAllocator()); + rapidjson::Value remote_read_time; + num_str = std::to_string(remote_read_time_us.load()); + remote_read_time.SetString(num_str.c_str(), static_cast(num_str.length()), + root.GetAllocator()); + root.AddMember("compaction remote read time us", remote_read_time, root.GetAllocator()); // print all rowsets' version as an array rapidjson::Document versions_arr; diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index ee3fdc35c6a2c5..f3af8b09b27869 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -251,6 +251,10 @@ class CloudTablet final : public BaseTablet { int64_t last_cumu_no_suitable_version_ms = 0; int64_t last_access_time_ms = 0; + std::atomic local_read_time_us = 0; + std::atomic remote_read_time_us = 0; + std::atomic exec_compaction_time_us = 0; + // Return merged extended schema TabletSchemaSPtr merged_tablet_schema() const override; diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index dccf56e9c9b725..7e312c5847f1dd 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1359,6 +1359,12 @@ Status CloudCompactionMixin::execute_compact_impl(int64_t permits) { // 4. modify rowsets in memory RETURN_IF_ERROR(modify_rowsets()); + // update compaction status data + auto tablet = std::static_pointer_cast(_tablet); + tablet->local_read_time_us.fetch_add(_stats.cloud_local_read_time); + tablet->remote_read_time_us.fetch_add(_stats.cloud_remote_read_time); + tablet->exec_compaction_time_us.fetch_add(watch.get_elapse_time_us()); + return Status::OK(); } diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index b213808af24a85..01f36d77dc2cc5 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -143,6 +143,9 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, stats_output->bytes_read_from_remote = reader.stats().file_cache_stats.bytes_read_from_remote; stats_output->cached_bytes_total = reader.stats().file_cache_stats.bytes_write_into_cache; + stats_output->cloud_local_read_time = reader.stats().file_cache_stats.local_io_timer / 1000; + stats_output->cloud_remote_read_time = + reader.stats().file_cache_stats.remote_io_timer / 1000; } RETURN_NOT_OK_STATUS_WITH_WARN(dst_rowset_writer->flush(), @@ -320,6 +323,9 @@ Status Merger::vertical_compact_one_group( stats_output->bytes_read_from_remote = reader.stats().file_cache_stats.bytes_read_from_remote; stats_output->cached_bytes_total = reader.stats().file_cache_stats.bytes_write_into_cache; + stats_output->cloud_local_read_time = reader.stats().file_cache_stats.local_io_timer / 1000; + stats_output->cloud_remote_read_time = + reader.stats().file_cache_stats.remote_io_timer / 1000; } RETURN_IF_ERROR(dst_rowset_writer->flush_columns(is_key)); diff --git a/be/src/olap/merger.h b/be/src/olap/merger.h index 76d053a7a79d6b..302e606371a7b5 100644 --- a/be/src/olap/merger.h +++ b/be/src/olap/merger.h @@ -43,6 +43,8 @@ class VerticalBlockReader; class Merger { public: struct Statistics { + int64_t cloud_local_read_time = 0; + int64_t cloud_remote_read_time = 0; // number of rows written to the destination rowset after merge int64_t output_rows = 0; int64_t merged_rows = 0; From 545f9977e91401751740726ec029e34be1aa0a98 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Jun 2025 09:47:21 +0800 Subject: [PATCH 069/572] branch-3.0: [improvement](nereids)Support GROUP BY ... WITH ROLLUP syntax #51948 (#51977) Cherry-picked from #51948 Co-authored-by: James --- .../org/apache/doris/nereids/DorisParser.g4 | 2 +- .../data/nereids_syntax_p0/grouping_sets.out | 29 +++++++++++++++++++ .../nereids_syntax_p0/grouping_sets.groovy | 9 ++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 67f3e7997fa9ef..7b3ecbafa7e27e 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1199,7 +1199,7 @@ groupingElement : ROLLUP LEFT_PAREN (expression (COMMA expression)*)? RIGHT_PAREN | CUBE LEFT_PAREN (expression (COMMA expression)*)? RIGHT_PAREN | GROUPING SETS LEFT_PAREN groupingSet (COMMA groupingSet)* RIGHT_PAREN - | expression (COMMA expression)* + | expression (COMMA expression)* (WITH ROLLUP)? ; groupingSet diff --git a/regression-test/data/nereids_syntax_p0/grouping_sets.out b/regression-test/data/nereids_syntax_p0/grouping_sets.out index bbc2997e3c8a0b..973909743943a7 100644 --- a/regression-test/data/nereids_syntax_p0/grouping_sets.out +++ b/regression-test/data/nereids_syntax_p0/grouping_sets.out @@ -132,6 +132,35 @@ 13 0 7 0 +-- !select_with_rollup1 -- +\N \N 36 +2 \N 6 +2 0 \N +2 1 6 +3 \N 12 +3 2 12 +4 \N 18 +4 0 \N +4 3 18 + +-- !select_with_rollup2 -- +\N 1 +2 0 +3 0 +4 0 + +-- !select_with_rollup3 -- +3 0 +8 0 +9 0 +20 1 + +-- !select_with_rollup4 -- +\N 1 +2 0 +3 0 +4 0 + -- !select -- \N \N 36 \N 0 \N diff --git a/regression-test/suites/nereids_syntax_p0/grouping_sets.groovy b/regression-test/suites/nereids_syntax_p0/grouping_sets.groovy index 8ca787fabfb802..99d2ddc7439eb2 100644 --- a/regression-test/suites/nereids_syntax_p0/grouping_sets.groovy +++ b/regression-test/suites/nereids_syntax_p0/grouping_sets.groovy @@ -110,6 +110,15 @@ suite("test_nereids_grouping_sets") { order_qt_select "select sum(k2+1), grouping_id(k1+1) from groupingSetsTable group by grouping sets((k1+1)) having (k1+1) > 1;"; order_qt_select "select sum(k2+1), grouping_id(k1+1) from groupingSetsTable group by grouping sets((k1+1), (k1)) having (k1+1) > 1;"; + // with rollup + qt_select_with_rollup1 """ + select (k1 + 1) k1_, k2, sum(k3) from groupingSetsTable group by + k1_, k2 with rollup order by k1_, k2 + """ + qt_select_with_rollup2 "select k1+1, grouping(k1+1) from groupingSetsTable group by (k1+1) with rollup order by k1+1;"; + qt_select_with_rollup3 "select sum(k2), grouping(k1+1) from groupingSetsTable group by k1+1 with rollup order by sum(k2)" + qt_select_with_rollup4 "select k1+1, grouping_id(k1) from groupingSetsTable group by k1 with rollup order by k1+1;" + // old grouping sets qt_select """ SELECT k1, k2, SUM(k3) FROM groupingSetsTable From 03692092da4dc6881a264977353c5d07ee48ec77 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Wed, 25 Jun 2025 09:56:14 +0800 Subject: [PATCH 070/572] [feat](test) Pick docker case for recycling CloudTablet unused rowsets (#52163) * Pick case from https://github.com/apache/doris/pull/51573 --- ...compaction_and_read_stale_cloud_docker.out | 36 ++ ...paction_and_read_stale_cloud_docker.groovy | 390 ++++++++++++++++++ ...egments_and_read_stale_cloud_docker.groovy | 347 ++++++++++++++++ 3 files changed, 773 insertions(+) create mode 100644 regression-test/data/compaction/test_filecache_compaction_and_read_stale_cloud_docker.out create mode 100644 regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy create mode 100644 regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy diff --git a/regression-test/data/compaction/test_filecache_compaction_and_read_stale_cloud_docker.out b/regression-test/data/compaction/test_filecache_compaction_and_read_stale_cloud_docker.out new file mode 100644 index 00000000000000..8701d535cf3b53 --- /dev/null +++ b/regression-test/data/compaction/test_filecache_compaction_and_read_stale_cloud_docker.out @@ -0,0 +1,36 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +1 99 +2 99 +3 99 +4 99 +5 99 + +-- !sql2 -- +1 99 +2 99 +3 99 +4 99 +5 99 + +-- !sql3 -- +1 99 +2 99 +3 99 +4 99 +5 99 + +-- !sql4 -- +1 99 +2 99 +3 99 +4 99 +5 100 + +-- !sql5 -- +1 99 +2 99 +3 99 +4 99 +5 100 + diff --git a/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy new file mode 100644 index 00000000000000..c1dc7f6dbc7345 --- /dev/null +++ b/regression-test/suites/compaction/test_filecache_compaction_and_read_stale_cloud_docker.groovy @@ -0,0 +1,390 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.atomic.AtomicBoolean +import org.apache.doris.regression.suite.ClusterOptions +import org.codehaus.groovy.runtime.IOGroovyMethods +import org.apache.doris.regression.util.Http + +suite("test_filecache_compaction_and_read_stale_cloud_docker", "docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.enableDebugPoints() + options.feConfigs.add("enable_workload_group=false") + options.beConfigs.add('compaction_promotion_version_count=5') + options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') + options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') + options.beConfigs.add('enable_java_support=false') + + def dbName = "" + def testTable = "test_filecache_compaction_and_read_stale_cloud_docker" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + def backendId_to_backendBrpcPort = [:] + + def triggerCompaction = { tablet -> + def compact_type = "cumulative" + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + if (compact_type == "cumulative") { + def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } else if (compact_type == "full") { + def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) + assertEquals(code_2, 0) + return out_2 + } else { + assertFalse(True) + } + } + + def getTabletStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def waitForCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def running = true + do { + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + def getLocalDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + boolean running = true + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_local?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + def getMsDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + boolean running = true + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_ms?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get ms delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + docker(options) { + def fes = sql_return_maparray "show frontends" + logger.info("frontends: ${fes}") + def url = "jdbc:mysql://${fes[0].Host}:${fes[0].QueryPort}/" + logger.info("url: " + url) + AtomicBoolean query_result = new AtomicBoolean(true) + def query = { + connect( context.config.jdbcUser, context.config.jdbcPassword, url) { + logger.info("query start") + def results = sql_return_maparray """ select * from ${dbName}.${testTable}; """ + logger.info("query result: " + results) + Set keys = new HashSet<>() + for (final def result in results) { + if (keys.contains(result.k)) { + logger.info("find duplicate key: " + result.k) + query_result.set(false) + break + } + keys.add(result.k) + } + logger.info("query finish. query_result: " + query_result.get()) + } + } + + def result = sql 'SELECT DATABASE()' + dbName = result[0][0] + + sql """ DROP TABLE IF EXISTS ${testTable} """ + sql """ + create table ${testTable} (`k` int NOT NULL, `v` int NOT NULL) + UNIQUE KEY(`k`) + DISTRIBUTED BY HASH(`k`) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + // getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + getBackendIpHttpAndBrpcPort(backendId_to_backendIP, backendId_to_backendHttpPort, backendId_to_backendBrpcPort); + + def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + try { + Set all_history_stale_rowsets = new HashSet<>(); + + // write some data + sql """ INSERT INTO ${testTable} VALUES (1,99); """ + sql """ INSERT INTO ${testTable} VALUES (2,99); """ + sql """ INSERT INTO ${testTable} VALUES (3,99); """ + sql """ INSERT INTO ${testTable} VALUES (4,99); """ + sql """ INSERT INTO ${testTable} VALUES (5,99); """ + sql "sync" + order_qt_sql1 """ select * from ${testTable}; """ + + def tablet_status = getTabletStatus(tablet) + + // after compaction, [1-6] versions will become stale rowsets + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + + // trigger compaction to generate base rowset + assertTrue(triggerCompaction(tablet).contains("Success")) + waitForCompaction(tablet) + tablet_status = getTabletStatus(tablet) + assertEquals(2, tablet_status["rowsets"].size()) + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + + def ms_dm = getMsDeleteBitmapStatus(tablet) + assertEquals(0, ms_dm["delete_bitmap_count"]) + order_qt_sql2 "select * from ${testTable}" + + // write some data + sql """ INSERT INTO ${testTable} VALUES (1,99); """ + sql """ INSERT INTO ${testTable} VALUES (2,99); """ + sql """ INSERT INTO ${testTable} VALUES (3,99); """ + sql """ INSERT INTO ${testTable} VALUES (4,99); """ + sql """ INSERT INTO ${testTable} VALUES (5,99); """ + sql """ sync """ + order_qt_sql3 "select * from ${testTable}" + tablet_status = getTabletStatus(tablet) + assertEquals(7, tablet_status["rowsets"].size()) + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + ms_dm = getMsDeleteBitmapStatus(tablet) + assertEquals(5, ms_dm["delete_bitmap_count"]) + + // trigger and block one query + GetDebugPoint().enableDebugPointForAllBEs("NewOlapScanner::_init_tablet_reader_params.block") + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset") + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", + [tablet_id: "${tablet_id}", start_version: 7, end_version: 11]); + Thread query_thread = new Thread(() -> query()) + query_thread.start() + sleep(100) + + // trigger compaction + // getTabletStatus(tablet) + assertTrue(triggerCompaction(tablet).contains("Success")) + waitForCompaction(tablet) + logger.info("compaction2 finished") + // check rowset count + tablet_status = getTabletStatus(tablet) + assertEquals(3, tablet_status["rowsets"].size()) + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + // check ms delete bitmap count + ms_dm = getMsDeleteBitmapStatus(tablet) + assertEquals(5, ms_dm["delete_bitmap_count"]) + assertEquals(5, ms_dm["cardinality"]) + // check local delete bitmap count + def local_dm = getLocalDeleteBitmapStatus(tablet) + assertEquals(5, local_dm["delete_bitmap_count"]) + assertEquals(5, local_dm["cardinality"]) + + // wait for stale rowsets are deleted + boolean is_stale_rowsets_deleted = false + for (int i = 0; i < 100; i++) { + tablet_status = getTabletStatus(tablet) + if (tablet_status["stale_rowsets"].size() == 0) { + is_stale_rowsets_deleted = true + break + } + sleep(500) + } + assertTrue(is_stale_rowsets_deleted, "stale rowsets are not deleted") + // check to delete bitmap of stale rowsets is not deleted + sleep(1000) + def local_dm_status = getLocalDeleteBitmapStatus(tablet) + assertEquals(5, local_dm_status["delete_bitmap_count"]) + + // unnlock query and check no duplicated keys + GetDebugPoint().disableDebugPointForAllBEs("NewOlapScanner::_init_tablet_reader_params.block") + query_thread.join() + assertTrue(query_result.get(), "find duplicated keys") + + // check delete bitmap of compaction2 stale rowsets are deleted + // write some data + sql """ INSERT INTO ${testTable} VALUES (1,99); """ + sql """ INSERT INTO ${testTable} VALUES (2,99); """ + sql """ INSERT INTO ${testTable} VALUES (3,99); """ + sql """ INSERT INTO ${testTable} VALUES (4,99); """ + sql """ INSERT INTO ${testTable} VALUES (5,100); """ + sql "sync" + order_qt_sql4 "select * from ${testTable}" + logger.info("order_qt_sql4 finished") + tablet_status = getTabletStatus(tablet) + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + getMsDeleteBitmapStatus(tablet) + // trigger compaction + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", + [tablet_id: "${tablet_id}", start_version: 12, end_version: 16]); + tablet_status = getTabletStatus(tablet) + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + assertTrue(triggerCompaction(tablet).contains("Success")) + waitForCompaction(tablet) + boolean is_compaction_finished = false + for (int i = 0; i < 100; i++) { + tablet_status = getTabletStatus(tablet) + all_history_stale_rowsets.addAll(tablet_status["rowsets"]) + if (tablet_status["rowsets"].size() == 4) { + logger.info("final tablet status: ${tablet_status}") + is_compaction_finished = true + break + } + sleep(500) + } + assertTrue(is_compaction_finished, "compaction is not finished") + logger.info("compaction3 finished") + // check ms delete bitmap count + ms_dm = getMsDeleteBitmapStatus(tablet) + assertEquals(10, ms_dm["delete_bitmap_count"]) + assertEquals(10, ms_dm["cardinality"]) + // check delete bitmap count + logger.info("check local delete bitmap is deleted") + boolean is_local_dm_deleted = false + for (int i = 0; i < 100; i++) { + local_dm_status = getLocalDeleteBitmapStatus(tablet) + if (local_dm_status["delete_bitmap_count"] == 10) { + assertEquals(10, local_dm_status["cardinality"]) + is_local_dm_deleted = true + break + } + sleep(500) + } + assertTrue(is_local_dm_deleted, "delete bitmap of compaction2 stale rowsets are not deleted") + order_qt_sql5 "select * from ${testTable}" + + tablet_status = getTabletStatus(tablet) + def final_rowsets = tablet_status["rowsets"] + + // sleep for vacuum_stale_rowsets_interval_s=10 seconds to wait for unused rowsets are deleted + sleep(21000) + + def be_host = backendId_to_backendIP[tablet.BackendId] + def be_http_port = backendId_to_backendHttpPort[tablet.BackendId] + for (int i = 0; i < all_history_stale_rowsets.size(); i++) { + def rowsetStr = all_history_stale_rowsets[i] + // [12-12] 1 DATA NONOVERLAPPING 02000000000000124843c92c13625daa8296c20957119893 1011.00 B + def start_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[0].toInteger() + def end_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[1].toInteger() + def rowset_id = rowsetStr.split(" ")[4] + if (start_version == 0 || start_version != end_version) { + continue + } + + logger.info("rowset ${i}, start: ${start_version}, end: ${end_version}, id: ${rowset_id}") + def data = Http.GET("http://${be_host}:${be_http_port}/api/file_cache?op=list_cache&value=${rowset_id}_0.dat", true) + logger.info("file cache data: ${data}") + assertTrue(data.size() == 0) + } + + for (int i = 0; i < final_rowsets.size(); i++) { + def rowsetStr = final_rowsets[i] + def start_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[0].toInteger() + def end_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[1].toInteger() + def rowset_id = rowsetStr.split(" ")[4] + if (start_version == 0) { + continue + } + + logger.info("final rowset ${i}, start: ${start_version}, end: ${end_version}, id: ${rowset_id}") + def data = Http.GET("http://${be_host}:${be_http_port}/api/file_cache?op=list_cache&value=${rowset_id}_0.dat", true) + logger.info("file cache data: ${data}") + assertTrue(data.size() > 0) + } + + def (code_0, out_0, err_0) = curl("GET", "http://${be_host}:${backendId_to_backendBrpcPort[tablet.BackendId]}/vars/unused_rowsets_count") + logger.info("out_0: ${out_0}") + def unusedRowsetsCount = out_0.trim().split(":")[1].trim().toInteger() + assertEquals(0, unusedRowsetsCount) + + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + } + } +} diff --git a/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy b/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy new file mode 100644 index 00000000000000..4b6613160907d4 --- /dev/null +++ b/regression-test/suites/compaction/test_filecache_compaction_multisegments_and_read_stale_cloud_docker.groovy @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.atomic.AtomicBoolean +import org.apache.doris.regression.suite.ClusterOptions +import org.codehaus.groovy.runtime.IOGroovyMethods +import org.apache.doris.regression.util.Http + +suite("test_filecache_compaction_multisegments_and_read_stale_cloud_docker", "docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + options.enableDebugPoints() + options.feConfigs.add("enable_workload_group=false") + options.beConfigs.add('compaction_promotion_version_count=5') + options.beConfigs.add('tablet_rowset_stale_sweep_time_sec=0') + options.beConfigs.add('vacuum_stale_rowsets_interval_s=10') + options.beConfigs.add('enable_java_support=false') + options.beConfigs.add('doris_scanner_row_bytes=1') + + def dbName = "" + def testTable = "test_filecache_multisegments_and_read_stale" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + def backendId_to_backendBrpcPort = [:] + + def triggerCompaction = { tablet -> + def compact_type = "cumulative" + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + if (compact_type == "cumulative") { + def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } else if (compact_type == "full") { + def (code_2, out_2, err_2) = be_run_full_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_2 + ", out=" + out_2 + ", err=" + err_2) + assertEquals(code_2, 0) + return out_2 + } else { + assertFalse(True) + } + } + + def getTabletStatus = { tablet, rowsetIndex, lastRowsetSegmentNum, enableAssert = false, outputRowsets = null -> + String compactionUrl = tablet["CompactionStatus"] + def (code, out, err) = curl("GET", compactionUrl) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + if (outputRowsets != null) { + outputRowsets.addAll(tabletJson.rowsets) + } + + assertTrue(tabletJson.rowsets.size() >= rowsetIndex) + def rowset = tabletJson.rowsets.get(rowsetIndex - 1) + logger.info("rowset: ${rowset}") + int start_index = rowset.indexOf("]") + int end_index = rowset.indexOf("DATA") + def segmentNumStr = rowset.substring(start_index + 1, end_index).trim() + logger.info("segmentNumStr: ${segmentNumStr}") + if (enableAssert) { + assertEquals(lastRowsetSegmentNum, Integer.parseInt(segmentNumStr)) + } else { + return lastRowsetSegmentNum == Integer.parseInt(segmentNumStr); + } + } + + def waitForCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def running = true + do { + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + def getLocalDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + boolean running = true + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_local?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + def getMsDeleteBitmapStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + boolean running = true + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/delete_bitmap/count_ms?verbose=true&tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get ms delete bitmap count status: =" + code + ", out=" + out) + assertEquals(code, 0) + def deleteBitmapStatus = parseJson(out.trim()) + return deleteBitmapStatus + } + + docker(options) { + def fes = sql_return_maparray "show frontends" + logger.info("frontends: ${fes}") + def url = "jdbc:mysql://${fes[0].Host}:${fes[0].QueryPort}/" + logger.info("url: " + url) + AtomicBoolean query_result = new AtomicBoolean(true) + def query = { + connect( context.config.jdbcUser, context.config.jdbcPassword, url) { + logger.info("query start") + def results = sql_return_maparray """ select * from ${dbName}.${testTable}; """ + logger.info("query result: " + results) + Set keys = new HashSet<>() + for (final def result in results) { + if (keys.contains(result.k)) { + logger.info("find duplicate key: " + result.k) + query_result.set(false) + break + } + keys.add(result.k) + } + logger.info("query finish. query_result: " + query_result.get()) + } + } + + def result = sql 'SELECT DATABASE()' + dbName = result[0][0] + + sql """ DROP TABLE IF EXISTS ${testTable} """ + sql """ CREATE TABLE IF NOT EXISTS ${testTable} ( + `k1` int(11) NULL, + `k2` int(11) NULL, + `v3` int(11) NULL, + `v4` int(11) NULL + ) unique KEY(`k1`, `k2`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "disable_auto_compaction" = "true" + ); + """ + + // getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + getBackendIpHttpAndBrpcPort(backendId_to_backendIP, backendId_to_backendHttpPort, backendId_to_backendBrpcPort); + + def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + def backend_id = tablet.BackendId + + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") + GetDebugPoint().enableDebugPointForAllBEs("CumulativeCompaction.modify_rowsets.delete_expired_stale_rowset") + GetDebugPoint().enableDebugPointForAllBEs("Tablet.delete_expired_stale_rowset.start_delete_unused_rowset") + + Set all_history_stale_rowsets = new HashSet<>(); + try { + // load 1 + streamLoad { + table "${testTable}" + set 'column_separator', ',' + set 'compress_type', 'GZ' + file 'test_schema_change_add_key_column.csv.gz' + time 10000 // limit inflight 10s + + check { res, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(res) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(8192, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + def rowCount1 = sql """ select count() from ${testTable}; """ + logger.info("rowCount1: ${rowCount1}") + // check generate 3 segments + getTabletStatus(tablet, 2, 3, true, all_history_stale_rowsets) + + // trigger compaction + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", + [tablet_id: "${tablet.TabletId}", start_version: 2, end_version: 2]) + def (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + logger.info("compact json: " + compactJson) + // check generate 1 segments + for (int i = 0; i < 20; i++) { + if (getTabletStatus(tablet, 2, 1, false, all_history_stale_rowsets)) { + break + } + sleep(100) + } + getTabletStatus(tablet, 2, 1, false, all_history_stale_rowsets) + sql """ select * from ${testTable} limit 1; """ + + // load 2 + streamLoad { + table "${testTable}" + set 'column_separator', ',' + set 'compress_type', 'GZ' + file 'test_schema_change_add_key_column1.csv.gz' + time 10000 // limit inflight 10s + + check { res, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(res) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20480, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + } + } + sql "sync" + def rowCount2 = sql """ select count() from ${testTable}; """ + logger.info("rowCount2: ${rowCount2}") + // check generate 3 segments + getTabletStatus(tablet, 3, 6, false, all_history_stale_rowsets) + def local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local delete bitmap 1: " + local_dm) + + // trigger compaction for load 2 + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", + [tablet_id: "${tablet.TabletId}", start_version: 3, end_version: 3]) + (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + compactJson = parseJson(out.trim()) + logger.info("compact json: " + compactJson) + waitForCompaction(tablet) + // check generate 1 segments + for (int i = 0; i < 20; i++) { + if (getTabletStatus(tablet, 3, 1, false, all_history_stale_rowsets)) { + break + } + sleep(100) + } + getTabletStatus(tablet, 3, 1, false, all_history_stale_rowsets) + + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.vacuum_stale_rowsets") // cloud + GetDebugPoint().enableDebugPointForAllBEs("DeleteBitmapAction._handle_show_local_delete_bitmap_count.start_delete_unused_rowset") // local + local_dm = getLocalDeleteBitmapStatus(tablet) + logger.info("local delete bitmap 2: " + local_dm) + assertEquals(1, local_dm["delete_bitmap_count"]) + + + // sleep for vacuum_stale_rowsets_interval_s=10 seconds to wait for unused rowsets are deleted + sleep(21000) + + def be_host = backendId_to_backendIP[tablet.BackendId] + def be_http_port = backendId_to_backendHttpPort[tablet.BackendId] + logger.info("be_host: ${be_host}, be_http_port: ${be_http_port}, BrpcPort: ${backendId_to_backendBrpcPort[tablet.BackendId]}") + + for (int i = 0; i < all_history_stale_rowsets.size(); i++) { + def rowsetStr = all_history_stale_rowsets[i] + // [12-12] 1 DATA NONOVERLAPPING 02000000000000124843c92c13625daa8296c20957119893 1011.00 B + def start_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[0].toInteger() + def end_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[1].toInteger() + def rowset_id = rowsetStr.split(" ")[4] + if (start_version == 0) { + continue + } + + int start_index = rowsetStr.indexOf("]") + int end_index = rowsetStr.indexOf("DATA") + def segmentNum = rowsetStr.substring(start_index + 1, end_index).trim().toInteger() + + logger.info("rowset ${i}, start: ${start_version}, end: ${end_version}, id: ${rowset_id}, segment: ${segmentNum}") + def data = Http.GET("http://${be_host}:${be_http_port}/api/file_cache?op=list_cache&value=${rowset_id}_0.dat", true) + logger.info("file cache data: ${data}") + if (segmentNum <= 1) { + assertTrue(data.size() > 0) + } else { + assertTrue(data.size() == 0) + } + } + + def (code_0, out_0, err_0) = curl("GET", "http://${be_host}:${backendId_to_backendBrpcPort[tablet.BackendId]}/vars/unused_rowsets_count") + logger.info("out_0: ${out_0}") + def unusedRowsetsCount = out_0.trim().split(":")[1].trim().toInteger() + assertEquals(0, unusedRowsetsCount) + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + } + } +} From 294db353cec46c2dacc609d4e81bbba41ea4a7ea Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 25 Jun 2025 20:44:42 +0800 Subject: [PATCH 071/572] [feat](warmup) display tables in SHOW WARM UP JOB results (#51594) (#52291) backport #51594 --- .../java/org/apache/doris/common/Triple.java | 109 ++++++++++++++++++ .../doris/analysis/ShowCloudWarmUpStmt.java | 1 + .../doris/analysis/WarmUpClusterStmt.java | 6 +- .../doris/cloud/CacheHotspotManager.java | 10 +- .../apache/doris/cloud/CloudWarmUpJob.java | 22 ++++ .../cloud/cache/CacheHotspotManagerTest.java | 2 +- .../warm_up/table/test_warm_up_table.groovy | 6 + .../warm_up/table/test_warm_up_tables.groovy | 8 ++ 8 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 fe/fe-common/src/main/java/org/apache/doris/common/Triple.java diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Triple.java b/fe/fe-common/src/main/java/org/apache/doris/common/Triple.java new file mode 100644 index 00000000000000..6e5291354f064d --- /dev/null +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Triple.java @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/Pair.java +// and modified by Doris + +package org.apache.doris.common; + +import com.google.gson.annotations.SerializedName; + +import java.util.Comparator; +import java.util.Objects; + +/** + * The equivalent of a {@link Pair} but with three elements: left, middle, and right. + *

+ * Notice: When using Triple for persistence, users need to guarantee that L, M, and R can be serialized through Gson + */ +public class Triple { + public static TripleComparator> TRIPLE_VALUE_COMPARATOR = new TripleComparator<>(); + + @SerializedName(value = "left") + public L left; + @SerializedName(value = "middle") + public M middle; + @SerializedName(value = "right") + public R right; + + private Triple(L left, M middle, R right) { + this.left = left; + this.middle = middle; + this.right = right; + } + + public static Triple ofSame(K same) { + return new Triple<>(same, same, same); + } + + public static Triple of(L left, M middle, R right) { + return new Triple<>(left, middle, right); + } + + public L getLeft() { + return left; + } + + public M getMiddle() { + return middle; + } + + public R getRight() { + return right; + } + + /** + * A triple is equal if all three parts are equal(). + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o instanceof Triple) { + Triple other = (Triple) o; + + boolean leftEqual = Objects.isNull(left) ? other.left == null : left.equals(other.left); + boolean middleEqual = Objects.isNull(middle) ? other.middle == null : middle.equals(other.middle); + boolean rightEqual = Objects.isNull(right) ? other.right == null : right.equals(other.right); + + return leftEqual && middleEqual && rightEqual; + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(left, middle, right); + } + + @Override + public String toString() { + String leftStr = Objects.nonNull(left) ? left.toString() : ""; + String middleStr = Objects.nonNull(middle) ? middle.toString() : ""; + String rightStr = Objects.nonNull(right) ? right.toString() : ""; + return leftStr + ":" + middleStr + ":" + rightStr; + } + + public static class TripleComparator> implements Comparator { + @Override + public int compare(T o1, T o2) { + return o1.right.compareTo(o2.right); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java index f823aeb9c15636..9ec063d3f76164 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java @@ -44,6 +44,7 @@ public class ShowCloudWarmUpStmt extends ShowStmt implements NotFallbackInParser .add("AllBatch") .add("FinishTime") .add("ErrMsg") + .add("Tables") .build(); public ShowCloudWarmUpStmt(Expr whereClause) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java index 9f386a686a298a..cca21d5c259c06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java @@ -26,11 +26,10 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.Triple; import org.apache.doris.common.UserException; import com.google.common.base.Strings; -import org.apache.commons.lang3.tuple.ImmutableTriple; -import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -108,8 +107,7 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (partitionName.length() != 0 && !table.containsPartition(partitionName)) { throw new AnalysisException("The partition " + partitionName + " doesn't exist"); } - Triple part = - new ImmutableTriple<>(dbName, tableName.getTbl(), partitionName); + Triple part = Triple.of(dbName, tableName.getTbl(), partitionName); tables.add(part); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index 1e86fda15db9af..b73e467836d91c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -38,6 +38,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.common.Triple; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.rpc.RpcException; @@ -52,7 +53,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.thrift.TException; @@ -633,7 +633,13 @@ public long createJob(WarmUpClusterStmt stmt) throws AnalysisException { Map>> beToTabletIdBatches = splitBatch(beToWarmUpTablets); CloudWarmUpJob.JobType jobType = stmt.isWarmUpWithTable() ? JobType.TABLE : JobType.CLUSTER; - CloudWarmUpJob warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType); + CloudWarmUpJob warmUpJob; + if (jobType == JobType.TABLE) { + warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType, + stmt.getTables(), stmt.isForce()); + } else { + warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType); + } addCloudWarmUpJob(warmUpJob); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(warmUpJob); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java index 79c00e322c2509..463a37c4635dc5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Triple; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.TimeUtils; @@ -42,6 +43,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -52,6 +54,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class CloudWarmUpJob implements Writable { private static final Logger LOG = LogManager.getLogger(CloudWarmUpJob.class); @@ -99,6 +102,12 @@ public enum JobType { @SerializedName(value = "JobType") protected JobType jobType; + @SerializedName(value = "tables") + protected List> tables = new ArrayList<>(); + + @SerializedName(value = "force") + protected boolean force = false; + private Map beToClient; private Map beToAddr; @@ -128,6 +137,14 @@ public CloudWarmUpJob(long jobId, String cloudClusterName, } } + public CloudWarmUpJob(long jobId, String cloudClusterName, + Map>> beToTabletIdBatches, JobType jobType, + List> tables, boolean force) { + this(jobId, cloudClusterName, beToTabletIdBatches, jobType); + this.tables = tables; + this.force = force; + } + public long getJobId() { return jobId; } @@ -182,6 +199,11 @@ public List getJobInfo() { info.add(Long.toString(maxBatchSize)); info.add(TimeUtils.longToTimeStringWithms(finishedTimeMs)); info.add(errMsg); + info.add(tables.stream() + .map(t -> StringUtils.isEmpty(t.getRight()) + ? t.getLeft() + "." + t.getMiddle() + : t.getLeft() + "." + t.getMiddle() + "." + t.getRight()) + .collect(Collectors.joining(", "))); return info; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/cache/CacheHotspotManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/cache/CacheHotspotManagerTest.java index ff42ea31bcb83f..04155b250994a6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/cache/CacheHotspotManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/cache/CacheHotspotManagerTest.java @@ -23,11 +23,11 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.cloud.CacheHotspotManager; import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Triple; import org.apache.doris.system.Backend; import mockit.Mock; import mockit.MockUp; -import org.apache.commons.lang3.tuple.Triple; import org.junit.Assert; import org.junit.Test; diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy index 3f9dc93d550071..b7eb8761951049 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy @@ -23,6 +23,10 @@ suite("test_warm_up_table") { def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ return jobStateResult[0][2] } + def getTablesFromShowCommand = { jobId -> + def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ + return jobStateResult[0][9] + } List ipList = new ArrayList<>(); List hbPortList = new ArrayList<>() @@ -154,6 +158,8 @@ suite("test_warm_up_table") { sql "cancel warm up job where id = ${jobId[0][0]}" assertTrue(false); } + def tablesString = getTablesFromShowCommand(jobId[0][0]) + assertTrue(tablesString.contains("customer"), tablesString) sleep(30000) long ttl_cache_size = 0 getMetricsMethod.call(ipList[0], brpcPortList[0]) { diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy index bf39e922802576..77286717117578 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy @@ -23,6 +23,10 @@ suite("test_warm_up_tables") { def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ return jobStateResult[0][2] } + def getTablesFromShowCommand = { jobId -> + def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ + return jobStateResult[0][9] + } List ipList = new ArrayList<>(); List hbPortList = new ArrayList<>() @@ -164,6 +168,10 @@ suite("test_warm_up_tables") { jobId_ = sql "warm up cluster regression_cluster_name1 with table customer partition p3 and table supplier;" waitJobDone(jobId_); + def tablesString = getTablesFromShowCommand(jobId_[0][0]) + assertTrue(tablesString.contains("customer.p3"), tablesString) + assertTrue(tablesString.contains("supplier"), tablesString) + sleep(30000) long ttl_cache_size = 0 getMetricsMethod.call(ipList[0], brpcPortList[0]) { From 81ccfe0eec668408644b3556afb21ffeb6fd10c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 09:50:00 +0800 Subject: [PATCH 072/572] branch-3.0: [opt](checker) Add inverted index file check for checker #51591 (#52318) Cherry-picked from #51591 Co-authored-by: Uniqueyou --- cloud/src/recycler/checker.cpp | 144 +++++++++++++++++++++++++- cloud/test/CMakeLists.txt | 4 +- cloud/test/mock_accessor.cpp | 183 --------------------------------- cloud/test/mock_accessor.h | 158 ++++++++++++++++++++++++++++ cloud/test/recycler_test.cpp | 147 +++++++++++++++++++++++++- 5 files changed, 448 insertions(+), 188 deletions(-) delete mode 100644 cloud/test/mock_accessor.cpp diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 60b6b7fc5eefea..6a191b021b2a8e 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -26,10 +26,12 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -422,7 +424,8 @@ int InstanceChecker::init_storage_vault_accessors(const InstanceInfoPB& instance LOG(WARNING) << "malformed storage vault, unable to deserialize key=" << hex(k); return -1; } - + TEST_SYNC_POINT_CALLBACK("InstanceRecycler::init_storage_vault_accessors.mock_vault", + &accessor_map_, &vault); if (vault.has_hdfs_info()) { auto accessor = std::make_shared(vault.hdfs_info()); int ret = accessor->init(); @@ -536,6 +539,7 @@ int InstanceChecker::do_check() { bool data_loss = false; for (int i = 0; i < rs_meta.num_segments(); ++i) { auto path = segment_path(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), i); + if (tablet_files_cache.files.contains(path)) { continue; } @@ -549,6 +553,57 @@ int InstanceChecker::do_check() { LOG(WARNING) << "object not exist, path=" << path << " key=" << hex(key); } + std::vector> index_ids; + for (const auto& i : rs_meta.tablet_schema().index()) { + if (i.has_index_type() && i.index_type() == IndexType::INVERTED) { + index_ids.emplace_back(i.index_id(), i.index_suffix_name()); + } + } + std::string tablet_idx_key = meta_tablet_idx_key({instance_id_, rs_meta.tablet_id()}); + if (!key_exist(txn_kv_.get(), tablet_idx_key)) { + for (int i = 0; i < rs_meta.num_segments(); ++i) { + std::vector index_path_v; + std::vector loss_file_path; + if (rs_meta.tablet_schema().inverted_index_storage_format() == + InvertedIndexStorageFormatPB::V1) { + for (const auto& index_id : index_ids) { + LOG(INFO) << "check inverted index, tablet_id=" << rs_meta.tablet_id() + << " rowset_id=" << rs_meta.rowset_id_v2() + << " segment_index=" << i << " index_id=" << index_id.first + << " index_suffix_name=" << index_id.second; + index_path_v.emplace_back( + inverted_index_path_v1(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), + i, index_id.first, index_id.second)); + } + } else { + index_path_v.emplace_back( + inverted_index_path_v2(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), i)); + } + + if (!index_path_v.empty()) { + if (std::all_of(index_path_v.begin(), index_path_v.end(), + [&](const auto& idx_file_path) { + if (!tablet_files_cache.files.contains(idx_file_path)) { + loss_file_path.emplace_back(idx_file_path); + return false; + } + return true; + })) { + continue; + } + } + + data_loss = true; + LOG(WARNING) << "object not exist, path=" + << std::accumulate(loss_file_path.begin(), loss_file_path.end(), + std::string(), + [](const auto& a, const auto& b) { + return a.empty() ? b : a + ", " + b; + }) + << " key=" << hex(tablet_idx_key); + } + } + if (data_loss) { ++num_rowset_loss; } @@ -647,6 +702,12 @@ int InstanceChecker::do_inverted_check() { }; TabletRowsets tablet_rowsets_cache; + struct TabletIndexes { + int64_t tablet_id {0}; + std::unordered_set index_ids; + }; + TabletIndexes tablet_indexes_cache; + // Return 0 if check success, return 1 if file is garbage data, negative if error occurred auto check_segment_file = [&](const std::string& obj_key) { std::vector str; @@ -724,8 +785,77 @@ int InstanceChecker::do_inverted_check() { return 0; }; + auto check_inverted_index_file = [&](const std::string& obj_key) { + std::vector str; + butil::SplitString(obj_key, '/', &str); + // data/{tablet_id}/{rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx + if (str.size() < 3) { + return -1; + } + + int64_t tablet_id = atol(str[1].c_str()); + if (tablet_id <= 0) { + LOG(WARNING) << "failed to parse tablet_id, key=" << obj_key; + return -1; + } + + if (!str.back().ends_with(".idx")) { + return 0; // Not an index file + } + + int64_t index_id; + + size_t pos = str.back().find_last_of('_'); + if (pos == std::string::npos || pos + 1 >= str.back().size() - 4) { + LOG(WARNING) << "Invalid index_id format, key=" << obj_key; + return -1; + } + index_id = atol(str.back().substr(pos + 1, str.back().size() - 4).c_str()); - // TODO(Xiaocc): Currently we haven't implemented one generator-like s3 accessor list function + if (tablet_indexes_cache.tablet_id == tablet_id) { + if (tablet_indexes_cache.index_ids.contains(index_id)) { + return 0; + } else { + LOG(WARNING) << "index not exists, key=" << obj_key; + return -1; + } + } + // Get all index id of this tablet + tablet_indexes_cache.tablet_id = tablet_id; + tablet_indexes_cache.index_ids.clear(); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to create txn"; + return -1; + } + auto tablet_idx_key = meta_tablet_idx_key({instance_id_, tablet_id}); + std::string tablet_idx_val; + err = txn->get(tablet_idx_key, &tablet_idx_val); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to get tablet idx," + << " key=" << hex(tablet_idx_key) << " err=" << err; + return -1; + } + + TabletIndexPB tablet_idx_pb; + if (!tablet_idx_pb.ParseFromArray(tablet_idx_val.data(), tablet_idx_val.size())) { + LOG(WARNING) << "malformed index meta value, key=" << hex(tablet_idx_key); + return -1; + } + if (!tablet_idx_pb.has_index_id()) { + LOG(WARNING) << "tablet index meta does not have index_id, key=" << hex(tablet_idx_key); + return -1; + } + tablet_indexes_cache.index_ids.insert(tablet_idx_pb.index_id()); + + if (!tablet_indexes_cache.index_ids.contains(index_id)) { + LOG(WARNING) << "index should be recycled, key=" << obj_key; + return 1; + } + + return 0; + }; // so we choose to skip here. TEST_SYNC_POINT_RETURN_WITH_VALUE("InstanceChecker::do_inverted_check", (int)0); @@ -748,6 +878,16 @@ int InstanceChecker::do_inverted_check() { check_ret = -1; } } + ret = check_inverted_index_file(file->path); + if (ret != 0) { + LOG(WARNING) << "failed to check index file, uri=" << accessor->uri() + << " path=" << file->path; + if (ret == 1) { + ++num_file_leak; + } else { + check_ret = -1; + } + } } if (!list_iter->is_valid()) { diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt index 65c9cde561b85d..8378a33ddfd2dd 100644 --- a/cloud/test/CMakeLists.txt +++ b/cloud/test/CMakeLists.txt @@ -17,7 +17,7 @@ add_executable(doris_txn_test doris_txn_test.cpp) add_executable(txn_kv_test txn_kv_test.cpp) set_target_properties(txn_kv_test PROPERTIES COMPILE_FLAGS "-fno-access-control") -add_executable(recycler_test recycler_test.cpp mock_accessor.cpp) +add_executable(recycler_test recycler_test.cpp) add_executable(mem_txn_kv_test mem_txn_kv_test.cpp) @@ -59,7 +59,7 @@ add_executable(util_test util_test.cpp) add_executable(network_util_test network_util_test.cpp) -add_executable(txn_lazy_commit_test txn_lazy_commit_test.cpp mock_accessor.cpp) +add_executable(txn_lazy_commit_test txn_lazy_commit_test.cpp) message("Meta-service test dependencies: ${TEST_LINK_LIBS}") #target_link_libraries(sync_point_test ${TEST_LINK_LIBS}) diff --git a/cloud/test/mock_accessor.cpp b/cloud/test/mock_accessor.cpp deleted file mode 100644 index f11c5969321fc9..00000000000000 --- a/cloud/test/mock_accessor.cpp +++ /dev/null @@ -1,183 +0,0 @@ - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "mock_accessor.h" - -#include - -#include -#include -#include - -#include "common/logging.h" -#include "common/string_util.h" -#include "cpp/sync_point.h" -#include "recycler/storage_vault_accessor.h" - -namespace doris::cloud { - -class MockListIterator final : public ListIterator { -public: - MockListIterator(std::vector entries) : entries_(std::move(entries)) {} - ~MockListIterator() override = default; - - bool is_valid() override { return true; } - - bool has_next() override { return !entries_.empty(); } - - std::optional next() override { - std::optional ret; - if (has_next()) { - ret = FileMeta {.path = std::move(entries_.back())}; - entries_.pop_back(); - } - - return ret; - } - -private: - std::vector entries_; -}; - -MockAccessor::MockAccessor() : StorageVaultAccessor(AccessorType::MOCK) { - uri_ = "mock"; -} - -MockAccessor::~MockAccessor() = default; - -auto MockAccessor::get_prefix_range(const std::string& path_prefix) { - auto begin = objects_.lower_bound(path_prefix); - if (begin == objects_.end()) { - return std::make_pair(begin, begin); - } - - auto path1 = path_prefix; - path1.back() += 1; - auto end = objects_.lower_bound(path1); - return std::make_pair(begin, end); -} - -int MockAccessor::delete_prefix_impl(const std::string& path_prefix) { - TEST_SYNC_POINT("MockAccessor::delete_prefix"); - LOG(INFO) << "delete object of prefix=" << path_prefix; - std::lock_guard lock(mtx_); - - auto [begin, end] = get_prefix_range(path_prefix); - if (begin == end) { - return 0; - } - - objects_.erase(begin, end); - return 0; -} - -int MockAccessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { - auto norm_path_prefix = path_prefix; - strip_leading(norm_path_prefix, "/"); - if (norm_path_prefix.empty()) { - LOG_WARNING("invalid dir_path {}", path_prefix); - return -1; - } - - return delete_prefix_impl(norm_path_prefix); -} - -int MockAccessor::delete_directory(const std::string& dir_path) { - auto norm_dir_path = dir_path; - strip_leading(norm_dir_path, "/"); - if (norm_dir_path.empty()) { - LOG_WARNING("invalid dir_path {}", dir_path); - return -1; - } - - return delete_prefix_impl(!norm_dir_path.ends_with('/') ? norm_dir_path + '/' : norm_dir_path); -} - -int MockAccessor::delete_all(int64_t expiration_time) { - std::lock_guard lock(mtx_); - objects_.clear(); - return 0; -} - -int MockAccessor::delete_files(const std::vector& paths) { - TEST_SYNC_POINT_RETURN_WITH_VALUE("MockAccessor::delete_files", (int)0); - - for (auto&& path : paths) { - delete_file(path); - } - return 0; -} - -int MockAccessor::delete_file(const std::string& path) { - LOG(INFO) << "delete object path=" << path; - std::lock_guard lock(mtx_); - objects_.erase(path); - return 0; -} - -int MockAccessor::put_file(const std::string& path, const std::string& content) { - std::lock_guard lock(mtx_); - objects_.insert(path); - return 0; -} - -int MockAccessor::list_all(std::unique_ptr* res) { - std::vector entries; - - { - std::lock_guard lock(mtx_); - entries.reserve(objects_.size()); - entries.assign(objects_.rbegin(), objects_.rend()); - } - - *res = std::make_unique(std::move(entries)); - - return 0; -} - -int MockAccessor::list_directory(const std::string& dir_path, std::unique_ptr* res) { - auto norm_dir_path = dir_path; - strip_leading(norm_dir_path, "/"); - if (norm_dir_path.empty()) { - LOG_WARNING("invalid dir_path {}", dir_path); - return -1; - } - - std::vector entries; - - { - std::lock_guard lock(mtx_); - auto [begin, end] = get_prefix_range(norm_dir_path); - if (begin != end) { - entries.reserve(std::distance(begin, end)); - std::ranges::copy(std::ranges::subrange(begin, end) | std::ranges::views::reverse, - std::back_inserter(entries)); - } - } - - *res = std::make_unique(std::move(entries)); - - return 0; -} - -int MockAccessor::exists(const std::string& path) { - std::lock_guard lock(mtx_); - return !objects_.contains(path); -} - -} // namespace doris::cloud diff --git a/cloud/test/mock_accessor.h b/cloud/test/mock_accessor.h index ba8ede324622d5..4e209d93261e6e 100644 --- a/cloud/test/mock_accessor.h +++ b/cloud/test/mock_accessor.h @@ -17,12 +17,43 @@ #pragma once +#include + +#include #include +#include #include +#include +#include "common/logging.h" +#include "common/string_util.h" +#include "cpp/sync_point.h" +#include "mock_accessor.h" #include "recycler/storage_vault_accessor.h" namespace doris::cloud { +class MockListIterator final : public ListIterator { +public: + MockListIterator(std::vector entries) : entries_(std::move(entries)) {} + ~MockListIterator() override = default; + + bool is_valid() override { return true; } + + bool has_next() override { return !entries_.empty(); } + + std::optional next() override { + std::optional ret; + if (has_next()) { + ret = FileMeta {.path = std::move(entries_.back())}; + entries_.pop_back(); + } + + return ret; + } + +private: + std::vector entries_; +}; class MockAccessor final : public StorageVaultAccessor { public: @@ -57,4 +88,131 @@ class MockAccessor final : public StorageVaultAccessor { std::set objects_; }; +inline MockAccessor::MockAccessor() : StorageVaultAccessor(AccessorType::MOCK) { + uri_ = "mock"; +} + +inline MockAccessor::~MockAccessor() = default; + +inline auto MockAccessor::get_prefix_range(const std::string& path_prefix) { + auto begin = objects_.lower_bound(path_prefix); + if (begin == objects_.end()) { + return std::make_pair(begin, begin); + } + + auto path1 = path_prefix; + path1.back() += 1; + auto end = objects_.lower_bound(path1); + return std::make_pair(begin, end); +} + +inline int MockAccessor::delete_prefix_impl(const std::string& path_prefix) { + TEST_SYNC_POINT("MockAccessor::delete_prefix"); + LOG(INFO) << "delete object of prefix=" << path_prefix; + std::lock_guard lock(mtx_); + + auto [begin, end] = get_prefix_range(path_prefix); + if (begin == end) { + return 0; + } + + objects_.erase(begin, end); + return 0; +} + +inline int MockAccessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { + auto norm_path_prefix = path_prefix; + strip_leading(norm_path_prefix, "/"); + if (norm_path_prefix.empty()) { + LOG_WARNING("invalid dir_path {}", path_prefix); + return -1; + } + + return delete_prefix_impl(norm_path_prefix); +} + +inline int MockAccessor::delete_directory(const std::string& dir_path) { + auto norm_dir_path = dir_path; + strip_leading(norm_dir_path, "/"); + if (norm_dir_path.empty()) { + LOG_WARNING("invalid dir_path {}", dir_path); + return -1; + } + + return delete_prefix_impl(!norm_dir_path.ends_with('/') ? norm_dir_path + '/' : norm_dir_path); +} + +inline int MockAccessor::delete_all(int64_t expiration_time) { + std::lock_guard lock(mtx_); + objects_.clear(); + return 0; +} + +inline int MockAccessor::delete_files(const std::vector& paths) { + TEST_SYNC_POINT_RETURN_WITH_VALUE("MockAccessor::delete_files", (int)0); + + for (auto&& path : paths) { + delete_file(path); + } + return 0; +} + +inline int MockAccessor::delete_file(const std::string& path) { + LOG(INFO) << "delete object path=" << path; + std::lock_guard lock(mtx_); + objects_.erase(path); + return 0; +} + +inline int MockAccessor::put_file(const std::string& path, const std::string& content) { + std::lock_guard lock(mtx_); + objects_.insert(path); + return 0; +} + +inline int MockAccessor::list_all(std::unique_ptr* res) { + std::vector entries; + + { + std::lock_guard lock(mtx_); + entries.reserve(objects_.size()); + entries.assign(objects_.rbegin(), objects_.rend()); + } + + *res = std::make_unique(std::move(entries)); + + return 0; +} + +inline int MockAccessor::list_directory(const std::string& dir_path, + std::unique_ptr* res) { + auto norm_dir_path = dir_path; + strip_leading(norm_dir_path, "/"); + if (norm_dir_path.empty()) { + LOG_WARNING("invalid dir_path {}", dir_path); + return -1; + } + + std::vector entries; + + { + std::lock_guard lock(mtx_); + auto [begin, end] = get_prefix_range(norm_dir_path); + if (begin != end) { + entries.reserve(std::distance(begin, end)); + std::ranges::copy(std::ranges::subrange(begin, end) | std::ranges::views::reverse, + std::back_inserter(entries)); + } + } + + *res = std::make_unique(std::move(entries)); + + return 0; +} + +inline int MockAccessor::exists(const std::string& path) { + std::lock_guard lock(mtx_); + return !objects_.contains(path); +} + } // namespace doris::cloud diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index bcd7dd39160651..da5dcc1556b97c 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -17,6 +17,7 @@ #include "recycler/recycler.h" +#include #include #include #include @@ -258,8 +259,11 @@ static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor rowset_pb.set_creation_time(current_time); if (num_inverted_indexes > 0) { auto schema = rowset_pb.mutable_tablet_schema(); + schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); for (int i = 0; i < num_inverted_indexes; ++i) { - schema->add_index()->set_index_id(i); + auto index = schema->add_index(); + index->set_index_id(i); + index->set_index_type(IndexType::INVERTED); } } rowset_pb.SerializeToString(&val); @@ -277,6 +281,24 @@ static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor auto path = segment_path(tablet_id, rowset_id, i); accessor->put_file(path, ""); for (int j = 0; j < num_inverted_indexes; ++j) { + std::string key1; + std::string val1; + MetaTabletIdxKeyInfo key_info1 {instance_id, tablet_id}; + meta_tablet_idx_key(key_info1, &key1); + TabletIndexPB tablet_table; + tablet_table.set_db_id(db_id); + tablet_table.set_index_id(j); + tablet_table.set_tablet_id(tablet_id); + if (!tablet_table.SerializeToString(&val1)) { + return -1; + } + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(key1, val1); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } auto path = inverted_index_path_v1(tablet_id, rowset_id, i, j, ""); accessor->put_file(path, ""); } @@ -2470,6 +2492,129 @@ TEST(CheckerTest, DISABLED_abnormal_inverted_check) { ASSERT_NE(checker.do_inverted_check(), 0); } +TEST(CheckerTest, inverted_check_recycle_idx_file) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->disable_processing(); + }); + + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("CheckerTest"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->enable_processing(); + + for (int t = 10001; t <= 10100; ++t) { + for (int v = 0; v < 10; ++v) { + int ret = create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1, 3); + ASSERT_EQ(ret, 0) << "Failed to create committed rs: " << ret; + } + } + std::unique_ptr list_iter; + int ret = accessor->list_directory("data", &list_iter); + ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; + + int64_t tablet_id_to_delete_index = -1; + for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { + std::vector str; + butil::SplitString(file->path, '/', &str); + int64_t tablet_id = atol(str[1].c_str()); + + // only delete one index files of ever tablet for mock recycle + // The reason for not select "delete all idx file" is that inverted checking cannot handle this case + // forward checking is required. + if (file->path.ends_with(".idx") && tablet_id_to_delete_index != tablet_id) { + accessor->delete_file(file->path); + tablet_id_to_delete_index = tablet_id; + } + } + ASSERT_EQ(checker.do_inverted_check(), 1); +} + +TEST(CheckerTest, forward_check_recycle_idx_file) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->disable_processing(); + }); + + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("CheckerTest"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->enable_processing(); + + for (int t = 10001; t <= 10100; ++t) { + for (int v = 0; v < 10; ++v) { + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1, 3); + } + } + std::unique_ptr list_iter; + int ret = accessor->list_directory("data", &list_iter); + ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; + + for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { + // delete all index files of ever tablet for mock recycle + if (file->path.ends_with(".idx")) { + accessor->delete_file(file->path); + } + } + ASSERT_EQ(checker.do_check(), 1); +} + TEST(CheckerTest, normal) { auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0); From 1c02fdbbecece77fad7fb8afffa5dc59c9f1eaed Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:06:25 +0800 Subject: [PATCH 073/572] branch-3.0: [fix](cloud) Fix `ConcurrentModificationException` in cloud rebalance #52013 (#52309) Cherry-picked from #52013 Co-authored-by: deardeng --- .../cloud/catalog/CloudTabletRebalancer.java | 18 +- .../multi_cluster/test_rebalance.groovy | 292 +++++++++++------- 2 files changed, 191 insertions(+), 119 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index b93d4fe2cff464..1b014caea82779 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -35,6 +35,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.rpc.RpcException; import org.apache.doris.system.Backend; @@ -55,6 +56,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -418,11 +420,22 @@ public void checkInflghtWarmUpCacheAsync() { for (Map.Entry> entry : beToInfightTasks.entrySet()) { LOG.info("before pre cache check dest be {} inflight task num {}", entry.getKey(), entry.getValue().size()); Backend destBackend = cloudSystemInfoService.getBackend(entry.getKey()); + if (DebugPointUtil.isEnable("CloudTabletRebalancer.checkInflghtWarmUpCacheAsync.beNull")) { + LOG.info("debug point CloudTabletRebalancer.checkInflghtWarmUpCacheAsync.beNull, be {}", destBackend); + destBackend = null; + } if (destBackend == null || (!destBackend.isAlive() && destBackend.getLastUpdateMs() < needRehashDeadTime)) { + List toRemove = new LinkedList<>(); for (InfightTask task : entry.getValue()) { for (InfightTablet key : tabletToInfightTask.keySet()) { - tabletToInfightTask.remove(new InfightTablet(task.pickedTablet.getId(), key.clusterId)); + toRemove.add(new InfightTablet(task.pickedTablet.getId(), key.clusterId)); + } + } + for (InfightTablet key : toRemove) { + if (LOG.isDebugEnabled()) { + LOG.debug("remove tablet {}-{}", key.getClusterId(), key.getTabletId()); } + tabletToInfightTask.remove(key); } continue; } @@ -447,6 +460,9 @@ public void checkInflghtWarmUpCacheAsync() { LOG.info("{} pre cache timeout, forced to change the mapping", result.getKey()); } updateClusterToBeMap(task.pickedTablet, task.destBe, clusterId, infos); + if (LOG.isDebugEnabled()) { + LOG.debug("remove tablet {}-{}", clusterId, task.pickedTablet.getId()); + } tabletToInfightTask.remove(new InfightTablet(task.pickedTablet.getId(), clusterId)); } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy index 7a76533b0a47d2..c15157308c4903 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy @@ -21,142 +21,198 @@ suite('test_rebalance_in_cloud', 'multi_cluster,docker') { if (!isCloudMode()) { return; } - def options = new ClusterOptions() - options.feConfigs += [ - 'cloud_cluster_check_interval_second=1', - 'enable_cloud_warm_up_for_rebalance=false', - 'cloud_tablet_rebalancer_interval_second=1', - 'cloud_balance_tablet_percent_per_run=0.5', - 'cloud_pre_heating_time_limit_sec=1', - 'sys_log_verbose_modules=org', + + def clusterOptions = [ + new ClusterOptions(), + new ClusterOptions(), ] - options.setFeNum(3) - options.setBeNum(1) - options.cloudMode = true - options.connectToFollower = true - options.enableDebugPoints() - - docker(options) { - sql """ - CREATE TABLE table100 ( - class INT, - id INT, - score INT SUM - ) - AGGREGATE KEY(class, id) - DISTRIBUTED BY HASH(class) BUCKETS 48 - """ - - sql """ - CREATE TABLE table_p2 ( k1 int(11) NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) - AGGREGATE KEY(k1, k2) - PARTITION BY RANGE(k1) ( - PARTITION p1992 VALUES [("-2147483648"), ("19930101")), - PARTITION p1993 VALUES [("19930101"), ("19940101")), - PARTITION p1994 VALUES [("19940101"), ("19950101")), - PARTITION p1995 VALUES [("19950101"), ("19960101")), - PARTITION p1996 VALUES [("19960101"), ("19970101")), - PARTITION p1997 VALUES [("19970101"), ("19980101")), - PARTITION p1998 VALUES [("19980101"), ("19990101"))) - DISTRIBUTED BY HASH(k1) BUCKETS 3 - """ - GetDebugPoint().enableDebugPointForAllFEs("CloudReplica.getBackendIdImpl.primaryClusterToBackends"); - sql """set global forward_to_master=false""" - - // add a be - cluster.addBackend(1, null) - - dockerAwaitUntil(30) { - def bes = sql """show backends""" - log.info("bes: {}", bes) - bes.size() == 2 - } + for (options in clusterOptions) { + options.setFeNum(3) + options.setBeNum(1) + options.cloudMode = true + options.connectToFollower = true + options.enableDebugPoints() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + 'cloud_balance_tablet_percent_per_run=0.5', - dockerAwaitUntil(5) { - def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" - log.info("replica distribution table100: {}", ret) - ret.size() == 2 - } + 'sys_log_verbose_modules=org', + ] + } + clusterOptions[0].feConfigs += ['enable_cloud_warm_up_for_rebalance=true', 'cloud_pre_heating_time_limit_sec=300'] + clusterOptions[1].feConfigs += ['enable_cloud_warm_up_for_rebalance=false'] - def result = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM table100; """ - assertEquals(2, result.size()) - int replicaNum = 0 - - for (def row : result) { - log.info("replica distribution: ${row} ".toString()) - replicaNum = Integer.valueOf((String) row.ReplicaNum) - if (replicaNum == 0) { - // due to debug point, observer not hash replica - } else { - assertTrue(replicaNum <= 25 && replicaNum >= 23) + + for (int i = 0; i < clusterOptions.size(); i++) { + log.info("begin warm up {}", i == 0 ? "ON" : "OFF") + docker(clusterOptions[i]) { + sql """ + CREATE TABLE table100 ( + class INT, + id INT, + score INT SUM + ) + AGGREGATE KEY(class, id) + DISTRIBUTED BY HASH(class) BUCKETS 48 + """ + + sql """ + CREATE TABLE table_p2 ( k1 int(11) NOT NULL, k2 varchar(20) NOT NULL, k3 int sum NOT NULL ) + AGGREGATE KEY(k1, k2) + PARTITION BY RANGE(k1) ( + PARTITION p1992 VALUES [("-2147483648"), ("19930101")), + PARTITION p1993 VALUES [("19930101"), ("19940101")), + PARTITION p1994 VALUES [("19940101"), ("19950101")), + PARTITION p1995 VALUES [("19950101"), ("19960101")), + PARTITION p1996 VALUES [("19960101"), ("19970101")), + PARTITION p1997 VALUES [("19970101"), ("19980101")), + PARTITION p1998 VALUES [("19980101"), ("19990101"))) + DISTRIBUTED BY HASH(k1) BUCKETS 3 + """ + GetDebugPoint().enableDebugPointForAllFEs("CloudReplica.getBackendIdImpl.primaryClusterToBackends"); + sql """set global forward_to_master=false""" + + // add a be + cluster.addBackend(1, null) + + dockerAwaitUntil(30) { + def bes = sql """show backends""" + log.info("bes: {}", bes) + bes.size() == 2 } - } - dockerAwaitUntil(5) { - def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1992)""" - log.info("replica distribution table_p2: {}", ret) - ret.size() == 2 - } + dockerAwaitUntil(5) { + def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" + log.info("replica distribution table100: {}", ret) + ret.size() == 2 + } + def result = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM table100; """ + assertEquals(2, result.size()) + int replicaNum = 0 - result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1992) """ - assertEquals(2, result.size()) - for (def row : result) { - replicaNum = Integer.valueOf((String) row.ReplicaNum) - log.info("replica distribution: ${row} ".toString()) - if (replicaNum != 0) { - assertTrue(replicaNum <= 2 && replicaNum >= 1) + for (def row : result) { + log.info("replica distribution: ${row} ".toString()) + replicaNum = Integer.valueOf((String) row.ReplicaNum) + if (replicaNum == 0) { + // due to debug point, observer not hash replica + } else { + assertTrue(replicaNum <= 25 && replicaNum >= 23) + } } - } - result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1993) """ - assertEquals(2, result.size()) - for (def row : result) { - replicaNum = Integer.valueOf((String) row.ReplicaNum) - log.info("replica distribution: ${row} ".toString()) - if (replicaNum != 0) { - assertTrue(replicaNum <= 2 && replicaNum >= 1) + dockerAwaitUntil(5) { + def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1992)""" + log.info("replica distribution table_p2: {}", ret) + ret.size() == 2 } - } - result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1994) """ - assertEquals(2, result.size()) - for (def row : result) { - replicaNum = Integer.valueOf((String) row.ReplicaNum) - log.info("replica distribution: ${row} ".toString()) - if (replicaNum != 0) { - assertTrue(replicaNum <= 2 && replicaNum >= 1) + + result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1992) """ + assertEquals(2, result.size()) + for (def row : result) { + replicaNum = Integer.valueOf((String) row.ReplicaNum) + log.info("replica distribution: ${row} ".toString()) + if (replicaNum != 0) { + assertTrue(replicaNum <= 2 && replicaNum >= 1) + } } - } - result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1995) """ - assertEquals(2, result.size()) - for (def row : result) { - replicaNum = Integer.valueOf((String) row.ReplicaNum) - log.info("replica distribution: ${row} ".toString()) - if (replicaNum != 0) { - assertTrue(replicaNum <= 2 && replicaNum >= 1) + result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1993) """ + assertEquals(2, result.size()) + for (def row : result) { + replicaNum = Integer.valueOf((String) row.ReplicaNum) + log.info("replica distribution: ${row} ".toString()) + if (replicaNum != 0) { + assertTrue(replicaNum <= 2 && replicaNum >= 1) + } } - } - result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1996) """ - assertEquals(2, result.size()) - for (def row : result) { - replicaNum = Integer.valueOf((String) row.ReplicaNum) - log.info("replica distribution: ${row} ".toString()) - if (replicaNum != 0) { - assertTrue(replicaNum <= 2 && replicaNum >= 1) + result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1994) """ + assertEquals(2, result.size()) + for (def row : result) { + replicaNum = Integer.valueOf((String) row.ReplicaNum) + log.info("replica distribution: ${row} ".toString()) + if (replicaNum != 0) { + assertTrue(replicaNum <= 2 && replicaNum >= 1) + } } - } - result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1997) """ - assertEquals(2, result.size()) - for (def row : result) { - replicaNum = Integer.valueOf((String) row.ReplicaNum) - log.info("replica distribution: ${row} ".toString()) - if (replicaNum != 0) { - assertTrue(replicaNum <= 2 && replicaNum >= 1) + result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1995) """ + assertEquals(2, result.size()) + for (def row : result) { + replicaNum = Integer.valueOf((String) row.ReplicaNum) + log.info("replica distribution: ${row} ".toString()) + if (replicaNum != 0) { + assertTrue(replicaNum <= 2 && replicaNum >= 1) + } + } + + result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1996) """ + assertEquals(2, result.size()) + for (def row : result) { + replicaNum = Integer.valueOf((String) row.ReplicaNum) + log.info("replica distribution: ${row} ".toString()) + if (replicaNum != 0) { + assertTrue(replicaNum <= 2 && replicaNum >= 1) + } + } + + result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1997) """ + assertEquals(2, result.size()) + for (def row : result) { + replicaNum = Integer.valueOf((String) row.ReplicaNum) + log.info("replica distribution: ${row} ".toString()) + if (replicaNum != 0) { + assertTrue(replicaNum <= 2 && replicaNum >= 1) + } + } + + if (i == 1) { + // just test warm up + return + } + + GetDebugPoint().enableDebugPointForAllFEs("CloudTabletRebalancer.checkInflghtWarmUpCacheAsync.beNull"); + // add a be + cluster.addBackend(1, null) + // warm up + sql """admin set frontend config("enable_cloud_warm_up_for_rebalance"="true")""" + + // test rebalance thread still work + dockerAwaitUntil(30) { + def bes = sql """show backends""" + log.info("bes: {}", bes) + bes.size() == 3 + } + + dockerAwaitUntil(5) { + def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" + log.info("replica distribution table100: {}", ret) + ret.size() == 3 + } + + result = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM table100; """ + assertEquals(3, result.size()) + log.info("replica distribution: ${result} ".toString()) + + // test 10s not balance, due to debug point + for (int j = 0; j < 10; j++) { + assertTrue(result.any { row -> + Integer.valueOf((String) row.ReplicaNum) == 0 + }) + sleep(1 * 1000) + } + GetDebugPoint().disableDebugPointForAllFEs("CloudTabletRebalancer.checkInflghtWarmUpCacheAsync.beNull"); + dockerAwaitUntil(10) { + def ret = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" + log.info("replica distribution table100: {}", ret) + ret.any { row -> + Integer.valueOf((String) row.ReplicaNum) == 16 + } } } + logger.info("Successfully run {} times", i + 1) } } From b4e65e72741151a1636c1aca338b06c746317383 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:17:59 +0800 Subject: [PATCH 074/572] =?UTF-8?q?branch-3.0:=20[Fix](JsonReader)=20Fix?= =?UTF-8?q?=20the=20issue=20where=20the=20null=20bitmap=20of=20the=20JSON?= =?UTF-8?q?=20reader=20was=20not=20initialized=20when=20the=20JSON=20path?= =?UTF-8?q?=20is=20specified=20as=20'$.=E2=80=99=20#52211=20(#52267)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #52211 Co-authored-by: lihangyu --- .../vec/exec/format/json/new_json_reader.cpp | 1 + .../load_p0/stream_load/test_json_load.out | 9 ++++++- .../load_p0/stream_load/test_json_load.groovy | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index 37ab5041216f9f..c8969c6d4c314e 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -2018,6 +2018,7 @@ Status NewJsonReader::_simdjson_write_columns_by_jsonpath( if (slot_desc->is_nullable()) { nullable_column = assert_cast(column_ptr); target_column_ptr = &nullable_column->get_nested_column(); + nullable_column->get_null_map_data().push_back(0); } auto* column_string = assert_cast(target_column_ptr); column_string->insert_data(_simdjson_ondemand_padding_buffer.data(), diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out index 1d6777bb21e7ab..3ef9ecb5be941f 100644 --- a/regression-test/data/load_p0/stream_load/test_json_load.out +++ b/regression-test/data/load_p0/stream_load/test_json_load.out @@ -259,4 +259,11 @@ test k2_value -- !select31 -- 789 beijing haidian -1111 \N \N \ No newline at end of file +1111 \N \N + +-- !select30 -- +12345 {"k1":12345,"k2":"11111","k3":111111,"k4":[11111]} {"k1":12345,"k2":"11111","k3":111111,"k4":[11111]} 111111 +12346 {"k1":12346,"k2":"22222","k4":[22222]} {"k1":12346,"k2":"22222","k4":[22222]} \N +12347 {"k1":12347,"k3":"33333","k4":[22222]} {"k1":12347,"k3":"33333","k4":[22222]} 33333 +12348 {"k1":12348,"k3":"33333","k5":{"k51":1024,"xxxx":[11111]}} {"k1":12348,"k3":"33333","k5":{"k51":1024,"xxxx":[11111]}} 33333 + diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index 1042b5a3d71cdd..cbecb831cfcad7 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -932,4 +932,31 @@ suite("test_json_load", "p0,nonConcurrent") { } finally { // try_sql("DROP TABLE IF EXISTS ${testTable}") } + + // support read "$." as root with json type + try { + sql "DROP TABLE IF EXISTS ${testTable}" + sql """CREATE TABLE IF NOT EXISTS ${testTable} + ( + `k1` varchar(1024) NULL, + `k2` json NULL, + `k3` json NULL, + `k4` json NULL + ) + DUPLICATE KEY(`k1`) + COMMENT '' + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + load_json_data.call("${testTable}", "${testTable}_case30", 'false', 'true', 'json', '', '[\"$.k1\",\"$.\", \"$.\", \"$.k3\"]', + '', '', '', 'test_read_root_path.json') + + sql "sync" + qt_select30 "select * from ${testTable} order by k1" + + } finally { + // try_sql("DROP TABLE IF EXISTS ${testTable}") + } } From 1c2ba48423b93cc2d2b3f6dacfd26e38b5789572 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Thu, 26 Jun 2025 10:20:51 +0800 Subject: [PATCH 075/572] brach-3.0 cherry-pick [Fix](Variant) fix serialize with json key contains `.` as name (#51930) cherry-pick from #51857 --- .../segment_v2/hierarchical_data_reader.h | 8 +++++++ be/src/vec/columns/column_object.cpp | 8 ++----- be/src/vec/core/field.h | 9 ++----- be/src/vec/data_types/data_type_object.cpp | 10 ++++++-- .../vec/data_types/serde/data_type_serde.cpp | 2 +- be/src/vec/json/json_parser.h | 10 ++++++++ be/src/vec/json/path_in_data.h | 18 -------------- be/test/vec/columns/column_object_test.cpp | 1 + gensrc/proto/data.proto | 1 + .../data/variant_p0/column_name.out | 24 ++++++++++++++----- .../suites/variant_p0/column_name.groovy | 15 ++++++++++-- 11 files changed, 64 insertions(+), 42 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h index c6f325aaceab94..6b0d4d1234541d 100644 --- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h +++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h @@ -45,6 +45,14 @@ namespace doris::segment_v2 { +struct PathWithColumnAndType { + vectorized::PathInData path; + vectorized::ColumnPtr column; + vectorized::DataTypePtr type; +}; + +using PathsWithColumnAndType = std::vector; + // Reader for hierarchical data for variant, merge with root(sparse encoded columns) class HierarchicalDataReader : public ColumnIterator { public: diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index 4badf5bfaefa6b..f14f0c62debddc 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -907,11 +907,7 @@ void ColumnObject::try_insert(const Field& field) { } const auto& object = field.get(); size_t old_size = size(); - for (const auto& [key_str, value] : object) { - PathInData key; - if (!key_str.empty()) { - key = PathInData(key_str); - } + for (const auto& [key, value] : object) { if (!has_subcolumn(key)) { bool succ = add_sub_column(key, old_size); if (!succ) { @@ -1004,7 +1000,7 @@ void ColumnObject::get(size_t n, Field& res) const { entry->data.get(n, field); // Notice: we treat null as empty field, since we do not distinguish null and empty for Variant type. if (field.get_type() != Field::Types::Null) { - object.try_emplace(entry->path.get_path(), field); + object.try_emplace(entry->path, field); } } if (object.empty()) { diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h index 8113dc602fbd4e..922f9abb13e03e 100644 --- a/be/src/vec/core/field.h +++ b/be/src/vec/core/field.h @@ -43,6 +43,7 @@ #include "util/quantile_state.h" #include "vec/common/uint128.h" #include "vec/core/types.h" +#include "vec/json/path_in_data.h" namespace doris { namespace vectorized { @@ -153,13 +154,7 @@ DEFINE_FIELD_VECTOR(Tuple); DEFINE_FIELD_VECTOR(Map); #undef DEFINE_FIELD_VECTOR -using FieldMap = std::map>; -#define DEFINE_FIELD_MAP(X) \ - struct X : public FieldMap { \ - using FieldMap::FieldMap; \ - } -DEFINE_FIELD_MAP(VariantMap); -#undef DEFINE_FIELD_MAP +using VariantMap = std::map; class JsonbField { public: diff --git a/be/src/vec/data_types/data_type_object.cpp b/be/src/vec/data_types/data_type_object.cpp index 6dde7df44d5ba1..1eab54fd5e16fd 100644 --- a/be/src/vec/data_types/data_type_object.cpp +++ b/be/src/vec/data_types/data_type_object.cpp @@ -69,6 +69,7 @@ int64_t DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column, } PColumnMeta column_meta_pb; column_meta_pb.set_name(entry->path.get_path()); + entry->path.to_protobuf(column_meta_pb.mutable_column_path(), -1 /*not used here*/); type->to_pb_column_meta(&column_meta_pb); std::string meta_binary; column_meta_pb.SerializeToString(&meta_binary); @@ -112,6 +113,7 @@ char* DataTypeObject::serialize(const IColumn& column, char* buf, int be_exec_ve ++num_of_columns; PColumnMeta column_meta_pb; column_meta_pb.set_name(entry->path.get_path()); + entry->path.to_protobuf(column_meta_pb.mutable_column_path(), -1 /*not used here*/); type->to_pb_column_meta(&column_meta_pb); std::string meta_binary; column_meta_pb.SerializeToString(&meta_binary); @@ -157,11 +159,15 @@ const char* DataTypeObject::deserialize(const char* buf, MutableColumnPtr* colum MutableColumnPtr sub_column = type->create_column(); buf = type->deserialize(buf, &sub_column, be_exec_version); - // add subcolumn to column_object PathInData key; - if (!column_meta_pb.name().empty()) { + if (column_meta_pb.has_column_path()) { + // init from path pb + key.from_protobuf(column_meta_pb.column_path()); + } else if (!column_meta_pb.name().empty()) { + // init from name for compatible key = PathInData {column_meta_pb.name()}; } + // add subcolumn to column_object column_object->add_sub_column(key, std::move(sub_column), type); } size_t num_rows = 0; diff --git a/be/src/vec/data_types/serde/data_type_serde.cpp b/be/src/vec/data_types/serde/data_type_serde.cpp index 0709df03bb733d..d412ce18139d84 100644 --- a/be/src/vec/data_types/serde/data_type_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_serde.cpp @@ -55,7 +55,7 @@ void DataTypeSerDe::convert_variant_map_to_rapidjson( continue; } rapidjson::Value key; - key.SetString(item.first.data(), item.first.size()); + key.SetString(item.first.get_path().data(), item.first.get_path().size()); rapidjson::Value val; convert_field_to_rapidjson(item.second, val, allocator); if (val.IsNull() && item.first.empty()) { diff --git a/be/src/vec/json/json_parser.h b/be/src/vec/json/json_parser.h index af2e452dddc0c1..c1815ae5be4d50 100644 --- a/be/src/vec/json/json_parser.h +++ b/be/src/vec/json/json_parser.h @@ -28,6 +28,7 @@ #include #include +#include "runtime/primitive_type.h" #include "util/jsonb_writer.h" #include "vec/columns/column.h" #include "vec/common/string_ref.h" @@ -124,6 +125,15 @@ enum class ExtractType { struct ParseConfig { bool enable_flatten_nested = false; }; + +/// Result of parsing of a document. +/// Contains all paths extracted from document +/// and values which are related to them. +struct ParseResult { + std::vector paths; + std::vector values; +}; + template class JSONDataParser { public: diff --git a/be/src/vec/json/path_in_data.h b/be/src/vec/json/path_in_data.h index 8d94b02f37ac2b..74ae345855421f 100644 --- a/be/src/vec/json/path_in_data.h +++ b/be/src/vec/json/path_in_data.h @@ -29,11 +29,8 @@ #include #include "gen_cpp/segment_v2.pb.h" -#include "vec/columns/column.h" #include "vec/common/uint128.h" -#include "vec/core/field.h" #include "vec/core/types.h" -#include "vec/data_types/data_type.h" namespace doris::vectorized { @@ -129,13 +126,6 @@ class PathInDataBuilder { size_t current_anonymous_array_level = 0; }; using PathsInData = std::vector; -/// Result of parsing of a document. -/// Contains all paths extracted from document -/// and values which are related to them. -struct ParseResult { - std::vector paths; - std::vector values; -}; struct PathInDataRef { const PathInData* ref; @@ -148,12 +138,4 @@ struct PathInDataRef { bool operator==(const PathInDataRef& other) const { return *this->ref == *other.ref; } }; -struct PathWithColumnAndType { - PathInData path; - ColumnPtr column; - DataTypePtr type; -}; - -using PathsWithColumnAndType = std::vector; - } // namespace doris::vectorized diff --git a/be/test/vec/columns/column_object_test.cpp b/be/test/vec/columns/column_object_test.cpp index 06d987e414e1b3..a7498e82e87824 100644 --- a/be/test/vec/columns/column_object_test.cpp +++ b/be/test/vec/columns/column_object_test.cpp @@ -22,6 +22,7 @@ #include #include "vec/columns/common_column_test.h" +#include "vec/json/path_in_data.h" namespace doris::vectorized { diff --git a/gensrc/proto/data.proto b/gensrc/proto/data.proto index 9b3824db3dc17d..7cb126e973ba66 100644 --- a/gensrc/proto/data.proto +++ b/gensrc/proto/data.proto @@ -63,6 +63,7 @@ message PColumnMeta { optional bool result_is_nullable = 6; optional string function_name = 7; optional int32 be_exec_version = 8; + optional segment_v2.ColumnPathInfo column_path = 9; } message PBlock { diff --git a/regression-test/data/variant_p0/column_name.out b/regression-test/data/variant_p0/column_name.out index 6ac882d29225d2..0f54df05d91076 100644 --- a/regression-test/data/variant_p0/column_name.out +++ b/regression-test/data/variant_p0/column_name.out @@ -37,13 +37,25 @@ UPPER CASE lower case \N \N \N -"" -"" + + 1234566 16 8888888 -"UPPER CASE" -"dkdkdkdkdkd" -"ooaoaaaaaaa" -"xmxxmmmmmm" +UPPER CASE +dkdkdkdkdkd +ooaoaaaaaaa +xmxxmmmmmm + +-- !sql_cnt_1 -- +128 + +-- !sql_cnt_2 -- +128 + +-- !sql_cnt_3 -- +128 + +-- !sql_cnt_4 -- +128 diff --git a/regression-test/suites/variant_p0/column_name.groovy b/regression-test/suites/variant_p0/column_name.groovy index 7962112ff75f4d..7cf7fe198b154e 100644 --- a/regression-test/suites/variant_p0/column_name.groovy +++ b/regression-test/suites/variant_p0/column_name.groovy @@ -25,7 +25,7 @@ suite("regression_test_variant_column_name", "variant_type"){ ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "true"); + properties("replication_num" = "1", "disable_auto_compaction" = "false"); """ sql """insert into ${table_name} values (1, '{"中文" : "中文", "\\\u4E2C\\\u6587": "unicode"}')""" @@ -61,7 +61,18 @@ suite("regression_test_variant_column_name", "variant_type"){ sql """insert into var_column_name values (7, '{"": 1234566}')""" sql """insert into var_column_name values (7, '{"": 8888888}')""" - qt_sql "select Tags[''] from var_column_name order by cast(Tags[''] as string)" + qt_sql "select cast(Tags[''] as text) from var_column_name order by cast(Tags[''] as string)" + + // name with `.` + sql "truncate table var_column_name" + sql """insert into var_column_name values (7, '{"a.b": "UPPER CASE", "a.c": "lower case", "a" : {"b" : 123}, "a" : {"c" : 456}}')""" + for (int i = 0; i < 7; i++) { + sql """insert into var_column_name select * from var_column_name""" + } + qt_sql_cnt_1 "select count(Tags['a.b']) from var_column_name" + qt_sql_cnt_2 "select count(Tags['a.c']) from var_column_name" + qt_sql_cnt_3 "select count(Tags['a']['b']) from var_column_name" + qt_sql_cnt_4 "select count(Tags['a']['c']) from var_column_name" try { sql """insert into var_column_name values (7, '{"": "UPPER CASE", "": "lower case"}')""" From ea5f549443b45481ce4d41534b7f3907ff9bb66d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:21:44 +0800 Subject: [PATCH 076/572] branch-3.0: [file cache]add some stats for file_cache_statistics #51484 (#51900) Cherry-picked from #51484 Co-authored-by: Wen Zhenghu --- be/src/io/cache/block_file_cache.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 72bbd708506734..1e16d79dc5c057 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -1789,6 +1789,7 @@ void BlockFileCache::run_background_monitor() { check_need_evict_cache_in_advance(); } else { _need_evict_cache_in_advance = false; + _need_evict_cache_in_advance_metrics->set_value(0); } { @@ -2174,6 +2175,12 @@ std::map BlockFileCache::get_stats() { stats["disposable_queue_curr_elements"] = (double)_cur_disposable_queue_element_count_metrics->get_value(); + stats["total_removed_counts"] = (double)_num_removed_blocks->get_value(); + stats["total_hit_counts"] = (double)_num_hit_blocks->get_value(); + stats["total_read_counts"] = (double)_num_read_blocks->get_value(); + stats["need_evict_cache_in_advance"] = (double)_need_evict_cache_in_advance; + stats["disk_resource_limit_mode"] = (double)_disk_resource_limit_mode; + return stats; } From 3a3c25e4317b8baa2bcded287ac6fc644dde0b51 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 26 Jun 2025 10:23:19 +0800 Subject: [PATCH 077/572] =?UTF-8?q?branch-3.0:[fix](planner)Fix=20the=20co?= =?UTF-8?q?ncurrency=20issue=20caused=20by=20Expr=20setting=20member?= =?UTF-8?q?=E2=80=A6=20(#52203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … variables (#51389) pick: https://github.com/apache/doris/pull/51389 --- .../apache/doris/analysis/AnalyticExpr.java | 36 +++++ .../apache/doris/analysis/AnalyticWindow.java | 34 +++++ .../apache/doris/analysis/ArithmeticExpr.java | 13 ++ .../apache/doris/analysis/ArrayLiteral.java | 11 ++ .../doris/analysis/BetweenPredicate.java | 11 ++ .../doris/analysis/BinaryPredicate.java | 9 ++ .../doris/analysis/BitmapFilterPredicate.java | 10 ++ .../apache/doris/analysis/BoolLiteral.java | 8 ++ .../org/apache/doris/analysis/CaseExpr.java | 25 ++++ .../org/apache/doris/analysis/CastExpr.java | 19 ++- .../apache/doris/analysis/ColumnRefExpr.java | 8 ++ .../doris/analysis/CompoundPredicate.java | 14 ++ .../analysis/CreateMaterializedViewStmt.java | 13 +- .../apache/doris/analysis/DateLiteral.java | 8 ++ .../apache/doris/analysis/DecimalLiteral.java | 8 ++ .../doris/analysis/DefaultValueExpr.java | 8 ++ .../apache/doris/analysis/EncryptKeyRef.java | 10 ++ .../doris/analysis/ExistsPredicate.java | 15 ++ .../java/org/apache/doris/analysis/Expr.java | 36 ++--- .../apache/doris/analysis/FloatLiteral.java | 8 ++ .../doris/analysis/FunctionCallExpr.java | 134 ++++++++++++++++++ .../apache/doris/analysis/IPv4Literal.java | 8 ++ .../apache/doris/analysis/IPv6Literal.java | 8 ++ .../doris/analysis/ImportColumnDesc.java | 2 +- .../apache/doris/analysis/InPredicate.java | 17 +++ .../doris/analysis/InformationFunction.java | 8 ++ .../org/apache/doris/analysis/IntLiteral.java | 8 ++ .../doris/analysis/IsNullPredicate.java | 9 ++ .../apache/doris/analysis/JsonLiteral.java | 8 ++ .../analysis/LambdaFunctionCallExpr.java | 47 ++++++ .../doris/analysis/LambdaFunctionExpr.java | 22 +++ .../doris/analysis/LargeIntLiteral.java | 8 ++ .../apache/doris/analysis/LikePredicate.java | 9 ++ .../apache/doris/analysis/MVColumnItem.java | 2 +- .../org/apache/doris/analysis/MapLiteral.java | 14 ++ .../apache/doris/analysis/MatchPredicate.java | 9 ++ .../org/apache/doris/analysis/MaxLiteral.java | 8 ++ .../apache/doris/analysis/NullLiteral.java | 8 ++ .../apache/doris/analysis/OrderByElement.java | 25 ++++ .../doris/analysis/PlaceHolderExpr.java | 11 ++ .../org/apache/doris/analysis/SlotRef.java | 45 +++++- .../apache/doris/analysis/StringLiteral.java | 8 ++ .../apache/doris/analysis/StructLiteral.java | 10 ++ .../org/apache/doris/analysis/Subquery.java | 8 ++ .../analysis/TimestampArithmeticExpr.java | 41 ++++++ .../doris/analysis/TupleIsNullPredicate.java | 8 ++ .../apache/doris/analysis/VariableExpr.java | 18 +++ .../apache/doris/catalog/AliasFunction.java | 4 +- .../doris/catalog/MaterializedIndexMeta.java | 2 +- .../load/routineload/RoutineLoadJob.java | 12 +- .../AbstractSelectMaterializedIndexRule.java | 9 +- .../SelectMaterializedIndexWithAggregate.java | 2 +- .../expressions/functions/udf/AliasUdf.java | 2 +- .../CreateMaterializedViewStmtTest.java | 2 +- .../org/apache/doris/analysis/ExprTest.java | 44 ++++-- .../rules/rewrite/mv/SelectMvIndexTest.java | 14 +- .../apache/doris/rewrite/FEFunctionsTest.java | 5 +- 57 files changed, 829 insertions(+), 74 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java index 87681a636e21a1..92f303efdb08a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java @@ -26,6 +26,8 @@ import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.TreeNode; @@ -908,6 +910,40 @@ public String toSqlImpl() { return sb.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + if (sqlString != null) { + return sqlString; + } + StringBuilder sb = new StringBuilder(); + sb.append(fnCall.toSql(disableTableName, needExternalSql, tableType, table)).append(" OVER ("); + boolean needsSpace = false; + if (!partitionExprs.isEmpty()) { + sb.append("PARTITION BY ").append(exprListToSql(partitionExprs)); + needsSpace = true; + } + if (!orderByElements.isEmpty()) { + List orderByStrings = Lists.newArrayList(); + for (OrderByElement e : orderByElements) { + orderByStrings.add(e.toSql(disableTableName, needExternalSql, tableType, table)); + } + if (needsSpace) { + sb.append(" "); + } + sb.append("ORDER BY ").append(Joiner.on(", ").join(orderByStrings)); + needsSpace = true; + } + if (window != null) { + if (needsSpace) { + sb.append(" "); + } + sb.append(window.toSql(disableTableName, needExternalSql, tableType, table)); + } + sb.append(")"); + return sb.toString(); + } + @Override public String toDigestImpl() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java index 47e756578d8ea1..c2ee5176d5e071 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java @@ -20,6 +20,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TAnalyticWindow; import org.apache.doris.thrift.TAnalyticWindowBoundary; @@ -172,6 +174,18 @@ public String toSql() { return sb.toString(); } + public String toSql(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder sb = new StringBuilder(); + + if (expr != null) { + sb.append(expr.toSql(disableTableName, needExternalSql, tableType, table)).append(" "); + } + + sb.append(type.toString()); + return sb.toString(); + } + public String toDigest() { StringBuilder sb = new StringBuilder(); @@ -320,6 +334,26 @@ public String toSql() { return sb.toString(); } + public String toSql(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + if (toSqlString != null) { + return toSqlString; + } + + StringBuilder sb = new StringBuilder(); + sb.append(type.toString()).append(" "); + + if (rightBoundary == null) { + sb.append(leftBoundary.toSql(disableTableName, needExternalSql, tableType, table)); + } else { + sb.append("BETWEEN ").append(leftBoundary.toSql(disableTableName, needExternalSql, tableType, table)) + .append(" AND "); + sb.append(rightBoundary.toSql(disableTableName, needExternalSql, tableType, table)); + } + + return sb.toString(); + } + public String toDigest() { StringBuilder sb = new StringBuilder(); sb.append(type.toString()).append(" "); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java index 6f6deafb09af48..a3405462b93c57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java @@ -26,6 +26,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -269,6 +271,17 @@ public String toSqlImpl() { } } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + if (children.size() == 1) { + return op.toString() + " " + getChild(0).toSql(disableTableName, needExternalSql, tableType, table); + } else { + return "(" + getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + op.toString() + + " " + getChild(1).toSql(disableTableName, needExternalSql, tableType, table) + ")"; + } + } + @Override public String toDigestImpl() { if (children.size() == 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java index b798f2f15bea85..7cb84743c041d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java @@ -18,6 +18,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -117,6 +119,15 @@ protected String toSqlImpl() { return "[" + StringUtils.join(list, ", ") + "]"; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + List list = new ArrayList<>(children.size()); + children.forEach(v -> list.add(v.toSqlImpl(disableTableName, needExternalSql, tableType, table))); + + return "[" + StringUtils.join(list, ", ") + "]"; + } + @Override public String toDigestImpl() { List list = new ArrayList<>(children.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java index 5ebe1ce3e939bb..ad011d34519b85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java @@ -20,6 +20,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -94,6 +96,15 @@ public String toSqlImpl() { + children.get(1).toSql() + " AND " + children.get(2).toSql(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + String notStr = (isNotBetween) ? "NOT " : ""; + return children.get(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + notStr + "BETWEEN " + + children.get(1).toSql(disableTableName, needExternalSql, tableType, table) + " AND " + children.get(2) + .toSql(disableTableName, needExternalSql, tableType, table); + } + @Override public String toDigestImpl() { String notStr = (isNotBetween) ? "NOT " : ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java index 0d2c664533bbd4..968f1fa38f0ee3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java @@ -26,6 +26,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.TypeUtils; import org.apache.doris.common.AnalysisException; @@ -285,6 +287,13 @@ public String toSqlImpl() { return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + op.toString() + " " + + getChild(1).toSql(disableTableName, needExternalSql, tableType, table); + } + @Override public String toDigestImpl() { return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java index be455ebea1359d..c9a0bb40677b2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BitmapFilterPredicate.java @@ -18,6 +18,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExprNode; @@ -84,6 +86,14 @@ protected String toSqlImpl() { .toSql() + ")"; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return (notIn ? "not " : "") + "BitmapFilterPredicate(" + children.get(0) + .toSql(disableTableName, needExternalSql, tableType, table) + ", " + children.get(1) + .toSql(disableTableName, needExternalSql, tableType, table) + ")"; + } + @Override protected void toThrift(TExprNode msg) { Preconditions.checkArgument(false, "`toThrift` in BitmapFilterPredicate should not be reached!"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java index 7008872ababc4b..246b6921add3ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BoolLiteral.java @@ -21,6 +21,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -106,6 +108,12 @@ public String toSqlImpl() { return value ? "TRUE" : "FALSE"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return value ? "TRUE" : "FALSE"; + } + @Override public String getStringValue() { return value ? "1" : "0"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java index 3eae897f30d762..856b011b93fc62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java @@ -20,6 +20,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TCaseExpr; @@ -152,6 +154,29 @@ public String toSqlImpl() { return output.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder output = new StringBuilder("CASE"); + int childIdx = 0; + if (hasCaseExpr) { + output.append(' ') + .append(children.get(childIdx++).toSql(disableTableName, needExternalSql, tableType, table)); + } + while (childIdx + 2 <= children.size()) { + output.append( + " WHEN " + children.get(childIdx++).toSql(disableTableName, needExternalSql, tableType, table)); + output.append( + " THEN " + children.get(childIdx++).toSql(disableTableName, needExternalSql, tableType, table)); + } + if (hasElseExpr) { + output.append(" ELSE " + children.get(children.size() - 1) + .toSql(disableTableName, needExternalSql, tableType, table)); + } + output.append(" END"); + return output.toString(); + } + @Override public String toDigestImpl() { StringBuilder sb = new StringBuilder("CASE"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java index aab6a9dbec0747..e6aa47c6334bf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java @@ -27,6 +27,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarFunction; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.catalog.TypeUtils; import org.apache.doris.common.AnalysisException; @@ -217,9 +219,6 @@ public Expr clone() { @Override public String toSqlImpl() { - if (needExternalSql) { - return getChild(0).toSql(); - } if (isAnalyzed) { return "CAST(" + getChild(0).toSql() + " AS " + type.toSql() + ")"; } else { @@ -228,6 +227,20 @@ public String toSqlImpl() { } } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, TableIf table) { + if (needExternalSql) { + return getChild(0).toSql(disableTableName, needExternalSql, tableType, table); + } + if (isAnalyzed) { + return "CAST(" + getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " AS " + + type.toSql() + ")"; + } else { + return "CAST(" + getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " AS " + + (isImplicit ? type.toString() : targetTypeDef.toSql()) + ")"; + } + } + @Override public String toDigestImpl() { boolean isVerbose = ConnectContext.get() != null diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java index 35529ea6798658..5314f0637862d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnRefExpr.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.nereids.util.Utils; import org.apache.doris.thrift.TColumnRef; @@ -93,6 +95,12 @@ protected String toSqlImpl() { return columnName; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return columnName; + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.COLUMN_REF; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java index dcd67598dd4122..1f47022f41f252 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java @@ -22,6 +22,8 @@ import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.ScalarFunction; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -100,6 +102,18 @@ public String toSqlImpl() { } } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + if (children.size() == 1) { + Preconditions.checkState(op == Operator.NOT); + return "NOT " + getChild(0).toSql(disableTableName, needExternalSql, tableType, table); + } else { + return getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + op.toString() + " " + + getChild(1).toSql(disableTableName, needExternalSql, tableType, table); + } + } + @Override public String toDigestImpl() { if (children.size() == 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java index 166ee53276bfb8..e1404e251fca56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java @@ -272,11 +272,11 @@ public void analyzeSelectClause(Analyzer analyzer) throws AnalysisException { SelectListItem selectListItem = selectList.getItems().get(i); Expr selectListItemExpr = selectListItem.getExpr(); - selectListItemExpr.setDisableTableName(true); + selectListItemExpr.disableTableName(); if (!(selectListItemExpr instanceof SlotRef) && !(selectListItemExpr instanceof FunctionCallExpr) && !(selectListItemExpr instanceof ArithmeticExpr)) { throw new AnalysisException("The materialized view only support the single column or function expr. " - + "Error column: " + selectListItemExpr.toSql()); + + "Error column: " + selectListItemExpr.toSqlWithoutTbl()); } if (!isReplay && selectListItemExpr.hasAutoInc()) { @@ -297,14 +297,14 @@ public void analyzeSelectClause(Analyzer analyzer) throws AnalysisException { if (!isReplay && selectListItemExpr.containsAggregate()) { throw new AnalysisException( "The materialized view's expr calculations cannot be included outside aggregate functions" - + ", expr: " + selectListItemExpr.toSql()); + + ", expr: " + selectListItemExpr.toSqlWithoutTbl()); } List slots = new ArrayList<>(); selectListItemExpr.collect(SlotRef.class, slots); if (!isReplay && slots.size() == 0) { throw new AnalysisException( "The materialized view contain constant expr is disallowed, expr: " - + selectListItemExpr.toSql()); + + selectListItemExpr.toSqlWithoutTbl()); } if (meetAggregate) { throw new AnalysisException("The aggregate column should be after the single column"); @@ -423,7 +423,8 @@ private void analyzeOrderByClause() throws AnalysisException { if (!mvColumnItem.getDefineExpr().equals(orderByElement)) { throw new AnalysisException("The order of columns in order by clause must be same as " - + "the order of columns in select list, " + mvColumnItem.getDefineExpr().toSql() + " vs " + + "the order of columns in select list, " + mvColumnItem.getDefineExpr().toSqlWithoutTbl() + + " vs " + orderByElement.toSql()); } Preconditions.checkState(mvColumnItem.getAggregationType() == null); @@ -607,7 +608,7 @@ public Map parseDefineExpr(Analyzer analyzer) throws AnalysisExcep for (SelectListItem selectListItem : selectList.getItems()) { Expr selectListItemExpr = selectListItem.getExpr(); Expr expr = selectListItemExpr; - String name = mvColumnBuilder(MaterializedIndexMeta.normalizeName(expr.toSql())); + String name = mvColumnBuilder(MaterializedIndexMeta.normalizeName(expr.toSqlWithoutTbl())); if (selectListItemExpr instanceof FunctionCallExpr) { FunctionCallExpr functionCallExpr = (FunctionCallExpr) selectListItemExpr; switch (functionCallExpr.getFnName().getFunction().toLowerCase()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 0a9eb389ac5854..4ad13af1f8ca41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -22,6 +22,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -677,6 +679,12 @@ public String toSqlImpl() { return "'" + getStringValue() + "'"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "'" + getStringValue() + "'"; + } + private void fillPaddedValue(char[] buffer, int start, long value, int length) { int end = start + length; for (int i = end - 1; i >= start; i--) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java index 41d64c27ae6ea6..e13fc019fd8e36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java @@ -19,6 +19,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -275,6 +277,12 @@ public String toSqlImpl() { return getStringValue(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getStringValue(); + } + @Override public String getStringValue() { return value.toPlainString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java index 0a84657fc7b510..ca746d45ba1741 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExpr.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -31,6 +33,12 @@ protected String toSqlImpl() { return null; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return null; + } + @Override protected void toThrift(TExprNode msg) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyRef.java index 7950fe37c4b304..6b00e64a2f5628 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/EncryptKeyRef.java @@ -19,6 +19,8 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.EncryptKey; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; @@ -89,6 +91,14 @@ protected String toSqlImpl() { return sb.toString(); } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder sb = new StringBuilder(); + sb.append(encryptKeyName.toSql()); + return sb.toString(); + } + @Override protected void toThrift(TExprNode msg) { // no operation diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java index 9fd4b18980e8d3..548c6163e86097 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.thrift.TExprNode; import com.google.common.base.Preconditions; @@ -70,6 +72,19 @@ public String toSqlImpl() { return strBuilder.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder strBuilder = new StringBuilder(); + if (notExists) { + strBuilder.append("NOT "); + + } + strBuilder.append("EXISTS "); + strBuilder.append(getChild(0).toSql(disableTableName, needExternalSql, tableType, table)); + return strBuilder.toString(); + } + @Override public String toDigestImpl() { StringBuilder strBuilder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index a7cfad25d465fc..ee3fdbe9027d18 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -95,9 +95,6 @@ public abstract class Expr extends TreeNode implements ParseNode, Cloneabl public static final String DEFAULT_EXPR_NAME = "expr"; protected boolean disableTableName = false; - protected boolean needExternalSql = false; - protected TableType tableType = null; - protected TableIf inputTable = null; // to be used where we can't come up with a better estimate public static final double DEFAULT_SELECTIVITY = 0.1; @@ -922,31 +919,26 @@ public static long getNumDistinctValues(List exprs) { } public String toSql() { + if (disableTableName) { + return toSqlWithoutTbl(); + } return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl(); } - public void setDisableTableName(boolean value) { - disableTableName = value; - for (Expr child : children) { - child.setDisableTableName(value); - } + public String toSql(boolean disableTableName, boolean needExternalSql, TableType tableType, TableIf table) { + return (printSqlInParens) ? "(" + toSqlImpl(disableTableName, needExternalSql, tableType, table) + ")" + : toSqlImpl(disableTableName, needExternalSql, tableType, table); } - public void setExternalContext(boolean needExternalSql, TableType tableType, TableIf inputTable) { - this.needExternalSql = needExternalSql; - this.tableType = tableType; - this.inputTable = inputTable; - + public void disableTableName() { + disableTableName = true; for (Expr child : children) { - child.setExternalContext(needExternalSql, tableType, inputTable); + child.disableTableName(); } } public String toSqlWithoutTbl() { - setDisableTableName(true); - String result = toSql(); - setDisableTableName(false); - return result; + return toSql(true, false, null, null); } public String toDigest() { @@ -959,6 +951,9 @@ public String toDigest() { */ protected abstract String toSqlImpl(); + protected abstract String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table); + /** * !!!!!! Important !!!!!! * Subclasses should override this method if @@ -969,10 +964,7 @@ protected String toDigestImpl() { } public String toExternalSql(TableType tableType, TableIf table) { - setExternalContext(true, tableType, table); - String result = toSql(); - setExternalContext(false, null, null); - return result; + return toSql(false, true, tableType, table); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 074fc250967820..074b0eea5ec727 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -19,6 +19,8 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -137,6 +139,12 @@ public String toSqlImpl() { return getStringValue(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getStringValue(); + } + @Override public String getStringValue() { // TODO: Here is weird use float to represent TIME type diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 03f6762b4384ae..62353ba32879c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -35,6 +35,8 @@ import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -669,6 +671,92 @@ private String paramsToSql() { return sb.toString(); } + private String paramsToSql(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder sb = new StringBuilder(); + sb.append("("); + + if (fnParams.isStar()) { + sb.append("*"); + } + if (fnParams.isDistinct()) { + sb.append("DISTINCT "); + } + int len = children.size(); + + if (fnName.getFunction().equalsIgnoreCase("char")) { + for (int i = 1; i < len; ++i) { + sb.append(children.get(i).toSql(disableTableName, needExternalSql, tableType, table)); + if (i < len - 1) { + sb.append(", "); + } + } + sb.append(" using "); + String encodeType = children.get(0).toSql(disableTableName, needExternalSql, tableType, table); + if (encodeType.charAt(0) == '\'') { + encodeType = encodeType.substring(1, encodeType.length()); + } + if (encodeType.charAt(encodeType.length() - 1) == '\'') { + encodeType = encodeType.substring(0, encodeType.length() - 1); + } + sb.append(encodeType).append(")"); + return sb.toString(); + } + + // XXX_diff are used by nereids only + if (fnName.getFunction().equalsIgnoreCase("years_diff") || fnName.getFunction().equalsIgnoreCase("months_diff") + || fnName.getFunction().equalsIgnoreCase("days_diff") + || fnName.getFunction().equalsIgnoreCase("hours_diff") + || fnName.getFunction().equalsIgnoreCase("minutes_diff") + || fnName.getFunction().equalsIgnoreCase("seconds_diff") + || fnName.getFunction().equalsIgnoreCase("milliseconds_diff") + || fnName.getFunction().equalsIgnoreCase("microseconds_diff")) { + sb.append(children.get(0).toSql(disableTableName, needExternalSql, tableType, table)).append(", "); + sb.append(children.get(1).toSql(disableTableName, needExternalSql, tableType, table)).append(")"); + return sb.toString(); + } + // used by nereids END + + if (fnName.getFunction().equalsIgnoreCase("json_array") + || fnName.getFunction().equalsIgnoreCase("json_object") + || fnName.getFunction().equalsIgnoreCase("json_insert") + || fnName.getFunction().equalsIgnoreCase("json_replace") + || fnName.getFunction().equalsIgnoreCase("json_set")) { + len = len - 1; + } + + for (int i = 0; i < len; ++i) { + if (i != 0) { + if (fnName.getFunction().equalsIgnoreCase("group_concat") + && orderByElements.size() > 0 && i == len - orderByElements.size()) { + sb.append(" "); + } else { + sb.append(", "); + } + } + if (ConnectContext.get() != null && ConnectContext.get().getState().isQuery() && i == 1 + && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") + || fnName.getFunction().equalsIgnoreCase("aes_encrypt") + || fnName.getFunction().equalsIgnoreCase("sm4_decrypt") + || fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) { + sb.append("\'***\'"); + continue; + } else if (orderByElements.size() > 0 && i == len - orderByElements.size()) { + sb.append("ORDER BY "); + } + sb.append(children.get(i).toSql(disableTableName, needExternalSql, tableType, table)); + if (orderByElements.size() > 0 && i >= len - orderByElements.size()) { + if (orderByElements.get(i - len + orderByElements.size()).getIsAsc()) { + sb.append(" ASC"); + } else { + sb.append(" DESC"); + } + } + } + sb.append(")"); + return sb.toString(); + } + @Override public String toSqlImpl() { Expr expr; @@ -714,6 +802,52 @@ public String toSqlImpl() { return sb.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + Expr expr; + if (originStmtFnExpr != null) { + expr = originStmtFnExpr; + } else { + expr = this; + } + StringBuilder sb = new StringBuilder(); + + // when function is like or regexp, the expr generated sql should be like this + // eg: child1 like child2 + if (fnName.getFunction().equalsIgnoreCase("like") + || fnName.getFunction().equalsIgnoreCase("regexp")) { + sb.append(children.get(0).toSql(disableTableName, needExternalSql, tableType, table)); + sb.append(" "); + sb.append(((FunctionCallExpr) expr).fnName); + sb.append(" "); + sb.append(children.get(1).toSql(disableTableName, needExternalSql, tableType, table)); + } else if (fnName.getFunction().equalsIgnoreCase("encryptkeyref")) { + sb.append("key "); + for (int i = 0; i < children.size(); i++) { + String str = ((StringLiteral) children.get(i)).getValue(); + if (str.isEmpty()) { + continue; + } + sb.append(str); + sb.append("."); + } + sb.deleteCharAt(sb.length() - 1); + } else { + sb.append(((FunctionCallExpr) expr).fnName); + sb.append(paramsToSql(disableTableName, needExternalSql, tableType, table)); + if (fnName.getFunction().equalsIgnoreCase("json_quote") + || fnName.getFunction().equalsIgnoreCase("json_array") + || fnName.getFunction().equalsIgnoreCase("json_object") + || fnName.getFunction().equalsIgnoreCase("json_insert") + || fnName.getFunction().equalsIgnoreCase("json_replace") + || fnName.getFunction().equalsIgnoreCase("json_set")) { + return forJSON(sb.toString()); + } + } + return sb.toString(); + } + private String paramsToDigest() { StringBuilder sb = new StringBuilder(); sb.append("("); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java index 03b5d46eb2c149..ab81ebdb85b115 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv4Literal.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -118,6 +120,12 @@ protected String toSqlImpl() { return "\"" + getStringValue() + "\""; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "\"" + getStringValue() + "\""; + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.IPV4_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java index 1acf5902b7ad9d..7d719720442505 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -83,6 +85,12 @@ protected String toSqlImpl() { return "\"" + getStringValue() + "\""; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "\"" + getStringValue() + "\""; + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.IPV6_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java index e62fca2c27f7ad..2d4aec8974a482 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ImportColumnDesc.java @@ -80,7 +80,7 @@ public String toString() { StringBuilder sb = new StringBuilder(); sb.append(columnName); if (expr != null) { - sb.append("=").append(expr.toSql()); + sb.append("=").append(expr.toSqlWithoutTbl()); } return sb.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java index 128fac0effd1ec..f07016fc8bbee8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java @@ -25,6 +25,8 @@ import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarFunction; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Reference; @@ -285,6 +287,21 @@ public String toSqlImpl() { return strBuilder.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder strBuilder = new StringBuilder(); + String notStr = (isNotIn) ? "NOT " : ""; + strBuilder.append( + getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + notStr + "IN ("); + for (int i = 1; i < children.size(); ++i) { + strBuilder.append(getChild(i).toSql(disableTableName, needExternalSql, tableType, table)); + strBuilder.append((i + 1 != children.size()) ? ", " : ""); + } + strBuilder.append(")"); + return strBuilder.toString(); + } + @Override public String toDigestImpl() { StringBuilder strBuilder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java index f5aecde5d0f07c..5590a4eeb6baae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InformationFunction.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; @@ -98,6 +100,12 @@ public String toSqlImpl() { return funcType + "()"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return funcType + "()"; + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java index a33a1b741ec8a2..22180b378dcadd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java @@ -18,6 +18,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; @@ -298,6 +300,12 @@ public String toSqlImpl() { return getStringValue(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getStringValue(); + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.INT_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java index 1b2d626f3c558e..a6b183c72e65a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java @@ -24,6 +24,8 @@ import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.ScalarFunction; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -112,6 +114,13 @@ public String toSqlImpl() { return getChild(0).toSql() + (isNotNull ? " IS NOT NULL" : " IS NULL"); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + (isNotNull ? " IS NOT NULL" + : " IS NULL"); + } + @Override public String toDigestImpl() { return getChild(0).toDigest() + (isNotNull ? " IS NOT NULL" : " IS NULL"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java index 050ec16a3304e9..a22c2581cad904 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/JsonLiteral.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -84,6 +86,12 @@ public boolean isMinValue() { return false; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "'" + value.replaceAll("'", "''") + "'"; + } + @Override public String toSqlImpl() { return "'" + value.replaceAll("'", "''") + "'"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java index 500f0585dc858c..b5bc74d85f31f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionCallExpr.java @@ -20,6 +20,8 @@ import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.Function; import org.apache.doris.catalog.Function.NullableMode; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -351,4 +353,49 @@ public String toSqlImpl() { sb.append(")"); return sb.toString(); } + + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder sb = new StringBuilder(); + + String fnName = getFnName().getFunction(); + if (fn != null) { + // `array_last` will be replaced with `element_at` function after analysis. + // At this moment, using the name `array_last` would generate invalid SQL. + fnName = fn.getFunctionName().getFunction(); + } + sb.append(fnName); + sb.append("("); + int childSize = children.size(); + Expr lastExpr = getChild(childSize - 1); + // eg: select array_map(x->x>10, k1) from table, + // but we need analyze each param, so change the function like this in parser + // array_map(x->x>10, k1) ---> array_map(k1, x->x>10), + // so maybe the lambda expr is the end position. and need this check. + boolean lastIsLambdaExpr = (lastExpr instanceof LambdaFunctionExpr); + if (lastIsLambdaExpr) { + sb.append(lastExpr.toSql(disableTableName, needExternalSql, tableType, table)); + sb.append(", "); + } + for (int i = 0; i < childSize - 1; ++i) { + sb.append(getChild(i).toSql(disableTableName, needExternalSql, tableType, table)); + if (i != childSize - 2) { + sb.append(", "); + } + } + // and some functions is only implement as a normal array function; + // but also want use as lambda function, select array_sortby(x->x,['b','a','c']); + // so we convert to: array_sortby(array('b', 'a', 'c'), array_map(x -> `x`, array('b', 'a', 'c'))) + if (!lastIsLambdaExpr) { + if (childSize > 1) { + // some functions don't have lambda expr, so don't need to add "," + // such as array_exists(array_map(x->x>3, [1,2,3,6,34,3,11])) + sb.append(", "); + } + sb.append(lastExpr.toSql(disableTableName, needExternalSql, tableType, table)); + } + sb.append(")"); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java index c171bf18c274ab..39a98ff49de626 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LambdaFunctionExpr.java @@ -18,6 +18,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.ArrayType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -146,6 +148,26 @@ protected String toSqlImpl() { return res; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + String nameStr = ""; + Expr lambdaExpr = slotExprs.get(0); + int exprSize = names.size(); + for (int i = 0; i < exprSize; ++i) { + nameStr = nameStr + names.get(i); + if (i != exprSize - 1) { + nameStr = nameStr + ","; + } + } + if (exprSize > 1) { + nameStr = "(" + nameStr + ")"; + } + String res = String.format("%s -> %s", nameStr, + lambdaExpr.toSql(disableTableName, needExternalSql, tableType, table)); + return res; + } + @Override protected void toThrift(TExprNode msg) { msg.setNodeType(TExprNodeType.LAMBDA_FUNCTION_EXPR); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java index 00cd043ab1fffe..b5f78eef955512 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java @@ -18,6 +18,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.io.Text; @@ -211,6 +213,12 @@ public String toSqlImpl() { return getStringValue(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getStringValue(); + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.LARGE_INT_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java index 7982c877513c0a..099453f9889421 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java @@ -23,6 +23,8 @@ import org.apache.doris.catalog.Function; import org.apache.doris.catalog.FunctionSet; import org.apache.doris.catalog.ScalarFunction; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -114,6 +116,13 @@ public String toSqlImpl() { return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + op.toString() + " " + + getChild(1).toSql(disableTableName, needExternalSql, tableType, table); + } + @Override public String toDigestImpl() { return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java index a9e11458582547..e1439450d646bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MVColumnItem.java @@ -87,7 +87,7 @@ public MVColumnItem(String name, Type type) { } public MVColumnItem(Expr defineExpr) throws AnalysisException { - this.name = CreateMaterializedViewStmt.mvColumnBuilder(defineExpr.toSql()); + this.name = CreateMaterializedViewStmt.mvColumnBuilder(defineExpr.toSqlWithoutTbl()); if (this.name == null) { throw new AnalysisException("defineExpr.toSql() is null"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java index 396883699c5f65..d15ab9b2685e9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java @@ -18,6 +18,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.MapType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -196,6 +198,18 @@ protected String toSqlImpl() { return "MAP{" + StringUtils.join(list, ", ") + "}"; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + List list = new ArrayList<>(children.size()); + for (int i = 0; i < children.size() && i + 1 < children.size(); i += 2) { + list.add( + children.get(i).toSqlImpl(disableTableName, needExternalSql, tableType, table) + ":" + children.get( + i + 1).toSqlImpl(disableTableName, needExternalSql, tableType, table)); + } + return "MAP{" + StringUtils.join(list, ", ") + "}"; + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.MAP_LITERAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MatchPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MatchPredicate.java index 3b3d9ad7ee0865..c104fd10920f72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MatchPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MatchPredicate.java @@ -24,6 +24,8 @@ import org.apache.doris.catalog.Index; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.ScalarFunction; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.thrift.TExprNode; @@ -223,6 +225,13 @@ public String toSqlImpl() { return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getChild(0).toSql(disableTableName, needExternalSql, tableType, table) + " " + op.toString() + " " + + getChild(1).toSql(disableTableName, needExternalSql, tableType, table); + } + @Override protected void toThrift(TExprNode msg) { msg.node_type = TExprNodeType.MATCH_PRED; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java index 03c4653833d5d6..5753b24ab55fce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MaxLiteral.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.FormatOptions; import org.apache.doris.thrift.TExprNode; @@ -58,6 +60,12 @@ public String toSqlImpl() { return "MAXVALUE"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "MAXVALUE"; + } + @Override public String toString() { return toSql(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index 70840bb4fdb648..fa5675c15cefe8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -21,6 +21,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; @@ -101,6 +103,12 @@ public String toSqlImpl() { return getStringValue(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return getStringValue(); + } + @Override public String getStringValue() { return "NULL"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java index 39b8bd328d39c2..ca5c3a782b73be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java @@ -20,6 +20,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Lists; @@ -137,6 +139,29 @@ public String toSql() { return strBuilder.toString(); } + public String toSql(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append(expr.toSql(disableTableName, needExternalSql, tableType, table)); + strBuilder.append(isAsc ? " ASC" : " DESC"); + + // When ASC and NULLS LAST or DESC and NULLS FIRST, we do not print NULLS FIRST/LAST + // because it is the default behavior and we want to avoid printing NULLS FIRST/LAST + // whenever possible as it is incompatible with Hive (SQL compatibility with Hive is + // important for views). + if (nullsFirstParam != null) { + if (isAsc && nullsFirstParam) { + // If ascending, nulls are last by default, so only add if nulls first. + strBuilder.append(" NULLS FIRST"); + } else if (!isAsc && !nullsFirstParam) { + // If descending, nulls are first by default, so only add if nulls last. + strBuilder.append(" NULLS LAST"); + } + } + + return strBuilder.toString(); + } + public String toDigest() { StringBuilder strBuilder = new StringBuilder(); strBuilder.append(expr.toDigest()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index eda712552e86e1..a7cf5c3cd438f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -19,6 +19,8 @@ import org.apache.doris.catalog.MysqlColType; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -183,6 +185,15 @@ public String toSqlImpl() { return "_placeholder_(" + this.lExpr.toSqlImpl() + ")"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + if (this.lExpr == null) { + return "?"; + } + return "_placeholder_(" + this.lExpr.toSqlImpl(disableTableName, needExternalSql, tableType, table) + ")"; + } + // @Override public Expr reset() { this.lExpr = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index 7078c90f1ed5be..3e8117e365b950 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -242,8 +242,49 @@ public String debugString() { @Override public String toSqlImpl() { + StringBuilder sb = new StringBuilder(); + String subColumnPaths = ""; + if (subColPath != null && !subColPath.isEmpty()) { + subColumnPaths = "." + String.join(".", subColPath); + } + if (tblName != null) { + return tblName.toSql() + "." + label + subColumnPaths; + } else if (label != null) { + if (ConnectContext.get() != null + && ConnectContext.get().getState().isNereids() + && !ConnectContext.get().getState().isQuery() + && ConnectContext.get().getSessionVariable() != null + && desc != null) { + return label + "[#" + desc.getId().asInt() + "]"; + } else { + return label; + } + } else if (desc == null) { + // virtual slot of an alias function + // when we try to translate an alias function to Nereids style, the desc in the place holding slotRef + // is null, and we just need the name of col. + return "`" + col + "`"; + } else if (desc.getSourceExprs() != null) { + if ((ToSqlContext.get() == null || ToSqlContext.get().isNeedSlotRefId())) { + if (desc.getId().asInt() != 1) { + sb.append(""); + } + } + for (Expr expr : desc.getSourceExprs()) { + sb.append(" "); + sb.append(expr.toSql()); + } + return sb.toString(); + } else { + return "" + sb.toString(); + } + } + + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf inputTable) { if (needExternalSql) { - return toExternalSqlImpl(); + return toExternalSqlImpl(tableType, inputTable); } if (disableTableName && label != null) { @@ -290,7 +331,7 @@ public String toSqlImpl() { } } - private String toExternalSqlImpl() { + private String toExternalSqlImpl(TableType tableType, TableIf inputTable) { if (col != null) { if (tableType.equals(TableType.JDBC_EXTERNAL_TABLE) || tableType.equals(TableType.JDBC) || tableType .equals(TableType.ODBC)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 51f1bb53b95ebe..02396e0a71272c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -21,6 +21,8 @@ package org.apache.doris.analysis; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -138,6 +140,12 @@ public String toSqlImpl() { return "'" + value.replaceAll("'", "''") + "'"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "'" + value.replaceAll("'", "''") + "'"; + } + @Override protected void toThrift(TExprNode msg) { if (value == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java index 78437c26ce07cc..90ee8438060a5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java @@ -19,6 +19,8 @@ import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FormatOptions; @@ -81,6 +83,14 @@ protected String toSqlImpl() { return "STRUCT(" + StringUtils.join(list, ", ") + ")"; } + @Override + protected String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + List list = new ArrayList<>(children.size()); + children.forEach(v -> list.add(v.toSqlImpl(disableTableName, needExternalSql, tableType, table))); + return "STRUCT(" + StringUtils.join(list, ", ") + ")"; + } + @Override public String toDigestImpl() { List list = new ArrayList<>(children.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java index 915230eff13c78..bb57c6b6c6e5c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java @@ -23,6 +23,8 @@ import org.apache.doris.catalog.MultiRowType; import org.apache.doris.catalog.StructField; import org.apache.doris.catalog.StructType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.thrift.TExprNode; @@ -61,6 +63,12 @@ public String toSqlImpl() { return "(" + stmt.toSql() + ")"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "(" + stmt.toSql() + ")"; + } + @Override public String toDigestImpl() { return "(" + stmt.toDigest() + ")"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java index 3b2e6fdde3a5b3..d88c13080c3c94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java @@ -22,6 +22,8 @@ import org.apache.doris.catalog.Function.NullableMode; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -411,6 +413,45 @@ public String toSqlImpl() { return strBuilder.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder strBuilder = new StringBuilder(); + if (funcName != null) { + if (funcName.equalsIgnoreCase("TIMESTAMPDIFF") || funcName.equalsIgnoreCase("TIMESTAMPADD")) { + strBuilder.append(funcName).append("("); + strBuilder.append(timeUnitIdent).append(", "); + strBuilder.append(getChild(1).toSql(disableTableName, needExternalSql, tableType, table)).append(", "); + strBuilder.append(getChild(0).toSql(disableTableName, needExternalSql, tableType, table)).append(")"); + return strBuilder.toString(); + } + // Function-call like version. + strBuilder.append(funcName).append("("); + strBuilder.append(getChild(0).toSql(disableTableName, needExternalSql, tableType, table)).append(", "); + strBuilder.append("INTERVAL "); + strBuilder.append(getChild(1).toSql(disableTableName, needExternalSql, tableType, table)); + strBuilder.append(" ").append(timeUnitIdent); + strBuilder.append(")"); + return strBuilder.toString(); + } + if (intervalFirst) { + // Non-function-call like version with interval as first operand. + strBuilder.append("INTERVAL "); + strBuilder.append(getChild(1).toSql(disableTableName, needExternalSql, tableType, table) + " "); + strBuilder.append(timeUnitIdent); + strBuilder.append(" ").append(op.toString()).append(" "); + strBuilder.append(getChild(0).toSql(disableTableName, needExternalSql, tableType, table)); + } else { + // Non-function-call like version with interval as second operand. + strBuilder.append(getChild(0).toSql(disableTableName, needExternalSql, tableType, table)); + strBuilder.append(" " + op.toString() + " "); + strBuilder.append("INTERVAL "); + strBuilder.append(getChild(1).toSql(disableTableName, needExternalSql, tableType, table) + " "); + strBuilder.append(timeUnitIdent); + } + return strBuilder.toString(); + } + @Override public String toDigestImpl() { StringBuilder strBuilder = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleIsNullPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleIsNullPredicate.java index b6af33108bdb31..285005d84c06ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleIsNullPredicate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TupleIsNullPredicate.java @@ -20,6 +20,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.thrift.TExprNode; @@ -192,6 +194,12 @@ public String toSqlImpl() { return "TupleIsNull(" + Joiner.on(",").join(tupleIds) + ")"; } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + return "TupleIsNull(" + Joiner.on(",").join(tupleIds) + ")"; + } + /** * Recursive function that replaces all 'IF(TupleIsNull(), NULL, e)' exprs in * 'expr' with e and returns the modified expr. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/VariableExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/VariableExpr.java index 90f6ffb0e02d55..29db739343a7f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/VariableExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/VariableExpr.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; @@ -203,6 +205,22 @@ public String toSqlImpl() { return sb.toString(); } + @Override + public String toSqlImpl(boolean disableTableName, boolean needExternalSql, TableType tableType, + TableIf table) { + StringBuilder sb = new StringBuilder(); + if (setType == SetType.USER) { + sb.append("@"); + } else { + sb.append("@@"); + if (setType == SetType.GLOBAL) { + sb.append("GLOBAL."); + } + } + sb.append(name); + return sb.toString(); + } + @Override public String toString() { return toSql(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java index 6ee9e2a4fec642..ca08250f6199a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AliasFunction.java @@ -251,7 +251,7 @@ public String toSql(boolean ifNotExists) { .append(" WITH PARAMETER(") .append(getParamsSting(parameters)) .append(") AS ") - .append(originFunction.toSql()) + .append(originFunction.toSqlWithoutTbl()) .append(";"); return sb.toString(); } @@ -271,7 +271,7 @@ public String getProperties() { Map properties = new HashMap<>(); properties.put("parameter", getParamsSting(parameters)); setSlotRefLabel(originFunction); - String functionStr = originFunction.toSql(); + String functionStr = originFunction.toSqlWithoutTbl(); functionStr = functionStr.replaceAll("'", "`"); properties.put("origin_function", functionStr); return new Gson().toJson(properties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 6a30e771a58bc8..0778459d0b1c69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -111,7 +111,7 @@ public MaterializedIndexMeta(long indexId, List schema, int schemaVersio public void setWhereClause(Expr whereClause) { this.whereClause = whereClause; if (this.whereClause != null) { - this.whereClause.setDisableTableName(true); + this.whereClause.disableTableName(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index d25450ae0d95a1..5cb1b6147a9ae7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -1741,7 +1741,7 @@ public String getShowCreateInfo() { } // 4.3.where_predicates if (whereExpr != null) { - sb.append("WHERE ").append(whereExpr.toSql()).append(",\n"); + sb.append("WHERE ").append(whereExpr.toSqlWithoutTbl()).append(",\n"); } // 4.4.partitions if (partitions != null) { @@ -1749,7 +1749,7 @@ public String getShowCreateInfo() { } // 4.5.delete_on_predicates if (deleteCondition != null) { - sb.append("DELETE ON ").append(deleteCondition.toSql()).append(",\n"); + sb.append("DELETE ON ").append(deleteCondition.toSqlWithoutTbl()).append(",\n"); } // 4.6.source_sequence if (sequenceCol != null) { @@ -1757,7 +1757,7 @@ public String getShowCreateInfo() { } // 4.7.preceding_predicates if (precedingFilter != null) { - sb.append("PRECEDING FILTER ").append(precedingFilter.toSql()).append(",\n"); + sb.append("PRECEDING FILTER ").append(precedingFilter.toSqlWithoutTbl()).append(",\n"); } // remove the last , if (sb.charAt(sb.length() - 2) == ',') { @@ -1850,8 +1850,8 @@ public String jobPropertiesToJsonString() { ? STAR_STRING : Joiner.on(",").join(partitions.getPartitionNames())); jobProperties.put("columnToColumnExpr", columnDescs == null ? STAR_STRING : Joiner.on(",").join(columnDescs.descs)); - jobProperties.put("precedingFilter", precedingFilter == null ? STAR_STRING : precedingFilter.toSql()); - jobProperties.put("whereExpr", whereExpr == null ? STAR_STRING : whereExpr.toSql()); + jobProperties.put("precedingFilter", precedingFilter == null ? STAR_STRING : precedingFilter.toSqlWithoutTbl()); + jobProperties.put("whereExpr", whereExpr == null ? STAR_STRING : whereExpr.toSqlWithoutTbl()); if (getFormat().equalsIgnoreCase("json")) { jobProperties.put(PROPS_FORMAT, "json"); } else { @@ -1872,7 +1872,7 @@ public String jobPropertiesToJsonString() { jobProperties.put(LoadStmt.EXEC_MEM_LIMIT, String.valueOf(execMemLimit)); jobProperties.put(LoadStmt.KEY_IN_PARAM_MERGE_TYPE, mergeType.toString()); jobProperties.put(LoadStmt.KEY_IN_PARAM_DELETE_CONDITION, - deleteCondition == null ? STAR_STRING : deleteCondition.toSql()); + deleteCondition == null ? STAR_STRING : deleteCondition.toSqlWithoutTbl()); jobProperties.putAll(this.jobProperties); Gson gson = new GsonBuilder().disableHtmlEscaping().create(); return gson.toJson(jobProperties); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index ceb496bc5c3ccc..e9e642b35fc64b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -166,9 +166,10 @@ protected static boolean containAllRequiredColumns(MaterializedIndex index, Logi // Here we use toSqlWithoutTbl because the output of toSql() is slot#[0] in Nereids Set indexConjuncts = PlanNode.splitAndCompoundPredicateToConjuncts(meta.getWhereClause()).stream() .map(e -> { - e.setDisableTableName(true); + e.disableTableName(); return e; - }).map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet()); + }).map(e -> new NereidsParser().parseExpression(e.toSqlWithoutTbl()).toSql()) + .collect(Collectors.toSet()); for (String indexConjunct : indexConjuncts) { if (predicateExprSql.contains(indexConjunct)) { @@ -540,10 +541,10 @@ protected SlotContext generateBaseScanExprToMvExpr(LogicalOlapScan mvPlan) { return new SlotContext(baseSlotToMvSlot, mvNameToMvSlot, PlanNode.splitAndCompoundPredicateToConjuncts(meta.getWhereClause()).stream() .map(e -> { - e.setDisableTableName(true); + e.disableTableName(); return e; }) - .map(e -> new NereidsParser().parseExpression(e.toSql())) + .map(e -> new NereidsParser().parseExpression(e.toSqlWithoutTbl())) .collect(Collectors.toSet())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index 42f3f8dbe40839..3090e37563700a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -1071,7 +1071,7 @@ private PreAggStatus checkGroupingExprs( private PreAggStatus checkPredicates(List predicates, CheckContext checkContext) { Set indexConjuncts = PlanNode .splitAndCompoundPredicateToConjuncts(checkContext.getMeta().getWhereClause()).stream() - .map(e -> new NereidsParser().parseExpression(e.toSql()).toSql()).collect(Collectors.toSet()); + .map(e -> new NereidsParser().parseExpression(e.toSqlWithoutTbl()).toSql()).collect(Collectors.toSet()); return disablePreAggIfContainsAnyValueColumn( predicates.stream().filter(e -> !indexConjuncts.contains(e.toSql())).collect(Collectors.toList()), checkContext, "Predicate %s contains value column %s"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java index c8b7a3c721ae4f..2b965ca90b2d31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/AliasUdf.java @@ -80,7 +80,7 @@ public boolean nullable() { * translate catalog alias function to nereids alias function */ public static void translateToNereidsFunction(String dbName, AliasFunction function) { - String functionSql = function.getOriginFunction().toSql(); + String functionSql = function.getOriginFunction().toSqlWithoutTbl(); Expression parsedFunction = new NereidsParser().parseExpression(functionSql); AliasUdf aliasUdf = new AliasUdf( diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java index 2692b35ab70a3e..0dd1239deeedba 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateMaterializedViewStmtTest.java @@ -356,7 +356,7 @@ public void testDuplicateColumn1(@Injectable SlotRef slotRef1, selectStmt.analyze(analyzer); selectStmt.getSelectList(); result = selectList; - slotRef1.toSql(); + slotRef1.toSqlWithoutTbl(); result = "k1"; } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java index 7bd16c260700bd..4ba5cb63879811 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ExprTest.java @@ -36,19 +36,23 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class ExprTest { private static final String internalCtl = InternalCatalog.INTERNAL_CATALOG_NAME; @Test public void testGetTableNameToColumnNames(@Mocked Analyzer analyzer, - @Injectable SlotDescriptor slotDesc1, - @Injectable SlotDescriptor slotDesc2, - @Injectable TupleDescriptor tupleDescriptor1, - @Injectable TupleDescriptor tupleDescriptor2, - @Injectable Table tableA, - @Injectable Table tableB) throws AnalysisException { + @Injectable SlotDescriptor slotDesc1, + @Injectable SlotDescriptor slotDesc2, + @Injectable TupleDescriptor tupleDescriptor1, + @Injectable TupleDescriptor tupleDescriptor2, + @Injectable Table tableA, + @Injectable Table tableB) throws AnalysisException { TableName tableAName = new TableName(internalCtl, "test", "tableA"); TableName tableBName = new TableName(internalCtl, "test", "tableB"); SlotRef tableAColumn1 = new SlotRef(tableAName, "c1"); @@ -182,7 +186,7 @@ public void testEqualSets() { Expr r7 = new DateLiteral(2020, 10, 23, 0, 0, 0, Type.DATETIMEV2); Expr r8 = new DateLiteral(2020, 10, 23, 0, 0, 0, ScalarType.createDatetimeV2Type(3)); - //list1 equal list2 + // list1 equal list2 List list1 = new ArrayList<>(); List list2 = new ArrayList<>(); list1.add(r1); @@ -201,7 +205,7 @@ public void testEqualSets() { list2.add(r8); Assert.assertTrue(Expr.equalSets(list1, list2)); - //list3 not equal list4 + // list3 not equal list4 list2.add(r4); Assert.assertFalse(Expr.equalSets(list1, list2)); } @@ -211,7 +215,7 @@ public void testUncheckedCastChildAvoidDoubleCast() throws AnalysisException { Expr cast = new CastExpr(Type.DATETIME, new IntLiteral(10000101)); FunctionCallExpr call = new FunctionCallExpr("leap", Lists.newArrayList(cast)); call.uncheckedCastChild(Type.DATETIME, 0); - //do not cast a castExpr + // do not cast a castExpr Assertions.assertTrue(call.getChild(0).getChild(0) instanceof IntLiteral); } @@ -233,4 +237,26 @@ public void testSrcSlotRef(@Injectable SlotDescriptor slotDescriptor) { Assert.assertTrue(srcSlotRef != null); Assert.assertTrue(srcSlotRef == slotRef); } + + @Test + public void testToSql() throws InterruptedException { + String toSqlValue = "`db1`.`table1`.`c1`"; + String toSqlWithoutTblValue = "`c1`"; + TableName tableName = new TableName(internalCtl, "db1", "table1"); + SlotRef slotRef = new SlotRef(tableName, "c1"); + ExecutorService executor = Executors.newFixedThreadPool(10); + + for (int i = 0; i < 100; i++) { + executor.submit(() -> { + Random random = new Random(); + if (random.nextInt() % 2 == 0) { + Assert.assertEquals(toSqlValue, slotRef.toSql()); + } else { + Assert.assertEquals(toSqlWithoutTblValue, slotRef.toSqlWithoutTbl()); + } + }); + } + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.MINUTES); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java index 41dfec8a6dd062..c769e1c210c804 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java @@ -168,21 +168,11 @@ void testFilterMV4() throws Exception { createMv(createMVSql); ConnectContext.get().getState().setNereids(true); Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException("db1") - .getOlapTableOrDdlException(EMPS_TABLE_NAME).getIndexIdToMeta() - .forEach((id, meta) -> { - if (meta.getWhereClause() != null) { - meta.getWhereClause().setDisableTableName(false); - } - }); + .getOlapTableOrDdlException(EMPS_TABLE_NAME).getIndexIdToMeta(); testMv(query1, EMPS_MV_NAME); ConnectContext.get().getState().setNereids(false); Env.getCurrentEnv().getCurrentCatalog().getDbOrAnalysisException("db1") - .getOlapTableOrDdlException(EMPS_TABLE_NAME).getIndexIdToMeta() - .forEach((id, meta) -> { - if (meta.getWhereClause() != null) { - meta.getWhereClause().setDisableTableName(true); - } - }); + .getOlapTableOrDdlException(EMPS_TABLE_NAME).getIndexIdToMeta(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java index b405d787219c7e..6f4035f9202605 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java @@ -708,8 +708,9 @@ public void divideDecimalV2Test() throws AnalysisException { @Test public void timeNowTest() throws AnalysisException { - String curTimeString = FEFunctions.curTime().toSqlImpl().replace("'", ""); - String currentTimestampString = FEFunctions.currentTimestamp().toSqlImpl().replace("'", ""); + String curTimeString = FEFunctions.curTime().toSqlImpl(false, false, null, null).replace("'", ""); + String currentTimestampString = FEFunctions.currentTimestamp().toSqlImpl(false, false, null, null) + .replace("'", ""); ZonedDateTime zonedDateTime = ZonedDateTime.now(TimeUtils.getTimeZone().toZoneId()); DateTimeFormatter formatter = null; From 2db6e5ed3690a4bdf44e4b158476bff8c74b4e93 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 26 Jun 2025 10:25:13 +0800 Subject: [PATCH 078/572] branch-3.0: [fix](Prepared Statment) Fix exec prepared insert stmt in non master error (#48689) (#52266) backport: https://github.com/apache/doris/pull/48689 Co-authored-by: Lijia Liu --- .../trees/plans/commands/PrepareCommand.java | 2 +- .../org/apache/doris/qe/ConnectContext.java | 7 +++++ .../org/apache/doris/qe/ConnectProcessor.java | 28 ++++++++++++++++++- .../org/apache/doris/qe/MasterOpExecutor.java | 7 +++++ .../doris/qe/MysqlConnectProcessor.java | 28 +++++++++++++++++-- .../org/apache/doris/qe/StmtExecutor.java | 17 ++++++++--- gensrc/thrift/FrontendService.thrift | 1 + .../suites/query_p0/test_forward_qeury.groovy | 7 ++++- 8 files changed, 87 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java index 40c0d8b567ba12..d0d2fffa278b2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java @@ -110,7 +110,7 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } ctx.addPreparedStatementContext(name, new PreparedStatementContext(this, ctx, ctx.getStatementContext(), name)); - if (ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE) { + if (ctx.getCommand() == MysqlCommand.COM_STMT_PREPARE && !ctx.isProxy()) { executor.sendStmtPrepareOK(Integer.parseInt(name), labels); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 54de7ed1339ffb..6d86cc7f037f40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -76,6 +76,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import io.netty.util.concurrent.FastThreadLocal; +import lombok.Getter; +import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -83,6 +85,7 @@ import org.xnio.StreamConnection; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -248,6 +251,10 @@ public enum ConnectType { // it's default thread-safe private boolean isProxy = false; + @Getter + @Setter + private ByteBuffer prepareExecuteBuffer; + private MysqlHandshakePacket mysqlHandshakePacket; public void setUserQueryTimeout(int queryTimeout) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index cbc0aee98a5c8d..06f5397d3dccd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -47,6 +47,7 @@ import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mysql.MysqlChannel; +import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.mysql.MysqlPacket; import org.apache.doris.mysql.MysqlSerializer; import org.apache.doris.mysql.MysqlServerStatusFlag; @@ -60,6 +61,7 @@ import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; +import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; import org.apache.doris.plugin.DialectConverterPlugin; @@ -86,6 +88,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -714,7 +717,24 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException UUID uuid = UUID.randomUUID(); queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); } - executor.queryRetry(queryId); + if (request.isSetPrepareExecuteBuffer()) { + ctx.setCommand(MysqlCommand.COM_STMT_PREPARE); + executor.execute(); + ctx.setCommand(MysqlCommand.COM_STMT_EXECUTE); + String preparedStmtId = executor.getPrepareStmtName(); + PreparedStatementContext preparedStatementContext = ctx.getPreparedStementContext(preparedStmtId); + if (preparedStatementContext == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("Something error, just support nereids preparedStmtId:{}", preparedStmtId); + } + throw new RuntimeException("Prepare failed when proxy execute"); + } + handleExecute(preparedStatementContext.command, Long.parseLong(preparedStmtId), + preparedStatementContext, + ByteBuffer.wrap(request.getPrepareExecuteBuffer()).order(ByteOrder.LITTLE_ENDIAN), queryId); + } else { + executor.queryRetry(queryId); + } } catch (IOException e) { // Client failed. LOG.warn("Process one query failed because IOException: ", e); @@ -784,4 +804,10 @@ private Map userVariableFromThrift(Map t throw new TException(e.getMessage()); } } + + + protected void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx, + ByteBuffer packetBuf, TUniqueId queryId) { + throw new NotSupportedException("Just MysqlConnectProcessor support execute"); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java index 1f7d87bdfe35b3..285a752cb32305 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MasterOpExecutor.java @@ -25,6 +25,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; +import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.thrift.FrontendService; import org.apache.doris.thrift.TExpr; import org.apache.doris.thrift.TExprNode; @@ -236,6 +237,12 @@ private TMasterOpRequest buildStmtForwardParams() throws AnalysisException { if (ctx.isTxnModel()) { params.setTxnLoadInfo(ctx.getTxnEntry().getTxnLoadInfoInObserver()); } + + if (ctx.getCommand() == MysqlCommand.COM_STMT_EXECUTE) { + if (null != ctx.getPrepareExecuteBuffer()) { + params.setPrepareExecuteBuffer(ctx.getPrepareExecuteBuffer()); + } + } return params; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 97b5061a212907..50990a753c35fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -43,6 +43,7 @@ import org.apache.doris.nereids.trees.plans.PlaceholderId; import org.apache.doris.nereids.trees.plans.commands.ExecuteCommand; import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; +import org.apache.doris.thrift.TUniqueId; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -100,7 +101,18 @@ private void debugPacket() { } } - private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx) { + private String getHexStr(ByteBuffer packetBuf) { + byte[] bytes = packetBuf.array(); + StringBuilder hex = new StringBuilder(); + for (int i = packetBuf.position(); i < packetBuf.limit(); ++i) { + hex.append(String.format("%02X ", bytes[i])); + } + return hex.toString(); + } + + @Override + protected void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx, + ByteBuffer packetBuf, TUniqueId queryId) { int paramCount = prepareCommand.placeholderCount(); LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount); // null bitmap @@ -108,6 +120,12 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS try { StatementContext statementContext = prepCtx.statementContext; if (paramCount > 0) { + if (LOG.isDebugEnabled()) { + LOG.debug("execute param buf: {}, array: {}", packetBuf, getHexStr(packetBuf)); + } + if (!ctx.isProxy()) { + ctx.setPrepareExecuteBuffer(packetBuf.duplicate()); + } byte[] nullbitmapData = new byte[(paramCount + 7) / 8]; packetBuf.get(nullbitmapData); // new_params_bind_flag @@ -148,7 +166,11 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS stmt.setOrigStmt(prepareCommand.getOriginalStmt()); executor = new StmtExecutor(ctx, stmt); ctx.setExecutor(executor); - executor.execute(); + if (null != queryId) { + executor.execute(queryId); + } else { + executor.execute(); + } if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) { stmtStr = executeStmt.toSql(); stmtStr = stmtStr + " /*originalSql = " + prepareCommand.getOriginalStmt().originStmt + "*/"; @@ -191,7 +213,7 @@ private void handleExecute() { "msg: Not supported such prepared statement"); return; } - handleExecute(preparedStatementContext.command, stmtId, preparedStatementContext); + handleExecute(preparedStatementContext.command, stmtId, preparedStatementContext, packetBuf, null); } // Process COM_QUERY statement, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 7555337756352b..368e3f94ab1298 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -145,6 +145,7 @@ import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -279,6 +280,7 @@ public class StmtExecutor { private Data.PQueryStatistics.Builder statisticsForAuditLog; private boolean isCached; private String stmtName; + private String prepareStmtName; // for prox private String mysqlLoadId; // Handle selects that fe can do without be private boolean isHandleQueryInFe = false; @@ -701,8 +703,12 @@ private void executeByNereids(TUniqueId queryId) throws Exception { } long stmtId = Config.prepared_stmt_start_id > 0 ? Config.prepared_stmt_start_id : context.getPreparedStmtId(); - logicalPlan = new PrepareCommand(String.valueOf(stmtId), - logicalPlan, statementContext.getPlaceholders(), originStmt); + this.prepareStmtName = String.valueOf(stmtId); + // When proxy executing, this.statementContext is created in constructor. + // But context.statementContext is created in LogicalPlanBuilder. + List placeholders = context == null + ? statementContext.getPlaceholders() : context.getStatementContext().getPlaceholders(); + logicalPlan = new PrepareCommand(prepareStmtName, logicalPlan, placeholders, originStmt); } // when we in transaction mode, we only support insert into command and transaction command if (context.isTxnModel()) { @@ -723,8 +729,7 @@ private void executeByNereids(TUniqueId queryId) throws Exception { if (logicalPlan instanceof InsertIntoTableCommand) { profileType = ProfileType.LOAD; } - if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE - || context.getCommand() == MysqlCommand.COM_STMT_EXECUTE) { + if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { throw new UserException("Forward master command is not supported for prepare statement"); } if (isProxy) { @@ -3687,4 +3692,8 @@ public void sendProxyQueryResult() throws IOException { context.getMysqlChannel().sendOnePacket(byteBuffer); } } + + public String getPrepareStmtName() { + return this.prepareStmtName; + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 92205c5ae0f011..0723de2cf2f8d3 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -597,6 +597,7 @@ struct TMasterOpRequest { // transaction load 29: optional TTxnLoadInfo txnLoadInfo 30: optional TGroupCommitInfo groupCommitInfo + 31: optional binary prepareExecuteBuffer // selectdb cloud 1000: optional string cloud_cluster diff --git a/regression-test/suites/query_p0/test_forward_qeury.groovy b/regression-test/suites/query_p0/test_forward_qeury.groovy index d4761c835a26e0..e2b11e9535ff93 100644 --- a/regression-test/suites/query_p0/test_forward_qeury.groovy +++ b/regression-test/suites/query_p0/test_forward_qeury.groovy @@ -43,7 +43,12 @@ suite("test_forward_query", 'docker') { cluster.injectDebugPoints(NodeType.FE, ['StmtExecutor.forward_all_queries' : [forwardAllQueries:true, execute:1]]) - def ret = sql """ SELECT * FROM ${tbl} """ + def stmt = prepareStatement("""INSERT INTO ${tbl} VALUES(?);""") + stmt.setInt(1, 2) + stmt.executeUpdate() + + def ret = sql """ SELECT * FROM ${tbl} order by k1""" assertEquals(ret[0][0], 1) + assertEquals(ret[1][0], 2) } } From 8710bea95d9eabb003f42b6dfc03d4646f1de81f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:29:17 +0800 Subject: [PATCH 079/572] branch-3.0: [feat](storage vault) Check storage vault connectivity for BE when starting #51175 (#52319) Cherry-picked from #51175 Co-authored-by: Lei Zhang --- be/src/cloud/cloud_storage_engine.cpp | 54 ++++++++++++++++++++------- be/src/cloud/cloud_storage_engine.h | 3 +- be/src/cloud/config.cpp | 3 ++ be/src/cloud/config.h | 2 + be/src/io/fs/s3_file_system.cpp | 2 + 5 files changed, 50 insertions(+), 14 deletions(-) diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 9b8528c1a21871..1c07c4924a9c86 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -119,12 +119,25 @@ static Status vault_process_error(std::string_view id, } struct VaultCreateFSVisitor { - VaultCreateFSVisitor(const std::string& id, const cloud::StorageVaultPB_PathFormat& path_format) - : id(id), path_format(path_format) {} + VaultCreateFSVisitor(const std::string& id, const cloud::StorageVaultPB_PathFormat& path_format, + bool check_fs) + : id(id), path_format(path_format), check_fs(check_fs) {} Status operator()(const S3Conf& s3_conf) const { - LOG(INFO) << "get new s3 info: " << s3_conf.to_string() << " resource_id=" << id; + LOG(INFO) << "get new s3 info: " << s3_conf.to_string() << " resource_id=" << id + << " check_fs: " << check_fs; auto fs = DORIS_TRY(io::S3FileSystem::create(s3_conf, id)); + if (check_fs && !s3_conf.client_conf.role_arn.empty()) { + bool res = false; + // just check connectivity, not care object if exist + auto st = fs->exists("not_exist_object", &res); + if (!st.ok()) { + LOG(FATAL) << "failed to check s3 fs, resource_id: " << id << " st: " << st + << "s3_conf: " << s3_conf.to_string() + << "add enable_check_storage_vault=false to be.conf to skip the check"; + } + } + put_storage_resource(id, {std::move(fs), path_format}, 0); LOG_INFO("successfully create s3 vault, vault id {}", id); return Status::OK(); @@ -142,6 +155,7 @@ struct VaultCreateFSVisitor { const std::string& id; const cloud::StorageVaultPB_PathFormat& path_format; + bool check_fs; }; struct RefreshFSVaultVisitor { @@ -176,7 +190,7 @@ struct RefreshFSVaultVisitor { }; Status CloudStorageEngine::open() { - sync_storage_vault(); + sync_storage_vault(config::enable_check_storage_vault); // TODO(plat1ko): DeleteBitmapTxnManager @@ -321,13 +335,25 @@ Status CloudStorageEngine::start_bg_threads() { return Status::OK(); } -void CloudStorageEngine::sync_storage_vault() { +void CloudStorageEngine::sync_storage_vault(bool check_storage_vault) { cloud::StorageVaultInfos vault_infos; bool enable_storage_vault = false; - auto st = _meta_mgr->get_storage_vault_info(&vault_infos, &enable_storage_vault); - if (!st.ok()) { - LOG(WARNING) << "failed to get storage vault info. err=" << st; - return; + auto st = Status::OK(); + while (true) { + st = _meta_mgr->get_storage_vault_info(&vault_infos, &enable_storage_vault); + if (st.ok()) { + break; + } + + if (!check_storage_vault) { + LOG(WARNING) << "failed to get storage vault info. err=" << st; + return; + } + + LOG(WARNING) << "failed to get storage vault info from ms, err=" << st + << " sleep 200ms retry or add enable_check_storage_vault=false to be.conf" + << " to skip the check."; + std::this_thread::sleep_for(std::chrono::milliseconds(200)); } if (vault_infos.empty()) { @@ -337,10 +363,12 @@ void CloudStorageEngine::sync_storage_vault() { for (auto& [id, vault_info, path_format] : vault_infos) { auto fs = get_filesystem(id); - auto status = (fs == nullptr) - ? std::visit(VaultCreateFSVisitor {id, path_format}, vault_info) - : std::visit(RefreshFSVaultVisitor {id, std::move(fs), path_format}, - vault_info); + auto status = + (fs == nullptr) + ? std::visit(VaultCreateFSVisitor {id, path_format, check_storage_vault}, + vault_info) + : std::visit(RefreshFSVaultVisitor {id, std::move(fs), path_format}, + vault_info); if (!status.ok()) [[unlikely]] { LOG(WARNING) << vault_process_error(id, vault_info, std::move(st)); } diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index 107e9c9867755a..9e63be6c36c47f 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -24,6 +24,7 @@ //#include "cloud/cloud_full_compaction.h" #include "cloud/cloud_cumulative_compaction_policy.h" #include "cloud/cloud_tablet.h" +#include "cloud/config.h" #include "cloud/schema_cloud_dictionary_cache.h" #include "cloud_txn_delete_bitmap_cache.h" #include "io/cache/block_file_cache_factory.h" @@ -136,7 +137,7 @@ class CloudStorageEngine final : public BaseStorageEngine { std::shared_ptr cumu_compaction_policy( std::string_view compaction_policy); - void sync_storage_vault(); + void sync_storage_vault(bool check = false); io::FileCacheBlockDownloader& file_cache_block_downloader() const { return *_file_cache_block_downloader; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 7f26988a68b4e5..800df55b3d6c6b 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -85,5 +85,8 @@ DEFINE_mInt32(delete_bitmap_rpc_retry_times, "25"); DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "5000"); DEFINE_mInt32(meta_service_conflict_error_retry_times, "10"); + +DEFINE_Bool(enable_check_storage_vault, "true"); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 9d2027b965b2c5..76e37163828006 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -122,5 +122,7 @@ DECLARE_mInt64(meta_service_rpc_reconnect_interval_ms); DECLARE_mInt32(meta_service_conflict_error_retry_times); +DECLARE_Bool(enable_check_storage_vault); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index ec1c63c91067eb..224368bf8d59a2 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -267,6 +267,8 @@ Status S3FileSystem::exists_impl(const Path& path, bool* res) const { CHECK_S3_CLIENT(client); auto key = DORIS_TRY(get_key(path)); + VLOG_DEBUG << "key:" << key << " path:" << path; + auto resp = client->head_object({.bucket = _bucket, .key = key}); if (resp.resp.status.code == ErrorCode::OK) { From c8b1ed87c3e8c5284a085deb96644b2c16e8a6c6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 10:29:50 +0800 Subject: [PATCH 080/572] branch-3.0: [fix](load) resolve UBSan error when printing unique IDs #52042 (#52261) Cherry-picked from #52042 Co-authored-by: Kaijie Chen --- be/src/runtime/load_stream.cpp | 2 +- be/src/util/uid_util.cpp | 4 ++++ be/src/util/uid_util.h | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp index 0bd045d46c16d3..eb28daaa86fd20 100644 --- a/be/src/runtime/load_stream.cpp +++ b/be/src/runtime/load_stream.cpp @@ -72,7 +72,7 @@ TabletStream::TabletStream(PUniqueId load_id, int64_t id, int64_t txn_id, } inline std::ostream& operator<<(std::ostream& ostr, const TabletStream& tablet_stream) { - ostr << "load_id=" << tablet_stream._load_id << ", txn_id=" << tablet_stream._txn_id + ostr << "load_id=" << print_id(tablet_stream._load_id) << ", txn_id=" << tablet_stream._txn_id << ", tablet_id=" << tablet_stream._id << ", status=" << tablet_stream._status.status(); return ostr; } diff --git a/be/src/util/uid_util.cpp b/be/src/util/uid_util.cpp index 0f93f437ab6cc4..fb767f09bd7e5b 100644 --- a/be/src/util/uid_util.cpp +++ b/be/src/util/uid_util.cpp @@ -44,6 +44,10 @@ std::ostream& operator<<(std::ostream& os, const UniqueId& uid) { return os; } +std::string print_id(const UniqueId& id) { + return id.to_string(); +} + std::string print_id(const TUniqueId& id) { return fmt::format(FMT_COMPILE("{:x}-{:x}"), static_cast(id.hi), static_cast(id.lo)); diff --git a/be/src/util/uid_util.h b/be/src/util/uid_util.h index 5c0b5fb72fb3cd..7f300c561b04ef 100644 --- a/be/src/util/uid_util.h +++ b/be/src/util/uid_util.h @@ -168,6 +168,7 @@ inline TUniqueId generate_uuid() { std::ostream& operator<<(std::ostream& os, const UniqueId& uid); +std::string print_id(const UniqueId& id); std::string print_id(const TUniqueId& id); std::string print_id(const PUniqueId& id); From 86686da7bc765b0d419fddf3060a551c8f0880ee Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 26 Jun 2025 10:48:02 +0800 Subject: [PATCH 081/572] [branch-3.0]Remove cpu/memory sum check limit (#52057) ### What problem does this PR solve? 1 Remove cpu/memory sum check limit. 2 remove cpu_hard_limit switch, support cpu hard/soft limit at same time. --- be/src/agent/workload_group_listener.cpp | 7 +----- .../runtime/workload_group/workload_group.cpp | 25 ++----------------- .../runtime/workload_group/workload_group.h | 1 - .../workload_group/workload_group_manager.h | 6 ----- .../java/org/apache/doris/common/Config.java | 6 ++--- .../resource/workloadgroup/WorkloadGroup.java | 9 ------- .../workloadgroup/WorkloadGroupMgr.java | 13 +--------- 7 files changed, 7 insertions(+), 60 deletions(-) diff --git a/be/src/agent/workload_group_listener.cpp b/be/src/agent/workload_group_listener.cpp index f0f57869f2545a..50a8b8e509c9ec 100644 --- a/be/src/agent/workload_group_listener.cpp +++ b/be/src/agent/workload_group_listener.cpp @@ -54,10 +54,6 @@ void WorkloadGroupListener::handle_topic_info(const std::vector& topi auto wg = _exec_env->workload_group_mgr()->get_or_create_workload_group(workload_group_info); - // 3 set cpu soft hard limit switch - _exec_env->workload_group_mgr()->_enable_cpu_hard_limit.store( - workload_group_info.enable_cpu_hard_limit); - // 4 create and update task scheduler wg->upsert_task_scheduler(&workload_group_info, _exec_env); @@ -65,8 +61,7 @@ void WorkloadGroupListener::handle_topic_info(const std::vector& topi wg->upsert_scan_io_throttle(&workload_group_info); LOG(INFO) << "[topic_publish_wg]update workload group finish, wg info=" - << wg->debug_string() << ", enable_cpu_hard_limit=" - << (_exec_env->workload_group_mgr()->enable_cpu_hard_limit() ? "true" : "false") + << wg->debug_string() << ", cgroup cpu_shares=" << workload_group_info.cgroup_cpu_shares << ", cgroup cpu_hard_limit=" << workload_group_info.cgroup_cpu_hard_limit << ", cgroup home path=" << config::doris_cgroup_cpu_path diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index b07548f9970eca..d2018b0b28ea6b 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -363,12 +363,6 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info( enable_memory_overcommit = tworkload_group_info.enable_memory_overcommit; } - // 8 cpu soft limit or hard limit - bool enable_cpu_hard_limit = false; - if (tworkload_group_info.__isset.enable_cpu_hard_limit) { - enable_cpu_hard_limit = tworkload_group_info.enable_cpu_hard_limit; - } - // 9 scan thread num int scan_thread_num = config::doris_scanner_thread_pool_thread_num; if (tworkload_group_info.__isset.scan_thread_num && tworkload_group_info.scan_thread_num > 0) { @@ -422,7 +416,6 @@ WorkloadGroupInfo WorkloadGroupInfo::parse_topic_info( .enable_memory_overcommit = enable_memory_overcommit, .version = version, .cpu_hard_limit = cpu_hard_limit, - .enable_cpu_hard_limit = enable_cpu_hard_limit, .scan_thread_num = scan_thread_num, .max_remote_scan_thread_num = max_remote_scan_thread_num, .min_remote_scan_thread_num = min_remote_scan_thread_num, @@ -437,7 +430,6 @@ void WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e std::string tg_name = tg_info->name; int cpu_hard_limit = tg_info->cpu_hard_limit; uint64_t cpu_shares = tg_info->cpu_share; - bool enable_cpu_hard_limit = tg_info->enable_cpu_hard_limit; int scan_thread_num = tg_info->scan_thread_num; int max_remote_scan_thread_num = tg_info->max_remote_scan_thread_num; int min_remote_scan_thread_num = tg_info->min_remote_scan_thread_num; @@ -551,21 +543,8 @@ void WorkloadGroup::upsert_task_scheduler(WorkloadGroupInfo* tg_info, ExecEnv* e // step 6: update cgroup cpu if needed if (_cgroup_cpu_ctl) { - if (enable_cpu_hard_limit) { - if (cpu_hard_limit > 0) { - _cgroup_cpu_ctl->update_cpu_hard_limit(cpu_hard_limit); - _cgroup_cpu_ctl->update_cpu_soft_limit( - CgroupCpuCtl::cpu_soft_limit_default_value()); - } else { - LOG(INFO) << "[upsert wg thread pool] enable cpu hard limit but value is " - "illegal: " - << cpu_hard_limit << ", gid=" << tg_id; - } - } else { - _cgroup_cpu_ctl->update_cpu_soft_limit(cpu_shares); - _cgroup_cpu_ctl->update_cpu_hard_limit( - CPU_HARD_LIMIT_DEFAULT_VALUE); // disable cpu hard limit - } + _cgroup_cpu_ctl->update_cpu_hard_limit(cpu_hard_limit); + _cgroup_cpu_ctl->update_cpu_soft_limit(cpu_shares); _cgroup_cpu_ctl->get_cgroup_cpu_info(&(tg_info->cgroup_cpu_shares), &(tg_info->cgroup_cpu_hard_limit)); } diff --git a/be/src/runtime/workload_group/workload_group.h b/be/src/runtime/workload_group/workload_group.h index b4d1406a94d7bc..97d90300a81419 100644 --- a/be/src/runtime/workload_group/workload_group.h +++ b/be/src/runtime/workload_group/workload_group.h @@ -264,7 +264,6 @@ struct WorkloadGroupInfo { const bool enable_memory_overcommit = false; const int64_t version = 0; const int cpu_hard_limit = 0; - const bool enable_cpu_hard_limit = false; const int scan_thread_num = 0; const int max_remote_scan_thread_num = 0; const int min_remote_scan_thread_num = 0; diff --git a/be/src/runtime/workload_group/workload_group_manager.h b/be/src/runtime/workload_group/workload_group_manager.h index db2444e0bc1191..3cd00ebdb44d91 100644 --- a/be/src/runtime/workload_group/workload_group_manager.h +++ b/be/src/runtime/workload_group/workload_group_manager.h @@ -54,12 +54,6 @@ class WorkloadGroupMgr { void stop(); - std::atomic _enable_cpu_hard_limit = false; - - bool enable_cpu_soft_limit() { return !_enable_cpu_hard_limit.load(); } - - bool enable_cpu_hard_limit() { return _enable_cpu_hard_limit.load(); } - void refresh_wg_weighted_memory_limit(); void get_wg_resource_usage(vectorized::Block* block); diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 0ff3300c6fab21..3fd035fdcb921c 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1920,6 +1920,9 @@ public class Config extends ConfigBase { @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL) public static boolean enable_workload_group = true; + @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL) + public static boolean enable_wg_memory_sum_limit = true; + @ConfField(mutable = true) public static boolean enable_query_queue = true; @@ -1930,9 +1933,6 @@ public class Config extends ConfigBase { @ConfField(mutable = true) public static long query_queue_update_interval_ms = 5000; - @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL) - public static boolean enable_cpu_hard_limit = false; - @ConfField(mutable = true, description = { "当BE内存用量大于该值时，查询会进入排队逻辑，默认值为-1，代表该值不生效。取值范围0~1的小数", "When be memory usage bigger than this value, query could queue, " diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java index 408895dd12f3cc..4fff8388dece66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroup.java @@ -19,7 +19,6 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.Pair; @@ -593,14 +592,6 @@ public TopicInfo toTopicInfo() { if (memOvercommitStr != null) { tWorkloadGroupInfo.setEnableMemoryOvercommit(Boolean.valueOf(memOvercommitStr)); } - // enable_cpu_hard_limit = true, using cpu hard limit - // enable_cpu_hard_limit = false, using cpu soft limit - tWorkloadGroupInfo.setEnableCpuHardLimit(Config.enable_cpu_hard_limit); - - if (Config.enable_cpu_hard_limit && cpuHardLimit <= 0) { - LOG.warn("enable_cpu_hard_limit=true but cpuHardLimit value not illegal," - + "id=" + id + ",name=" + name); - } String scanThreadNumStr = properties.get(SCAN_THREAD_NUM); if (scanThreadNumStr != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java index 442b1bd783c56d..1e81c91bcea169 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java @@ -390,7 +390,6 @@ private void checkGlobalUnlock(WorkloadGroup newWg, WorkloadGroup oldWg) throws for (String newWgOneTag : newWgTagSet) { double sumOfAllMemLimit = 0; - int sumOfAllCpuHardLimit = 0; // 1 get sum value of all wg which has same tag without current wg for (Map.Entry entry : idToWorkloadGroup.entrySet()) { @@ -409,9 +408,6 @@ private void checkGlobalUnlock(WorkloadGroup newWg, WorkloadGroup oldWg) throws continue; } - if (wg.getCpuHardLimitWhenCalSum() > 0) { - sumOfAllCpuHardLimit += wg.getCpuHardLimitWhenCalSum(); - } if (wg.getMemoryLimitPercentWhenCalSum() > 0) { sumOfAllMemLimit += wg.getMemoryLimitPercentWhenCalSum(); } @@ -419,22 +415,15 @@ private void checkGlobalUnlock(WorkloadGroup newWg, WorkloadGroup oldWg) throws // 2 sum current wg value sumOfAllMemLimit += newWg.getMemoryLimitPercentWhenCalSum(); - sumOfAllCpuHardLimit += newWg.getCpuHardLimitWhenCalSum(); // 3 check total sum - if (sumOfAllMemLimit > 100.0 + 1e-6) { + if (Config.enable_wg_memory_sum_limit && sumOfAllMemLimit > 100.0 + 1e-6) { throw new DdlException( "The sum of all workload group " + WorkloadGroup.MEMORY_LIMIT + " within tag " + ( newWgTag.isPresent() ? newWgTag.get() : "") + " cannot be greater than 100.0%. current sum val:" + sumOfAllMemLimit); } - if (sumOfAllCpuHardLimit > 100) { - throw new DdlException( - "sum of all workload group " + WorkloadGroup.CPU_HARD_LIMIT + " within tag " + ( - newWgTag.isPresent() - ? newWgTag.get() : "") + " can not be greater than 100% "); - } } } From bf42b594fdbca98ca89841e33508b4e98bcfbd18 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 14:11:32 +0800 Subject: [PATCH 082/572] branch-3.0: [fix](hive) ignore all hidden dir and files #52286 (#52323) Cherry-picked from #52286 Co-authored-by: Mingyu Chen (Rayner) --- .../datasource/hive/HiveMetaStoreCache.java | 18 ++++-------------- .../doris/datasource/PathVisibleTest.java | 7 +++++-- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index de867bfc2b928c..314162d3f0d0df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -1056,34 +1056,24 @@ public static boolean isFileVisible(Path path) { return false; } String pathStr = path.toUri().toString(); - if (containsHiddenPath(pathStr) || path.getName().startsWith("_")) { + if (containsHiddenPath(pathStr)) { return false; } - for (String name : pathStr.split("/")) { - if (isGeneratedPath(name)) { - return false; - } - } return true; } private static boolean containsHiddenPath(String path) { - if (path.startsWith(".")) { + // Hive ignores files starting with _ and . + if (path.startsWith(".") || path.startsWith("_")) { return true; } for (int i = 0; i < path.length() - 1; i++) { - if (path.charAt(i) == '/' && path.charAt(i + 1) == '.') { + if (path.charAt(i) == '/' && (path.charAt(i + 1) == '.' || path.charAt(i + 1) == '_')) { return true; } } return false; } - - private static boolean isGeneratedPath(String name) { - return "_temporary".equals(name) // generated by spark - || "_imapala_insert_staging".equals(name) // generated by impala - || name.startsWith("."); // generated by hive or hidden file - } } @Data diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java index 0937bbc3cc856c..6610a5f25218c2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PathVisibleTest.java @@ -31,17 +31,20 @@ public void shouldReturnFalseWhenPathIsNull() { Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/.hidden/path"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible/path/.file"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_temporary_xx"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_imapala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path/_impala_insert_staging"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible//.hidden/path"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("s3://visible/.hidden/path"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("///visible/path/.file"))); Assert.assertFalse(FileCacheValue.isFileVisible(new Path("/visible/path///_temporary_xx"))); - Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_imapala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible(new Path("hdfs://visible//path/_impala_insert_staging"))); + Assert.assertFalse(FileCacheValue.isFileVisible( + new Path("hdfs://hacluster/user/hive/warehouse/db1.db/tbl1/_spark_metadata/"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("s3://visible/path"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("path"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://visible/path./1.txt"))); Assert.assertTrue(FileCacheValue.isFileVisible(new Path("/1.txt"))); + Assert.assertTrue(FileCacheValue.isFileVisible(new Path("hdfs://vis_ible_/pa.th./1_.txt__"))); } } From 3488cd9e4fbadc59232bc61c703d427329ad326b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 15:59:20 +0800 Subject: [PATCH 083/572] branch-3.0: [fix](ci) disable build ui in performance #52351 (#52355) Cherry-picked from #52351 Co-authored-by: Dongyang Li --- regression-test/pipeline/performance/conf/custom_env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/pipeline/performance/conf/custom_env.sh b/regression-test/pipeline/performance/conf/custom_env.sh index 27a4544655eff3..1671733f91d911 100644 --- a/regression-test/pipeline/performance/conf/custom_env.sh +++ b/regression-test/pipeline/performance/conf/custom_env.sh @@ -18,3 +18,4 @@ export DORIS_TOOLCHAIN=gcc export BUILD_TYPE=release +export DISABLE_BUILD_UI=ON From 2755729917307a92e443498e879b002060b2f34f Mon Sep 17 00:00:00 2001 From: walter Date: Thu, 26 Jun 2025 19:15:59 +0800 Subject: [PATCH 084/572] branch-3.0: [chore](cloud) rename cloud::put/get to blob_put/blob_get #51762 (#52368) cherry pick from #51762 --- cloud/src/common/util.cpp | 12 ++++++------ cloud/src/common/util.h | 12 ++++++------ cloud/src/meta-service/http_encode_key.cpp | 6 +++--- cloud/src/meta-service/meta_service.cpp | 12 ++++++------ cloud/src/meta-service/meta_service_schema.cpp | 10 +++++----- cloud/src/recycler/meta_checker.cpp | 2 +- cloud/src/recycler/recycler.cpp | 2 +- cloud/test/recycler_test.cpp | 4 ++-- cloud/test/txn_kv_test.cpp | 14 +++++++------- 9 files changed, 37 insertions(+), 37 deletions(-) diff --git a/cloud/src/common/util.cpp b/cloud/src/common/util.cpp index 50f29afb0ba548..377f83674ce4b2 100644 --- a/cloud/src/common/util.cpp +++ b/cloud/src/common/util.cpp @@ -331,7 +331,7 @@ TxnErrorCode ValueBuf::get(Transaction* txn, std::string_view key, bool snapshot return TxnErrorCode::TXN_OK; } -TxnErrorCode get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot) { +TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot) { return val->get(txn, key, snapshot); } @@ -346,16 +346,16 @@ TxnErrorCode key_exists(Transaction* txn, std::string_view key, bool snapshot) { return it->has_next() ? TxnErrorCode::TXN_OK : TxnErrorCode::TXN_KEY_NOT_FOUND; } -void put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, uint8_t ver, - size_t split_size) { +void blob_put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, + uint8_t ver, size_t split_size) { std::string value; bool ret = pb.SerializeToString(&value); // Always success DCHECK(ret) << hex(key) << ' ' << pb.ShortDebugString(); - put(txn, key, value, ver, split_size); + blob_put(txn, key, value, ver, split_size); } -void put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, - size_t split_size) { +void blob_put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, + size_t split_size) { auto split_vec = split_string(value, split_size); int64_t suffix_base = ver; suffix_base <<= 56; diff --git a/cloud/src/common/util.h b/cloud/src/common/util.h index 8f2e8aa077e200..838c683125aded 100644 --- a/cloud/src/common/util.h +++ b/cloud/src/common/util.h @@ -39,7 +39,7 @@ std::string unhex(std::string_view str); /** * Prettifies the given key, the first byte must be key space tag, say 0x01, and - * the remaining part must be the output of codec funtion family. + * the remaining part must be the output of codec function family. * * The result is like following: * @@ -107,7 +107,7 @@ struct ValueBuf { * @param snapshot if true, `key` will not be included in txn conflict detection this time * @return return TXN_OK for success get a key, TXN_KEY_NOT_FOUND for key not found, otherwise for error. */ -TxnErrorCode get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot = false); +TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot = false); /** * Test whether key exists @@ -127,8 +127,8 @@ TxnErrorCode key_exists(Transaction* txn, std::string_view key, bool snapshot = * @param ver value version * @param split_size how many byte sized fragments are the value split into */ -void put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, uint8_t ver, - size_t split_size = 90 * 1000); +void blob_put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, + uint8_t ver, size_t split_size = 90 * 1000); /** * Put a KV, it's value may be bigger than 100k @@ -138,7 +138,7 @@ void put(Transaction* txn, std::string_view key, const google::protobuf::Message * @param ver value version * @param split_size how many byte sized fragments are the value split into */ -void put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, - size_t split_size = 90 * 1000); +void blob_put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, + size_t split_size = 90 * 1000); } // namespace doris::cloud diff --git a/cloud/src/meta-service/http_encode_key.cpp b/cloud/src/meta-service/http_encode_key.cpp index 728b52df2d8d71..b851d0956f6bda 100644 --- a/cloud/src/meta-service/http_encode_key.cpp +++ b/cloud/src/meta-service/http_encode_key.cpp @@ -305,7 +305,7 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const brpc::URI& uri) { value.iters.push_back(std::move(it)); } while (more); } else { - err = cloud::get(txn.get(), key, &value, true); + err = cloud::blob_get(txn.get(), key, &value, true); } if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { @@ -371,7 +371,7 @@ HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) { // StatsTabletKey is special, it has a series of keys, we only handle the base stat key // MetaSchemaPBDictionaryKey, MetaSchemaKey, MetaDeleteBitmapKey are splited in to multiple KV ValueBuf value; - err = cloud::get(txn.get(), key, &value, true); + err = cloud::blob_get(txn.get(), key, &value, true); bool kv_found = true; if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { @@ -413,7 +413,7 @@ HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) { // and the number of final keys may be less than the initial number of keys if (kv_found) value.remove(txn.get()); - // TODO(gavin): use cloud::put() to deal with split-multi-kv and special keys + // TODO(gavin): use cloud::blob_put() to deal with split-multi-kv and special keys // StatsTabletKey is special, it has a series of keys, we only handle the base stat key // MetaSchemaPBDictionaryKey, MetaSchemaKey, MetaDeleteBitmapKey are splited in to multiple KV txn->put(key, serialized_value_to_save); diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 66dec09662f430..c43364a4b81e76 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -702,7 +702,7 @@ void internal_get_tablet(MetaServiceCode& code, std::string& msg, const std::str auto key = meta_schema_key( {instance_id, tablet_meta->index_id(), tablet_meta->schema_version()}); ValueBuf val_buf; - err = cloud::get(txn, key, &val_buf); + err = cloud::blob_get(txn, key, &val_buf); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to get schema, err={}", err == TxnErrorCode::TXN_KEY_NOT_FOUND @@ -780,7 +780,7 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle auto key = meta_schema_key( {instance_id, tablet_meta.index_id(), tablet_meta.schema_version()}); ValueBuf val_buf; - err = cloud::get(txn.get(), key, &val_buf); + err = cloud::blob_get(txn.get(), key, &val_buf); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to get schema, err={}", @@ -952,7 +952,7 @@ static void set_schema_in_existed_rowset(MetaServiceCode& code, std::string& msg std::string schema_key = meta_schema_key({instance_id, existed_rowset_meta.index_id(), existed_rowset_meta.schema_version()}); ValueBuf val_buf; - TxnErrorCode err = cloud::get(txn, schema_key, &val_buf, true); + TxnErrorCode err = cloud::blob_get(txn, schema_key, &val_buf, true); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format( @@ -1666,7 +1666,7 @@ static bool try_fetch_and_parse_schema(Transaction* txn, RowsetMetaCloudPB& rows const std::string& key, MetaServiceCode& code, std::string& msg) { ValueBuf val_buf; - TxnErrorCode err = cloud::get(txn, key, &val_buf); + TxnErrorCode err = cloud::blob_get(txn, key, &val_buf); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to get schema, schema_version={}, rowset_version=[{}-{}]: {}", @@ -2260,7 +2260,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont } } // splitting large values (>90*1000) into multiple KVs - cloud::put(txn.get(), key, val, 0); + cloud::blob_put(txn.get(), key, val, 0); current_key_count++; current_value_count += val.size(); total_key_count++; @@ -3034,7 +3034,7 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control std::string dict_key = meta_schema_pb_dictionary_key({instance_id, request->index_id()}); ValueBuf dict_val; - err = cloud::get(txn.get(), dict_key, &dict_val); + err = cloud::blob_get(txn.get(), dict_key, &dict_val); LOG(INFO) << "Retrieved column pb dictionary, index_id=" << request->index_id() << " key=" << hex(dict_key) << " error=" << err; if (err != TxnErrorCode::TXN_KEY_NOT_FOUND && err != TxnErrorCode::TXN_OK) { diff --git a/cloud/src/meta-service/meta_service_schema.cpp b/cloud/src/meta-service/meta_service_schema.cpp index bbbedd1e3abe8e..31c067faea4991 100644 --- a/cloud/src/meta-service/meta_service_schema.cpp +++ b/cloud/src/meta-service/meta_service_schema.cpp @@ -56,7 +56,7 @@ void put_schema_kv(MetaServiceCode& code, std::string& msg, Transaction* txn, return type; }; ValueBuf buf; - auto err = cloud::get(txn, schema_key, &buf); + auto err = cloud::blob_get(txn, schema_key, &buf); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to get schema, err=" << err; return false; @@ -119,7 +119,7 @@ void put_schema_kv(MetaServiceCode& code, std::string& msg, Transaction* txn, LOG_INFO("put schema kv").tag("key", hex(schema_key)); uint8_t ver = config::meta_schema_value_version; if (ver > 0) { - cloud::put(txn, schema_key, schema, ver); + cloud::blob_put(txn, schema_key, schema, ver); } else { auto schema_value = schema.SerializeAsString(); txn->put(schema_key, schema_value); @@ -221,7 +221,7 @@ void write_schema_dict(MetaServiceCode& code, std::string& msg, const std::strin SchemaCloudDictionary dict; std::string dict_key = meta_schema_pb_dictionary_key({instance_id, rowset_meta->index_id()}); ValueBuf dict_val; - auto err = cloud::get(txn, dict_key, &dict_val); + auto err = cloud::blob_get(txn, dict_key, &dict_val); LOG(INFO) << "Retrieved column pb dictionary, index_id=" << rowset_meta->index_id() << " key=" << hex(dict_key) << " error=" << err; if (err != TxnErrorCode::TXN_KEY_NOT_FOUND && err != TxnErrorCode::TXN_OK) { @@ -312,7 +312,7 @@ void write_schema_dict(MetaServiceCode& code, std::string& msg, const std::strin return; } // splitting large values (>90*1000) into multiple KVs - cloud::put(txn, dict_key, dict_val, 0); + cloud::blob_put(txn, dict_key, dict_val, 0); LOG(INFO) << "Dictionary saved, key=" << hex(dict_key) << " txn_id=" << rowset_meta->txn_id() << " Dict size=" << dict.column_dict_size() << ", index_id=" << rowset_meta->index_id() @@ -331,7 +331,7 @@ void read_schema_dict(MetaServiceCode& code, std::string& msg, const std::string SchemaCloudDictionary dict; std::string column_dict_key = meta_schema_pb_dictionary_key({instance_id, index_id}); ValueBuf dict_val; - auto err = cloud::get(txn, column_dict_key, &dict_val); + auto err = cloud::blob_get(txn, column_dict_key, &dict_val); if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) { code = cast_as(err); ss << "internal error, failed to get dict, err=" << err; diff --git a/cloud/src/recycler/meta_checker.cpp b/cloud/src/recycler/meta_checker.cpp index f1223068d4be6e..3718c2f2099951 100644 --- a/cloud/src/recycler/meta_checker.cpp +++ b/cloud/src/recycler/meta_checker.cpp @@ -347,7 +347,7 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) { meta_schema_key({instance_id_, tablet_info.index_id, tablet_info.schema_version}, &schema_key); ValueBuf val_buf; - err = cloud::get(txn.get(), schema_key, &val_buf); + err = cloud::blob_get(txn.get(), schema_key, &val_buf); if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { LOG(WARNING) << "tablet schema not found: " << tablet_info.debug_string(); return false; diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index 78b6ec3cf6a2c7..0ec4fc917171b1 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -442,7 +442,7 @@ class InstanceRecycler::InvertedIndexIdCache { } auto schema_key = meta_schema_key({instance_id_, index_id, schema_version}); ValueBuf val_buf; - err = cloud::get(txn.get(), schema_key, &val_buf); + err = cloud::blob_get(txn.get(), schema_key, &val_buf); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to get schema, err=" << err; return static_cast(err); diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index da5dcc1556b97c..6b2fac0efe5342 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -2901,7 +2901,7 @@ TEST(CheckerTest, delete_bitmap_inverted_check_normal) { auto delete_bitmap_key = meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0}); std::string delete_bitmap_val(1000, 'A'); - cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0, 300); + cloud::blob_put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0, 300); } } } @@ -2995,7 +2995,7 @@ TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) { auto delete_bitmap_key = meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, ver, 0}); std::string delete_bitmap_val(1000, 'A'); - cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0, 300); + cloud::blob_put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0, 300); } } } diff --git a/cloud/test/txn_kv_test.cpp b/cloud/test/txn_kv_test.cpp index 7f91e70f9110ce..27088f3da72a4a 100644 --- a/cloud/test/txn_kv_test.cpp +++ b/cloud/test/txn_kv_test.cpp @@ -250,7 +250,7 @@ TEST(TxnKvTest, CompatibleGetTest) { ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); ASSERT_EQ(doris::cloud::key_exists(txn.get(), key), TxnErrorCode::TXN_KEY_NOT_FOUND); ValueBuf val_buf; - ASSERT_EQ(doris::cloud::get(txn.get(), key, &val_buf), TxnErrorCode::TXN_KEY_NOT_FOUND); + ASSERT_EQ(doris::cloud::blob_get(txn.get(), key, &val_buf), TxnErrorCode::TXN_KEY_NOT_FOUND); txn->put(key, val); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); @@ -259,7 +259,7 @@ TEST(TxnKvTest, CompatibleGetTest) { ASSERT_EQ(err, TxnErrorCode::TXN_OK); err = doris::cloud::key_exists(txn.get(), key); ASSERT_EQ(err, TxnErrorCode::TXN_OK); - err = doris::cloud::get(txn.get(), key, &val_buf); + err = doris::cloud::blob_get(txn.get(), key, &val_buf); ASSERT_EQ(err, TxnErrorCode::TXN_OK); EXPECT_EQ(val_buf.ver, 0); doris::TabletSchemaCloudPB saved_schema; @@ -277,7 +277,7 @@ TEST(TxnKvTest, CompatibleGetTest) { // Check remove ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); ASSERT_EQ(doris::cloud::key_exists(txn.get(), key), TxnErrorCode::TXN_KEY_NOT_FOUND); - ASSERT_EQ(doris::cloud::get(txn.get(), key, &val_buf), TxnErrorCode::TXN_KEY_NOT_FOUND); + ASSERT_EQ(doris::cloud::blob_get(txn.get(), key, &val_buf), TxnErrorCode::TXN_KEY_NOT_FOUND); } TEST(TxnKvTest, PutLargeValueTest) { @@ -305,7 +305,7 @@ TEST(TxnKvTest, PutLargeValueTest) { auto key = meta_schema_key({instance_id, 10005, 1}); std::unique_ptr txn; ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); - doris::cloud::put(txn.get(), key, schema, 1, 100); + doris::cloud::blob_put(txn.get(), key, schema, 1, 100); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); // Check get @@ -313,7 +313,7 @@ TEST(TxnKvTest, PutLargeValueTest) { ValueBuf val_buf; doris::TabletSchemaCloudPB saved_schema; ASSERT_EQ(doris::cloud::key_exists(txn.get(), key), TxnErrorCode::TXN_OK); - TxnErrorCode err = doris::cloud::get(txn.get(), key, &val_buf); + TxnErrorCode err = doris::cloud::blob_get(txn.get(), key, &val_buf); ASSERT_EQ(err, TxnErrorCode::TXN_OK); std::cout << "num iterators=" << val_buf.iters.size() << std::endl; EXPECT_EQ(val_buf.ver, 1); @@ -329,7 +329,7 @@ TEST(TxnKvTest, PutLargeValueTest) { auto* limit = doris::try_any_cast(args[0]); *limit = 100; }); - err = doris::cloud::get(txn.get(), key, &val_buf); + err = doris::cloud::blob_get(txn.get(), key, &val_buf); ASSERT_EQ(err, TxnErrorCode::TXN_OK); std::cout << "num iterators=" << val_buf.iters.size() << std::endl; EXPECT_EQ(val_buf.ver, 1); @@ -364,7 +364,7 @@ TEST(TxnKvTest, PutLargeValueTest) { // Check remove ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); ASSERT_EQ(doris::cloud::key_exists(txn.get(), key), TxnErrorCode::TXN_KEY_NOT_FOUND); - ASSERT_EQ(doris::cloud::get(txn.get(), key, &val_buf), TxnErrorCode::TXN_KEY_NOT_FOUND); + ASSERT_EQ(doris::cloud::blob_get(txn.get(), key, &val_buf), TxnErrorCode::TXN_KEY_NOT_FOUND); } TEST(TxnKvTest, RangeGetIteratorContinue) { From 6f08907d2d4c7b6c3b7f95eac11c50aafdd59963 Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 26 Jun 2025 19:17:48 +0800 Subject: [PATCH 085/572] [branch-3.0]Fix workload group regression test (#52343) --- .../workload_manager_p0/test_curd_wlg.groovy | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy index ff394bcbe4adde..e5e13d3df830fa 100644 --- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy +++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy @@ -171,12 +171,6 @@ suite("test_crud_wlg") { sql "alter workload group test_group properties ( 'cpu_hard_limit'='99%' );" - test { - sql "alter workload group normal properties ( 'cpu_hard_limit'='2%' );" - - exception "can not be greater than 100%" - } - sql "alter workload group test_group properties ( 'cpu_hard_limit'='20%' );" qt_cpu_hard_limit_1 """ select count(1) from ${table_name} """ qt_cpu_hard_limit_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num from information_schema.workload_groups where name in ('normal','test_group') order by name;" @@ -289,18 +283,6 @@ suite("test_crud_wlg") { exception "a positive integer between 1 and 100" } - test { - sql "create workload group if not exists test_group2 " + - "properties ( " + - " 'cpu_share'='10', " + - " 'memory_limit'='3%', " + - " 'enable_memory_overcommit'='true', " + - " 'cpu_hard_limit'='99%' " + - ");" - - exception "can not be greater than 100%" - } - // test show workload groups qt_select_tvf_1 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num from information_schema.workload_groups where name in ('normal','test_group') order by name;" @@ -507,27 +489,14 @@ suite("test_crud_wlg") { sql "create workload group if not exists tag1_wg1 properties ( 'cpu_hard_limit'='10%', 'tag'='tag1');" - test { - sql "create workload group if not exists tag1_wg2 properties ( 'cpu_hard_limit'='91%', 'tag'='tag1');" - exception "can not be greater than 100%" - } - sql "create workload group if not exists tag1_wg2 properties ( 'cpu_hard_limit'='10%', 'tag'='tag1');" sql "create workload group if not exists tag2_wg1 properties ( 'cpu_hard_limit'='91%', 'tag'='tag2');" - test { - sql "alter workload group tag2_wg1 properties ( 'tag'='tag1' );" - exception "can not be greater than 100% " - } sql "alter workload group tag2_wg1 properties ( 'cpu_hard_limit'='10%' );" sql "alter workload group tag2_wg1 properties ( 'tag'='tag1' );" - test { - sql "create workload group if not exists tag1_wg3 properties ( 'cpu_hard_limit'='80%', 'tag'='tag1');" - exception "can not be greater than 100% " - } sql "drop workload group tag2_wg1;" sql "create workload group if not exists tag1_wg3 properties ( 'cpu_hard_limit'='80%', 'tag'='tag1');" From eaa74b3af26c5176b5518999d764205dfcbfb4c4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 22:05:09 +0800 Subject: [PATCH 086/572] branch-3.0: [chore](conf) change max_running_txn_num_per_db to 10000 #51367 (#52380) Cherry-picked from #51367 Co-authored-by: Yongqiang YANG Co-authored-by: Yongqiang YANG --- .../src/main/java/org/apache/doris/common/Config.java | 2 +- .../apache/doris/transaction/DatabaseTransactionMgr.java | 6 ++++++ .../src/test/java/org/apache/doris/catalog/FakeEnv.java | 5 +++++ .../java/org/apache/doris/common/proc/DbsProcDirTest.java | 5 +++-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 3fd035fdcb921c..de2604ad011a13 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -712,7 +712,7 @@ public class Config extends ConfigBase { "单个数据库最大并发运行的事务数，包括 prepare 和 commit 事务。", "Maximum concurrent running txn num including prepare, commit txns under a single db.", "Txn manager will reject coming txns."}) - public static int max_running_txn_num_per_db = 1000; + public static int max_running_txn_num_per_db = 10000; @ConfField(masterOnly = true, description = {"pending load task 执行线程数。这个配置可以限制当前等待的导入作业数。" + "并且应小于 `max_running_txn_num_per_db`。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 59d5bc571f09a3..119571b7b29f47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -36,6 +36,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DuplicatedRequestException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.LoadException; @@ -314,6 +315,11 @@ public long beginTransaction(List tableIdList, String label, TUniqueId req long listenerId, long timeoutSecond) throws DuplicatedRequestException, LabelAlreadyUsedException, BeginTransactionException, AnalysisException, QuotaExceedException, MetaNotFoundException { + + if (!Env.getCurrentEnv().isMaster() && !FeConstants.runningUnitTest) { + throw new BeginTransactionException("FE is not master"); + } + Database db = env.getInternalCatalog().getDbOrMetaException(dbId); if (!coordinator.isFromInternal) { InternalDatabaseUtil.checkDatabase(db.getFullName(), ConnectContext.get()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEnv.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEnv.java index 32f32cf9e7ebb3..d4fc9b5d8b0f0e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEnv.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/FakeEnv.java @@ -60,4 +60,9 @@ public static SystemInfoService getCurrentSystemInfo() { return systemInfo; } + @Mock + public boolean isMaster() { + return true; + } + } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java index 1746f2d6d8c443..0d04afb130e74d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.transaction.GlobalTransactionMgr; @@ -213,9 +214,9 @@ public void testFetchResultNormal() throws AnalysisException { "LastUpdateTime"), result.getColumnNames()); List> rows = Lists.newArrayList(); rows.add(Arrays.asList(String.valueOf(db1.getId()), db1.getFullName(), "0", "0.000 ", "8388608.000 TB", - FeConstants.null_string, "0", "1073741824", "10", "1000", FeConstants.null_string)); + FeConstants.null_string, "0", "1073741824", "10", String.valueOf(Config.max_running_txn_num_per_db), FeConstants.null_string)); rows.add(Arrays.asList(String.valueOf(db2.getId()), db2.getFullName(), "0", "0.000 ", "8388608.000 TB", - FeConstants.null_string, "0", "1073741824", "20", "1000", FeConstants.null_string)); + FeConstants.null_string, "0", "1073741824", "20", String.valueOf(Config.max_running_txn_num_per_db), FeConstants.null_string)); Assert.assertEquals(rows, result.getRows()); } From 7e3ff0723ac0db162b17cb5d0555d27495f8d69a Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Thu, 26 Jun 2025 22:06:11 +0800 Subject: [PATCH 087/572] [fix](missing version) not catchup is worse than lastFailed (#50094) (#52382) pick #50094 --- .../src/main/java/org/apache/doris/catalog/Tablet.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index ca7f51da823576..ea76273e6dfbfa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -318,16 +318,14 @@ public List getQueryableReplicas(long visibleVersion, Map 0) { - mayMissingVersionReplica.add(replica); - continue; - } - if (!replica.checkVersionCatchUp(visibleVersion, false)) { notCatchupReplica.add(replica); continue; } + if (replica.getLastFailedVersion() > 0) { + mayMissingVersionReplica.add(replica); + continue; + } Set thisBeAlivePaths = backendAlivePathHashs.get(replica.getBackendIdWithoutException()); ReplicaState state = replica.getState(); From d1ecb320a9d46d7e41c1fde0814af2cb8ababdfc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Jun 2025 22:22:17 +0800 Subject: [PATCH 088/572] branch-3.0: [fix](schema-change) Fix single column table could not rename columns #47275 (#52340) Cherry-picked from #47275 Co-authored-by: Siyang Tang --- .../java/org/apache/doris/catalog/Env.java | 4 +- .../test_rename_single_col_tbl.out | 4 ++ .../test_rename_single_col_tbl.groovy | 37 +++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/schema_change_p0/test_rename_single_col_tbl.out create mode 100644 regression-test/suites/schema_change_p0/test_rename_single_col_tbl.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index eeb86251613ef3..ee74323ddfa6f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -5142,8 +5142,8 @@ private void renameColumn(Database db, OlapTable table, String colName, Map indexIdToMeta = table.getIndexIdToMeta(); for (Map.Entry entry : indexIdToMeta.entrySet()) { - // rename column is not implemented for table without column unique id. - if (entry.getValue().getMaxColUniqueId() <= 0) { + // rename column is not implemented for non-light-schema-change table. + if (!table.getEnableLightSchemaChange()) { throw new DdlException("not implemented for table without column unique id," + " which are created with property 'light_schema_change'."); } diff --git a/regression-test/data/schema_change_p0/test_rename_single_col_tbl.out b/regression-test/data/schema_change_p0/test_rename_single_col_tbl.out new file mode 100644 index 00000000000000..aea2129c772424 --- /dev/null +++ b/regression-test/data/schema_change_p0/test_rename_single_col_tbl.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !desc -- +test_rename_single_col_tbl DUP_KEYS rename_partition_col DATE datev2 No true \N true + diff --git a/regression-test/suites/schema_change_p0/test_rename_single_col_tbl.groovy b/regression-test/suites/schema_change_p0/test_rename_single_col_tbl.groovy new file mode 100644 index 00000000000000..8183b2878a17d4 --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_rename_single_col_tbl.groovy @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_rename_single_col_tbl") { + def tblName = "test_rename_single_col_tbl" + sql """ DROP TABLE IF EXISTS ${tblName} """ + sql """ + CREATE TABLE ${tblName} + ( + col0 DATE NOT NULL, + ) + DUPLICATE KEY(col0) + DISTRIBUTED BY HASH(col0) BUCKETS 4 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + ALTER TABLE ${tblName} RENAME COLUMN col0 rename_partition_col + """ + sql """ SYNC """ + qt_desc """ DESC ${tblName} ALL """ +} \ No newline at end of file From 8cb58d8bb3850d08d438bb75c791677c21ff8f19 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 10:04:57 +0800 Subject: [PATCH 089/572] branch-3.0: [Enhancement](checker) Add more log for checker #51951 (#52387) Cherry-picked from #51951 Co-authored-by: abmdocrt --- cloud/src/recycler/checker.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 6a191b021b2a8e..954a3332d7566e 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -550,7 +550,8 @@ int InstanceChecker::do_check() { } data_loss = true; TEST_SYNC_POINT_CALLBACK("InstanceChecker.do_check1", &path); - LOG(WARNING) << "object not exist, path=" << path << " key=" << hex(key); + LOG(WARNING) << "object not exist, path=" << path + << ", rs_meta=" << rs_meta.ShortDebugString() << " key=" << hex(key); } std::vector> index_ids; From fc323dfac936b64d1c8ebdc6574cf3b09e23e1a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 10:06:02 +0800 Subject: [PATCH 090/572] branch-3.0: [Fix](auto-partition) Fix false positive check of max_auto_partition_num #52270 (#52367) Cherry-picked from #52270 Co-authored-by: zclllyybb --- .../doris/service/FrontendServiceImpl.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 4a7def9ce166a4..92cfe98e535f5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -3640,20 +3640,6 @@ public TCreatePartitionResult createPartition(TCreatePartitionRequest request) t return result; } - // check partition's number limit. - int partitionNum = olapTable.getPartitionNum() + addPartitionClauseMap.size(); - if (partitionNum > Config.max_auto_partition_num) { - String errorMessage = String.format( - "create partition failed. partition numbers %d will exceed limit variable " - + "max_auto_partition_num %d", - partitionNum, Config.max_auto_partition_num); - LOG.warn(errorMessage); - errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage)); - result.setStatus(errorStatus); - LOG.warn("send create partition error status: {}", result); - return result; - } - for (AddPartitionClause addPartitionClause : addPartitionClauseMap.values()) { try { // here maybe check and limit created partitions num @@ -3668,6 +3654,20 @@ public TCreatePartitionResult createPartition(TCreatePartitionRequest request) t } } + // check partition's number limit. because partitions in addPartitionClauseMap may be duplicated with existing + // partitions, which would lead to false positive. so we should check the partition number AFTER adding new + // partitions using its ACTUAL NUMBER, rather than the sum of existing and requested partitions. + if (olapTable.getPartitionNum() > Config.max_auto_partition_num) { + String errorMessage = String.format( + "partition numbers %d exceeded limit of variable max_auto_partition_num %d", + olapTable.getPartitionNum(), Config.max_auto_partition_num); + LOG.warn(errorMessage); + errorStatus.setErrorMsgs(Lists.newArrayList(errorMessage)); + result.setStatus(errorStatus); + LOG.warn("send create partition error status: {}", result); + return result; + } + // build partition & tablets List partitions = Lists.newArrayList(); List tablets = Lists.newArrayList(); From 35a2633ab921195250a25902d217ea6b7986bd87 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 27 Jun 2025 10:15:47 +0800 Subject: [PATCH 091/572] =?UTF-8?q?[fix](pipeline)=20premature=20exit=20ca?= =?UTF-8?q?using=20core=20dump=20during=20concurrent=20pr=E2=80=A6=20(#523?= =?UTF-8?q?65)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …epare execution (#51492) Issue Number: close #51491 Problem Summary: When the queue of the FragmentMgrAsync thread pool is full, newly submitted tasks are rejected and return early. However, previously submitted tasks may still be scheduled for execution later. This can lead to premature destruction of objects such as PipelineFragmentContext and TPipelineFragmentParams that are referenced by those tasks, resulting in null pointer exceptions during task execution and ultimately causing a coredump. The pr policy is to wait until all previously submitted tasks are completed before returning. ``` *** SIGSEGV address not mapped to object (@0x1c8) received by PID 3941201 (TID 2115617 OR 0xfe1685bb97f0) from PID 456; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/jenkins_agent/workspace/BigDataComponent_doris-unified-arm-release/be/src/common/signal_handler.h:421 1# os::Linux::chained_handler(int, siginfo_t*, void*) in /usr/jdk64/current/jre/lib/aarch64/server/libjvm.so 2# JVM_handle_linux_signal in /usr/jdk64/current/jre/lib/aarch64/server/libjvm.so 3# signalHandler(int, siginfo_t*, void*) in /usr/jdk64/current/jre/lib/aarch64/server/libjvm.so 4# 0x0000FFFF6B2A07C0 in linux-vdso.so.1 5# doris::TUniqueId::TUniqueId(doris::TUniqueId const&) at /home/jenkins_agent/workspace/BigDataComponent_doris-unified-arm-release/gensrc/build/gen_cpp/Types_types.cpp:2354 6# doris::AttachTask::AttachTask(doris::QueryContext*) at /home/jenkins_agent/workspace/BigDataComponent_doris-unified-arm-release/be/src/runtime/thread_context.cpp:60 7# std::_Function_handler::_M_invoke(std::_Any_data const&) at /usr/lib/gcc/aarch64-linux-gnu/13/../../../../include/c++/13/bits/std_function.h:290 8# doris::ThreadPool::dispatch_thread() at /home/jenkins_agent/workspace/BigDataComponent_doris-unified-arm-release/be/src/util/threadpool.cpp:552 9# doris::Thread::supervise_thread(void*) at /home/jenkins_agent/workspace/BigDataComponent_doris-unified-arm-release/be/src/util/thread.cpp:499 10# 0x0000FFFF6AF187AC in /lib64/libpthread.so.0 11# 0x0000FFFF6B16548C in /lib64/libc.so.6 ``` Co-authored-by: XLPE Co-authored-by: XLPE --- be/src/pipeline/pipeline_fragment_context.cpp | 38 ++++++++++--------- be/src/util/countdown_latch.h | 7 ++++ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index 81a84451e89c0a..a8e62f4f6d3524 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -106,6 +106,7 @@ #include "runtime/thread_context.h" #include "service/backend_options.h" #include "util/container_util.hpp" +#include "util/countdown_latch.h" #include "util/debug_util.h" #include "util/uid_util.h" #include "vec/common/sort/heap_sorter.h" @@ -513,27 +514,28 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag target_size > _runtime_state->query_options().parallel_prepare_threshold)) { // If instances parallelism is big enough ( > parallel_prepare_threshold), we will prepare all tasks by multi-threads std::vector prepare_status(target_size); - std::mutex m; - std::condition_variable cv; - int prepare_done = 0; + int submitted_tasks = 0; + Status submit_status; + CountDownLatch latch((int)target_size); for (size_t i = 0; i < target_size; i++) { - RETURN_IF_ERROR(thread_pool->submit_func([&, i]() { + submit_status = thread_pool->submit_func([&, i]() { SCOPED_ATTACH_TASK(_query_ctx.get()); prepare_status[i] = pre_and_submit(i, this); - std::unique_lock lock(m); - prepare_done++; - if (prepare_done == target_size) { - cv.notify_one(); - } - })); - } - std::unique_lock lock(m); - if (prepare_done != target_size) { - cv.wait(lock); - for (size_t i = 0; i < target_size; i++) { - if (!prepare_status[i].ok()) { - return prepare_status[i]; - } + latch.count_down(); + }); + if (LIKELY(submit_status.ok())) { + submitted_tasks++; + } else { + break; + } + } + latch.arrive_and_wait(target_size - submitted_tasks); + if (UNLIKELY(!submit_status.ok())) { + return submit_status; + } + for (int i = 0; i < submitted_tasks; i++) { + if (!prepare_status[i].ok()) { + return prepare_status[i]; } } } else { diff --git a/be/src/util/countdown_latch.h b/be/src/util/countdown_latch.h index a41a417d20f707..b27737e8bb1d16 100644 --- a/be/src/util/countdown_latch.h +++ b/be/src/util/countdown_latch.h @@ -91,6 +91,13 @@ class CountDownLatch { } } + // decrements the internal counter by n and blocks the calling thread until the counter reaches zero. + void arrive_and_wait(uint64_t n) { + DCHECK_GE(n, 0); + count_down(n); + wait(); + } + uint64_t count() const { std::lock_guard lock(_lock); return _count; From 9930b4da40656510d62feef9ff73e70a445854b2 Mon Sep 17 00:00:00 2001 From: deardeng Date: Fri, 27 Jun 2025 10:29:18 +0800 Subject: [PATCH 092/572] branch-3.0: [Fix](cloud) Fix getVisibleVersion returning RuntimeException #51044 (#52028) Cherry-pick from #51044 --- .../org/apache/doris/catalog/OlapTable.java | 75 ++++++++----------- .../doris/common/NereidsSqlCacheManager.java | 19 ++++- .../doris/common/proc/TablesProcDir.java | 13 +++- .../doris/datasource/InternalCatalog.java | 4 +- .../apache/doris/nereids/SqlCacheContext.java | 18 ++++- .../plans/physical/PhysicalOlapScan.java | 4 +- .../apache/doris/qe/cache/CacheAnalyzer.java | 12 ++- .../doris/statistics/AnalysisManager.java | 11 ++- .../statistics/StatisticsAutoCollector.java | 11 ++- .../doris/statistics/util/StatisticsUtil.java | 9 ++- .../transaction/GlobalTransactionMgrTest.java | 11 ++- 11 files changed, 127 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 06e29ae84c22a4..7e6457f29409af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -49,7 +49,6 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.mtmv.MTMVRefreshContext; import org.apache.doris.mtmv.MTMVRelatedTableIf; @@ -1222,19 +1221,16 @@ public Partition getPartitionOrAnalysisException(long partitionId) throws Analys return partition; } - public void getVersionInBatchForCloudMode(Collection partitionIds) { - if (Config.isCloudMode()) { // do nothing for non-cloud mode - List partitions = partitionIds.stream() - .sorted() - .map(this::getPartition) - .map(partition -> (CloudPartition) partition) - .collect(Collectors.toList()); - try { - CloudPartition.getSnapshotVisibleVersion(partitions); - } catch (RpcException e) { - throw new RuntimeException(e); - } + public void getVersionInBatchForCloudMode(Collection partitionIds) throws RpcException { + if (Config.isNotCloudMode()) { + return; } + List partitions = partitionIds.stream() + .sorted() + .map(this::getPartition) + .map(partition -> (CloudPartition) partition) + .collect(Collectors.toList()); + CloudPartition.getSnapshotVisibleVersion(partitions); } // select the non-empty partition ids belonging to this table. @@ -3034,18 +3030,22 @@ public void updateVisibleVersionAndTime(long visibleVersion, long visibleVersion // During `getNextVersion` and `updateVisibleVersionAndTime` period, // the write lock on the table should be held continuously public long getNextVersion() { - if (!Config.isCloudMode()) { + if (Config.isNotCloudMode()) { return tableAttributes.getNextVersion(); - } else { - // cloud mode should not reach here - if (LOG.isDebugEnabled()) { - LOG.debug("getNextVersion in Cloud mode in OlapTable {} ", getName()); - } + } + // cloud mode should not reach here + if (LOG.isDebugEnabled()) { + LOG.debug("getNextVersion in Cloud mode in OlapTable {} ", getName()); + } + try { return getVisibleVersion() + 1; + } catch (RpcException e) { + LOG.warn("getNextVersion in Cloud mode in OlapTable {}", getName(), e); + throw new RuntimeException(e); } } - public long getVisibleVersion() { + public long getVisibleVersion() throws RpcException { if (Config.isNotCloudMode()) { return tableAttributes.getVisibleVersion(); } @@ -3075,28 +3075,9 @@ public long getVisibleVersion() { } return version; } catch (RpcException e) { - throw new RuntimeException("get version from meta service failed", e); - } - } - - // Get the table versions in batch. - public static List getVisibleVersionByTableIds(Collection tableIds) { - List tables = new ArrayList<>(); - - InternalCatalog catalog = Env.getCurrentEnv().getInternalCatalog(); - for (long tableId : tableIds) { - Table table = catalog.getTableByTableId(tableId); - if (table == null) { - throw new RuntimeException("get table visible version failed, no such table " + tableId + " exists"); - } - if (table.getType() != TableType.OLAP) { - throw new RuntimeException( - "get table visible version failed, table " + tableId + " is not a OLAP table"); - } - tables.add((OlapTable) table); + LOG.warn("get version from meta service failed", e); + throw e; } - - return getVisibleVersionInBatch(tables); } // Get the table versions in batch. @@ -3227,10 +3208,16 @@ public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCont } @Override - public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) { + public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) + throws AnalysisException { Map tableVersions = context.getBaseVersions().getTableVersions(); - long visibleVersion = tableVersions.containsKey(id) ? tableVersions.get(id) : getVisibleVersion(); - return new MTMVVersionSnapshot(visibleVersion, id); + try { + long visibleVersion = tableVersions.containsKey(id) ? tableVersions.get(id) : getVisibleVersion(); + return new MTMVVersionSnapshot(visibleVersion, id); + } catch (RpcException e) { + LOG.warn("getVisibleVersion failed", e); + throw new AnalysisException("getVisibleVersion failed " + e.getMessage()); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index 62d052f18b67ad..ba4b465d4b0852 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -60,12 +60,15 @@ import org.apache.doris.qe.ResultSet; import org.apache.doris.qe.cache.CacheAnalyzer; import org.apache.doris.qe.cache.SqlCache; +import org.apache.doris.rpc.RpcException; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.lang.reflect.Field; import java.time.Duration; @@ -80,6 +83,7 @@ * NereidsSqlCacheManager */ public class NereidsSqlCacheManager { + private static final Logger LOG = LogManager.getLogger(NereidsSqlCacheManager.class); // key: :: // value: SqlCacheContext private volatile Cache sqlCaches; @@ -349,7 +353,13 @@ private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) { } OlapTable olapTable = (OlapTable) tableIf; - long currentTableVersion = olapTable.getVisibleVersion(); + long currentTableVersion = 0L; + try { + currentTableVersion = olapTable.getVisibleVersion(); + } catch (RpcException e) { + LOG.warn("table {}, in cloud getVisibleVersion exception", olapTable.getName(), e); + return true; + } long cacheTableVersion = tableVersion.version; // some partitions have been dropped, or delete or updated or replaced, or insert rows into new partition? if (currentTableVersion != cacheTableVersion) { @@ -366,7 +376,12 @@ private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) { } OlapTable olapTable = (OlapTable) tableIf; Collection partitionIds = scanTable.getScanPartitions(); - olapTable.getVersionInBatchForCloudMode(partitionIds); + try { + olapTable.getVersionInBatchForCloudMode(partitionIds); + } catch (RpcException e) { + LOG.warn("failed to get version in batch for table {}", fullTableName, e); + return true; + } for (Long scanPartitionId : scanTable.getScanPartitions()) { Partition partition = olapTable.getPartition(scanPartitionId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java index f4e39080abe4c3..f5021cc95350f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java @@ -28,11 +28,14 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.rpc.RpcException; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.Collections; @@ -45,6 +48,7 @@ * show table family groups' info within a db */ public class TablesProcDir implements ProcDirInterface { + private static final Logger LOG = LogManager.getLogger(ProcDirInterface.class); public static final ImmutableList TITLE_NAMES = new ImmutableList.Builder() .add("TableId").add("TableName").add("IndexNum").add("PartitionColumnName") .add("PartitionNum").add("State").add("Type").add("LastConsistencyCheckTime").add("ReplicaCount") @@ -119,6 +123,13 @@ public ProcResult fetchResult() throws AnalysisException { ++idx; } } + long version = 0; + try { + version = ((OlapTable) table).getVisibleVersion(); + } catch (RpcException e) { + LOG.warn("table {}, in cloud getVisibleVersion exception", table.getName(), e); + throw new AnalysisException(e.getMessage()); + } replicaCount = olapTable.getReplicaCount(); tableInfo.add(table.getId()); tableInfo.add(table.getName()); @@ -130,7 +141,7 @@ public ProcResult fetchResult() throws AnalysisException { // last check time tableInfo.add(TimeUtils.longToTimeString(olapTable.getLastCheckTime())); tableInfo.add(replicaCount); - tableInfo.add(olapTable.getVisibleVersion()); + tableInfo.add(version); tableInfo.add(olapTable.getVisibleVersionTime()); } else { tableInfo.add(table.getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 3657915ae4fbc9..ce6a9b76e8c389 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2016,7 +2016,9 @@ public void dropPartitionWithoutCheck(Database db, OlapTable olapTable, String p } } - long version = olapTable.getVisibleVersion(); + // In cloud mode, the internal partition deletion logic will update the table version, + // so here we only need to handle non-cloud mode. + long version = 0L; long versionTime = olapTable.getVisibleVersionTime(); // Only update table version if drop a non-empty partition if (partition != null && partition.hasData()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index 2278436888b2f6..0794d0aca0b2cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -35,6 +35,7 @@ import org.apache.doris.proto.Types.PUniqueId; import org.apache.doris.qe.ResultSet; import org.apache.doris.qe.cache.CacheProxy; +import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TUniqueId; import com.google.common.collect.ImmutableList; @@ -43,6 +44,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Collections; import java.util.List; @@ -55,6 +58,7 @@ /** SqlCacheContext */ public class SqlCacheContext { + private static final Logger LOG = LogManager.getLogger(SqlCacheContext.class); private final UserIdentity userIdentity; private final TUniqueId queryId; // if contains udf/udaf/tableValuesFunction we can not process it and skip use sql cache @@ -140,11 +144,22 @@ public synchronized void addUsedTable(TableIf tableIf) { return; } + long version = 0; + try { + if (tableIf instanceof OlapTable) { + version = ((OlapTable) tableIf).getVisibleVersion(); + } + } catch (RpcException e) { + // in cloud, getVisibleVersion throw exception, disable sql cache temporary + setHasUnsupportedTables(true); + LOG.warn("table {}, in cloud getVisibleVersion exception", tableIf.getName(), e); + } + usedTables.put( new FullTableName(database.getCatalog().getName(), database.getFullName(), tableIf.getName()), new TableVersion( tableIf.getId(), - tableIf instanceof OlapTable ? ((OlapTable) tableIf).getVisibleVersion() : 0L, + version, tableIf.getType() ) ); @@ -460,7 +475,6 @@ public String toString() { @lombok.AllArgsConstructor public static class ScanTable { public final FullTableName fullTableName; - public final long latestVersion; public final List scanPartitions = Lists.newArrayList(); public void addScanPartition(Long partitionId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 698a189aa265ba..5e604c958e46b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -34,6 +34,8 @@ import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONObject; import java.util.List; @@ -44,7 +46,7 @@ * Physical olap scan plan. */ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapScan { - + private static final Logger LOG = LogManager.getLogger(PhysicalOlapScan.class); private final DistributionSpec distributionSpec; private final long selectedIndexId; private final ImmutableList selectedTabletIds; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 92aff2cc0ae39d..7bdffd707460d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -58,6 +58,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.RowBatch; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TUniqueId; import com.google.common.collect.Lists; @@ -701,12 +702,15 @@ private CacheTable buildCacheTableForOlapScanNode(OlapScanNode node) { DatabaseIf database = olapTable.getDatabase(); CatalogIf catalog = database.getCatalog(); ScanTable scanTable = new ScanTable( - new FullTableName(catalog.getName(), database.getFullName(), olapTable.getName()), - olapTable.getVisibleVersion()); + new FullTableName(catalog.getName(), database.getFullName(), olapTable.getName())); scanTables.add(scanTable); Collection partitionIds = node.getSelectedPartitionIds(); - olapTable.getVersionInBatchForCloudMode(partitionIds); + try { + olapTable.getVersionInBatchForCloudMode(partitionIds); + } catch (RpcException e) { + LOG.warn("Failed to get version in batch for cloud mode, partitions {}.", partitionIds, e); + } for (Long partitionId : node.getSelectedPartitionIds()) { Partition partition = olapTable.getPartition(partitionId); @@ -729,7 +733,7 @@ private CacheTable buildCacheTableForHiveScanNode(HiveScanNode node) { DatabaseIf database = tableIf.getDatabase(); CatalogIf catalog = database.getCatalog(); ScanTable scanTable = new ScanTable(new FullTableName( - catalog.getName(), database.getFullName(), tableIf.getName()), 0); + catalog.getName(), database.getFullName(), tableIf.getName())); scanTables.add(scanTable); return cacheTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 0bea8339dd55fd..0d6082ebb6fc90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -64,6 +64,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.AnalysisType; import org.apache.doris.statistics.AnalysisInfo.JobType; @@ -391,7 +392,15 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) { infoBuilder.setRowCount(rowCount); TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId()); infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get()); - infoBuilder.setTableVersion(table instanceof OlapTable ? ((OlapTable) table).getVisibleVersion() : 0); + long version = 0; + try { + if (table instanceof OlapTable) { + version = ((OlapTable) table).getVisibleVersion(); + } + } catch (RpcException e) { + LOG.warn("table {}, in cloud getVisibleVersion exception", table.getName(), e); + } + infoBuilder.setTableVersion(version); infoBuilder.setPriority(JobPriority.MANUAL); infoBuilder.setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows); infoBuilder.setEnablePartition(StatisticsUtil.enablePartitionAnalyze()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index 8420e86fa8ecb2..2f287cca035fc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -27,6 +27,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.AnalysisInfo.ScheduleType; @@ -226,6 +227,14 @@ protected AnalysisInfo createAnalyzeJobForTbl( for (Pair pair : jobColumns) { stringJoiner.add(pair.toString()); } + long version = 0; + try { + if (table instanceof OlapTable) { + version = ((OlapTable) table).getVisibleVersion(); + } + } catch (RpcException e) { + LOG.warn("table {}, in cloud getVisibleVersion exception", table.getName(), e); + } return new AnalysisInfoBuilder() .setJobId(Env.getCurrentEnv().getNextId()) .setCatalogId(table.getDatabase().getCatalog().getId()) @@ -246,7 +255,7 @@ protected AnalysisInfo createAnalyzeJobForTbl( .setTblUpdateTime(table.getUpdateTime()) .setRowCount(rowCount) .setUpdateRows(tableStatsStatus == null ? 0 : tableStatsStatus.updatedRows.get()) - .setTableVersion(table instanceof OlapTable ? ((OlapTable) table).getVisibleVersion() : 0) + .setTableVersion(version) .setPriority(priority) .setPartitionUpdateRows(tableStatsStatus == null ? null : tableStatsStatus.partitionUpdateRows) .setEnablePartition(StatisticsUtil.enablePartitionAnalyze()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 7f8a31ddcd308e..14865aa467b4af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -70,6 +70,7 @@ import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.qe.VariableMgr; +import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.AnalysisManager; import org.apache.doris.statistics.ColStatsMeta; @@ -1198,7 +1199,13 @@ public static boolean isLongTimeColumn(TableIf table, Pair colum // For olap table, if the table visible version and row count doesn't change since last analyze, // we don't need to analyze it because its data is not changed. OlapTable olapTable = (OlapTable) table; - return olapTable.getVisibleVersion() != columnStats.tableVersion + long version = 0; + try { + version = ((OlapTable) table).getVisibleVersion(); + } catch (RpcException e) { + LOG.warn("in cloud getVisibleVersion exception", e); + } + return version != columnStats.tableVersion || olapTable.getRowCount() != columnStats.rowCount; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java index 522021a97710d5..10e446348aff00 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/transaction/GlobalTransactionMgrTest.java @@ -46,6 +46,7 @@ import org.apache.doris.load.routineload.RoutineLoadTaskInfo; import org.apache.doris.meta.MetaContext; import org.apache.doris.persist.EditLog; +import org.apache.doris.rpc.RpcException; import org.apache.doris.task.PublishVersionTask; import org.apache.doris.thrift.TKafkaRLTaskProgress; import org.apache.doris.thrift.TLoadSourceType; @@ -1175,9 +1176,15 @@ private static SubTransactionState generateSubTransactionState(TransactionState } private void checkTableVersion(OlapTable olapTable, long visibleVersion, long nextVersion) { - LOG.info("table={}, visibleVersion={}, nextVersion={}", olapTable.getName(), olapTable.getVisibleVersion(), + long version = 0; + try { + version = olapTable.getVisibleVersion(); + } catch (RpcException e) { + // ut do nothing + } + LOG.info("table={}, visibleVersion={}, nextVersion={}", olapTable.getName(), version, olapTable.getNextVersion()); - Assert.assertEquals(visibleVersion, olapTable.getVisibleVersion()); + Assert.assertEquals(visibleVersion, version); Assert.assertEquals(nextVersion, olapTable.getNextVersion()); } From ba7ae441c955fe9e91d98979a2aa0844273760b7 Mon Sep 17 00:00:00 2001 From: seawinde Date: Fri, 27 Jun 2025 11:37:53 +0800 Subject: [PATCH 093/572] branch-3.0: [fix](mtmv) Fix compensate union wrongly when direct query is empty relation #51700 (#51898) Cherry-picked #51700 --- .../doris/common/profile/SummaryProfile.java | 23 +++++++++- .../doris/mtmv/MTMVRelationManager.java | 3 +- .../apache/doris/nereids/CascadesContext.java | 5 -- .../apache/doris/nereids/NereidsPlanner.java | 18 -------- .../executor/TablePartitionCollector.java | 46 ------------------- .../mv/AbstractMaterializedViewRule.java | 16 ++++++- .../mv/InitMaterializationContextHook.java | 21 +++++++-- .../exploration/mv/MaterializedViewUtils.java | 10 ++++ .../exploration/mv/PartitionCompensator.java | 28 ++++++++--- .../rules/exploration/mv/StructInfo.java | 33 +++++++++---- .../rewrite/QueryPartitionCollector.java | 26 +++-------- .../mv/PartitionCompensatorTest.java | 13 ++++-- .../doris/nereids/util/PlanChecker.java | 3 +- 13 files changed, 130 insertions(+), 115 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java index 70253613bfcd54..6662e1ac4e9ed3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/SummaryProfile.java @@ -21,6 +21,8 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TUnit; import org.apache.doris.transaction.TransactionType; @@ -231,6 +233,8 @@ public class SummaryProfile { @SerializedName(value = "nereidsCollectTablePartitionFinishTime") private long nereidsCollectTablePartitionFinishTime = -1; + @SerializedName(value = "nereidsCollectTablePartitionTime") + private long nereidsCollectTablePartitionTime = 0; @SerializedName(value = "nereidsAnalysisFinishTime") private long nereidsAnalysisFinishTime = -1; @SerializedName(value = "nereidsRewriteFinishTime") @@ -524,6 +528,10 @@ public void setNereidsCollectTablePartitionFinishTime() { this.nereidsCollectTablePartitionFinishTime = TimeUtils.getStartTimeMs(); } + public void addCollectTablePartitionTime(long elapsed) { + nereidsCollectTablePartitionTime += elapsed; + } + public void setNereidsAnalysisTime() { this.nereidsAnalysisFinishTime = TimeUtils.getStartTimeMs(); } @@ -800,7 +808,9 @@ public String getPrettyNereidsRewriteTime() { public String getPrettyNereidsCollectTablePartitionTime() { - return getPrettyTime(nereidsCollectTablePartitionFinishTime, nereidsRewriteFinishTime, TUnit.TIME_MS); + long totalTime = nereidsCollectTablePartitionFinishTime + - nereidsRewriteFinishTime + nereidsCollectTablePartitionTime; + return RuntimeProfile.printCounter(totalTime, TUnit.TIME_MS); } public String getPrettyNereidsOptimizeTime() { @@ -966,4 +976,15 @@ public void setExecutedByFrontend(boolean executedByFrontend) { public void write(DataOutput output) throws IOException { Text.writeString(output, GsonUtils.GSON.toJson(this)); } + + public static SummaryProfile getSummaryProfile(ConnectContext connectContext) { + ConnectContext ctx = connectContext == null ? ConnectContext.get() : connectContext; + if (ctx != null) { + StmtExecutor executor = ctx.getExecutor(); + if (executor != null) { + return executor.getSummaryProfile(); + } + } + return null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index bd9244af61e71f..c45558ec8cfdf3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -45,6 +45,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.BitSet; import java.util.Collection; import java.util.List; import java.util.Map; @@ -86,7 +87,7 @@ public Set getAvailableMTMVs(List tableInfos, ConnectContex Set res = Sets.newLinkedHashSet(); Set mvInfos = getMTMVInfos(tableInfos); Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( - ctx.getStatementContext()); + ctx.getStatementContext(), new BitSet()); for (BaseTableInfo tableInfo : mvInfos) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index d431fc545396b7..1486f03e269b13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.jobs.executor.Analyzer; import org.apache.doris.nereids.jobs.executor.TableCollectAndHookInitializer; -import org.apache.doris.nereids.jobs.executor.TablePartitionCollector; import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob; import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob.RootRewriteJobContext; @@ -229,10 +228,6 @@ public TableCollectAndHookInitializer newTableCollector() { return new TableCollectAndHookInitializer(this); } - public TablePartitionCollector newTablePartitionCollector() { - return new TablePartitionCollector(this); - } - public Analyzer newAnalyzer() { return new Analyzer(this); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 4e0fabc05c0b46..584b0e67c1330f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -358,21 +358,6 @@ protected void collectAndLockTable(boolean showPlanProcess) { } } - protected void collectTableUsedPartitions(boolean showPlanProcess) { - if (LOG.isDebugEnabled()) { - LOG.debug("Start to collect table used partition"); - } - keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newTablePartitionCollector().execute()); - NereidsTracer.logImportantTime("EndCollectTablePartitions"); - if (LOG.isDebugEnabled()) { - LOG.debug("Start to collect table used partition"); - } - if (statementContext.getConnectContext().getExecutor() != null) { - statementContext.getConnectContext().getExecutor().getSummaryProfile() - .setNereidsCollectTablePartitionFinishTime(); - } - } - protected void analyze(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start analyze plan"); @@ -405,9 +390,6 @@ private void rewrite(boolean showPlanProcess) { if (statementContext.getConnectContext().getExecutor() != null) { statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsRewriteTime(); } - // collect partitions table used, this is for query rewrite by materialized view - // this is needed before init hook - collectTableUsedPartitions(showPlanProcess); cascadesContext.getStatementContext().getPlannerHooks().forEach(hook -> hook.afterRewrite(this)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java deleted file mode 100644 index e67b94d1314e89..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TablePartitionCollector.java +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.jobs.executor; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.jobs.rewrite.RewriteJob; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.rules.rewrite.QueryPartitionCollector; - -import java.util.List; - -/** - * Collect partitions which query used, this is useful for optimizing get available mvs, - * should collect after RBO - */ -public class TablePartitionCollector extends AbstractBatchJobExecutor { - public TablePartitionCollector(CascadesContext cascadesContext) { - super(cascadesContext); - } - - @Override - public List getJobs() { - return buildCollectorJobs(); - } - - private static List buildCollectorJobs() { - return jobs( - custom(RuleType.COLLECT_PARTITIONS, QueryPartitionCollector::new) - ); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 49f657911d2681..40a261f33fb3ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -22,6 +22,8 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Id; import org.apache.doris.common.Pair; +import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.common.util.TimeUtils; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -291,7 +293,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( - cascadesContext.getConnectContext().getStatementContext()); + cascadesContext.getStatementContext(), queryStructInfo.getRelationIdBitSet()); Set relateTableUsedPartitions = queryUsedPartitions.get(relatedTableInfo.toList()); if (relateTableUsedPartitions == null) { materializationContext.recordFailReason(queryStructInfo, @@ -413,6 +415,18 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca logicalProperties, queryPlan.getLogicalProperties())); continue; } + // need to collect table partition again, because the rewritten plan would contain new relation + // and the rewritten plan would part in rewritten later , the table used partition info is needed + // for later rewrite + long startTimeMs = TimeUtils.getStartTimeMs(); + try { + MaterializedViewUtils.collectTableUsedPartitions(rewrittenPlan, cascadesContext); + } finally { + SummaryProfile summaryProfile = SummaryProfile.getSummaryProfile(cascadesContext.getConnectContext()); + if (summaryProfile != null) { + summaryProfile.addCollectTablePartitionTime(TimeUtils.getElapsedTimeMs(startTimeMs)); + } + } trySetStatistics(materializationContext, cascadesContext); rewriteResults.add(rewrittenPlan); recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 5baeff7c5852ce..6270dc9572ec8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.PlannerHook; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.qe.ConnectContext; @@ -62,6 +63,15 @@ public class InitMaterializationContextHook implements PlannerHook { @Override public void afterRewrite(NereidsPlanner planner) { + CascadesContext cascadesContext = planner.getCascadesContext(); + // collect partitions table used, this is for query rewrite by materialized view + // this is needed before init hook, because compare partition version in init hook would use this + MaterializedViewUtils.collectTableUsedPartitions(cascadesContext.getRewritePlan(), cascadesContext); + StatementContext statementContext = cascadesContext.getStatementContext(); + if (statementContext.getConnectContext().getExecutor() != null) { + statementContext.getConnectContext().getExecutor().getSummaryProfile() + .setNereidsCollectTablePartitionFinishTime(); + } initMaterializationContext(planner.getCascadesContext()); } @@ -145,12 +155,17 @@ private List createAsyncMaterializationContext(CascadesC // so regenerate the struct info table bitset StructInfo mvStructInfo = mtmvCache.getStructInfo(); BitSet tableBitSetInCurrentCascadesContext = new BitSet(); - mvStructInfo.getRelations().forEach(relation -> tableBitSetInCurrentCascadesContext.set( - cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt())); + BitSet relationIdBitSetInCurrentCascadesContext = new BitSet(); + mvStructInfo.getRelations().forEach(relation -> { + tableBitSetInCurrentCascadesContext.set( + cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt()); + relationIdBitSetInCurrentCascadesContext.set(relation.getRelationId().asInt()); + }); asyncMaterializationContext.add(new AsyncMaterializationContext(materializedView, mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), ImmutableList.of(), ImmutableList.of(), cascadesContext, - mtmvCache.getStructInfo().withTableBitSet(tableBitSetInCurrentCascadesContext))); + mtmvCache.getStructInfo().withTableBitSet(tableBitSetInCurrentCascadesContext, + relationIdBitSetInCurrentCascadesContext))); } catch (Exception e) { LOG.warn(String.format("MaterializationContext init mv cache generate fail, current queryId is %s", cascadesContext.getConnectContext().getQueryIdentifier()), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index b504298bba6e3b..cda4af0d42e088 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.rules.analysis.BindRelation; import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; +import org.apache.doris.nereids.rules.rewrite.QueryPartitionCollector; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; @@ -299,6 +300,15 @@ public static List extractNondeterministicFunction(Plan plan) { return nondeterministicFunctions; } + /** + * Collect table used partitions, this is used for mv rewrite partition union + * can not cumulative, if called multi times, should clean firstly + */ + public static void collectTableUsedPartitions(Plan plan, CascadesContext cascadesContext) { + // the recorded partition is based on relation id + plan.accept(new QueryPartitionCollector(), cascadesContext); + } + /** * Check the query if Contains query operator * Such sql as following should return true diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 98629f86028b81..0ee88ce82af070 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -40,6 +40,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.BitSet; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -164,8 +165,12 @@ public static boolean needUnionRewrite(MaterializationContext materializationCon /** * Get query used partitions * this is calculated from tableUsedPartitionNameMap and tables in statementContext - * */ - public static Map, Set> getQueryUsedPartitions(StatementContext statementContext) { + * + * @param customRelationIdSet if union compensate occurs, the new query used partitions is changed, + * so need to get used partitions by relation id set + */ + public static Map, Set> getQueryUsedPartitions(StatementContext statementContext, + BitSet customRelationIdSet) { // get table used partitions // if table is not in statementContext().getTables() which means the table is partition prune as empty relation Multimap, Pair>> tableUsedPartitionNameMap = statementContext @@ -174,7 +179,7 @@ public static Map, Set> getQueryUsedPartitions(StatementCon // if value is null, means query all partitions // if value is not empty, means query some partitions Map, Set> queryUsedRelatedTablePartitionsMap = new HashMap<>(); - outer: + tableLoop: for (Map.Entry, TableIf> queryUsedTableEntry : statementContext.getTables().entrySet()) { Set usedPartitionSet = new HashSet<>(); Collection>> tableUsedPartitions = @@ -185,11 +190,20 @@ public static Map, Set> getQueryUsedPartitions(StatementCon continue; } for (Pair> partitionPair : tableUsedPartitions) { - if (ALL_PARTITIONS.equals(partitionPair)) { - queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), null); - continue outer; + if (!customRelationIdSet.isEmpty()) { + if (ALL_PARTITIONS.equals(partitionPair)) { + continue; + } + if (customRelationIdSet.get(partitionPair.key().asInt())) { + usedPartitionSet.addAll(partitionPair.value()); + } + } else { + if (ALL_PARTITIONS.equals(partitionPair)) { + queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), null); + continue tableLoop; + } + usedPartitionSet.addAll(partitionPair.value()); } - usedPartitionSet.addAll(partitionPair.value()); } } queryUsedRelatedTablePartitionsMap.put(queryUsedTableEntry.getKey(), usedPartitionSet); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 2003e7c12ae113..45cd0950ff9b56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -100,6 +100,7 @@ public class StructInfo { // So if the cascadesContext currently is different form the cascadesContext which generated it. // Should regenerate the tableBitSet by current cascadesContext and call withTableBitSet method private final BitSet tableBitSet; + private final BitSet relationIdBitSet; // this is for LogicalCompatibilityContext later private final Map relationIdStructInfoNodeMap; // this recorde the predicates which can pull up, not shuttled @@ -137,6 +138,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG shuttledExpressionsToExpressionsMap, Map> expressionToShuttledExpressionToMap, BitSet tableIdSet, + BitSet relationIdSet, SplitPredicate splitPredicate, EquivalenceClass equivalenceClass, List planOutputShuttledExpressions) { @@ -148,6 +150,7 @@ private StructInfo(Plan originalPlan, ObjectId originalPlanId, HyperGraph hyperG this.bottomPlan = bottomPlan; this.relations = relations; this.tableBitSet = tableIdSet; + this.relationIdBitSet = relationIdSet; this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap; this.predicates = predicates; this.splitPredicate = splitPredicate; @@ -164,17 +167,19 @@ public StructInfo withPredicates(Predicates predicates) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, predicates, this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, - this.tableBitSet, null, null, this.planOutputShuttledExpressions); + this.tableBitSet, this.relationIdBitSet, null, null, + this.planOutputShuttledExpressions); } /** * Construct StructInfo with new tableBitSet */ - public StructInfo withTableBitSet(BitSet tableBitSet) { + public StructInfo withTableBitSet(BitSet tableBitSet, BitSet relationIdBitSet) { return new StructInfo(this.originalPlan, this.originalPlanId, this.hyperGraph, this.valid, this.topPlan, this.bottomPlan, this.relations, this.relationIdStructInfoNodeMap, this.predicates, this.shuttledExpressionsToExpressionsMap, this.expressionToShuttledExpressionToMap, - tableBitSet, this.splitPredicate, this.equivalenceClass, this.planOutputShuttledExpressions); + tableBitSet, relationIdBitSet, this.splitPredicate, this.equivalenceClass, + this.planOutputShuttledExpressions); } private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, @@ -185,6 +190,7 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, List relations, Map relationIdStructInfoNodeMap, BitSet hyperTableBitSet, + BitSet relationBitSet, CascadesContext cascadesContext) { // Collect relations from hyper graph which in the bottom plan firstly @@ -194,8 +200,11 @@ private static boolean collectStructInfoFromGraph(HyperGraph hyperGraph, List nodeRelations = new ArrayList<>(); nodePlan.accept(RELATION_COLLECTOR, nodeRelations); relations.addAll(nodeRelations); - nodeRelations.forEach(relation -> hyperTableBitSet.set( - cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt())); + nodeRelations.forEach(relation -> { + hyperTableBitSet.set( + cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt()); + relationBitSet.set(relation.getRelationId().asInt()); + }); // plan relation collector and set to map StructInfoNode structInfoNode = (StructInfoNode) node; // record expressions in node @@ -314,12 +323,14 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable Map>> shuttledHashConjunctsToConjunctsMap = new LinkedHashMap<>(); BitSet tableBitSet = new BitSet(); + BitSet relationBitSet = new BitSet(); Map> expressionToShuttledExpressionToMap = new HashMap<>(); boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan, shuttledHashConjunctsToConjunctsMap, expressionToShuttledExpressionToMap, relationList, relationIdStructInfoNodeMap, tableBitSet, + relationBitSet, cascadesContext); valid = valid && hyperGraph.getNodes().stream().allMatch(n -> ((StructInfoNode) n).getExpressions() != null); @@ -338,8 +349,7 @@ public static StructInfo of(Plan originalPlan, @Nullable Plan topPlan, @Nullable return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid, topPlan, bottomPlan, relationList, relationIdStructInfoNodeMap, predicates, shuttledHashConjunctsToConjunctsMap, expressionToShuttledExpressionToMap, - tableBitSet, null, null, - planOutputShuttledExpressions); + tableBitSet, relationBitSet, null, null, planOutputShuttledExpressions); } public List getRelations() { @@ -443,6 +453,10 @@ public BitSet getTableBitSet() { return tableBitSet; } + public BitSet getRelationIdBitSet() { + return relationIdBitSet; + } + public List getPlanOutputShuttledExpressions() { return planOutputShuttledExpressions; } @@ -752,10 +766,11 @@ public static Pair addFilterOnTableScan(Plan queryPlan, Map { + Plan filterAddedPlan = MaterializedViewUtils.rewriteByRules(parentCascadesContext, context -> { Rewriter.getWholeTreeRewriter(context).execute(); return context.getRewritePlan(); - }, queryPlanWithUnionFilter, queryPlan), true); + }, queryPlanWithUnionFilter, queryPlan); + return Pair.of(filterAddedPlan, true); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java index 2ad993b361d43a..cfe5d6863c387b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/QueryPartitionCollector.java @@ -20,17 +20,14 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.PartitionCompensator; -import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; -import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; -import org.apache.doris.qe.ConnectContext; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import com.google.common.collect.Multimap; import org.apache.logging.log4j.LogManager; @@ -43,29 +40,18 @@ /** * Used to collect query partitions, only collect once * */ -public class QueryPartitionCollector extends DefaultPlanRewriter implements CustomRewriter { +public class QueryPartitionCollector extends DefaultPlanVisitor { public static final Logger LOG = LogManager.getLogger(QueryPartitionCollector.class); @Override - public Plan rewriteRoot(Plan plan, JobContext jobContext) { - - ConnectContext connectContext = ConnectContext.get(); - if (connectContext != null && connectContext.getSessionVariable().internalSession) { - return plan; - } - plan.accept(this, connectContext); - return plan; - } - - @Override - public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, ConnectContext context) { + public Void visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, CascadesContext context) { TableIf table = catalogRelation.getTable(); if (table.getDatabase() == null) { LOG.error("QueryPartitionCollector visitLogicalCatalogRelation database is null, table is " + table.getName()); - return catalogRelation; + return null; } Multimap, Pair>> tableUsedPartitionNameMap = context.getStatementContext() .getTableUsedPartitionNameMap(); @@ -90,6 +76,6 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation catalogRelation, // not support get partition scene, we consider query all partitions from table tableUsedPartitionNameMap.put(table.getFullQualifiers(), PartitionCompensator.ALL_PARTITIONS); } - return catalogRelation; + return null; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java index 76246e52f9d542..25c0a679d8d388 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensatorTest.java @@ -28,6 +28,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.BitSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -107,9 +108,11 @@ public void testGetQueryTableUsedPartition() { + "left outer join orders_list_partition\n" + "on l1.l_shipdate = o_orderdate\n", nereidsPlanner -> { + MaterializedViewUtils.collectTableUsedPartitions(nereidsPlanner.getRewrittenPlan(), + nereidsPlanner.getCascadesContext()); Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( - nereidsPlanner.getCascadesContext().getStatementContext()); + nereidsPlanner.getCascadesContext().getStatementContext(), new BitSet()); List itmeQualifier = ImmutableList.of( "internal", "partition_compensate_test", "lineitem_list_partition"); @@ -131,6 +134,8 @@ public void testGetAllTableUsedPartition() { + "left outer join orders_list_partition\n" + "on l1.l_shipdate = o_orderdate\n", nereidsPlanner -> { + MaterializedViewUtils.collectTableUsedPartitions(nereidsPlanner.getRewrittenPlan(), + nereidsPlanner.getCascadesContext()); List qualifier = ImmutableList.of( "internal", "partition_compensate_test", "lineitem_list_partition"); @@ -140,7 +145,7 @@ public void testGetAllTableUsedPartition() { Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( - nereidsPlanner.getCascadesContext().getStatementContext()); + nereidsPlanner.getCascadesContext().getStatementContext(), new BitSet()); Set queryTableUsedPartition = queryUsedPartitions.get(qualifier); // if tableUsedPartitionNameMap contain any PartitionCompensator.ALL_PARTITIONS // consider query all partitions from table @@ -161,6 +166,8 @@ public void testGetAllTableUsedPartitionList() { + "left outer join orders_list_partition\n" + "on l1.l_shipdate = o_orderdate\n", nereidsPlanner -> { + MaterializedViewUtils.collectTableUsedPartitions(nereidsPlanner.getRewrittenPlan(), + nereidsPlanner.getCascadesContext()); List qualifier = ImmutableList.of( "internal", "partition_compensate_test", "lineitem_list_partition"); @@ -171,7 +178,7 @@ public void testGetAllTableUsedPartitionList() { Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( - nereidsPlanner.getCascadesContext().getStatementContext()); + nereidsPlanner.getCascadesContext().getStatementContext(), new BitSet()); Set queryTableUsedPartition = queryUsedPartitions.get(qualifier); // if tableUsedPartitionNameMap contain only PartitionCompensator.ALL_PARTITIONS // consider query all partitions from table diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index e900937d2baa68..e535e3a8ac5e25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -50,6 +50,7 @@ import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; @@ -244,7 +245,7 @@ public Rule build() { public PlanChecker rewrite() { Rewriter.getWholeTreeRewriter(cascadesContext).execute(); - cascadesContext.newTablePartitionCollector().execute(); + MaterializedViewUtils.collectTableUsedPartitions(cascadesContext.getRewritePlan(), cascadesContext); InitMaterializationContextHook.INSTANCE.initMaterializationContext(this.cascadesContext); cascadesContext.toMemo(); return this; From e0302c141c82264f32bd3d533885bd723da9fd95 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 27 Jun 2025 12:19:22 +0800 Subject: [PATCH 094/572] branch-3.0: [fix](nereids) correct the log class in the command (#43896) (#52254) backport: https://github.com/apache/doris/pull/43896 Co-authored-by: Yao-MR --- .../doris/nereids/trees/plans/commands/PrepareCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java index d0d2fffa278b2c..42274c5eaf8d4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java @@ -39,7 +39,7 @@ * Prepared Statement */ public class PrepareCommand extends Command { - private static final Logger LOG = LogManager.getLogger(StmtExecutor.class); + private static final Logger LOG = LogManager.getLogger(PrepareCommand.class); private final List placeholders = new ArrayList<>(); private final LogicalPlan logicalPlan; From 3d4314529951e3a0850f3e59f3180faa2e6498cc Mon Sep 17 00:00:00 2001 From: zzzxl Date: Fri, 27 Jun 2025 17:26:05 +0800 Subject: [PATCH 095/572] [opt](inverted index) enhance inverted index profile (#50876) pick https://github.com/apache/doris/pull/47504 pick https://github.com/apache/doris/pull/48826 pick https://github.com/apache/doris/pull/48950 --- be/src/clucene | 2 +- be/src/io/cache/block_file_cache_profile.h | 40 ++++++- be/src/io/cache/cached_remote_file_reader.cpp | 15 ++- be/src/io/io_common.h | 9 +- be/src/olap/inverted_index_profile.h | 57 ++++++++++ be/src/olap/inverted_index_stats.h | 34 ++++++ be/src/olap/olap_common.h | 4 + .../query/phrase_edge_query.cpp | 3 +- .../inverted_index/query/phrase_edge_query.h | 1 + .../query/phrase_prefix_query.cpp | 8 +- .../query/phrase_prefix_query.h | 3 + .../inverted_index/query/prefix_query.cpp | 4 +- .../inverted_index/query/prefix_query.h | 17 ++- .../inverted_index/query/regexp_query.cpp | 3 +- .../inverted_index/query/regexp_query.h | 1 + .../inverted_index_fs_directory.cpp | 57 ++++++---- .../segment_v2/inverted_index_fs_directory.h | 2 + .../segment_v2/inverted_index_reader.cpp | 63 +++++++---- .../rowset/segment_v2/inverted_index_reader.h | 12 ++- be/src/pipeline/exec/olap_scan_operator.cpp | 8 ++ be/src/pipeline/exec/olap_scan_operator.h | 3 + be/src/vec/exec/scan/new_olap_scanner.cpp | 8 ++ be/test/olap/inverted_index_profile_test.cpp | 44 ++++++++ .../util/index_compaction_utils.cpp | 2 +- .../inverted_index_fs_directory_test.cpp | 102 ++++++++++++++++++ 25 files changed, 428 insertions(+), 74 deletions(-) create mode 100644 be/src/olap/inverted_index_profile.h create mode 100644 be/src/olap/inverted_index_stats.h create mode 100644 be/test/olap/inverted_index_profile_test.cpp create mode 100644 be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp diff --git a/be/src/clucene b/be/src/clucene index 317e50714884fe..569398a5c96b4c 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit 317e50714884fe338e0f44e870d841df3a91689f +Subproject commit 569398a5c96b4c626251ccbe81257945a3d2aef4 diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index f9d9df0939f017..583e1287c05ad2 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -75,7 +75,6 @@ struct FileCacheProfile { struct FileCacheProfileReporter { RuntimeProfile::Counter* num_local_io_total = nullptr; RuntimeProfile::Counter* num_remote_io_total = nullptr; - RuntimeProfile::Counter* num_inverted_index_remote_io_total = nullptr; RuntimeProfile::Counter* local_io_timer = nullptr; RuntimeProfile::Counter* bytes_scanned_from_cache = nullptr; RuntimeProfile::Counter* bytes_scanned_from_remote = nullptr; @@ -89,6 +88,14 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* get_timer = nullptr; RuntimeProfile::Counter* set_timer = nullptr; + RuntimeProfile::Counter* inverted_index_num_local_io_total = nullptr; + RuntimeProfile::Counter* inverted_index_num_remote_io_total = nullptr; + RuntimeProfile::Counter* inverted_index_bytes_scanned_from_cache = nullptr; + RuntimeProfile::Counter* inverted_index_bytes_scanned_from_remote = nullptr; + RuntimeProfile::Counter* inverted_index_local_io_timer = nullptr; + RuntimeProfile::Counter* inverted_index_remote_io_timer = nullptr; + RuntimeProfile::Counter* inverted_index_io_timer = nullptr; + FileCacheProfileReporter(RuntimeProfile* profile) { static const char* cache_profile = "FileCache"; ADD_TIMER_WITH_LEVEL(profile, cache_profile, 1); @@ -96,8 +103,6 @@ struct FileCacheProfileReporter { cache_profile, 1); num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT, cache_profile, 1); - num_inverted_index_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "NumInvertedIndexRemoteIOTotal", TUnit::UNIT, cache_profile, 1); local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LocalIOUseTimer", cache_profile, 1); remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RemoteIOUseTimer", cache_profile, 1); write_cache_io_timer = @@ -117,13 +122,26 @@ struct FileCacheProfileReporter { lock_wait_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LockWaitTimer", cache_profile, 1); get_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "GetTimer", cache_profile, 1); set_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "SetTimer", cache_profile, 1); + + inverted_index_num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexNumLocalIOTotal", TUnit::UNIT, cache_profile, 1); + inverted_index_num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexNumRemoteIOTotal", TUnit::UNIT, cache_profile, 1); + inverted_index_bytes_scanned_from_cache = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexBytesScannedFromCache", TUnit::BYTES, cache_profile, 1); + inverted_index_bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "InvertedIndexBytesScannedFromRemote", TUnit::BYTES, cache_profile, 1); + inverted_index_local_io_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "InvertedIndexLocalIOUseTimer", cache_profile, 1); + inverted_index_remote_io_timer = ADD_CHILD_TIMER_WITH_LEVEL( + profile, "InvertedIndexRemoteIOUseTimer", cache_profile, 1); + inverted_index_io_timer = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "InvertedIndexIOTimer", cache_profile, 1); } void update(const FileCacheStatistics* statistics) const { COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); - COUNTER_UPDATE(num_inverted_index_remote_io_total, - statistics->num_inverted_index_remote_io_total); COUNTER_UPDATE(local_io_timer, statistics->local_io_timer); COUNTER_UPDATE(remote_io_timer, statistics->remote_io_timer); COUNTER_UPDATE(write_cache_io_timer, statistics->write_cache_io_timer); @@ -136,6 +154,18 @@ struct FileCacheProfileReporter { COUNTER_UPDATE(lock_wait_timer, statistics->lock_wait_timer); COUNTER_UPDATE(get_timer, statistics->get_timer); COUNTER_UPDATE(set_timer, statistics->set_timer); + + COUNTER_UPDATE(inverted_index_num_local_io_total, + statistics->inverted_index_num_local_io_total); + COUNTER_UPDATE(inverted_index_num_remote_io_total, + statistics->inverted_index_num_remote_io_total); + COUNTER_UPDATE(inverted_index_bytes_scanned_from_cache, + statistics->inverted_index_bytes_read_from_local); + COUNTER_UPDATE(inverted_index_bytes_scanned_from_remote, + statistics->inverted_index_bytes_read_from_remote); + COUNTER_UPDATE(inverted_index_local_io_timer, statistics->inverted_index_local_io_timer); + COUNTER_UPDATE(inverted_index_remote_io_timer, statistics->inverted_index_remote_io_timer); + COUNTER_UPDATE(inverted_index_io_timer, statistics->inverted_index_io_timer); } }; diff --git a/be/src/io/cache/cached_remote_file_reader.cpp b/be/src/io/cache/cached_remote_file_reader.cpp index c7476b7ab7476c..b89bdcf2f6de0a 100644 --- a/be/src/io/cache/cached_remote_file_reader.cpp +++ b/be/src/io/cache/cached_remote_file_reader.cpp @@ -339,9 +339,6 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->num_local_io_total++; statis->bytes_read_from_local += read_stats.bytes_read; } else { - if (is_inverted_index) { - statis->num_inverted_index_remote_io_total++; - } statis->num_remote_io_total++; statis->bytes_read_from_remote += read_stats.bytes_read; } @@ -357,6 +354,18 @@ void CachedRemoteFileReader::_update_stats(const ReadStatistics& read_stats, statis->get_timer += read_stats.get_timer; statis->set_timer += read_stats.set_timer; + if (is_inverted_index) { + if (read_stats.hit_cache) { + statis->inverted_index_num_local_io_total++; + statis->inverted_index_bytes_read_from_local += read_stats.bytes_read; + } else { + statis->inverted_index_num_remote_io_total++; + statis->inverted_index_bytes_read_from_remote += read_stats.bytes_read; + } + statis->inverted_index_local_io_timer += read_stats.local_read_timer; + statis->inverted_index_remote_io_timer += read_stats.remote_read_timer; + } + g_skip_cache_num << read_stats.skip_cache; g_skip_cache_sum << read_stats.skip_cache; } diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h index d4a4e26a7c1840..909941181d3bcb 100644 --- a/be/src/io/io_common.h +++ b/be/src/io/io_common.h @@ -38,7 +38,6 @@ namespace io { struct FileCacheStatistics { int64_t num_local_io_total = 0; int64_t num_remote_io_total = 0; - int64_t num_inverted_index_remote_io_total = 0; int64_t local_io_timer = 0; int64_t bytes_read_from_local = 0; int64_t bytes_read_from_remote = 0; @@ -51,6 +50,14 @@ struct FileCacheStatistics { int64_t lock_wait_timer = 0; int64_t get_timer = 0; int64_t set_timer = 0; + + int64_t inverted_index_num_local_io_total = 0; + int64_t inverted_index_num_remote_io_total = 0; + int64_t inverted_index_bytes_read_from_local = 0; + int64_t inverted_index_bytes_read_from_remote = 0; + int64_t inverted_index_local_io_timer = 0; + int64_t inverted_index_remote_io_timer = 0; + int64_t inverted_index_io_timer = 0; }; struct IOContext { diff --git a/be/src/olap/inverted_index_profile.h b/be/src/olap/inverted_index_profile.h new file mode 100644 index 00000000000000..4b3855b5c8ae68 --- /dev/null +++ b/be/src/olap/inverted_index_profile.h @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "olap/inverted_index_stats.h" +#include "util/runtime_profile.h" + +namespace doris { + +class InvertedIndexProfileReporter { +public: + InvertedIndexProfileReporter() = default; + ~InvertedIndexProfileReporter() = default; + + void update(RuntimeProfile* profile, const InvertedIndexStatistics* statistics) { + // Determine the iteration limit: the smaller of 20 or the size of statistics->stats + size_t iteration_limit = std::min(20, statistics->stats.size()); + + for (size_t i = 0; i < iteration_limit; ++i) { + const auto& stats = statistics->stats[i]; + + ADD_TIMER_WITH_LEVEL(profile, hit_rows_name, 1); + auto* hit_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HitRows_" + stats.column_name, + TUnit::UNIT, hit_rows_name, 1); + COUNTER_UPDATE(hit_rows, stats.hit_rows); + + ADD_TIMER_WITH_LEVEL(profile, exec_time_name, 1); + auto* exec_time = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ExecTime_" + stats.column_name, + TUnit::TIME_NS, exec_time_name, 1); + COUNTER_UPDATE(exec_time, stats.exec_time); + } + } + +private: + static constexpr const char* hit_rows_name = "HitRows"; + static constexpr const char* exec_time_name = "ExecTime"; +}; + +} // namespace doris diff --git a/be/src/olap/inverted_index_stats.h b/be/src/olap/inverted_index_stats.h new file mode 100644 index 00000000000000..b82b230f41d71e --- /dev/null +++ b/be/src/olap/inverted_index_stats.h @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + +struct InvertedIndexQueryStatistics { + std::string column_name; + int64_t hit_rows = 0; + int64_t exec_time = 0; +}; + +struct InvertedIndexStatistics { + std::vector stats; +}; + +} // namespace doris diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 9ffac13022d169..f792e3ac6f90f2 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -37,6 +37,7 @@ #include "common/config.h" #include "io/io_common.h" +#include "olap/inverted_index_stats.h" #include "olap/olap_define.h" #include "olap/rowset/rowset_fwd.h" #include "util/hash_util.hpp" @@ -373,9 +374,12 @@ struct OlapReaderStatistics { int64_t inverted_index_query_bitmap_copy_timer = 0; int64_t inverted_index_searcher_open_timer = 0; int64_t inverted_index_searcher_search_timer = 0; + int64_t inverted_index_searcher_search_init_timer = 0; + int64_t inverted_index_searcher_search_exec_timer = 0; int64_t inverted_index_searcher_cache_hit = 0; int64_t inverted_index_searcher_cache_miss = 0; int64_t inverted_index_downgrade_count = 0; + InvertedIndexStatistics inverted_index_stats; int64_t output_index_result_column_timer = 0; // number of segment filtered by column stat when creating seg iterator diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.cpp b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.cpp index f82433826e9581..2df4aa5929bcb3 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.cpp @@ -32,6 +32,7 @@ namespace doris::segment_v2 { PhraseEdgeQuery::PhraseEdgeQuery(const std::shared_ptr& searcher, const TQueryOptions& query_options, const io::IOContext* io_ctx) : _searcher(searcher), + _io_ctx(io_ctx), _query(std::make_unique()), _max_expansions(query_options.inverted_index_max_expansions) {} @@ -143,7 +144,7 @@ void PhraseEdgeQuery::find_words(const std::function& cb) { Term* term = nullptr; TermEnum* enumerator = nullptr; try { - enumerator = _searcher->getReader()->terms(); + enumerator = _searcher->getReader()->terms(nullptr, _io_ctx); while (enumerator->next()) { term = enumerator->term(); cb(term); diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.h b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.h index 9eb3bd57c4a916..bdf962c9f48b7b 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.h +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query.h @@ -48,6 +48,7 @@ class PhraseEdgeQuery : public Query { private: std::shared_ptr _searcher; + const io::IOContext* _io_ctx = nullptr; std::wstring _field_name; std::vector _terms; diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.cpp b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.cpp index 88bb3c1171fa30..ca45108008a89f 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.cpp @@ -18,7 +18,6 @@ #include "phrase_prefix_query.h" #include "CLucene/util/stringUtil.h" -#include "olap/rowset//segment_v2/inverted_index/query/prefix_query.h" namespace doris::segment_v2 { @@ -27,7 +26,8 @@ PhrasePrefixQuery::PhrasePrefixQuery(const std::shared_ptr()), - _max_expansions(query_options.inverted_index_max_expansions) {} + _max_expansions(query_options.inverted_index_max_expansions), + _prefix_query(io_ctx) {} void PhrasePrefixQuery::add(const std::wstring& field_name, const std::vector& terms) { if (terms.empty()) { @@ -42,8 +42,8 @@ void PhrasePrefixQuery::add(const std::wstring& field_name, const std::vector prefix_terms; - PrefixQuery::get_prefix_terms(_searcher->getReader(), field_name, terms[i], - prefix_terms, _max_expansions); + _prefix_query.get_prefix_terms(_searcher->getReader(), field_name, terms[i], + prefix_terms, _max_expansions); if (prefix_terms.empty()) { std::wstring ws_term = StringUtil::string_to_wstring(terms[i]); Term* t = _CLNEW Term(field_name.c_str(), ws_term.c_str()); diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.h b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.h index 5cac597951eac7..d813faf8340c07 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.h +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query.h @@ -24,6 +24,8 @@ #include "CLucene/search/MultiPhraseQuery.h" // clang-format on +#include "olap/rowset/segment_v2/inverted_index/query/prefix_query.h" + CL_NS_USE(search) namespace doris::segment_v2 { @@ -42,6 +44,7 @@ class PhrasePrefixQuery : public Query { std::unique_ptr _query; int32_t _max_expansions = 50; + PrefixQuery _prefix_query; }; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.cpp b/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.cpp index 1400622735214f..2ba7b94966c486 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.cpp @@ -19,6 +19,8 @@ namespace doris::segment_v2 { +PrefixQuery::PrefixQuery(const io::IOContext* io_ctx) : _io_ctx(io_ctx) {} + void PrefixQuery::get_prefix_terms(IndexReader* reader, const std::wstring& field_name, const std::string& prefix, std::vector& prefix_terms, @@ -26,7 +28,7 @@ void PrefixQuery::get_prefix_terms(IndexReader* reader, const std::wstring& fiel std::wstring ws_prefix = StringUtil::string_to_wstring(prefix); Term* prefix_term = _CLNEW Term(field_name.c_str(), ws_prefix.c_str()); - TermEnum* enumerator = reader->terms(prefix_term); + TermEnum* enumerator = reader->terms(prefix_term, _io_ctx); int32_t count = 0; Term* lastTerm = nullptr; diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.h b/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.h index 9a33b13dd8da76..bff2719455c05a 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.h +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/prefix_query.h @@ -17,10 +17,7 @@ #pragma once -#include -#include - -#include +#include "olap/rowset/segment_v2/inverted_index/query/query.h" CL_NS_USE(index) @@ -28,13 +25,15 @@ namespace doris::segment_v2 { class PrefixQuery { public: - PrefixQuery() = default; + PrefixQuery(const io::IOContext* io_ctx); virtual ~PrefixQuery() = default; - static void get_prefix_terms(IndexReader* reader, const std::wstring& field_name, - const std::string& prefix, - std::vector& prefix_terms, - int32_t max_expansions = 50); + void get_prefix_terms(IndexReader* reader, const std::wstring& field_name, + const std::string& prefix, std::vector& prefix_terms, + int32_t max_expansions = 50); + +private: + const io::IOContext* _io_ctx = nullptr; }; } // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp b/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp index 69de4b7818b870..5838e1c373741a 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp @@ -27,6 +27,7 @@ namespace doris::segment_v2 { RegexpQuery::RegexpQuery(const std::shared_ptr& searcher, const TQueryOptions& query_options, const io::IOContext* io_ctx) : _searcher(searcher), + _io_ctx(io_ctx), _max_expansions(query_options.inverted_index_max_expansions), _query(searcher, query_options, io_ctx) {} @@ -66,7 +67,7 @@ void RegexpQuery::add(const std::wstring& field_name, const std::vectorgetReader()->terms(); + enumerator = _searcher->getReader()->terms(nullptr, _io_ctx); while (enumerator->next()) { term = enumerator->term(); std::string input = lucene_wcstoutf8string(term->text(), term->textLength()); diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.h b/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.h index 650ad2bf10b002..782e492c47302f 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.h +++ b/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.h @@ -36,6 +36,7 @@ class RegexpQuery : public Query { private: std::shared_ptr _searcher; + const io::IOContext* _io_ctx = nullptr; int32_t _max_expansions = 50; DisjunctionQuery _query; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp index 759cc0b430ba41..c633d29a7fc0c0 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp @@ -237,35 +237,46 @@ void DorisFSDirectory::FSIndexInput::seekInternal(const int64_t position) { void DorisFSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len) { CND_PRECONDITION(_handle != nullptr, "shared file handle has closed"); CND_PRECONDITION(_handle->_reader != nullptr, "file is not open"); - std::lock_guard wlock(_handle->_shared_lock); - int64_t position = getFilePointer(); - if (_pos != position) { - _pos = position; - } + int64_t inverted_index_io_timer = 0; + { + SCOPED_RAW_TIMER(&inverted_index_io_timer); + + std::lock_guard wlock(_handle->_shared_lock); + + int64_t position = getFilePointer(); + if (_pos != position) { + _pos = position; + } + + if (_handle->_fpos != _pos) { + _handle->_fpos = _pos; + } - if (_handle->_fpos != _pos) { + Slice result {b, (size_t)len}; + size_t bytes_read = 0; + Status st = _handle->_reader->read_at(_pos, result, &bytes_read, &_io_ctx); + DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexInput::readInternal_reader_read_at_error", { + st = Status::InternalError( + "debug point: " + "DorisFSDirectory::FSIndexInput::readInternal_reader_read_at_error"); + }) + if (!st.ok()) { + _CLTHROWA(CL_ERR_IO, "read past EOF"); + } + bufferLength = len; + DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexInput::readInternal_bytes_read_error", + { bytes_read = len + 10; }) + if (bytes_read != len) { + _CLTHROWA(CL_ERR_IO, "read error"); + } + _pos += bufferLength; _handle->_fpos = _pos; } - Slice result {b, (size_t)len}; - size_t bytes_read = 0; - Status st = _handle->_reader->read_at(_pos, result, &bytes_read, &_io_ctx); - DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexInput::readInternal_reader_read_at_error", { - st = Status::InternalError( - "debug point: DorisFSDirectory::FSIndexInput::readInternal_reader_read_at_error"); - }) - if (!st.ok()) { - _CLTHROWA(CL_ERR_IO, "read past EOF"); - } - bufferLength = len; - DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexInput::readInternal_bytes_read_error", - { bytes_read = len + 10; }) - if (bytes_read != len) { - _CLTHROWA(CL_ERR_IO, "read error"); + if (_io_ctx.file_cache_stats != nullptr) { + _io_ctx.file_cache_stats->inverted_index_io_timer += inverted_index_io_timer; } - _pos += bufferLength; - _handle->_fpos = _pos; } void DorisFSDirectory::FSIndexOutput::init(const io::FileSystemSPtr& fs, const char* path) { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h index 991ef52d046137..0bba5b49756070 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h @@ -209,6 +209,8 @@ class DorisFSDirectory::FSIndexInput : public lucene::store::BufferedIndexInput void seekInternal(const int64_t position) override; // IndexInput methods void readInternal(uint8_t* b, const int32_t len) override; + + friend class DorisFSDirectoryTest; }; /** diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 6f53857156a476..662c930ead2a93 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -256,8 +256,14 @@ Status InvertedIndexReader::match_index_search( return Status::Error( "query type " + query_type_to_string(query_type) + ", query is nullptr"); } - query->add(query_info); - query->search(*term_match_bitmap); + { + SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_init_timer); + query->add(query_info); + } + { + SCOPED_RAW_TIMER(&stats->inverted_index_searcher_search_exec_timer); + query->search(*term_match_bitmap); + } } catch (const CLuceneError& e) { return Status::Error("CLuceneError occured: {}", e.what()); @@ -542,7 +548,7 @@ Status BkdIndexReader::construct_bkd_query_value(const void* query_value, return Status::OK(); } -Status BkdIndexReader::invoke_bkd_try_query(const void* query_value, +Status BkdIndexReader::invoke_bkd_try_query(const io::IOContext* io_ctx, const void* query_value, InvertedIndexQueryType query_type, std::shared_ptr r, uint32_t* count) { @@ -550,7 +556,7 @@ Status BkdIndexReader::invoke_bkd_try_query(const void* query_value, case InvertedIndexQueryType::LESS_THAN_QUERY: { auto visitor = std::make_unique>( - r.get(), nullptr, true); + io_ctx, r.get(), nullptr, true); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); *count = r->estimate_point_count(visitor.get()); break; @@ -558,7 +564,7 @@ Status BkdIndexReader::invoke_bkd_try_query(const void* query_value, case InvertedIndexQueryType::LESS_EQUAL_QUERY: { auto visitor = std::make_unique>( - r.get(), nullptr, true); + io_ctx, r.get(), nullptr, true); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); *count = r->estimate_point_count(visitor.get()); break; @@ -566,7 +572,7 @@ Status BkdIndexReader::invoke_bkd_try_query(const void* query_value, case InvertedIndexQueryType::GREATER_THAN_QUERY: { auto visitor = std::make_unique>( - r.get(), nullptr, true); + io_ctx, r.get(), nullptr, true); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); *count = r->estimate_point_count(visitor.get()); break; @@ -574,14 +580,14 @@ Status BkdIndexReader::invoke_bkd_try_query(const void* query_value, case InvertedIndexQueryType::GREATER_EQUAL_QUERY: { auto visitor = std::make_unique>( - r.get(), nullptr, true); + io_ctx, r.get(), nullptr, true); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); *count = r->estimate_point_count(visitor.get()); break; } case InvertedIndexQueryType::EQUAL_QUERY: { auto visitor = std::make_unique>( - r.get(), nullptr, true); + io_ctx, r.get(), nullptr, true); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); *count = r->estimate_point_count(visitor.get()); break; @@ -592,14 +598,15 @@ Status BkdIndexReader::invoke_bkd_try_query(const void* query_value, return Status::OK(); } -Status BkdIndexReader::invoke_bkd_query(const void* query_value, InvertedIndexQueryType query_type, +Status BkdIndexReader::invoke_bkd_query(const io::IOContext* io_ctx, const void* query_value, + InvertedIndexQueryType query_type, std::shared_ptr r, std::shared_ptr& bit_map) { switch (query_type) { case InvertedIndexQueryType::LESS_THAN_QUERY: { auto visitor = std::make_unique>( - r.get(), bit_map.get()); + io_ctx, r.get(), bit_map.get()); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); r->intersect(visitor.get()); break; @@ -607,7 +614,7 @@ Status BkdIndexReader::invoke_bkd_query(const void* query_value, InvertedIndexQu case InvertedIndexQueryType::LESS_EQUAL_QUERY: { auto visitor = std::make_unique>( - r.get(), bit_map.get()); + io_ctx, r.get(), bit_map.get()); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); r->intersect(visitor.get()); break; @@ -615,7 +622,7 @@ Status BkdIndexReader::invoke_bkd_query(const void* query_value, InvertedIndexQu case InvertedIndexQueryType::GREATER_THAN_QUERY: { auto visitor = std::make_unique>( - r.get(), bit_map.get()); + io_ctx, r.get(), bit_map.get()); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); r->intersect(visitor.get()); break; @@ -623,14 +630,14 @@ Status BkdIndexReader::invoke_bkd_query(const void* query_value, InvertedIndexQu case InvertedIndexQueryType::GREATER_EQUAL_QUERY: { auto visitor = std::make_unique>( - r.get(), bit_map.get()); + io_ctx, r.get(), bit_map.get()); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); r->intersect(visitor.get()); break; } case InvertedIndexQueryType::EQUAL_QUERY: { auto visitor = std::make_unique>( - r.get(), bit_map.get()); + io_ctx, r.get(), bit_map.get()); RETURN_IF_ERROR(construct_bkd_query_value(query_value, r, visitor.get())); r->intersect(visitor.get()); break; @@ -668,7 +675,7 @@ Status BkdIndexReader::try_query(const io::IOContext* io_ctx, OlapReaderStatisti return Status::OK(); } - return invoke_bkd_try_query(query_value, query_type, r, count); + return invoke_bkd_try_query(io_ctx, query_value, query_type, r, count); } catch (const CLuceneError& e) { return Status::Error( "BKD Query CLuceneError Occurred, error msg: {}", e.what()); @@ -706,7 +713,7 @@ Status BkdIndexReader::query(const io::IOContext* io_ctx, OlapReaderStatistics* return Status::OK(); } - RETURN_IF_ERROR(invoke_bkd_query(query_value, query_type, r, bit_map)); + RETURN_IF_ERROR(invoke_bkd_query(io_ctx, query_value, query_type, r, bit_map)); bit_map->runOptimize(); cache->insert(cache_key, bit_map, &cache_handler); @@ -751,9 +758,9 @@ InvertedIndexReaderType BkdIndexReader::type() { } template -InvertedIndexVisitor::InvertedIndexVisitor(lucene::util::bkd::bkd_reader* r, +InvertedIndexVisitor::InvertedIndexVisitor(const void* io_ctx, lucene::util::bkd::bkd_reader* r, roaring::Roaring* h, bool only_count) - : _hits(h), _num_hits(0), _only_count(only_count), _reader(r) {} + : _io_ctx(io_ctx), _hits(h), _num_hits(0), _only_count(only_count), _reader(r) {} template int InvertedIndexVisitor::matches(uint8_t* packed_value) { @@ -1156,8 +1163,24 @@ Status InvertedIndexIterator::read_from_inverted_index( } } - RETURN_IF_ERROR(_reader->query(&_io_ctx, _stats, _runtime_state, column_name, query_value, - query_type, bit_map)); + auto execute_query = [&]() { + return _reader->query(&_io_ctx, _stats, _runtime_state, column_name, query_value, + query_type, bit_map); + }; + + if (_runtime_state->query_options().enable_profile) { + InvertedIndexQueryStatistics query_stats; + { + SCOPED_RAW_TIMER(&query_stats.exec_time); + RETURN_IF_ERROR(execute_query()); + } + query_stats.column_name = column_name; + query_stats.hit_rows = bit_map->cardinality(); + _stats->inverted_index_stats.stats.emplace_back(query_stats); + } else { + RETURN_IF_ERROR(execute_query()); + } + return Status::OK(); } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 390928493f6689..410e5f32e57fc4 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -299,6 +299,7 @@ class StringTypeInvertedIndexReader : public InvertedIndexReader { template class InvertedIndexVisitor : public lucene::util::bkd::bkd_reader::intersect_visitor { private: + const void* _io_ctx = nullptr; roaring::Roaring* _hits = nullptr; uint32_t _num_hits; bool _only_count; @@ -309,8 +310,8 @@ class InvertedIndexVisitor : public lucene::util::bkd::bkd_reader::intersect_vis std::string query_max; public: - InvertedIndexVisitor(lucene::util::bkd::bkd_reader* r, roaring::Roaring* hits, - bool only_count = false); + InvertedIndexVisitor(const void* io_ctx, lucene::util::bkd::bkd_reader* r, + roaring::Roaring* hits, bool only_count = false); ~InvertedIndexVisitor() override = default; void set_reader(lucene::util::bkd::bkd_reader* r) { _reader = r; } @@ -329,6 +330,7 @@ class InvertedIndexVisitor : public lucene::util::bkd::bkd_reader::intersect_vis std::vector& max_packed) override; lucene::util::bkd::relation compare_prefix(std::vector& prefix) override; uint32_t get_num_hits() const { return _num_hits; } + const void* get_io_context() override { return _io_ctx; } }; class BkdIndexReader : public InvertedIndexReader { @@ -351,9 +353,11 @@ class BkdIndexReader : public InvertedIndexReader { Status try_query(const io::IOContext* io_ctx, OlapReaderStatistics* stats, const std::string& column_name, const void* query_value, InvertedIndexQueryType query_type, uint32_t* count) override; - Status invoke_bkd_try_query(const void* query_value, InvertedIndexQueryType query_type, + Status invoke_bkd_try_query(const io::IOContext* io_ctx, const void* query_value, + InvertedIndexQueryType query_type, std::shared_ptr r, uint32_t* count); - Status invoke_bkd_query(const void* query_value, InvertedIndexQueryType query_type, + Status invoke_bkd_query(const io::IOContext* io_ctx, const void* query_value, + InvertedIndexQueryType query_type, std::shared_ptr r, std::shared_ptr& bit_map); template diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp index 1434a16184e0ab..53a7d8cd0f3490 100644 --- a/be/src/pipeline/exec/olap_scan_operator.cpp +++ b/be/src/pipeline/exec/olap_scan_operator.cpp @@ -174,6 +174,10 @@ Status OlapScanLocalState::_init_profile() { ADD_TIMER(_segment_profile, "InvertedIndexSearcherOpenTime"); _inverted_index_searcher_search_timer = ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchTime"); + _inverted_index_searcher_search_init_timer = + ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchInitTime"); + _inverted_index_searcher_search_exec_timer = + ADD_TIMER(_segment_profile, "InvertedIndexSearcherSearchExecTime"); _inverted_index_searcher_cache_hit_counter = ADD_COUNTER(_segment_profile, "InvertedIndexSearcherCacheHit", TUnit::UNIT); _inverted_index_searcher_cache_miss_counter = @@ -228,6 +232,10 @@ Status OlapScanLocalState::_init_profile() { _segment_create_column_readers_timer = ADD_TIMER(_scanner_profile, "SegmentCreateColumnReadersTimer"); _segment_load_index_timer = ADD_TIMER(_scanner_profile, "SegmentLoadIndexTimer"); + + _index_filter_profile = std::make_unique("IndexFilter"); + _scanner_profile->add_child(_index_filter_profile.get(), true, nullptr); + return Status::OK(); } diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h index e6bc345ffd6ff0..24a1b1b876a354 100644 --- a/be/src/pipeline/exec/olap_scan_operator.h +++ b/be/src/pipeline/exec/olap_scan_operator.h @@ -98,6 +98,7 @@ class OlapScanLocalState final : public ScanLocalState { std::set _maybe_read_column_ids; std::unique_ptr _segment_profile; + std::unique_ptr _index_filter_profile; RuntimeProfile::Counter* _tablet_counter = nullptr; RuntimeProfile::Counter* _key_range_counter = nullptr; @@ -184,6 +185,8 @@ class OlapScanLocalState final : public ScanLocalState { RuntimeProfile::Counter* _inverted_index_query_bitmap_copy_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_open_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_search_timer = nullptr; + RuntimeProfile::Counter* _inverted_index_searcher_search_init_timer = nullptr; + RuntimeProfile::Counter* _inverted_index_searcher_search_exec_timer = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_cache_hit_counter = nullptr; RuntimeProfile::Counter* _inverted_index_searcher_cache_miss_counter = nullptr; RuntimeProfile::Counter* _inverted_index_downgrade_count_counter = nullptr; diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index 0970ded03e9850..e9c199074ecc4f 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -41,6 +41,7 @@ #include "exprs/function_filter.h" #include "io/cache/block_file_cache_profile.h" #include "io/io_common.h" +#include "olap/inverted_index_profile.h" #include "olap/olap_common.h" #include "olap/olap_tuple.h" #include "olap/rowset/rowset.h" @@ -634,12 +635,19 @@ void NewOlapScanner::_collect_profile_before_close() { stats.inverted_index_searcher_open_timer); \ COUNTER_UPDATE(Parent->_inverted_index_searcher_search_timer, \ stats.inverted_index_searcher_search_timer); \ + COUNTER_UPDATE(Parent->_inverted_index_searcher_search_init_timer, \ + stats.inverted_index_searcher_search_init_timer); \ + COUNTER_UPDATE(Parent->_inverted_index_searcher_search_exec_timer, \ + stats.inverted_index_searcher_search_exec_timer); \ COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_hit_counter, \ stats.inverted_index_searcher_cache_hit); \ COUNTER_UPDATE(Parent->_inverted_index_searcher_cache_miss_counter, \ stats.inverted_index_searcher_cache_miss); \ COUNTER_UPDATE(Parent->_inverted_index_downgrade_count_counter, \ stats.inverted_index_downgrade_count); \ + InvertedIndexProfileReporter inverted_index_profile; \ + inverted_index_profile.update(Parent->_index_filter_profile.get(), \ + &stats.inverted_index_stats); \ if (config::enable_file_cache) { \ io::FileCacheProfileReporter cache_profile(Parent->_segment_profile.get()); \ cache_profile.update(&stats.file_cache_stats); \ diff --git a/be/test/olap/inverted_index_profile_test.cpp b/be/test/olap/inverted_index_profile_test.cpp new file mode 100644 index 00000000000000..25dc63588c7b16 --- /dev/null +++ b/be/test/olap/inverted_index_profile_test.cpp @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/inverted_index_profile.h" + +#include + +#include + +#include "olap/inverted_index_stats.h" + +namespace doris { + +TEST(InvertedIndexProfileReporterTest, UpdateTest) { + auto runtime_profile = std::make_unique("test_profile"); + + InvertedIndexStatistics statistics; + statistics.stats.push_back({"test_column1", 101, 201}); + statistics.stats.push_back({"test_column2", 102, 202}); + + InvertedIndexProfileReporter reporter; + reporter.update(runtime_profile.get(), &statistics); + + ASSERT_EQ(runtime_profile->get_counter("HitRows_test_column1")->value(), 101); + ASSERT_EQ(runtime_profile->get_counter("ExecTime_test_column1")->value(), 201); + ASSERT_EQ(runtime_profile->get_counter("HitRows_test_column2")->value(), 102); + ASSERT_EQ(runtime_profile->get_counter("ExecTime_test_column2")->value(), 202); +} + +} // namespace doris \ No newline at end of file diff --git a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp index 02353fc54412c5..ae318571bbf394 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index/compaction/util/index_compaction_utils.cpp @@ -169,7 +169,7 @@ class IndexCompactionUtils { .ok()); auto result = std::make_shared(); EXPECT_TRUE(idx_reader - ->invoke_bkd_query(query_param->get_value(), + ->invoke_bkd_query(nullptr, query_param->get_value(), InvertedIndexQueryType::EQUAL_QUERY, *bkd_searcher, result) .ok()); diff --git a/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp new file mode 100644 index 00000000000000..fa2145544aff48 --- /dev/null +++ b/be/test/olap/rowset/segment_v2/inverted_index_fs_directory_test.cpp @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "olap/rowset/segment_v2/inverted_index_fs_directory.h" + +#include + +#include +#include +#include + +#include "common/config.h" +#include "io/fs/file_system.h" +#include "io/fs/local_file_system.h" + +namespace doris::segment_v2 { + +class DorisFSDirectoryTest : public ::testing::Test { +protected: + void SetUp() override { + // Enable debug points for testing + _original_enable_debug_points = config::enable_debug_points; + config::enable_debug_points = true; + + _tmp_dir = std::filesystem::temp_directory_path() / "doris_fs_directory_test"; + std::filesystem::remove_all(_tmp_dir); + std::filesystem::create_directories(_tmp_dir); + _fs = io::global_local_filesystem(); + _directory = std::make_unique(); + _directory->init(_fs, _tmp_dir.string().c_str()); + } + + void TearDown() override { + _directory.reset(); + std::filesystem::remove_all(_tmp_dir); + config::enable_debug_points = _original_enable_debug_points; + } + + std::filesystem::path _tmp_dir; + io::FileSystemSPtr _fs; + std::unique_ptr _directory; + bool _original_enable_debug_points; +}; + +TEST_F(DorisFSDirectoryTest, FSIndexInputReadInternalTimer) { + std::string file_name = "test_timer_file"; + std::filesystem::path test_file = _tmp_dir / file_name; + std::ofstream ofs(test_file); + std::string content = "some test content for timer"; + ofs << content; + ofs.close(); + + lucene::store::IndexInput* input1 = nullptr; + CLuceneError error; + bool result = + DorisFSDirectory::FSIndexInput::open(_fs, test_file.string().c_str(), input1, error); + EXPECT_TRUE(result); + ASSERT_NE(input1, nullptr); + + auto* fs_input1 = dynamic_cast(input1); + ASSERT_NE(fs_input1, nullptr); + + io::FileCacheStatistics stats; + fs_input1->_io_ctx.file_cache_stats = &stats; + + auto* input2 = fs_input1->clone(); + auto* fs_input2 = dynamic_cast(input2); + ASSERT_NE(fs_input2, nullptr); + + fs_input2->_io_ctx.file_cache_stats = &stats; + + uint8_t buffer1[10]; + input1->readBytes(buffer1, 10); + EXPECT_GT(stats.inverted_index_io_timer, 0); + int64_t old_time = stats.inverted_index_io_timer; + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + input2->seek(0); + uint8_t buffer2[10]; + input2->readBytes(buffer2, 10); + EXPECT_GT(stats.inverted_index_io_timer, old_time); + + _CLDELETE(input2); + _CLDELETE(input1); +} + +} // namespace doris::segment_v2 From a425e42f0434b05c72f3f4d09c92e23eb5ce0669 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 27 Jun 2025 17:29:35 +0800 Subject: [PATCH 096/572] branch-3.0: [bvar](cloud-mow) Add bvar for mow compaction get delete bitmap lock backoff sleep time (#52044) (#52297) pick https://github.com/apache/doris/pull/52044 --- be/src/cloud/cloud_meta_mgr.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index d1c6d4a5f2305b..dff85b3f639258 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -139,6 +139,8 @@ bvar::LatencyRecorder g_cloud_commit_txn_resp_redirect_latency("cloud_table_stat bvar::Adder g_cloud_meta_mgr_rpc_timeout_count("cloud_meta_mgr_rpc_timeout_count"); bvar::Window> g_cloud_ms_rpc_timeout_count_window( "cloud_meta_mgr_rpc_timeout_qps", &g_cloud_meta_mgr_rpc_timeout_count, 30); +bvar::LatencyRecorder g_cloud_be_mow_get_dbm_lock_backoff_sleep_time( + "cloud_be_mow_get_dbm_lock_backoff_sleep_time"); class MetaServiceProxy { public: @@ -1378,6 +1380,7 @@ Status CloudMetaMgr::get_delete_bitmap_update_lock(const CloudTablet& tablet, in Status st; std::default_random_engine rng = make_random_engine(); std::uniform_int_distribution u(500, 2000); + uint64_t backoff_sleep_time_ms {0}; do { st = retry_rpc("get delete bitmap update lock", req, &res, &MetaService_Stub::get_delete_bitmap_update_lock); @@ -1389,8 +1392,12 @@ Status CloudMetaMgr::get_delete_bitmap_update_lock(const CloudTablet& tablet, in LOG(WARNING) << "get delete bitmap lock conflict. " << debug_info(req) << " retry_times=" << retry_times << " sleep=" << duration_ms << "ms : " << res.status().msg(); + auto start = std::chrono::steady_clock::now(); bthread_usleep(duration_ms * 1000); + auto end = std::chrono::steady_clock::now(); + backoff_sleep_time_ms += duration_cast(end - start).count(); } while (++retry_times <= 100); + g_cloud_be_mow_get_dbm_lock_backoff_sleep_time << backoff_sleep_time_ms; if (res.status().code() == MetaServiceCode::KV_TXN_CONFLICT_RETRY_EXCEEDED_MAX_TIMES) { return Status::Error( "txn conflict when get delete bitmap update lock, table_id {}, lock_id {}, " From 547bfed4a42fd4e290a8c82a529dfc84bf0485b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 17:38:40 +0800 Subject: [PATCH 097/572] branch-3.0: [chore](dep)bump up parquet to 1.15.2 #51372 (#51601) Cherry-picked from #51372 Co-authored-by: Calvin Kirs --- fe/fe-common/pom.xml | 4 ++++ fe/pom.xml | 2 +- fe/spark-dpp/pom.xml | 12 ------------ 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index 8203b482410c5c..042f3c103c7c47 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -51,6 +51,10 @@ under the License. com.google.guava guava + + commons-io + commons-io + org.apache.thrift diff --git a/fe/pom.xml b/fe/pom.xml index 8343d5d138c8d3..215d887370cb79 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -224,7 +224,7 @@ under the License. 2.1.4 1.11.4 - 1.13.1 + 1.15.2 3.4.3 0.15.0 compile diff --git a/fe/spark-dpp/pom.xml b/fe/spark-dpp/pom.xml index df9549646f4a84..f59b4b7e55d845 100644 --- a/fe/spark-dpp/pom.xml +++ b/fe/spark-dpp/pom.xml @@ -74,18 +74,6 @@ under the License. org.apache.hadoop hadoop-common - - org.apache.parquet - parquet-column - - - org.apache.parquet - parquet-hadoop - - - org.apache.parquet - parquet-common - commons-collections commons-collections From 298548df1a84dc5d646f9f623d4c10c7af773c2f Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Fri, 27 Jun 2025 21:02:55 +0800 Subject: [PATCH 098/572] [branch-3.0] [Feature](recycler) Add recycler metrics for instance layer (#51448) (#52428) pick https://github.com/apache/doris/pull/51856 https://github.com/apache/doris/pull/51448 --- cloud/src/common/bvars.cpp | 40 +- cloud/src/common/bvars.h | 47 +- cloud/src/common/config.h | 4 + cloud/src/main.cpp | 3 +- cloud/src/recycler/recycler.cpp | 944 +++++++++++++++++++++++++++----- cloud/src/recycler/recycler.h | 134 ++++- cloud/test/recycler_test.cpp | 46 +- 7 files changed, 1019 insertions(+), 199 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 23e140d3c2d820..29a8d1447bc408 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -103,14 +104,41 @@ BvarStatusWithTag g_bvar_recycler_recycle_rowset_earlest_ts("recycler", BvarStatusWithTag g_bvar_recycler_recycle_tmp_rowset_earlest_ts("recycler", "recycle_tmp_rowset_earlest_ts"); BvarStatusWithTag g_bvar_recycler_recycle_expired_txn_label_earlest_ts("recycler", "recycle_expired_txn_label_earlest_ts"); bvar::Status g_bvar_recycler_task_max_concurrency("recycler_task_max_concurrency_num",0); -bvar::Adder g_bvar_recycler_task_concurrency; +// current concurrency of recycle task +bvar::Adder g_bvar_recycler_instance_recycle_task_concurrency; // recycler's mbvars -mBvarIntAdder g_bvar_recycler_instance_running("recycler_instance_running",{"instance_id"}); -mBvarLongStatus g_bvar_recycler_instance_last_recycle_duration("recycler_instance_last_recycle_duration_ms",{"instance_id"}); -mBvarLongStatus g_bvar_recycler_instance_next_time("recycler_instance_next_time_s",{"instance_id"}); -mBvarPairStatus g_bvar_recycler_instance_recycle_times("recycler_instance_recycle_times",{"instance_id"}); -mBvarLongStatus g_bvar_recycler_instance_recycle_last_success_times("recycler_instance_recycle_last_success_times",{"instance_id"}); +bvar::Adder g_bvar_recycler_instance_running_counter("recycler_instance_running_counter"); +// cost time of the last whole recycle process +mBvarStatus g_bvar_recycler_instance_last_recycle_duration("recycler_instance_last_recycle_duration",{"instance_id"}); +mBvarStatus g_bvar_recycler_instance_next_ts("recycler_instance_next_ts",{"instance_id"}); +// start and end timestamps of the recycle process +mBvarStatus g_bvar_recycler_instance_recycle_st_ts("recycler_instance_recycle_st_ts",{"instance_id"}); +mBvarStatus g_bvar_recycler_instance_recycle_ed_ts("recycler_instance_recycle_ed_ts",{"instance_id"}); +mBvarStatus g_bvar_recycler_instance_recycle_last_success_ts("recycler_instance_recycle_last_success_ts",{"instance_id"}); + +// recycler's mbvars +// instance_id: unique identifier for the instance +// resource_type: type of resource need to be recycled (index, partition, rowset, segment, tablet, etc.) +// resource_id: unique identifier for the repository +// status: status of the recycle task (normal, abnormal, etc.) +mBvarIntAdder g_bvar_recycler_vault_recycle_status("recycler_vault_recycle_status", {"instance_id", "resource_id", "status"}); +// current concurrency of vault delete task +mBvarIntAdder g_bvar_recycler_vault_recycle_task_concurrency("recycler_vault_recycle_task_concurrency", {"instance_id", "resource_type", "resource_id"}); +mBvarStatus g_bvar_recycler_instance_last_round_recycled_num("recycler_instance_last_round_recycled_num", {"instance_id", "resource_type"}); +mBvarStatus g_bvar_recycler_instance_last_round_to_recycle_num("recycler_instance_last_round_to_recycle_num", {"instance_id", "resource_type"}); +mBvarStatus g_bvar_recycler_instance_last_round_recycled_bytes("recycler_instance_last_round_recycled_bytes", {"instance_id", "resource_type"}); +mBvarStatus g_bvar_recycler_instance_last_round_to_recycle_bytes("recycler_instance_last_round_to_recycle_bytes", {"instance_id", "resource_type"}); +mBvarStatus g_bvar_recycler_instance_last_round_recycle_elpased_ts("recycler_instance_last_round_recycle_elpased_ts", {"instance_id", "resource_type"}); +// total recycled num and bytes of resources since recycler started +mBvarIntAdder g_bvar_recycler_instance_recycle_total_num_since_started("recycler_instance_recycle_total_num_since_started", {"instance_id", "resource_type"}); +mBvarIntAdder g_bvar_recycler_instance_recycle_total_bytes_since_started("recycler_instance_recycle_total_bytes_since_started", {"instance_id", "resource_type"}); +mBvarIntAdder g_bvar_recycler_instance_recycle_round("recycler_instance_recycle_round", {"instance_id", "resource_type"}); +// represents the ms required per resource to be recycled +// value of -1 means no resource recycled +mBvarStatus g_bvar_recycler_instance_recycle_time_per_resource("recycler_instance_recycle_time_per_resource", {"instance_id", "resource_type"}); +// represents the bytes of resources that can be recycled per ms +mBvarStatus g_bvar_recycler_instance_recycle_bytes_per_ms("recycler_instance_recycle_bytes_per_ms", {"instance_id", "resource_type"}); // txn_kv's bvars bvar::LatencyRecorder g_bvar_txn_kv_get("txn_kv", "get"); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 455d8caf45a1d4..e8b5a9a0f5bd9f 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -139,8 +140,7 @@ class mBvarWrapper { BvarType* stats = counter_.get_stats(std::list(dim_values)); if (stats) { if constexpr (std::is_same_v> || - std::is_same_v> || - is_pair_status::value) { + std::is_same_v>) { stats->set_value(value); } else { *stats << value; @@ -160,8 +160,6 @@ class mBvarWrapper { template struct is_valid_bvar_type : std::false_type {}; template - struct is_pair_status : std::false_type {}; - template struct is_valid_bvar_type> : std::true_type {}; template <> struct is_valid_bvar_type : std::true_type {}; @@ -169,8 +167,6 @@ class mBvarWrapper { struct is_valid_bvar_type> : std::true_type {}; template struct is_valid_bvar_type> : std::true_type {}; - template - struct is_pair_status>> : std::true_type {}; template <> struct is_valid_bvar_type : std::true_type {}; @@ -183,18 +179,8 @@ using mBvarIntRecorder = mBvarWrapper; using mBvarLatencyRecorder = mBvarWrapper; using mBvarIntMaxer = mBvarWrapper>; using mBvarDoubleMaxer = mBvarWrapper>; -using mBvarLongStatus = mBvarWrapper>; -using mBvarDoubleStatus = mBvarWrapper>; - -namespace std { -template -inline std::ostream& operator<<(std::ostream& os, const std::pair& p) { - return os << "{" << p.first << "," << p.second << "}"; -} -} // namespace std - template -using mBvarPairStatus = mBvarWrapper>>; +using mBvarStatus = mBvarWrapper>; // meta-service's bvars extern BvarLatencyRecorderWithTag g_bvar_ms_begin_txn; @@ -270,13 +256,28 @@ extern BvarStatusWithTag g_bvar_recycler_recycle_rowset_earlest_ts; extern BvarStatusWithTag g_bvar_recycler_recycle_tmp_rowset_earlest_ts; extern BvarStatusWithTag g_bvar_recycler_recycle_expired_txn_label_earlest_ts; +// recycler's mbvars extern bvar::Status g_bvar_recycler_task_max_concurrency; -extern bvar::Adder g_bvar_recycler_task_concurrency; -extern mBvarIntAdder g_bvar_recycler_instance_running; -extern mBvarLongStatus g_bvar_recycler_instance_last_recycle_duration; -extern mBvarLongStatus g_bvar_recycler_instance_next_time; -extern mBvarPairStatus g_bvar_recycler_instance_recycle_times; -extern mBvarLongStatus g_bvar_recycler_instance_recycle_last_success_times; +extern bvar::Adder g_bvar_recycler_instance_recycle_task_concurrency; +extern bvar::Adder g_bvar_recycler_instance_running_counter; +extern mBvarStatus g_bvar_recycler_instance_last_recycle_duration; +extern mBvarStatus g_bvar_recycler_instance_next_ts; +extern mBvarStatus g_bvar_recycler_instance_recycle_st_ts; +extern mBvarStatus g_bvar_recycler_instance_recycle_ed_ts; +extern mBvarStatus g_bvar_recycler_instance_recycle_last_success_ts; + +extern mBvarIntAdder g_bvar_recycler_vault_recycle_status; +extern mBvarIntAdder g_bvar_recycler_vault_recycle_task_concurrency; +extern mBvarStatus g_bvar_recycler_instance_last_round_recycled_num; +extern mBvarStatus g_bvar_recycler_instance_last_round_to_recycle_num; +extern mBvarStatus g_bvar_recycler_instance_last_round_recycled_bytes; +extern mBvarStatus g_bvar_recycler_instance_last_round_to_recycle_bytes; +extern mBvarStatus g_bvar_recycler_instance_last_round_recycle_elpased_ts; +extern mBvarIntAdder g_bvar_recycler_instance_recycle_total_num_since_started; +extern mBvarIntAdder g_bvar_recycler_instance_recycle_total_bytes_since_started; +extern mBvarIntAdder g_bvar_recycler_instance_recycle_round; +extern mBvarStatus g_bvar_recycler_instance_recycle_time_per_resource; +extern mBvarStatus g_bvar_recycler_instance_recycle_bytes_per_ms; // txn_kv's bvars extern bvar::LatencyRecorder g_bvar_txn_kv_get; diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 31607057a057f3..62cd5c9343b5b7 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -47,6 +47,8 @@ CONF_Int64(fdb_txn_timeout_ms, "10000"); CONF_Int64(brpc_max_body_size, "3147483648"); CONF_Int64(brpc_socket_max_unwritten_bytes, "1073741824"); +CONF_String(bvar_max_dump_multi_dimension_metric_num, "5000"); + // logging CONF_String(log_dir, "./log/"); CONF_String(log_level, "info"); // info warn error fatal @@ -102,6 +104,8 @@ CONF_mInt64(delete_bitmap_storage_optimize_check_version_gap, "1000"); CONF_mInt32(scan_instances_interval_seconds, "60"); // 1min // interval for check object CONF_mInt32(check_object_interval_seconds, "43200"); // 12hours +// enable recycler metrics statistics +CONF_Bool(enable_recycler_metrics, "false"); CONF_mInt64(check_recycle_task_interval_seconds, "600"); // 10min CONF_mInt64(recycler_sleep_before_scheduling_seconds, "60"); diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 18cf98720e9cb0..0aad97aab4d4c3 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -236,7 +236,8 @@ int main(int argc, char** argv) { std::cout << "try to start meta_service, recycler" << std::endl; } - google::SetCommandLineOption("bvar_max_dump_multi_dimension_metric_number", "2000"); + google::SetCommandLineOption("bvar_max_dump_multi_dimension_metric_number", + config::bvar_max_dump_multi_dimension_metric_num.c_str()); brpc::Server server; brpc::FLAGS_max_body_size = config::brpc_max_body_size; diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index 0ec4fc917171b1..c688097f83f71b 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -20,10 +20,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -64,6 +66,9 @@ namespace doris::cloud { using namespace std::chrono; +static RecyclerMetricsContext tablet_metrics_context_("global_recycler", "recycle_tablet"); +static RecyclerMetricsContext segment_metrics_context_("global_recycler", "recycle_segment"); + // return 0 for success get a key, 1 for key not found, negative for error [[maybe_unused]] static int txn_get(TxnKv* txn_kv, std::string_view key, std::string& val) { std::unique_ptr txn; @@ -165,7 +170,7 @@ static inline void check_recycle_task(const std::string& instance_id, const std: int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; if (cost > config::recycle_task_threshold_seconds) { - LOG_INFO("recycle task cost too much time cost={}s", cost) + LOG_WARNING("recycle task cost too much time cost={}s", cost) .tag("instance_id", instance_id) .tag("task", task_name) .tag("num_scanned", num_scanned) @@ -277,14 +282,18 @@ void Recycler::recycle_callback() { recycling_instance_map_.emplace(instance_id, instance_recycler); } if (stopped()) return; - LOG_INFO("begin to recycle instance").tag("instance_id", instance_id); + LOG_WARNING("begin to recycle instance").tag("instance_id", instance_id); auto ctime_ms = duration_cast(system_clock::now().time_since_epoch()).count(); - g_bvar_recycler_task_concurrency << 1; - g_bvar_recycler_instance_running.put({instance_id}, 1); - g_bvar_recycler_instance_recycle_times.put({instance_id}, std::make_pair(ctime_ms, -1)); + g_bvar_recycler_instance_recycle_task_concurrency << 1; + g_bvar_recycler_instance_running_counter << 1; + g_bvar_recycler_instance_recycle_st_ts.put({instance_id}, ctime_ms); + tablet_metrics_context_.reset(); + segment_metrics_context_.reset(); ret = instance_recycler->do_recycle(); - g_bvar_recycler_task_concurrency << -1; - g_bvar_recycler_instance_running.put({instance_id}, -1); + tablet_metrics_context_.finish_report(); + segment_metrics_context_.finish_report(); + g_bvar_recycler_instance_recycle_task_concurrency << -1; + g_bvar_recycler_instance_running_counter << -1; // If instance recycler has been aborted, don't finish this job if (!instance_recycler->stopped()) { finish_instance_recycle_job(txn_kv_.get(), recycle_job_key, instance_id, ip_port_, @@ -294,19 +303,20 @@ void Recycler::recycle_callback() { std::lock_guard lock(mtx_); recycling_instance_map_.erase(instance_id); } + auto now = duration_cast(system_clock::now().time_since_epoch()).count(); auto elpased_ms = now - ctime_ms; - g_bvar_recycler_instance_recycle_times.put({instance_id}, std::make_pair(ctime_ms, now)); + g_bvar_recycler_instance_recycle_ed_ts.put({instance_id}, now); g_bvar_recycler_instance_last_recycle_duration.put({instance_id}, elpased_ms); - g_bvar_recycler_instance_next_time.put({instance_id}, - now + config::recycle_interval_seconds * 1000); + g_bvar_recycler_instance_next_ts.put({instance_id}, + now + config::recycle_interval_seconds * 1000); LOG(INFO) << "recycle instance done, " << "instance_id=" << instance_id << " ret=" << ret << " ctime_ms: " << ctime_ms << " now: " << now; - g_bvar_recycler_instance_recycle_last_success_times.put({instance_id}, now); + g_bvar_recycler_instance_recycle_last_success_ts.put({instance_id}, now); - LOG_INFO("finish recycle instance") + LOG_WARNING("finish recycle instance") .tag("instance_id", instance_id) .tag("cost_ms", elpased_ms); } @@ -700,16 +710,16 @@ int InstanceRecycler::do_recycle() { * 3. remove instance kv */ int InstanceRecycler::recycle_deleted_instance() { - LOG_INFO("begin to recycle deleted instance").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle deleted instance").tag("instance_id", instance_id_); int ret = 0; auto start_time = steady_clock::now(); std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { auto cost = duration(steady_clock::now() - start_time).count(); - LOG(INFO) << (ret == 0 ? "successfully" : "failed to") - << " recycle deleted instance, cost=" << cost - << "s, instance_id=" << instance_id_; + LOG(WARNING) << (ret == 0 ? "successfully" : "failed to") + << " recycle deleted instance, cost=" << cost + << "s, instance_id=" << instance_id_; }); // delete all remote data @@ -805,6 +815,7 @@ int InstanceRecycler::recycle_indexes() { int64_t num_scanned = 0; int64_t num_expired = 0; int64_t num_recycled = 0; + RecyclerMetricsContext metrics_context(instance_id_, task_name); RecycleIndexKeyInfo index_key_info0 {instance_id_, 0}; RecycleIndexKeyInfo index_key_info1 {instance_id_, INT64_MAX}; @@ -813,7 +824,7 @@ int InstanceRecycler::recycle_indexes() { recycle_index_key(index_key_info0, &index_key0); recycle_index_key(index_key_info1, &index_key1); - LOG_INFO("begin to recycle indexes").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle indexes").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -822,7 +833,8 @@ int InstanceRecycler::recycle_indexes() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle indexes finished, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle indexes finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) @@ -905,19 +917,61 @@ int InstanceRecycler::recycle_indexes() { return -1; } } - if (recycle_tablets(index_pb.table_id(), index_id) != 0) { + if (recycle_tablets(index_pb.table_id(), index_id, metrics_context) != 0) { LOG_WARNING("failed to recycle tablets under index") .tag("table_id", index_pb.table_id()) .tag("instance_id", instance_id_) .tag("index_id", index_id); return -1; } - ++num_recycled; + metrics_context.total_recycled_num = ++num_recycled; + metrics_context.report(); check_recycle_task(instance_id_, task_name, num_scanned, num_recycled, start_time); index_keys.push_back(k); return 0; }; + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { + RecycleIndexPB index_pb; + if (!index_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t current_time = ::time(nullptr); + if (current_time < calc_expiration(index_pb)) { + return 0; + } + // decode index_id + auto k1 = k; + k1.remove_prefix(1); + std::vector, int, int>> out; + decode_key(&k1, &out); + // 0x01 "recycle" ${instance_id} "index" ${index_id} -> RecycleIndexPB + auto index_id = std::get(std::get<0>(out[3])); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::string val; + err = txn->get(k, &val); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return 0; + } + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + index_pb.Clear(); + if (!index_pb.ParseFromString(val)) { + return 0; + } + if (scan_tablets_and_statistics(index_pb.table_id(), index_id, metrics_context) != 0) { + return 0; + } + metrics_context.total_need_recycle_num++; + return 0; + }; + auto loop_done = [&index_keys, this]() -> int { if (index_keys.empty()) return 0; std::unique_ptr> defer((int*)0x01, @@ -929,7 +983,9 @@ int InstanceRecycler::recycle_indexes() { return 0; }; - return scan_and_recycle(index_key0, index_key1, std::move(recycle_func), std::move(loop_done)); + return scan_for_recycle_and_statistics(index_key0, index_key1, "indexes", metrics_context, + std::move(scan_and_statistics), std::move(recycle_func), + std::move(loop_done)); } bool check_lazy_txn_finished(std::shared_ptr txn_kv, const std::string instance_id, @@ -1022,6 +1078,7 @@ int InstanceRecycler::recycle_partitions() { int64_t num_scanned = 0; int64_t num_expired = 0; int64_t num_recycled = 0; + RecyclerMetricsContext metrics_context(instance_id_, task_name); RecyclePartKeyInfo part_key_info0 {instance_id_, 0}; RecyclePartKeyInfo part_key_info1 {instance_id_, INT64_MAX}; @@ -1030,7 +1087,7 @@ int InstanceRecycler::recycle_partitions() { recycle_partition_key(part_key_info0, &part_key0); recycle_partition_key(part_key_info1, &part_key1); - LOG_INFO("begin to recycle partitions").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle partitions").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -1039,7 +1096,8 @@ int InstanceRecycler::recycle_partitions() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle partitions finished, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle partitions finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) @@ -1129,7 +1187,8 @@ int InstanceRecycler::recycle_partitions() { int ret = 0; for (int64_t index_id : part_pb.index_id()) { - if (recycle_tablets(part_pb.table_id(), index_id, partition_id, is_empty_tablet) != 0) { + if (recycle_tablets(part_pb.table_id(), index_id, metrics_context, partition_id, + is_empty_tablet) != 0) { LOG_WARNING("failed to recycle tablets under partition") .tag("table_id", part_pb.table_id()) .tag("instance_id", instance_id_) @@ -1146,7 +1205,56 @@ int InstanceRecycler::recycle_partitions() { partition_version_keys.push_back(partition_version_key( {instance_id_, part_pb.db_id(), part_pb.table_id(), partition_id})); } + metrics_context.report(); + } + return ret; + }; + + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&, this](std::string_view k, std::string_view v) -> int { + RecyclePartitionPB part_pb; + if (!part_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t current_time = ::time(nullptr); + if (current_time < calc_expiration(part_pb)) { + return 0; + } + // decode partition_id + auto k1 = k; + k1.remove_prefix(1); + std::vector, int, int>> out; + decode_key(&k1, &out); + // 0x01 "recycle" ${instance_id} "partition" ${partition_id} -> RecyclePartitionPB + auto partition_id = std::get(std::get<0>(out[3])); + // Change state to RECYCLING + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::string val; + err = txn->get(k, &val); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return 0; + } + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + part_pb.Clear(); + if (!part_pb.ParseFromString(val)) { + return 0; + } + // Partitions with PREPARED state MUST have no data + bool is_empty_tablet = part_pb.state() == RecyclePartitionPB::PREPARED; + int ret = 0; + for (int64_t index_id : part_pb.index_id()) { + if (scan_tablets_and_statistics(part_pb.table_id(), index_id, metrics_context, + partition_id, is_empty_tablet) != 0) { + ret = 0; + } } + metrics_context.total_need_recycle_num++; return ret; }; @@ -1177,20 +1285,24 @@ int InstanceRecycler::recycle_partitions() { return 0; }; - return scan_and_recycle(part_key0, part_key1, std::move(recycle_func), std::move(loop_done)); + return scan_for_recycle_and_statistics(part_key0, part_key1, "partitions", metrics_context, + std::move(scan_and_statistics), std::move(recycle_func), + std::move(loop_done)); } int InstanceRecycler::recycle_versions() { int64_t num_scanned = 0; int64_t num_recycled = 0; + RecyclerMetricsContext metrics_context(instance_id_, "recycle_versions"); - LOG_INFO("begin to recycle table and partition versions").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle table and partition versions").tag("instance_id", instance_id_); auto start_time = steady_clock::now(); std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { auto cost = duration(steady_clock::now() - start_time).count(); - LOG_INFO("recycle table and partition versions finished, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle table and partition versions finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_recycled", num_recycled); @@ -1200,8 +1312,8 @@ int InstanceRecycler::recycle_versions() { auto version_key_end = partition_version_key({instance_id_, INT64_MAX, 0, 0}); int64_t last_scanned_table_id = 0; bool is_recycled = false; // Is last scanned kv recycled - auto recycle_func = [&num_scanned, &num_recycled, &last_scanned_table_id, &is_recycled, this]( - std::string_view k, std::string_view) { + auto recycle_func = [&num_scanned, &num_recycled, &last_scanned_table_id, &is_recycled, + &metrics_context, this](std::string_view k, std::string_view) { ++num_scanned; auto k1 = k; k1.remove_prefix(1); @@ -1249,15 +1361,58 @@ int InstanceRecycler::recycle_versions() { if (err != TxnErrorCode::TXN_OK) { return -1; } - ++num_recycled; + metrics_context.total_recycled_num = ++num_recycled; + metrics_context.report(); is_recycled = true; return 0; }; - return scan_and_recycle(version_key_begin, version_key_end, std::move(recycle_func)); + int64_t last_scanned_table_id_t = 0; + bool is_recycled_t = false; // Is last scanned kv recycled + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, &last_scanned_table_id_t, &is_recycled_t, this]( + std::string_view k, std::string_view) { + auto k1 = k; + k1.remove_prefix(1); + // 0x01 "version" ${instance_id} "partition" ${db_id} ${tbl_id} ${partition_id} + std::vector, int, int>> out; + decode_key(&k1, &out); + DCHECK_EQ(out.size(), 6) << k; + auto table_id = std::get(std::get<0>(out[4])); + if (table_id == last_scanned_table_id_t) { // Already handle kvs of this table + metrics_context.total_need_recycle_num += + is_recycled_t; // Version kv of this table has been recycled + return 0; + } + last_scanned_table_id_t = table_id; + is_recycled_t = false; + auto tablet_key_begin = stats_tablet_key({instance_id_, table_id, 0, 0, 0}); + auto tablet_key_end = stats_tablet_key({instance_id_, table_id, INT64_MAX, 0, 0}); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::unique_ptr iter; + err = txn->get(tablet_key_begin, tablet_key_end, &iter, false, 1); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + if (iter->has_next()) { // Table is useful, should not recycle table and partition versions + return 0; + } + metrics_context.total_need_recycle_num++; + is_recycled_t = true; + return 0; + }; + + return scan_for_recycle_and_statistics(version_key_begin, version_key_end, "versions", + metrics_context, std::move(scan_and_statistics), + std::move(recycle_func)); } -int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_t partition_id, +int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, + RecyclerMetricsContext& metrics_context, int64_t partition_id, bool is_empty_tablet) { int64_t num_scanned = 0; std::atomic_long num_recycled = 0; @@ -1336,8 +1491,8 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, int64_ tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id_, tablet_id})); if (!is_empty_tablet) { sync_executor.add([this, &num_recycled, tid = tablet_id, range_move = use_range_remove, - k]() mutable -> TabletKeyPair { - if (recycle_tablet(tid) != 0) { + &metrics_context, k]() mutable -> TabletKeyPair { + if (recycle_tablet(tid, metrics_context) != 0) { LOG_WARNING("failed to recycle tablet") .tag("instance_id", instance_id_) .tag("tablet_id", tid); @@ -1521,18 +1676,20 @@ int InstanceRecycler::delete_rowset_data(const doris::RowsetMetaCloudPB& rs_meta return accessor->delete_files(file_paths); } -int InstanceRecycler::delete_rowset_data(const std::vector& rowsets, - RowsetRecyclingState type) { +int InstanceRecycler::delete_rowset_data( + const std::map& rowsets, RowsetRecyclingState type, + RecyclerMetricsContext& metrics_context) { int ret = 0; // resource_id -> file_paths std::map> resource_file_paths; // (resource_id, tablet_id, rowset_id) std::vector> rowsets_delete_by_prefix; + bool is_formal_rowset = (type == RowsetRecyclingState::FORMAL_ROWSET); - for (const auto& rs : rowsets) { + for (const auto& [_, rs] : rowsets) { // we have to treat tmp rowset as "orphans" that may not related to any existing tablets // due to aborted schema change. - if (type == RowsetRecyclingState::FORMAL_ROWSET) { + if (is_formal_rowset) { std::lock_guard lock(recycled_tablets_mtx_); if (recycled_tablets_.count(rs.tablet_id())) { continue; // Rowset data has already been deleted @@ -1657,7 +1814,42 @@ int InstanceRecycler::delete_rowset_data(const std::vectordelete_files(*paths); + int ret = accessor->delete_files(*paths); + if (!ret) { + // deduplication of different files with the same rowset id + // 020000000000007fd045a62bc87a6587dd7ac274aa36e5a9_0.dat + //020000000000007fd045a62bc87a6587dd7ac274aa36e5a9_0.idx + std::set deleted_rowset_id; + + std::for_each( + paths->begin(), paths->end(), + [&metrics_context, &rowsets, &deleted_rowset_id](const std::string& path) { + std::vector str; + butil::SplitString(path, '/', &str); + std::string rowset_id; + if (auto pos = str.back().find('_'); pos != std::string::npos) { + rowset_id = str.back().substr(0, pos); + } else { + LOG(WARNING) << "failed to parse rowset_id, path=" << path; + return; + } + auto rs_meta = rowsets.find(rowset_id); + if (rs_meta != rowsets.end() && + !deleted_rowset_id.contains(rowset_id)) { + deleted_rowset_id.emplace(rowset_id); + metrics_context.total_recycled_data_size += + rs_meta->second.total_disk_size(); + segment_metrics_context_.total_recycled_num += + rs_meta->second.num_segments(); + segment_metrics_context_.total_recycled_data_size += + rs_meta->second.total_disk_size(); + metrics_context.total_recycled_num++; + } + }); + segment_metrics_context_.report(); + metrics_context.report(); + } + return ret; }); } for (const auto& [resource_id, tablet_id, rowset_id] : rowsets_delete_by_prefix) { @@ -1665,9 +1857,21 @@ int InstanceRecycler::delete_rowset_data(const std::vector int { return delete_rowset_data(resource_id, tablet_id, rowset_id); }); + concurrent_delete_executor.add([&]() -> int { + int ret = delete_rowset_data(resource_id, tablet_id, rowset_id); + if (!ret) { + auto rs = rowsets.at(rowset_id); + metrics_context.total_recycled_data_size += rs.total_disk_size(); + metrics_context.total_recycled_num++; + segment_metrics_context_.total_recycled_data_size += rs.total_disk_size(); + segment_metrics_context_.total_recycled_num += rs.num_segments(); + metrics_context.report(); + segment_metrics_context_.report(); + } + return ret; + }); } + bool finished = true; std::vector rets = concurrent_delete_executor.when_all(&finished); for (int r : rets) { @@ -1695,7 +1899,122 @@ int InstanceRecycler::delete_rowset_data(const std::string& resource_id, int64_t return accessor->delete_prefix(rowset_path_prefix(tablet_id, rowset_id)); } -int InstanceRecycler::recycle_tablet(int64_t tablet_id) { +int InstanceRecycler::scan_tablets_and_statistics(int64_t table_id, int64_t index_id, + RecyclerMetricsContext& metrics_context, + int64_t partition_id, bool is_empty_tablet) { + std::string tablet_key_begin, tablet_key_end; + + if (partition_id > 0) { + meta_tablet_key({instance_id_, table_id, index_id, partition_id, 0}, &tablet_key_begin); + meta_tablet_key({instance_id_, table_id, index_id, partition_id + 1, 0}, &tablet_key_end); + } else { + meta_tablet_key({instance_id_, table_id, index_id, 0, 0}, &tablet_key_begin); + meta_tablet_key({instance_id_, table_id, index_id + 1, 0, 0}, &tablet_key_end); + } + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&, is_empty_tablet, this](std::string_view k, + std::string_view v) -> int { + doris::TabletMetaCloudPB tablet_meta_pb; + if (!tablet_meta_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t tablet_id = tablet_meta_pb.tablet_id(); + + if (!check_lazy_txn_finished(txn_kv_, instance_id_, tablet_meta_pb.tablet_id())) { + return 0; + } + + if (!is_empty_tablet) { + if (scan_tablet_and_statistics(tablet_id, metrics_context) != 0) { + return 0; + } + tablet_metrics_context_.total_need_recycle_num++; + } + return 0; + }; + return scan_and_recycle(tablet_key_begin, tablet_key_end, std::move(scan_and_statistics)); +} + +int InstanceRecycler::scan_tablet_and_statistics(int64_t tablet_id, + RecyclerMetricsContext& metrics_context) { + int ret = 0; + std::map rowset_meta_map; + std::unique_ptr txn; + if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { + LOG_WARNING("failed to recycle tablet ") + .tag("tablet id", tablet_id) + .tag("instance_id", instance_id_) + .tag("reason", "failed to create txn"); + ret = -1; + } + GetRowsetResponse resp; + std::string msg; + MetaServiceCode code = MetaServiceCode::OK; + // get rowsets in tablet + internal_get_rowset(txn.get(), 0, std::numeric_limits::max() - 1, instance_id_, + tablet_id, code, msg, &resp); + if (code != MetaServiceCode::OK) { + LOG_WARNING("failed to get rowsets of tablet when recycle tablet") + .tag("tablet id", tablet_id) + .tag("msg", msg) + .tag("code", code) + .tag("instance id", instance_id_); + ret = -1; + } + for (const auto& rs_meta : resp.rowset_meta()) { + /* + * For compatibility, we skip the loop for [0-1] here. + * The purpose of this loop is to delete object files, + * and since [0-1] only has meta and doesn't have object files, + * skipping it doesn't affect system correctness. + * + * If not skipped, the check "if (!rs_meta.has_resource_id())" below + * would return error -1 directly, causing the recycle operation to fail. + * + * [0-1] doesn't have resource id is a bug. + * In the future, we will fix this problem, after that, + * we can remove this if statement. + * + * TODO(Yukang-Lian): remove this if statement when [0-1] has resource id in the future. + */ + + if (rs_meta.end_version() == 1) { + // Assert that [0-1] has no resource_id to make sure + // this if statement will not be forgetted to remove + // when the resource id bug is fixed + DCHECK(!rs_meta.has_resource_id()) << "rs_meta" << rs_meta.ShortDebugString(); + continue; + } + if (!rs_meta.has_resource_id()) { + LOG_WARNING("rowset meta does not have a resource id, impossible!") + .tag("rs_meta", rs_meta.ShortDebugString()) + .tag("instance_id", instance_id_) + .tag("tablet_id", tablet_id); + continue; + } + DCHECK(rs_meta.has_resource_id()) << "rs_meta" << rs_meta.ShortDebugString(); + auto it = accessor_map_.find(rs_meta.resource_id()); + // possible if the accessor is not initilized correctly + if (it == accessor_map_.end()) [[unlikely]] { + LOG_WARNING( + "failed to find resource id when recycle tablet, skip this vault accessor " + "recycle process") + .tag("tablet id", tablet_id) + .tag("instance_id", instance_id_) + .tag("resource_id", rs_meta.resource_id()) + .tag("rowset meta pb", rs_meta.ShortDebugString()); + continue; + } + + metrics_context.total_need_recycle_data_size += rs_meta.total_disk_size(); + tablet_metrics_context_.total_need_recycle_data_size += rs_meta.total_disk_size(); + segment_metrics_context_.total_need_recycle_data_size += rs_meta.total_disk_size(); + segment_metrics_context_.total_need_recycle_num += rs_meta.num_segments(); + } + return ret; +} + +int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& metrics_context) { LOG_INFO("begin to recycle rowsets in a dropped tablet") .tag("instance_id", instance_id_) .tag("tablet_id", tablet_id); @@ -1711,7 +2030,7 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { std::string recyc_rs_key0 = recycle_rowset_key({instance_id_, tablet_id, ""}); std::string recyc_rs_key1 = recycle_rowset_key({instance_id_, tablet_id + 1, ""}); - std::set resource_ids; + std::vector rowset_meta; int64_t recycle_rowsets_number = 0; int64_t recycle_segments_number = 0; int64_t recycle_rowsets_data_size = 0; @@ -1817,16 +2136,17 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { max_rowset_creation_time = std::max(max_rowset_creation_time, rs_meta.creation_time()); min_rowset_expiration_time = std::min(min_rowset_expiration_time, rs_meta.txn_expiration()); max_rowset_expiration_time = std::max(max_rowset_expiration_time, rs_meta.txn_expiration()); - resource_ids.emplace(rs_meta.resource_id()); + rowset_meta.emplace_back(rs_meta.resource_id()); + LOG(INFO) << "rs_meta.resource_id()=" << rs_meta.resource_id(); } LOG_INFO("recycle tablet start to delete object") .tag("instance id", instance_id_) .tag("tablet id", tablet_id) .tag("recycle tablet resource ids are", - std::accumulate(resource_ids.begin(), resource_ids.end(), std::string(), - [](const std::string& a, const std::string& b) { - return a.empty() ? b : a + "," + b; + std::accumulate(rowset_meta.begin(), rowset_meta.begin(), std::string(), + [](std::string acc, const auto& it) { + return acc.empty() ? it : acc + ", " + it; })); SyncExecutor concurrent_delete_executor( @@ -1838,13 +2158,24 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { // ATTN: there may be data leak if not all accessor initilized successfully // partial data deleted if the tablet is stored cross-storage vault // vault id is not attached to TabletMeta... - for (const auto& resource_id : resource_ids) { - concurrent_delete_executor.add([&, accessor_ptr = accessor_map_[resource_id]]() { - if (accessor_ptr->delete_directory(tablet_path_prefix(tablet_id)) != 0) { + for (const auto& resource_id : rowset_meta) { + concurrent_delete_executor.add([&, rs_id = resource_id, + accessor_ptr = accessor_map_[resource_id]]() { + std::unique_ptr> defer((int*)0x01, [&](int*) { + g_bvar_recycler_vault_recycle_task_concurrency.put( + {instance_id_, metrics_context.operation_type, rs_id}, -1); + metrics_context.report(); + }); + g_bvar_recycler_vault_recycle_task_concurrency.put( + {instance_id_, metrics_context.operation_type, rs_id}, 1); + int res = accessor_ptr->delete_directory(tablet_path_prefix(tablet_id)); + if (res != 0) { LOG(WARNING) << "failed to delete rowset data of tablet " << tablet_id << " path=" << accessor_ptr->uri(); + g_bvar_recycler_vault_recycle_status.put({instance_id_, rs_id, "abnormal"}, 1); return -1; } + g_bvar_recycler_vault_recycle_status.put({instance_id_, rs_id, "normal"}, 1); return 0; }); } @@ -1868,6 +2199,15 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) { return ret; } + tablet_metrics_context_.total_recycled_data_size += + recycle_rowsets_data_size + recycle_rowsets_index_size; + tablet_metrics_context_.total_recycled_num += 1; + segment_metrics_context_.total_recycled_num += recycle_segments_number; + segment_metrics_context_.total_recycled_data_size += + recycle_rowsets_data_size + recycle_rowsets_index_size; + tablet_metrics_context_.report(); + segment_metrics_context_.report(); + txn.reset(); if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { LOG_WARNING("failed to recycle tablet ") @@ -1913,6 +2253,7 @@ int InstanceRecycler::recycle_rowsets() { size_t total_rowset_value_size = 0; size_t expired_rowset_size = 0; std::atomic_long num_recycled = 0; + RecyclerMetricsContext metrics_context(instance_id_, task_name); RecycleRowsetKeyInfo recyc_rs_key_info0 {instance_id_, 0, ""}; RecycleRowsetKeyInfo recyc_rs_key_info1 {instance_id_, INT64_MAX, ""}; @@ -1921,7 +2262,7 @@ int InstanceRecycler::recycle_rowsets() { recycle_rowset_key(recyc_rs_key_info0, &recyc_rs_key0); recycle_rowset_key(recyc_rs_key_info1, &recyc_rs_key1); - LOG_INFO("begin to recycle rowsets").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle rowsets").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -1930,7 +2271,8 @@ int InstanceRecycler::recycle_rowsets() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle rowsets finished, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle rowsets finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) @@ -1944,7 +2286,9 @@ int InstanceRecycler::recycle_rowsets() { }); std::vector rowset_keys; - std::vector rowsets; + // rowset_id -> rowset_meta + // store rowset id and meta for statistics rs size when delete + std::map rowsets; // Store keys of rowset recycled by background workers std::mutex async_recycled_rowset_keys_mutex; @@ -2011,6 +2355,43 @@ int InstanceRecycler::recycle_rowsets() { return final_expiration; }; + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { + RecycleRowsetPB rowset; + if (!rowset.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t current_time = ::time(nullptr); + if (current_time < calc_expiration(rowset)) { // not expired + return 0; + } + if (!rowset.has_type()) { + if (!rowset.has_resource_id()) [[unlikely]] { + return 0; + } + if (rowset.resource_id().empty()) [[unlikely]] { + return 0; + } + return 0; + } + auto* rowset_meta = rowset.mutable_rowset_meta(); + if (!rowset_meta->has_resource_id()) [[unlikely]] { + if (rowset.type() == RecycleRowsetPB::PREPARE || rowset_meta->num_segments() != 0) { + return 0; + } + } + if (rowset.type() != RecycleRowsetPB::PREPARE) { + if (rowset_meta->num_segments() > 0) { + metrics_context.total_need_recycle_num++; + segment_metrics_context_.total_need_recycle_num += rowset_meta->num_segments(); + segment_metrics_context_.total_need_recycle_data_size += + rowset_meta->total_disk_size(); + metrics_context.total_need_recycle_data_size += rowset_meta->total_disk_size(); + } + } + return 0; + }; + auto handle_rowset_kv = [&](std::string_view k, std::string_view v) -> int { ++num_scanned; total_rowset_key_size += k.size(); @@ -2089,7 +2470,7 @@ int InstanceRecycler::recycle_rowsets() { num_compacted += rowset.type() == RecycleRowsetPB::COMPACT; rowset_keys.emplace_back(k); if (rowset_meta->num_segments() > 0) { // Skip empty rowset - rowsets.push_back(std::move(*rowset_meta)); + rowsets.emplace(rowset_meta->rowset_id_v2(), std::move(*rowset_meta)); } else { ++num_empty_rowset; } @@ -2099,12 +2480,15 @@ int InstanceRecycler::recycle_rowsets() { auto loop_done = [&]() -> int { std::vector rowset_keys_to_delete; - std::vector rowsets_to_delete; + // rowset_id -> rowset_meta + // store rowset id and meta for statistics rs size when delete + std::map rowsets_to_delete; rowset_keys_to_delete.swap(rowset_keys); rowsets_to_delete.swap(rowsets); worker_pool->submit([&, rowset_keys_to_delete = std::move(rowset_keys_to_delete), rowsets_to_delete = std::move(rowsets_to_delete)]() { - if (delete_rowset_data(rowsets_to_delete, RowsetRecyclingState::FORMAL_ROWSET) != 0) { + if (delete_rowset_data(rowsets_to_delete, RowsetRecyclingState::FORMAL_ROWSET, + metrics_context) != 0) { LOG(WARNING) << "failed to delete rowset data, instance_id=" << instance_id_; return; } @@ -2117,8 +2501,9 @@ int InstanceRecycler::recycle_rowsets() { return 0; }; - int ret = scan_and_recycle(recyc_rs_key0, recyc_rs_key1, std::move(handle_rowset_kv), - std::move(loop_done)); + int ret = scan_for_recycle_and_statistics(recyc_rs_key0, recyc_rs_key1, "rowsets", + metrics_context, std::move(scan_and_statistics), + std::move(handle_rowset_kv), std::move(loop_done)); worker_pool->stop(); if (!async_recycled_rowset_keys.empty()) { @@ -2224,6 +2609,7 @@ int InstanceRecycler::recycle_tmp_rowsets() { size_t expired_rowset_size = 0; size_t total_rowset_key_size = 0; size_t total_rowset_value_size = 0; + RecyclerMetricsContext metrics_context(instance_id_, task_name); MetaRowsetTmpKeyInfo tmp_rs_key_info0 {instance_id_, 0, 0}; MetaRowsetTmpKeyInfo tmp_rs_key_info1 {instance_id_, INT64_MAX, 0}; @@ -2232,7 +2618,7 @@ int InstanceRecycler::recycle_tmp_rowsets() { meta_rowset_tmp_key(tmp_rs_key_info0, &tmp_rs_key0); meta_rowset_tmp_key(tmp_rs_key_info1, &tmp_rs_key1); - LOG_INFO("begin to recycle tmp rowsets").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle tmp rowsets").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -2241,7 +2627,8 @@ int InstanceRecycler::recycle_tmp_rowsets() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle tmp rowsets finished, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle tmp rowsets finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) @@ -2253,7 +2640,9 @@ int InstanceRecycler::recycle_tmp_rowsets() { // Elements in `tmp_rowset_keys` has the same lifetime as `it` std::vector tmp_rowset_keys; - std::vector tmp_rowsets; + // rowset_id -> rowset_meta + // store tmp_rowset id and meta for statistics rs size when delete + std::map tmp_rowsets; int64_t earlest_ts = std::numeric_limits::max(); auto calc_expiration = [&earlest_ts, this](const doris::RowsetMetaCloudPB& rowset) { @@ -2327,17 +2716,54 @@ int InstanceRecycler::recycle_tmp_rowsets() { tmp_rowset_keys.push_back(k); if (rowset.num_segments() > 0) { // Skip empty rowset - tmp_rowsets.push_back(std::move(rowset)); + tmp_rowsets.emplace(rowset.rowset_id_v2(), std::move(rowset)); + } + return 0; + }; + + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { + doris::RowsetMetaCloudPB rowset; + if (!rowset.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t expiration = calc_expiration(rowset); + int64_t current_time = ::time(nullptr); + if (current_time < expiration) { + return 0; + } + + DCHECK_GT(rowset.txn_id(), 0) + << "txn_id=" << rowset.txn_id() << " rowset=" << rowset.ShortDebugString(); + if (!is_txn_finished(txn_kv_, instance_id_, rowset.txn_id())) { + return 0; + } + + if (!rowset.has_resource_id()) { + if (rowset.num_segments() > 0) [[unlikely]] { // impossible + return 0; + } + metrics_context.total_need_recycle_num++; + return 0; + } + + metrics_context.total_need_recycle_num++; + if (rowset.num_segments() > 0) { + metrics_context.total_need_recycle_data_size += rowset.total_disk_size(); + segment_metrics_context_.total_need_recycle_data_size += rowset.total_disk_size(); + segment_metrics_context_.total_need_recycle_num += rowset.num_segments(); } return 0; }; - auto loop_done = [&tmp_rowset_keys, &tmp_rowsets, &num_recycled, this]() -> int { + auto loop_done = [&tmp_rowset_keys, &tmp_rowsets, &num_recycled, &metrics_context, + this]() -> int { std::unique_ptr> defer((int*)0x01, [&](int*) { tmp_rowset_keys.clear(); tmp_rowsets.clear(); }); - if (delete_rowset_data(tmp_rowsets, RowsetRecyclingState::TMP_ROWSET) != 0) { + if (delete_rowset_data(tmp_rowsets, RowsetRecyclingState::TMP_ROWSET, metrics_context) != + 0) { LOG(WARNING) << "failed to delete tmp rowset data, instance_id=" << instance_id_; return -1; } @@ -2349,8 +2775,30 @@ int InstanceRecycler::recycle_tmp_rowsets() { return 0; }; - return scan_and_recycle(tmp_rs_key0, tmp_rs_key1, std::move(handle_rowset_kv), - std::move(loop_done)); + return scan_for_recycle_and_statistics(tmp_rs_key0, tmp_rs_key1, "tmp_rowsets", metrics_context, + std::move(scan_and_statistics), + std::move(handle_rowset_kv), std::move(loop_done)); +} + +int InstanceRecycler::scan_for_recycle_and_statistics( + std::string begin, std::string_view end, std::string task_name, + RecyclerMetricsContext& metrics_context, + std::function statistics_func, + std::function recycle_func, + std::function loop_done) { + if (config::enable_recycler_metrics) { + scan_and_recycle(begin, end, std::move(statistics_func)); + + // report to bvar + metrics_context.report(true); + tablet_metrics_context_.report(true); + segment_metrics_context_.report(true); + + int ret = scan_and_recycle(begin, end, std::move(recycle_func), std::move(loop_done)); + return ret; + } else { + return scan_and_recycle(begin, end, std::move(recycle_func), std::move(loop_done)); + } } int InstanceRecycler::scan_and_recycle( @@ -2419,6 +2867,7 @@ int InstanceRecycler::abort_timeout_txn() { int64_t num_timeout = 0; int64_t num_abort = 0; int64_t num_advance = 0; + RecyclerMetricsContext metrics_context(instance_id_, task_name); TxnRunningKeyInfo txn_running_key_info0 {instance_id_, 0, 0}; TxnRunningKeyInfo txn_running_key_info1 {instance_id_, INT64_MAX, INT64_MAX}; @@ -2427,7 +2876,7 @@ int InstanceRecycler::abort_timeout_txn() { txn_running_key(txn_running_key_info0, &begin_txn_running_key); txn_running_key(txn_running_key_info1, &end_txn_running_key); - LOG_INFO("begin to abort timeout txn").tag("instance_id", instance_id_); + LOG_WARNING("begin to abort timeout txn").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -2436,7 +2885,8 @@ int InstanceRecycler::abort_timeout_txn() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("end to abort timeout txn, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("end to abort timeout txn, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_timeout", num_timeout) @@ -2448,7 +2898,7 @@ int InstanceRecycler::abort_timeout_txn() { duration_cast(system_clock::now().time_since_epoch()).count(); auto handle_txn_running_kv = [&num_scanned, &num_timeout, &num_abort, &num_advance, - ¤t_time, + ¤t_time, &metrics_context, this](std::string_view k, std::string_view v) -> int { ++num_scanned; @@ -2543,14 +2993,57 @@ int InstanceRecycler::abort_timeout_txn() { .tag("txn_id", txn_id); return -1; } - ++num_abort; + metrics_context.total_recycled_num = ++num_abort; + metrics_context.report(); + } + + return 0; + }; + + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, ¤t_time, this](std::string_view k, + std::string_view v) -> int { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::string_view k1 = k; + k1.remove_prefix(1); + std::vector, int, int>> out; + if (decode_key(&k1, &out) != 0) { + return 0; + } + int64_t db_id = std::get(std::get<0>(out[3])); + int64_t txn_id = std::get(std::get<0>(out[4])); + // Update txn_info + std::string txn_inf_key, txn_inf_val; + txn_info_key({instance_id_, db_id, txn_id}, &txn_inf_key); + err = txn->get(txn_inf_key, &txn_inf_val); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + TxnInfoPB txn_info; + if (!txn_info.ParseFromString(txn_inf_val)) { + return 0; } + if (TxnStatusPB::TXN_STATUS_COMMITTED != txn_info.status()) { + TxnRunningPB txn_running_pb; + if (!txn_running_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + if (!config::force_immediate_recycle && txn_running_pb.timeout_time() > current_time) { + return 0; + } + metrics_context.total_need_recycle_num++; + } return 0; }; - return scan_and_recycle(begin_txn_running_key, end_txn_running_key, - std::move(handle_txn_running_kv)); + return scan_for_recycle_and_statistics( + begin_txn_running_key, end_txn_running_key, "abort_timeout_txns", metrics_context, + std::move(scan_and_statistics), std::move(handle_txn_running_kv)); } int InstanceRecycler::recycle_expired_txn_label() { @@ -2558,6 +3051,7 @@ int InstanceRecycler::recycle_expired_txn_label() { int64_t num_scanned = 0; int64_t num_expired = 0; int64_t num_recycled = 0; + RecyclerMetricsContext metrics_context(instance_id_, task_name); int ret = 0; RecycleTxnKeyInfo recycle_txn_key_info0 {instance_id_, 0, 0}; @@ -2568,7 +3062,7 @@ int InstanceRecycler::recycle_expired_txn_label() { recycle_txn_key(recycle_txn_key_info1, &end_recycle_txn_key); std::vector recycle_txn_info_keys; - LOG_INFO("begin to recycle expired txn").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle expired txn").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -2576,7 +3070,8 @@ int InstanceRecycler::recycle_expired_txn_label() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("end to recycle expired txn, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("end to recycle expired txn, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) @@ -2619,6 +3114,20 @@ int InstanceRecycler::recycle_expired_txn_label() { return 0; }; + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { + RecycleTxnPB recycle_txn_pb; + if (!recycle_txn_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + if ((config::force_immediate_recycle) || + (recycle_txn_pb.has_immediate() && recycle_txn_pb.immediate()) || + (calc_expiration(recycle_txn_pb) <= current_time_ms)) { + metrics_context.total_need_recycle_num++; + } + return 0; + }; + auto delete_recycle_txn_kv = [&](const std::string& k) -> int { std::string_view k1 = k; //RecycleTxnKeyInfo 0:instance_id 1:db_id 2:txn_id @@ -2698,7 +3207,9 @@ int InstanceRecycler::recycle_expired_txn_label() { LOG(WARNING) << "failed to delete expired txn, err=" << err << " key=" << hex(k); return -1; } - ++num_recycled; + metrics_context.total_recycled_num = ++num_recycled; + metrics_context.report(); + LOG(INFO) << "recycle expired txn, key=" << hex(k); return 0; }; @@ -2742,8 +3253,9 @@ int InstanceRecycler::recycle_expired_txn_label() { return ret; }; - return scan_and_recycle(begin_recycle_txn_key, end_recycle_txn_key, - std::move(handle_recycle_txn_kv), std::move(loop_done)); + return scan_for_recycle_and_statistics( + begin_recycle_txn_key, end_recycle_txn_key, "expired_txn_labels", metrics_context, + std::move(scan_and_statistics), std::move(handle_recycle_txn_kv), std::move(loop_done)); } struct CopyJobIdTuple { @@ -2805,9 +3317,11 @@ struct BatchObjStoreAccessor { paths_.clear(); copy_file_keys_.clear(); batch_count_++; + + LOG_WARNING("begin to delete {} internal stage objects in batch {}", paths_.size(), + batch_count_); }); - LOG_INFO("begin to delete {} internal stage objects in batch {}", paths_.size(), - batch_count_); + StopWatch sw; // TODO(yuejing): 在accessor的delete_objets的实现里可以考虑如果_paths数量不超过10个的话，就直接发10个delete objection operation而不是发post if (0 != accessor_->delete_files(paths_)) { @@ -2815,8 +3329,8 @@ struct BatchObjStoreAccessor { paths_.size(), batch_count_, sw.elapsed_us()); return; } - LOG_INFO("succeed to delete {} internal stage objects in batch {} and it takes {} us", - paths_.size(), batch_count_, sw.elapsed_us()); + LOG_WARNING("succeed to delete {} internal stage objects in batch {} and it takes {} us", + paths_.size(), batch_count_, sw.elapsed_us()); // delete fdb's keys for (auto& file_keys : copy_file_keys_) { auto& [log_trace, keys] = file_keys.second; @@ -2865,8 +3379,9 @@ int InstanceRecycler::recycle_copy_jobs() { // Used for INTERNAL stage's copy jobs to tag each batch for log trace uint64_t batch_count = 0; const std::string task_name = "recycle_copy_jobs"; + RecyclerMetricsContext metrics_context(instance_id_, task_name); - LOG_INFO("begin to recycle copy jobs").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle copy jobs").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -2875,7 +3390,8 @@ int InstanceRecycler::recycle_copy_jobs() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle copy jobs finished, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle copy jobs finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_finished", num_finished) @@ -2891,7 +3407,7 @@ int InstanceRecycler::recycle_copy_jobs() { copy_job_key(key_info1, &key1); std::unordered_map> stage_accessor_map; auto recycle_func = [&start_time, &num_scanned, &num_finished, &num_expired, &num_recycled, - &batch_count, &stage_accessor_map, &task_name, + &batch_count, &stage_accessor_map, &task_name, &metrics_context, this](std::string_view k, std::string_view v) -> int { ++num_scanned; CopyJobPB copy_job; @@ -2998,12 +3514,51 @@ int InstanceRecycler::recycle_copy_jobs() { return -1; } - ++num_recycled; + metrics_context.total_recycled_num = ++num_recycled; + metrics_context.report(); check_recycle_task(instance_id_, task_name, num_scanned, num_recycled, start_time); return 0; }; - return scan_and_recycle(key0, key1, std::move(recycle_func)); + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context](std::string_view k, std::string_view v) -> int { + CopyJobPB copy_job; + if (!copy_job.ParseFromArray(v.data(), v.size())) { + LOG_WARNING("malformed copy job").tag("key", hex(k)); + return 0; + } + + if (copy_job.job_status() == CopyJobPB::FINISH) { + if (copy_job.stage_type() == StagePB::EXTERNAL) { + int64_t current_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + if (copy_job.finish_time_ms() > 0) { + if (!config::force_immediate_recycle && + current_time < copy_job.finish_time_ms() + + config::copy_job_max_retention_second * 1000) { + return 0; + } + } else { + if (!config::force_immediate_recycle && + current_time < copy_job.start_time_ms() + + config::copy_job_max_retention_second * 1000) { + return 0; + } + } + } + } else if (copy_job.job_status() == CopyJobPB::LOADING) { + int64_t current_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + if (!config::force_immediate_recycle && current_time <= copy_job.timeout_time_ms()) { + return 0; + } + } + metrics_context.total_need_recycle_num++; + return 0; + }; + + return scan_for_recycle_and_statistics(key0, key1, "copy_jobs", metrics_context, + std::move(scan_and_statistics), std::move(recycle_func)); } int InstanceRecycler::init_copy_job_accessor(const std::string& stage_id, @@ -3105,9 +3660,10 @@ int InstanceRecycler::init_copy_job_accessor(const std::string& stage_id, int InstanceRecycler::recycle_stage() { int64_t num_scanned = 0; int64_t num_recycled = 0; + RecyclerMetricsContext metrics_context(instance_id_, "recycle_stage"); const std::string task_name = "recycle_stage"; - LOG_INFO("begin to recycle stage").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle stage").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); @@ -3116,7 +3672,8 @@ int InstanceRecycler::recycle_stage() { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle stage, cost={}s", cost) + metrics_context.finish_report(); + LOG_WARNING("recycle stage, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_recycled", num_recycled); @@ -3128,8 +3685,8 @@ int InstanceRecycler::recycle_stage() { std::string key1 = recycle_stage_key(key_info1); std::vector stage_keys; - auto recycle_func = [&start_time, &num_scanned, &num_recycled, &stage_keys, this]( - std::string_view k, std::string_view v) -> int { + auto recycle_func = [&start_time, &num_scanned, &num_recycled, &stage_keys, &metrics_context, + this](std::string_view k, std::string_view v) -> int { ++num_scanned; RecycleStagePB recycle_stage; if (!recycle_stage.ParseFromArray(v.data(), v.size())) { @@ -3169,7 +3726,7 @@ int InstanceRecycler::recycle_stage() { return ret; } - LOG_INFO("begin to delete objects of dropped internal stage") + LOG_WARNING("begin to delete objects of dropped internal stage") .tag("instance_id", instance_id_) .tag("stage_id", recycle_stage.stage().stage_id()) .tag("user_name", recycle_stage.stage().mysql_user_name()[0]) @@ -3184,12 +3741,58 @@ int InstanceRecycler::recycle_stage() { << ", ret=" << ret; return -1; } - ++num_recycled; + metrics_context.total_recycled_num = ++num_recycled; + metrics_context.report(); check_recycle_task(instance_id_, "recycle_stage", num_scanned, num_recycled, start_time); stage_keys.push_back(k); return 0; }; + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, this](std::string_view k, + std::string_view v) -> int { + RecycleStagePB recycle_stage; + if (!recycle_stage.ParseFromArray(v.data(), v.size())) { + LOG_WARNING("malformed recycle stage").tag("key", hex(k)); + return 0; + } + + int idx = stoi(recycle_stage.stage().obj_info().id()); + if (idx > instance_info_.obj_info().size() || idx < 1) { + LOG(WARNING) << "invalid idx: " << idx; + return 0; + } + + std::shared_ptr accessor; + int ret = SYNC_POINT_HOOK_RETURN_VALUE( + [&] { + auto& old_obj = instance_info_.obj_info()[idx - 1]; + auto s3_conf = S3Conf::from_obj_store_info(old_obj); + if (!s3_conf) { + return 0; + } + + s3_conf->prefix = recycle_stage.stage().obj_info().prefix(); + std::shared_ptr s3_accessor; + int ret = S3Accessor::create(std::move(s3_conf.value()), &s3_accessor); + if (ret != 0) { + return 0; + } + + accessor = std::move(s3_accessor); + return 0; + }(), + "recycle_stage:get_accessor", &accessor); + + if (ret != 0) { + LOG(WARNING) << "failed to init accessor ret=" << ret; + return 0; + } + + metrics_context.total_need_recycle_num++; + return 0; + }; + auto loop_done = [&stage_keys, this]() -> int { if (stage_keys.empty()) return 0; std::unique_ptr> defer((int*)0x01, @@ -3200,75 +3803,128 @@ int InstanceRecycler::recycle_stage() { } return 0; }; - - return scan_and_recycle(key0, key1, std::move(recycle_func), std::move(loop_done)); + return scan_for_recycle_and_statistics(key0, key1, "stages", metrics_context, + std::move(scan_and_statistics), std::move(recycle_func), + std::move(loop_done)); } int InstanceRecycler::recycle_expired_stage_objects() { - LOG_INFO("begin to recycle expired stage objects").tag("instance_id", instance_id_); + LOG_WARNING("begin to recycle expired stage objects").tag("instance_id", instance_id_); int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); + RecyclerMetricsContext metrics_context(instance_id_, "recycle_expired_stage_objects"); std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; - LOG_INFO("recycle expired stage objects, cost={}s", cost).tag("instance_id", instance_id_); + metrics_context.finish_report(); + LOG_WARNING("recycle expired stage objects, cost={}s", cost) + .tag("instance_id", instance_id_); }); + int ret = 0; - for (const auto& stage : instance_info_.stages()) { - std::stringstream ss; - ss << "instance_id=" << instance_id_ << ", stage_id=" << stage.stage_id() << ", user_name=" - << (stage.mysql_user_name().empty() ? "null" : stage.mysql_user_name().at(0)) - << ", user_id=" << (stage.mysql_user_id().empty() ? "null" : stage.mysql_user_id().at(0)) - << ", prefix=" << stage.obj_info().prefix(); - - if (stopped()) break; - if (stage.type() == StagePB::EXTERNAL) { - continue; - } - int idx = stoi(stage.obj_info().id()); - if (idx > instance_info_.obj_info().size() || idx < 1) { - LOG(WARNING) << "invalid idx: " << idx << ", id: " << stage.obj_info().id(); - continue; + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, this]() { + for (const auto& stage : instance_info_.stages()) { + if (stopped()) { + break; + } + if (stage.type() == StagePB::EXTERNAL) { + continue; + } + int idx = stoi(stage.obj_info().id()); + if (idx > instance_info_.obj_info().size() || idx < 1) { + continue; + } + const auto& old_obj = instance_info_.obj_info()[idx - 1]; + auto s3_conf = S3Conf::from_obj_store_info(old_obj); + if (!s3_conf) { + continue; + } + s3_conf->prefix = stage.obj_info().prefix(); + std::shared_ptr accessor; + int ret1 = S3Accessor::create(*s3_conf, &accessor); + if (ret1 != 0) { + continue; + } + if (s3_conf->prefix.find("/stage/") == std::string::npos) { + continue; + } + metrics_context.total_need_recycle_num++; } + }; - const auto& old_obj = instance_info_.obj_info()[idx - 1]; - auto s3_conf = S3Conf::from_obj_store_info(old_obj); - if (!s3_conf) { - LOG(WARNING) << "failed to init s3_conf with obj_info=" << old_obj.ShortDebugString(); - continue; - } + auto handle_recycle_func = [&, this]() { + for (const auto& stage : instance_info_.stages()) { + std::stringstream ss; + ss << "instance_id=" << instance_id_ << ", stage_id=" << stage.stage_id() + << ", user_name=" + << (stage.mysql_user_name().empty() ? "null" : stage.mysql_user_name().at(0)) + << ", user_id=" + << (stage.mysql_user_id().empty() ? "null" : stage.mysql_user_id().at(0)) + << ", prefix=" << stage.obj_info().prefix(); - s3_conf->prefix = stage.obj_info().prefix(); - std::shared_ptr accessor; - int ret1 = S3Accessor::create(*s3_conf, &accessor); - if (ret1 != 0) { - LOG(WARNING) << "failed to init s3 accessor ret=" << ret1 << " " << ss.str(); - ret = -1; - continue; - } + if (stopped()) { + break; + } + if (stage.type() == StagePB::EXTERNAL) { + continue; + } + int idx = stoi(stage.obj_info().id()); + if (idx > instance_info_.obj_info().size() || idx < 1) { + LOG(WARNING) << "invalid idx: " << idx << ", id: " << stage.obj_info().id(); + continue; + } - if (s3_conf->prefix.find("/stage/") == std::string::npos) { - LOG(WARNING) << "try to delete illegal prefix, which is catastrophic, " << ss.str(); - ret = -1; - continue; - } + const auto& old_obj = instance_info_.obj_info()[idx - 1]; + auto s3_conf = S3Conf::from_obj_store_info(old_obj); + if (!s3_conf) { + LOG(WARNING) << "failed to init s3_conf with obj_info=" + << old_obj.ShortDebugString(); + continue; + } - LOG(INFO) << "recycle expired stage objects, " << ss.str(); - int64_t expiration_time = - duration_cast(system_clock::now().time_since_epoch()).count() - - config::internal_stage_objects_expire_time_second; - if (config::force_immediate_recycle) { - expiration_time = INT64_MAX; - } - ret1 = accessor->delete_all(expiration_time); - if (ret1 != 0) { - LOG(WARNING) << "failed to recycle expired stage objects, ret=" << ret1 << " " - << ss.str(); - ret = -1; - continue; + s3_conf->prefix = stage.obj_info().prefix(); + std::shared_ptr accessor; + int ret1 = S3Accessor::create(*s3_conf, &accessor); + if (ret1 != 0) { + LOG(WARNING) << "failed to init s3 accessor ret=" << ret1 << " " << ss.str(); + ret = -1; + continue; + } + + if (s3_conf->prefix.find("/stage/") == std::string::npos) { + LOG(WARNING) << "try to delete illegal prefix, which is catastrophic, " << ss.str(); + ret = -1; + continue; + } + + LOG(INFO) << "recycle expired stage objects, " << ss.str(); + int64_t expiration_time = + duration_cast(system_clock::now().time_since_epoch()).count() - + config::internal_stage_objects_expire_time_second; + if (config::force_immediate_recycle) { + expiration_time = INT64_MAX; + } + ret1 = accessor->delete_all(expiration_time); + if (ret1 != 0) { + LOG(WARNING) << "failed to recycle expired stage objects, ret=" << ret1 << " " + << ss.str(); + ret = -1; + continue; + } + metrics_context.total_recycled_num++; + metrics_context.report(); } - } + }; + + // for calculate the total num or bytes of recyled objects + scan_and_statistics(); + + // report to bvar + metrics_context.report(true); + + handle_recycle_func(); return ret; } diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h index 84e4075e61be8d..d1ae8a056c857f 100644 --- a/cloud/src/recycler/recycler.h +++ b/cloud/src/recycler/recycler.h @@ -21,13 +21,16 @@ #include #include +#include #include #include #include #include #include #include +#include +#include "common/bvars.h" #include "meta-service/txn_lazy_committer.h" #include "recycler/storage_vault_accessor.h" #include "recycler/white_black_list.h" @@ -42,6 +45,7 @@ class InstanceRecycler; class StorageVaultAccessor; class Checker; class SimpleThreadPool; +class RecyclerMetricsContext; struct RecyclerThreadPoolGroup { RecyclerThreadPoolGroup() = default; RecyclerThreadPoolGroup(std::shared_ptr s3_producer_pool, @@ -165,15 +169,15 @@ class InstanceRecycler { * @param is_empty_tablet indicates whether the tablet has object files, can skip delete objects if tablet is empty * @return 0 for success otherwise error */ - int recycle_tablets(int64_t table_id, int64_t index_id, int64_t partition_id = -1, - bool is_empty_tablet = false); + int recycle_tablets(int64_t table_id, int64_t index_id, RecyclerMetricsContext& ctx, + int64_t partition_id = -1, bool is_empty_tablet = false); /** * recycle all rowsets belonging to the tablet specified by `tablet_id` * * @return 0 for success otherwise error */ - int recycle_tablet(int64_t tablet_id); + int recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& metrics_context); // scan and recycle useless partition version kv int recycle_versions(); @@ -218,6 +222,13 @@ class InstanceRecycler { std::function recycle_func, std::function loop_done = nullptr); + int scan_for_recycle_and_statistics( + std::string begin, std::string_view end, std::string task_name, + RecyclerMetricsContext& metrics_context, + std::function statistics_func, + std::function recycle_func, + std::function loop_done = nullptr); + // return 0 for success otherwise error int delete_rowset_data(const doris::RowsetMetaCloudPB& rs_meta_pb); @@ -227,8 +238,8 @@ class InstanceRecycler { const std::string& rowset_id); // return 0 for success otherwise error - int delete_rowset_data(const std::vector& rowsets, - RowsetRecyclingState type); + int delete_rowset_data(const std::map& rowsets, + RowsetRecyclingState type, RecyclerMetricsContext& metrics_context); /** * Get stage storage info from instance and init StorageVaultAccessor @@ -241,6 +252,14 @@ class InstanceRecycler { void unregister_recycle_task(const std::string& task_name); + // for scan all tablets and statistics metrics + int scan_tablets_and_statistics(int64_t tablet_id, int64_t index_id, + RecyclerMetricsContext& metrics_context, + int64_t partition_id = -1, bool is_empty_tablet = false); + + // for scan all rs of tablet and statistics metrics + int scan_tablet_and_statistics(int64_t tablet_id, RecyclerMetricsContext& metrics_context); + private: std::atomic_bool stopped_ {false}; std::shared_ptr txn_kv_; @@ -268,4 +287,109 @@ class InstanceRecycler { std::shared_ptr txn_lazy_committer_; }; +class RecyclerMetricsContext { +public: + RecyclerMetricsContext() = default; + + RecyclerMetricsContext(std::string instance_id, std::string operation_type) + : operation_type(std::move(operation_type)), instance_id(std::move(instance_id)) { + start(); + } + + ~RecyclerMetricsContext() = default; + + int64_t total_need_recycle_data_size = 0; + int64_t total_need_recycle_num = 0; + + std::atomic_long total_recycled_data_size {0}; + std::atomic_long total_recycled_num {0}; + + std::string operation_type {}; + std::string instance_id {}; + + double start_time = 0; + + void start() { + start_time = duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + + double duration() const { + return duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count() - + start_time; + } + + void reset() { + total_need_recycle_data_size = 0; + total_need_recycle_num = 0; + total_recycled_data_size.store(0); + total_recycled_num.store(0); + start_time = duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + } + + void finish_report() { + if (!operation_type.empty()) { + double cost = duration(); + g_bvar_recycler_instance_last_round_recycle_elpased_ts.put( + {instance_id, operation_type}, cost); + g_bvar_recycler_instance_recycle_round.put({instance_id, operation_type}, 1); + LOG(INFO) << "recycle instance: " << instance_id + << ", operation type: " << operation_type << ", cost: " << cost + << " ms, total recycled num: " << total_recycled_num.load() + << ", total recycled data size: " << total_recycled_data_size.load() + << " bytes"; + if (total_recycled_num.load()) { + g_bvar_recycler_instance_recycle_time_per_resource.put( + {instance_id, operation_type}, cost / total_recycled_num.load()); + } else { + g_bvar_recycler_instance_recycle_time_per_resource.put( + {instance_id, operation_type}, -1); + } + if (total_recycled_data_size.load()) { + g_bvar_recycler_instance_recycle_bytes_per_ms.put( + {instance_id, operation_type}, total_recycled_data_size.load() / cost); + } else { + g_bvar_recycler_instance_recycle_bytes_per_ms.put({instance_id, operation_type}, + -1); + } + } + } + + // `is_begin` is used to initialize total num of items need to be recycled + void report(bool is_begin = false) { + if (!operation_type.empty()) { + if (total_need_recycle_data_size > 0) { + // is init + if (is_begin) { + g_bvar_recycler_instance_last_round_to_recycle_bytes.put( + {instance_id, operation_type}, total_need_recycle_data_size); + } else { + g_bvar_recycler_instance_last_round_recycled_bytes.put( + {instance_id, operation_type}, total_recycled_data_size.load()); + g_bvar_recycler_instance_recycle_total_bytes_since_started.put( + {instance_id, operation_type}, total_recycled_data_size.load()); + } + } + + if (total_need_recycle_num > 0) { + // is init + if (is_begin) { + g_bvar_recycler_instance_last_round_to_recycle_num.put( + {instance_id, operation_type}, total_need_recycle_num); + } else { + g_bvar_recycler_instance_last_round_recycled_num.put( + {instance_id, operation_type}, total_recycled_num.load()); + g_bvar_recycler_instance_recycle_total_num_since_started.put( + {instance_id, operation_type}, total_recycled_num.load()); + } + } + } + } +}; + } // namespace doris::cloud diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index 6b2fac0efe5342..e70d54921826a8 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -54,6 +54,7 @@ using namespace doris; static const std::string instance_id = "instance_id_recycle_test"; static int64_t current_time = 0; static constexpr int64_t db_id = 1000; +static RecyclerMetricsContext ctx; static doris::cloud::RecyclerThreadPoolGroup thread_group; @@ -1083,7 +1084,7 @@ TEST(RecyclerTest, recycle_tablet) { ASSERT_EQ(create_partition_version_kv(txn_kv.get(), table_id, partition_id), 0); - ASSERT_EQ(0, recycler.recycle_tablets(table_id, index_id)); + ASSERT_EQ(0, recycler.recycle_tablets(table_id, index_id, ctx)); // check rowset does not exist on s3 std::unique_ptr list_iter; @@ -3279,16 +3280,17 @@ TEST(RecyclerTest, delete_rowset_data) { auto accessor = recycler.accessor_map_.begin()->second; // Delete multiple rowset files using one series of RowsetPB constexpr int index_id = 10001, tablet_id = 10002; - std::vector rowset_pbs; + std::map rowset_pbs; for (int i = 0; i < 10; ++i) { auto rowset = create_rowset(resource_id, tablet_id, index_id, 5, schemas[i % 5]); create_recycle_rowset( txn_kv.get(), accessor.get(), rowset, static_cast(i % (RecycleRowsetPB::Type_MAX + 1)), true); - rowset_pbs.emplace_back(std::move(rowset)); + rowset_pbs.emplace(rowset.rowset_id_v2(), std::move(rowset)); } - ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::FORMAL_ROWSET)); + ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::FORMAL_ROWSET, + ctx)); std::unique_ptr list_iter; ASSERT_EQ(0, accessor->list_all(&list_iter)); ASSERT_FALSE(list_iter->has_next()); @@ -3384,16 +3386,17 @@ TEST(RecyclerTest, delete_rowset_data_without_inverted_index_storage_format) { auto accessor = recycler.accessor_map_.begin()->second; // Delete multiple rowset files using one series of RowsetPB constexpr int index_id = 10001, tablet_id = 10002; - std::vector rowset_pbs; + std::map rowset_pbs; for (int i = 0; i < 10; ++i) { auto rowset = create_rowset(resource_id, tablet_id, index_id, 5, schemas[i % 5]); create_recycle_rowset( txn_kv.get(), accessor.get(), rowset, static_cast(i % (RecycleRowsetPB::Type_MAX + 1)), true); - rowset_pbs.emplace_back(std::move(rowset)); + rowset_pbs.emplace(rowset.rowset_id_v2(), std::move(rowset)); } - ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::FORMAL_ROWSET)); + ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::FORMAL_ROWSET, + ctx)); std::unique_ptr list_iter; ASSERT_EQ(0, accessor->list_all(&list_iter)); ASSERT_FALSE(list_iter->has_next()); @@ -3550,7 +3553,7 @@ TEST(RecyclerTest, init_vault_accessor_failed_test) { EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 0); // recycle tablet will fail because unuseful obj accessor can not connectted - EXPECT_EQ(recycler.recycle_tablet(0), -1); + EXPECT_EQ(recycler.recycle_tablet(0, ctx), -1); // however, useful mock accessor can recycle tablet EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 1); } @@ -3632,7 +3635,7 @@ TEST(RecyclerTest, recycle_tablet_without_resource_id) { EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 0); // recycle tablet will fail because unuseful obj accessor can not connectted - EXPECT_EQ(recycler.recycle_tablet(0), -1); + EXPECT_EQ(recycler.recycle_tablet(0, ctx), -1); // no resource id, cannot recycle EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 0); } @@ -3714,7 +3717,7 @@ TEST(RecyclerTest, recycle_tablet_with_wrong_resource_id) { EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 0); // recycle tablet will fail because unuseful obj accessor can not connectted - EXPECT_EQ(recycler.recycle_tablet(0), -1); + EXPECT_EQ(recycler.recycle_tablet(0, ctx), -1); // no resource id, cannot recycle EXPECT_EQ(recycler.accessor_map_.at("success_vault")->exists("data/0/test.csv"), 0); } @@ -3937,7 +3940,7 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) { std::make_shared(txn_kv)); ASSERT_EQ(recycler.init(), 0); auto accessor = recycler.accessor_map_.begin()->second; - std::vector rowset_pbs; + std::map rowset_pbs; doris::RowsetMetaCloudPB rowset; rowset.set_rowset_id(0); // useless but required rowset.set_rowset_id_v2("1"); @@ -3949,7 +3952,7 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) { rowset.mutable_tablet_schema()->CopyFrom(schema); create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1); rowset.clear_tablet_schema(); - rowset_pbs.emplace_back(rowset); + rowset_pbs.emplace(rowset.rowset_id_v2(), rowset); std::unordered_set list_files; std::unique_ptr iter; @@ -3964,7 +3967,8 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) { EXPECT_TRUE(list_files.contains("data/10000/1_0.dat")); EXPECT_TRUE(list_files.contains("data/10000/1_0_1.idx")); - ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET)); + ASSERT_EQ(0, + recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET, ctx)); list_files.clear(); iter.reset(); EXPECT_EQ(accessor->list_all(&iter), 0); @@ -4016,7 +4020,7 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) { std::make_shared(txn_kv)); ASSERT_EQ(recycler.init(), 0); auto accessor = recycler.accessor_map_.begin()->second; - std::vector rowset_pbs; + std::map rowset_pbs; doris::RowsetMetaCloudPB rowset; rowset.set_rowset_id(0); // useless but required rowset.set_rowset_id_v2("1"); @@ -4028,7 +4032,7 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) { rowset.mutable_tablet_schema()->CopyFrom(schema); create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true); rowset.clear_tablet_schema(); - rowset_pbs.emplace_back(rowset); + rowset_pbs.emplace(rowset.rowset_id_v2(), rowset); std::unordered_set list_files; std::unique_ptr iter; @@ -4043,7 +4047,8 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) { EXPECT_TRUE(list_files.contains("data/10000/1_0.dat")); EXPECT_TRUE(list_files.contains("data/10000/1_0.idx")); - ASSERT_EQ(0, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET)); + ASSERT_EQ(0, + recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET, ctx)); list_files.clear(); iter.reset(); EXPECT_EQ(accessor->list_all(&iter), 0); @@ -4100,7 +4105,7 @@ TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) { std::make_shared(txn_kv)); ASSERT_EQ(recycler.init(), 0); auto accessor = recycler.accessor_map_.begin()->second; - std::vector rowset_pbs; + std::map rowset_pbs; doris::RowsetMetaCloudPB rowset; rowset.set_rowset_id(0); // useless but required rowset.set_rowset_id_v2("1"); @@ -4112,7 +4117,7 @@ TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) { rowset.mutable_tablet_schema()->CopyFrom(schema); create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true); rowset.clear_tablet_schema(); - rowset_pbs.emplace_back(rowset); + rowset_pbs.emplace(rowset.rowset_id_v2(), rowset); rowset.set_rowset_id(0); // useless but required rowset.set_rowset_id_v2("2"); @@ -4124,7 +4129,7 @@ TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) { rowset.mutable_tablet_schema()->CopyFrom(schema); create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, 1, true); rowset.clear_tablet_schema(); - rowset_pbs.emplace_back(rowset); + rowset_pbs.emplace(rowset.rowset_id_v2(), rowset); std::unordered_set list_files; std::unique_ptr iter; @@ -4141,7 +4146,8 @@ TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) { EXPECT_TRUE(list_files.contains("data/20000/2_0.dat")); EXPECT_TRUE(list_files.contains("data/20000/2_0.idx")); - EXPECT_EQ(-1, recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET)); + EXPECT_EQ(-1, + recycler.delete_rowset_data(rowset_pbs, RowsetRecyclingState::TMP_ROWSET, ctx)); list_files.clear(); iter.reset(); EXPECT_EQ(accessor->list_all(&iter), 0); From 8e3fe1edcb000dbe13ed500671b5a791cc97815c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 21:59:56 +0800 Subject: [PATCH 099/572] branch-3.0: [fix](load) fix bad load id in injection #52339 (#52426) Cherry-picked from #52339 Co-authored-by: Kaijie Chen --- be/src/runtime/load_stream.cpp | 14 ++++++++------ be/src/runtime/load_stream.h | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp index eb28daaa86fd20..1a386341f891a7 100644 --- a/be/src/runtime/load_stream.cpp +++ b/be/src/runtime/load_stream.cpp @@ -57,7 +57,7 @@ bvar::Adder g_load_stream_cnt("load_stream_count"); bvar::LatencyRecorder g_load_stream_flush_wait_ms("load_stream_flush_wait_ms"); bvar::Adder g_load_stream_flush_running_threads("load_stream_flush_wait_threads"); -TabletStream::TabletStream(PUniqueId load_id, int64_t id, int64_t txn_id, +TabletStream::TabletStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile) : _id(id), _next_segid(0), @@ -332,7 +332,7 @@ Status TabletStream::close() { return _status.status(); } -IndexStream::IndexStream(PUniqueId load_id, int64_t id, int64_t txn_id, +IndexStream::IndexStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, std::shared_ptr schema, LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile) : _id(id), @@ -415,7 +415,8 @@ void IndexStream::close(const std::vector& tablets_to_commit, // TODO: Profile is temporary disabled, because: // 1. It's not being processed by the upstream for now // 2. There are some problems in _profile->to_thrift() -LoadStream::LoadStream(PUniqueId load_id, LoadStreamMgr* load_stream_mgr, bool enable_profile) +LoadStream::LoadStream(const PUniqueId& load_id, LoadStreamMgr* load_stream_mgr, + bool enable_profile) : _load_id(load_id), _enable_profile(false), _load_stream_mgr(load_stream_mgr) { g_load_stream_cnt << 1; _profile = std::make_unique("LoadStream"); @@ -640,9 +641,10 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* // otherwise the message will be ignored and causing close wait timeout if (hdr.opcode() != PStreamHeader::CLOSE_LOAD) { DBUG_EXECUTE_IF("LoadStream._dispatch.unknown_loadid", { - PUniqueId& load_id = const_cast(hdr.load_id()); - load_id.set_hi(UNKNOWN_ID_FOR_TEST); - load_id.set_lo(UNKNOWN_ID_FOR_TEST); + PStreamHeader& t_hdr = const_cast(hdr); + PUniqueId* load_id = t_hdr.mutable_load_id(); + load_id->set_hi(UNKNOWN_ID_FOR_TEST); + load_id->set_lo(UNKNOWN_ID_FOR_TEST); }); DBUG_EXECUTE_IF("LoadStream._dispatch.unknown_srcid", { PStreamHeader& t_hdr = const_cast(hdr); diff --git a/be/src/runtime/load_stream.h b/be/src/runtime/load_stream.h index 1eacbefb0523a5..301944baee15a9 100644 --- a/be/src/runtime/load_stream.h +++ b/be/src/runtime/load_stream.h @@ -44,8 +44,8 @@ using SegIdMapping = std::vector; using FailedTablets = std::vector>; class TabletStream { public: - TabletStream(PUniqueId load_id, int64_t id, int64_t txn_id, LoadStreamMgr* load_stream_mgr, - RuntimeProfile* profile); + TabletStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, + LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile); Status init(std::shared_ptr schema, int64_t index_id, int64_t partition_id); @@ -85,7 +85,7 @@ using TabletStreamSharedPtr = std::shared_ptr; class IndexStream { public: - IndexStream(PUniqueId load_id, int64_t id, int64_t txn_id, + IndexStream(const PUniqueId& load_id, int64_t id, int64_t txn_id, std::shared_ptr schema, LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile); @@ -116,7 +116,7 @@ using IndexStreamSharedPtr = std::shared_ptr; using StreamId = brpc::StreamId; class LoadStream : public brpc::StreamInputHandler { public: - LoadStream(PUniqueId load_id, LoadStreamMgr* load_stream_mgr, bool enable_profile); + LoadStream(const PUniqueId& load_id, LoadStreamMgr* load_stream_mgr, bool enable_profile); ~LoadStream() override; Status init(const POpenLoadStreamRequest* request); From 0875dbb43a79ad1c0674739b1a5e5c56830aabd4 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 27 Jun 2025 22:01:54 +0800 Subject: [PATCH 100/572] branch-3.0: [Opt](cloud-mow) Skip MS RPC retry's backoff when encounter fdb txn conflict when mow load get ms delete bitmap lock #52360 (#52439) pick https://github.com/apache/doris/pull/52360 --- cloud/src/meta-service/meta_service.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index c43364a4b81e76..660a7ef7320bc4 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -2567,6 +2567,14 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl << " initiators_size=" << lock_info.initiators_size(); err = txn->commit(); + + if (request->lock_id() > 0 && err == TxnErrorCode::TXN_CONFLICT) { + // For load, the only fdb txn conflict here is due to compaction(sc) job. + // We turn it into a lock conflict error to skip the MS RPC backoff becasue it's too long + // and totally let FE to control the retry backoff sleep time + code = MetaServiceCode::LOCK_CONFLICT; + } + if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); ss << "failed to get_delete_bitmap_update_lock, err=" << err; From c6a228e12d2ee45bb4d67abdea4b0da1c4e97ded Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 28 Jun 2025 17:25:54 +0800 Subject: [PATCH 101/572] branch-3.0: [Bug](distinct) fix distinct function with over return error result #51875 (#52312) Cherry-picked from #51875 Co-authored-by: zhangstar333 --- .../aggregate_function_distinct.h | 13 +++++++++ .../test_aggregate_window_functions.out | 28 +++++++++---------- .../window_functions/test_sum.out | 18 ++++++++++++ .../window_functions/test_sum.groovy | 22 +++++++++++++++ 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/be/src/vec/aggregate_functions/aggregate_function_distinct.h b/be/src/vec/aggregate_functions/aggregate_function_distinct.h index d4f1c18df454e9..e6befdf671f6cf 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_distinct.h +++ b/be/src/vec/aggregate_functions/aggregate_function_distinct.h @@ -63,6 +63,8 @@ struct AggregateFunctionDistinctSingleNumericData { using Self = AggregateFunctionDistinctSingleNumericData; Container data; + void clear() { data.clear(); } + void add(const IColumn** columns, size_t /* columns_num */, size_t row_num, Arena*) { const auto& vec = assert_cast&, TypeCheckOnRelease::DISABLE>(*columns[0]) @@ -132,6 +134,8 @@ struct AggregateFunctionDistinctGenericData { using Self = AggregateFunctionDistinctGenericData; Container data; + void clear() { data.clear(); } + void merge(const Self& rhs, Arena* arena) { DCHECK(!stable); if constexpr (!stable) { @@ -319,6 +323,15 @@ class AggregateFunctionDistinct nested_func->add_batch_single_place(arguments[0]->size(), get_nested_place(place), arguments_raw.data(), &arena); nested_func->insert_result_into(get_nested_place(place), to); + // for distinct agg function, the real calculate is add_batch_single_place at last step of insert_result_into function. + // but with distinct agg and over() window function together, the result will be inserted into many times with different rows + // so we need to clear the data, thus not to affect the next insert_result_into + this->data(place).clear(); + } + + void reset(AggregateDataPtr place) const override { + this->data(place).clear(); + nested_func->reset(get_nested_place(place)); } size_t size_of_data() const override { return prefix_size + nested_func->size_of_data(); } diff --git a/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out b/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out index ae6d1ad35a222f..006cea921b6835 100644 --- a/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out +++ b/regression-test/data/nereids_p0/sql_functions/aggregate_functions/test_aggregate_window_functions.out @@ -505,23 +505,23 @@ sichuan [{"cbe":{},"notnull":0,"null":1,"all":1}] 3 4 -- !agg_window_multi_distinct_sum -- -1 11320987615.0000000119 1 1617283945.0000000017 -1 3234567890.0000000034 -1 4851851835.0000000051 -1 6469135780.0000000068 -1 8086419725.0000000085 -1 9703703670.0000000102 +1 1617283945.0000000017 +1 1617283945.0000000017 +1 1617283945.0000000017 +1 1617283945.0000000017 +1 1617283945.0000000017 +1 1617283945.0000000017 +2 1217283945.0000000026 2 1217283945.0000000026 -2 2434567890.0000000052 -2 3651851835.0000000078 -2 4869135780.0000000104 -2 6086419725.0000000130 +2 1217283945.0000000026 +2 1217283945.0000000026 +2 1217283945.0000000026 +3 1093827157.0000000020 +3 1093827157.0000000020 +3 1093827157.0000000020 +3 1093827157.0000000020 3 1093827157.0000000020 -3 2187654314.0000000040 -3 3281481471.0000000060 -3 4375308628.0000000080 -3 5469135785.0000000100 -- !agg_window_bitmap_union -- 1 1,2,3 diff --git a/regression-test/data/query_p0/sql_functions/window_functions/test_sum.out b/regression-test/data/query_p0/sql_functions/window_functions/test_sum.out index 9185f64fa6e8ee..84f4bccb2d1049 100644 --- a/regression-test/data/query_p0/sql_functions/window_functions/test_sum.out +++ b/regression-test/data/query_p0/sql_functions/window_functions/test_sum.out @@ -4,3 +4,21 @@ 2 1243.500 3 24453.325 +-- !sql_window_muti1 -- +a +a +a +a + +-- !sql_window_muti2 -- +1 +1 +1 +1 + +-- !sql_window_muti3 -- +1 +1 +1 +1 + diff --git a/regression-test/suites/query_p0/sql_functions/window_functions/test_sum.groovy b/regression-test/suites/query_p0/sql_functions/window_functions/test_sum.groovy index 3611400568dcbe..e61f586181abb6 100644 --- a/regression-test/suites/query_p0/sql_functions/window_functions/test_sum.groovy +++ b/regression-test/suites/query_p0/sql_functions/window_functions/test_sum.groovy @@ -21,5 +21,27 @@ suite("test_sum") { (partition by k1 order by k3 range between current row and unbounded following) as w from test_query_db.test order by k1, w """ + + sql "create database if not exists multi_db" + sql "use multi_db" + sql "DROP TABLE IF EXISTS multi" + sql """ + CREATE TABLE multi ( + id int, + v1 int, + v2 varchar + ) ENGINE = OLAP + DUPLICATE KEY(id) COMMENT 'OLAP' + DISTRIBUTED BY HASH(id) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ + insert into multi values (1, 1, 'a'),(1, 1, 'a'), (2, 1, 'a'), (3, 1, 'a'); + """ + qt_sql_window_muti1 """ select multi_distinct_group_concat(v2) over() from multi; """ + qt_sql_window_muti2 """ select multi_distinct_sum(v1) over() from multi; """ + qt_sql_window_muti3 """ select multi_distinct_count(v1) over() from multi; """ } From f1c9c88f7e7869cf5b488d75177a9b983a68779f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 28 Jun 2025 17:26:53 +0800 Subject: [PATCH 102/572] branch-3.0: [fix](docker)Add docker-ps 'sudo' permissions #52395 (#52456) Cherry-picked from #52395 Co-authored-by: wuwenchi Co-authored-by: wuwenchi.wwc --- docker/thirdparties/run-thirdparties-docker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh index 6888c0189d0ac8..26580301dd941f 100755 --- a/docker/thirdparties/run-thirdparties-docker.sh +++ b/docker/thirdparties/run-thirdparties-docker.sh @@ -790,12 +790,12 @@ for compose in "${!pids[@]}"; do echo "" echo "print last 100 logs of the latest unhealthy container" - docker ps -a --latest --filter 'health=unhealthy' --format '{{.ID}}' | xargs -I '{}' sh -c 'echo "=== Logs of {} ===" && docker logs -t --tail 100 "{}"' + sudo docker ps -a --latest --filter 'health=unhealthy' --format '{{.ID}}' | xargs -I '{}' sh -c 'echo "=== Logs of {} ===" && docker logs -t --tail 100 "{}"' exit 1 fi done echo "docker started" -docker ps -a --format "{{.ID}} | {{.Image}} | {{.Status}}" +sudo docker ps -a --format "{{.ID}} | {{.Image}} | {{.Status}}" echo "all dockers started successfully" From 718ab8a0d8034216828d8d783d5fbeb407164af3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 28 Jun 2025 17:28:01 +0800 Subject: [PATCH 103/572] branch-3.0: [fix](map) fix createMap function #52231 (#52400) Cherry-picked from #52231 Co-authored-by: amory --- .../functions/scalar/CreateMap.java | 20 +++++- .../scalar_function/Map.out | 69 +++++++++++++++++++ .../scalar_function/Map.groovy | 27 ++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java index fe966f4e38df01..da18ad6480f2a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CreateMap.java @@ -24,11 +24,13 @@ import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.MapType; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; /** @@ -51,7 +53,23 @@ public CreateMap(Expression... varArgs) { @Override public DataType getDataType() { if (arity() >= 2) { - return MapType.of(child(0).getDataType(), child(1).getDataType()); + // use Array function to get the common key and value type + // first collect all key types in odd position, and value types in even position + // then get the common type of key and value + List keyExpressions = new ArrayList<>(); + List valueExpressions = new ArrayList<>(); + for (int i = 0; i < children.size(); i++) { + if (i % 2 == 0) { + keyExpressions.add(children.get(i)); + } else { + valueExpressions.add(children.get(i)); + } + } + Array keyArr = new Array().withChildren(keyExpressions); + DataType keyType = ((ArrayType) keyArr.getDataType()).getItemType(); + Array valueArr = new Array().withChildren(valueExpressions); + DataType valueType = ((ArrayType) valueArr.getDataType()).getItemType(); + return MapType.of(keyType, valueType); } return MapType.SYSTEM_DEFAULT; } diff --git a/regression-test/data/nereids_function_p0/scalar_function/Map.out b/regression-test/data/nereids_function_p0/scalar_function/Map.out index a272d3cf2c3440..1da11a27f96d98 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/Map.out +++ b/regression-test/data/nereids_function_p0/scalar_function/Map.out @@ -3653,3 +3653,72 @@ false false false +-- !map_constructor_basic -- +{"name":"John", "age":"30", "city":"NY", "zip":"10001"} + +-- !map_constructor_string_values -- +{"key1":"value1", "key2":"value2", "key3":"value3"} + +-- !map_constructor_mixed_types -- +{"string_key":"string_value", "int_key":"12345", "float_key":"3.14159"} + +-- !map_constructor_large_numbers -- +{"small":100, "medium":10000, "large":100000, "very_large":1000000} + +-- !map_constructor_json_strings -- +{"{"name":"John"}":"{"age":30}", "{"city":"NY"}":"{"zip":10001}"} + +-- !map_constructor_nested_json -- +{"{"user":{"name":"John","age":30}}":"{"address":{"city":"NY","zip":10001}}"} + +-- !map_constructor_special_chars -- +{"key with spaces":"value with spaces", "key-with-dashes":"value-with-dashes"} + +-- !map_constructor_unicode -- +{"中文键":"中文值", "key_中文":"value_中文"} + +-- !map_constructor_empty_strings -- +{"empty_key":"", "key_empty":"value", "empty_both":""} + +-- !map_constructor_null_values -- +{"null_key":null, "key_null":"value", "both_null":null} + +-- !map_constructor_boolean_values -- +{"true_key":1, "false_key":0, "mixed_key":123} + +-- !map_constructor_decimal_values -- +{"decimal_key":123.456, "integer_key":789.000, "mixed_decimal":10001.000} + +-- !map_constructor_date_values -- +{"date_key":"2023-01-01", "datetime_key":"2023-01-01 12:00:00", "string_key":"text"} + +-- !map_constructor_array_values -- +{"array_key":"[1,2,3]", "string_key":"text", "number_key":"10001"} + +-- !map_constructor_object_values -- +{"object_key":"{"nested":"value"}", "simple_key":"simple_value"} + +-- !map_constructor_long_strings -- +{"long_key":"this is a very long string value that should not be truncated", "short_key":"short"} + +-- !map_constructor_numeric_strings -- +{"numeric_string":"10001", "actual_number":"10001", "mixed":"text"} + +-- !map_constructor_escape_chars -- +{"quoted_key":"value with "quotes"", "backslash_key":"value with backslash"} + +-- !map_constructor_whitespace -- +{" spaced_key ":" spaced_value ", "normal_key":"normal_value"} + +-- !sql -- +{"{"name":"John"}":"{"age":30}", "{"city":"NY"}":"{"zip":10001}"} + +-- !sql -- +{"zip":10001, "code":10002, "number":10003} + +-- !sql -- +{"postal_code":10001, "area_code":10002, "zip_plus_4":10003} + +-- !sql -- +{"{"zip":10001}":"{"city":"NY"}", "{"code":10002}":"{"state":"NY"}"} + diff --git a/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy b/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy index f8b28c8da9a405..301841e8a44bfe 100644 --- a/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy +++ b/regression-test/suites/nereids_function_p0/scalar_function/Map.groovy @@ -286,5 +286,32 @@ suite("nereids_scalar_fn_map") { order_qt_map_contains_value_tint_date_notnull """ select map_contains_value(km_tint_date, kdt) from fn_test_not_nullable """ order_qt_map_contains_value_tint_dtm_notnull """ select map_contains_value(km_tint_dtm, kdtm) from fn_test_not_nullable """ + // map constructor function tests - bugfix for CreateMap with args not deducible + sql """ set enable_fold_constant_by_be=true;""" + sql """ set debug_skip_fold_constant=false;""" + qt_map_constructor_basic "select map('name', 'John', 'age', 30, 'city', 'NY', 'zip', 10001)" + qt_map_constructor_string_values "select map('key1', 'value1', 'key2', 'value2', 'key3', 'value3')" + qt_map_constructor_mixed_types "select map('string_key', 'string_value', 'int_key', 12345, 'float_key', 3.14159)" + qt_map_constructor_large_numbers "select map('small', 100, 'medium', 10000, 'large', 100000, 'very_large', 1000000)" + qt_map_constructor_json_strings "select map('{\"name\":\"John\"}', '{\"age\":30}', '{\"city\":\"NY\"}', '{\"zip\":10001}')" + qt_map_constructor_nested_json "select map('{\"user\":{\"name\":\"John\",\"age\":30}}', '{\"address\":{\"city\":\"NY\",\"zip\":10001}}')" + qt_map_constructor_special_chars "select map('key with spaces', 'value with spaces', 'key-with-dashes', 'value-with-dashes')" + qt_map_constructor_unicode "select map('中文键', '中文值', 'key_中文', 'value_中文')" + qt_map_constructor_empty_strings "select map('empty_key', '', 'key_empty', 'value', 'empty_both', '')" + qt_map_constructor_null_values "select map('null_key', null, 'key_null', 'value', 'both_null', null)" + qt_map_constructor_boolean_values "select map('true_key', true, 'false_key', false, 'mixed_key', 123)" + qt_map_constructor_decimal_values "select map('decimal_key', 123.456, 'integer_key', 789, 'mixed_decimal', 10001.0)" + qt_map_constructor_date_values "select map('date_key', '2023-01-01', 'datetime_key', '2023-01-01 12:00:00', 'string_key', 'text')" + qt_map_constructor_array_values "select map('array_key', '[1,2,3]', 'string_key', 'text', 'number_key', 10001)" + qt_map_constructor_object_values "select map('object_key', '{\"nested\":\"value\"}', 'simple_key', 'simple_value')" + qt_map_constructor_long_strings "select map('long_key', 'this is a very long string value that should not be truncated', 'short_key', 'short')" + qt_map_constructor_numeric_strings "select map('numeric_string', '10001', 'actual_number', 10001, 'mixed', 'text')" + qt_map_constructor_escape_chars "select map('quoted_key', 'value with \"quotes\"', 'backslash_key', 'value with \\ backslash')" + qt_map_constructor_whitespace "select map(' spaced_key ', ' spaced_value ', 'normal_key', 'normal_value')" + + qt_sql "select map('{\"name\":\"John\"}', '{\"age\":30}', '{\"city\":\"NY\"}', '{\"zip\":10001}')" + qt_sql "select map('zip', 10001, 'code', 10002, 'number', 10003)" + qt_sql "select map('postal_code', 10001, 'area_code', 10002, 'zip_plus_4', 10003)" + qt_sql "select map('{\"zip\":10001}', '{\"city\":\"NY\"}', '{\"code\":10002}', '{\"state\":\"NY\"}')" } \ No newline at end of file From f00bde37c7431b731e779559e307f6274d5c14bb Mon Sep 17 00:00:00 2001 From: Socrates Date: Sat, 28 Jun 2025 17:28:40 +0800 Subject: [PATCH 104/572] branch-3.0: [fix](csv) escape quote with double quote for csv format table (#50101) (#52403) bp: #50101 --- be/src/vec/data_types/serde/data_type_serde.h | 10 ++ .../data_types/serde/data_type_string_serde.h | 34 ++++ be/src/vec/exec/format/csv/csv_reader.cpp | 9 +- .../new_plain_text_line_reader.cpp | 19 +- .../file_reader/new_plain_text_line_reader.h | 3 + .../new_plain_text_line_reader_test.cpp | 168 ++++++++++++++++++ .../create_preinstalled_scripts/run76.hql | 10 ++ .../csv_json_table_simple.csv | 10 ++ .../hive/test_hive_serde_prop.out | 4 +- .../hive/test_open_csv_serde.out | 25 +++ ...est_csv_with_enclose_and_escapeS3_load.out | 3 +- .../enclose_with_escape_doublequote.csv | 1 + .../stream_load/enclose_with_escape_quote.csv | 1 + .../test_csv_with_enclose_and_escape.out | 2 + .../hive/test_open_csv_serde.groovy | 45 +++++ .../test_csv_with_enclose_and_escape.groovy | 3 +- 16 files changed, 335 insertions(+), 12 deletions(-) create mode 100644 be/test/vec/exec/format/file_reader/new_plain_text_line_reader_test.cpp create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/csv/csv_json_table_simple/csv_json_table_simple.csv create mode 100644 regression-test/data/external_table_p0/hive/test_open_csv_serde.out create mode 100644 regression-test/data/load_p0/stream_load/enclose_with_escape_doublequote.csv create mode 100644 regression-test/data/load_p0/stream_load/enclose_with_escape_quote.csv create mode 100644 regression-test/suites/external_table_p0/hive/test_open_csv_serde.groovy diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h index c46b3f311a2128..f7446d5c52a7aa 100644 --- a/be/src/vec/data_types/serde/data_type_serde.h +++ b/be/src/vec/data_types/serde/data_type_serde.h @@ -152,6 +152,8 @@ class DataTypeSerDe { */ bool converted_from_string = false; + char quote_char = '"'; + char escape_char = 0; /** * flags for each byte to indicate if escape is needed. @@ -271,6 +273,14 @@ class DataTypeSerDe { virtual Status deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const = 0; + + // In some cases, CSV and JSON deserialization behaviors may differ + // so we provide a default implementation that uses JSON deserialization + virtual Status deserialize_one_cell_from_csv(IColumn& column, Slice& slice, + const FormatOptions& options) const { + return deserialize_one_cell_from_json(column, slice, options); + } + // deserialize text vector is to avoid virtual function call in complex type nested loop virtual Status deserialize_column_from_json_vector(IColumn& column, std::vector& slices, int* num_deserialized, diff --git a/be/src/vec/data_types/serde/data_type_string_serde.h b/be/src/vec/data_types/serde/data_type_string_serde.h index a5e716096e19d1..340c1041b3cd73 100644 --- a/be/src/vec/data_types/serde/data_type_string_serde.h +++ b/be/src/vec/data_types/serde/data_type_string_serde.h @@ -63,6 +63,31 @@ inline void escape_string(const char* src, size_t& len, char escape_char) { len = dest_ptr - start; } +// specially escape quote with double quote +inline void escape_string_for_csv(const char* src, size_t& len, char escape_char, char quote_char) { + const char* start = src; + char* dest_ptr = const_cast(src); + const char* end = src + len; + bool escape_next_char = false; + + while (src < end) { + if ((src < end - 1 && *src == quote_char && *(src + 1) == quote_char) || + *src == escape_char) { + escape_next_char = !escape_next_char; + } else { + escape_next_char = false; + } + + if (escape_next_char) { + ++src; + } else { + *dest_ptr++ = *src++; + } + } + + len = dest_ptr - start; +} + template class DataTypeStringSerDeBase : public DataTypeSerDe { public: @@ -186,6 +211,15 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { return Status::OK(); } + Status deserialize_one_cell_from_csv(IColumn& column, Slice& slice, + const FormatOptions& options) const override { + if (options.escape_char != 0) { + escape_string_for_csv(slice.data, slice.size, options.escape_char, options.quote_char); + } + assert_cast(column).insert_data(slice.data, slice.size); + return Status::OK(); + } + Status deserialize_one_cell_from_hive_text( IColumn& column, Slice& slice, const FormatOptions& options, int hive_text_complex_type_delimiter_level = 1) const override { diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp index 5f2739ae2a6f7f..4c5adf6f4b15d7 100644 --- a/be/src/vec/exec/format/csv/csv_reader.cpp +++ b/be/src/vec/exec/format/csv/csv_reader.cpp @@ -339,7 +339,8 @@ Status CsvReader::init_reader(bool is_load) { (_state != nullptr && _state->trim_tailing_spaces_for_external_table_query()); _options.escape_char = _escape; - if (_params.file_attributes.text_params.collection_delimiter.size() == 0) { + _options.quote_char = _enclose; + if (_params.file_attributes.text_params.collection_delimiter.empty()) { switch (_text_serde_type) { case TTextSerdeType::JSON_TEXT_SERDE: _options.collection_delim = ','; @@ -626,8 +627,8 @@ Status CsvReader::deserialize_nullable_string(IColumn& column, Slice& slice) { } } static DataTypeStringSerDe stringSerDe; - auto st = stringSerDe.deserialize_one_cell_from_json(null_column.get_nested_column(), slice, - _options); + auto st = stringSerDe.deserialize_one_cell_from_csv(null_column.get_nested_column(), slice, + _options); if (!st.ok()) { // fill null if fail null_column.insert_data(nullptr, 0); // 0 is meaningless here @@ -681,7 +682,7 @@ Status CsvReader::_fill_dest_columns(const Slice& line, Block* block, switch (_text_serde_type) { case TTextSerdeType::JSON_TEXT_SERDE: RETURN_IF_ERROR( - _serdes[i]->deserialize_one_cell_from_json(*col_ptr, slice, _options)); + _serdes[i]->deserialize_one_cell_from_csv(*col_ptr, slice, _options)); break; case TTextSerdeType::HIVE_TEXT_SERDE: RETURN_IF_ERROR( diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp index 18bfd98892d6c0..94eaf2edadb55b 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp @@ -152,11 +152,22 @@ void EncloseCsvLineReaderContext::_on_pre_match_enclose(const uint8_t* start, si _should_escape = !_should_escape; } else if (_should_escape) [[unlikely]] { _should_escape = false; - } else if (start[_idx] == _enclose) [[unlikely]] { - _state.forward_to(ReaderState::MATCH_ENCLOSE); - ++_idx; - return; + } else if (_quote_escape) { + if (start[_idx] == _enclose) { + // double quote, escaped by quote + _quote_escape = false; + } else { + // match enclose + _quote_escape = false; + _state.forward_to(ReaderState::MATCH_ENCLOSE); + return; + } + } else if (start[_idx] == _enclose) { + _quote_escape = true; + } else { + _quote_escape = false; } + ++_idx; } while (_idx != len); diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h index c91b503cbe5c0d..66898128015c21 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.h @@ -222,6 +222,7 @@ class EncloseCsvLineReaderContext final inline void refresh_impl() { _idx = 0; _should_escape = false; + _quote_escape = false; _result = nullptr; _column_sep_positions.clear(); _state.reset(); @@ -256,6 +257,8 @@ class EncloseCsvLineReaderContext final size_t _idx = 0; bool _should_escape = false; + // quote is specially escaped by quote in csv format + bool _quote_escape = false; const std::string _column_sep; std::vector _column_sep_positions; diff --git a/be/test/vec/exec/format/file_reader/new_plain_text_line_reader_test.cpp b/be/test/vec/exec/format/file_reader/new_plain_text_line_reader_test.cpp new file mode 100644 index 00000000000000..2cc2796048e8c9 --- /dev/null +++ b/be/test/vec/exec/format/file_reader/new_plain_text_line_reader_test.cpp @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// be/test/vec/exec/format/file_reader/new_plain_text_line_reader_test.cpp + +#include "vec/exec/format/file_reader/new_plain_text_line_reader.h" + +#include + +namespace doris::vectorized { + +// Base test class for text line reader tests +class PlainTextLineReaderTest : public testing::Test { +protected: + // Helper function to verify line splitting results + void verify_split_result(const std::string& input, const std::string& line_delim, bool keep_cr, + const std::vector& expected_lines) { + PlainTextLineReaderCtx ctx(line_delim, line_delim.size(), keep_cr); + const auto* data = reinterpret_cast(input.c_str()); + size_t pos = 0; + size_t size = input.size(); + std::vector actual_lines; + + while (pos < size) { + ctx.refresh(); + const auto* line_end = ctx.read_line(data + pos, size - pos); + if (!line_end) { + actual_lines.emplace_back(reinterpret_cast(data + pos), size - pos); + break; + } + size_t line_len = line_end - (data + pos); + actual_lines.emplace_back(reinterpret_cast(data + pos), line_len); + pos += line_len + ctx.line_delimiter_length(); + } + + ASSERT_EQ(expected_lines, actual_lines); + } +}; + +// Test cases for PlainTextLineReaderCtx +TEST_F(PlainTextLineReaderTest, PlainTextBasic) { + verify_split_result("line1\nline2\nline3", "\n", false, {"line1", "line2", "line3"}); + + verify_split_result("line1\r\nline2\r\nline3", "\r\n", false, {"line1", "line2", "line3"}); + + verify_split_result("line1\r\nline2\r\nline3", "\n", true, {"line1\r", "line2\r", "line3"}); + + verify_split_result("line1\n\nline3", "\n", false, {"line1", "", "line3"}); + + verify_split_result("line1||line2||line3", "||", false, {"line1", "line2", "line3"}); +} + +// Test class for CSV line reader with enclosure support +class EncloseCsvLineReaderTest : public testing::Test { +protected: + // Helper function to verify CSV splitting results including column positions + void verify_csv_split(const std::string& input, const std::string& line_delim, + const std::string& col_sep, char enclose, char escape, bool keep_cr, + const std::vector& expected_lines, + const std::vector>& expected_col_positions) { + EncloseCsvLineReaderContext ctx(line_delim, line_delim.size(), col_sep, col_sep.size(), 10, + enclose, escape, keep_cr); + + const auto* data = reinterpret_cast(input.c_str()); + size_t pos = 0; + size_t size = input.size(); + std::vector actual_lines; + std::vector> actual_col_positions; + + while (pos < size) { + ctx.refresh(); + const uint8_t* line_end = ctx.read_line(data + pos, size - pos); + if (!line_end) { + actual_lines.emplace_back(reinterpret_cast(data + pos), size - pos); + actual_col_positions.push_back(ctx.column_sep_positions()); + break; + } + size_t line_len = line_end - (data + pos); + actual_lines.emplace_back(reinterpret_cast(data + pos), line_len); + actual_col_positions.push_back(ctx.column_sep_positions()); + pos += line_len + ctx.line_delimiter_length(); + } + + ASSERT_EQ(expected_lines, actual_lines); + ASSERT_EQ(expected_col_positions, actual_col_positions); + } +}; + +// Basic CSV format test cases +TEST_F(EncloseCsvLineReaderTest, CsvBasic) { + verify_csv_split("a,b,c\nd,e,f", "\n", ",", '"', '\\', false, {"a,b,c", "d,e,f"}, + {{1, 3}, {1, 3}}); + + verify_csv_split("\"a,x\",b,c\n\"d,y\",e,f", "\n", ",", '"', '\\', false, + {"\"a,x\",b,c", "\"d,y\",e,f"}, {{5, 7}, {5, 7}}); + + verify_csv_split("\"a\"\"x\",b,c\n\"d\\\"y\",e,f", "\n", ",", '"', '\\', false, + {R"("a""x",b,c)", R"("d\"y",e,f)"}, {{6, 8}, {6, 8}}); + + verify_csv_split("a||b||c\nd||e||f", "\n", "||", '"', '\\', false, {"a||b||c", "d||e||f"}, + {{1, 4}, {1, 4}}); +} + +// Edge cases and corner scenarios +TEST_F(EncloseCsvLineReaderTest, EdgeCases) { + verify_csv_split("\n\na,b,c", "\n", ",", '"', '\\', false, {"", "", "a,b,c"}, {{}, {}, {1, 3}}); + + verify_csv_split("\"abc,def\nghi,jkl", "\n", ",", '"', '\\', false, {"\"abc,def\nghi,jkl"}, + {{}}); + + verify_csv_split("a,b\r\nc,d\ne,f", "\r\n", ",", '"', '\\', false, {"a,b", "c,d\ne,f"}, + {{1}, {1, 5}}); + + verify_csv_split(R"(\,\"\n,b,c)", "\n", ",", '"', '\\', false, {R"(\,\"\n,b,c)"}, {{1, 6, 8}}); +} + +TEST_F(EncloseCsvLineReaderTest, QuoteEscaping) { + // Test multiple quoted fields with double-quote escaping in one line + verify_csv_split(R"("hello ""world\n""","foo ""bar""","test ""quote"" here")", "\n", ",", '"', + '\\', false, {R"("hello ""world\n""","foo ""bar""","test ""quote"" here")"}, + {{19, 33}}); + + // Test JSON-like string with escaped quotes + verify_csv_split( + R"({""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""1""}})", + "\n", ",", '"', '\\', false, + {R"({""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""1""}})"}, + {{18, 50}}); + + // Test custom enclose character + verify_csv_split(R"({|code|: |100|, |message|: |query success|, |data|: {|status|: |1|}})", + "\n", ",", '|', '\\', false, + {R"({|code|: |100|, |message|: |query success|, |data|: {|status|: |1|}})"}, + {{14, 42}}); +} + +TEST_F(EncloseCsvLineReaderTest, MultiCharDelimiters) { + // Test multi-character line delimiter + verify_csv_split("a,b,c\r\n\nd,e,f", "\r\n\n", ",", '"', '\\', false, {"a,b,c", "d,e,f"}, + {{1, 3}, {1, 3}}); + + // Test multi-character column delimiter + verify_csv_split("a|||b|||c\nd|||e|||f", "\n", "|||", '"', '\\', false, + {"a|||b|||c", "d|||e|||f"}, {{1, 5}, {1, 5}}); + + // Test both multi-character line and column delimiters + verify_csv_split("a|||b|||c\r\n\nd|||e|||f", "\r\n\n", "|||", '"', '\\', false, + {"a|||b|||c", "d|||e|||f"}, {{1, 5}, {1, 5}}); + + verify_csv_split("\"a|||b\"|||c\r\n\n\"d|||e\"|||f", "\r\n\n", "|||", '"', '\\', false, + {"\"a|||b\"|||c", "\"d|||e\"|||f"}, {{7}, {7}}); +} + +} // namespace doris::vectorized diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql index c003c9e7d50048..fe11a46067cffe 100755 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run76.hql @@ -1,3 +1,4 @@ +create database if not exists multi_catalog; use multi_catalog; CREATE TABLE text_table_normal_skip_header ( @@ -20,6 +21,15 @@ STORED AS TEXTFILE LOCATION '/user/doris/preinstalled_data/text/text_table_compressed_skip_header' TBLPROPERTIES ("skip.header.line.count"="5"); +CREATE TABLE csv_json_table_simple ( + id STRING, + status_json STRING +) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.OpenCSVSerde' +STORED AS TEXTFILE +LOCATION '/user/doris/preinstalled_data/csv/csv_json_table_simple'; + create database if not exists openx_json; use openx_json; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/csv/csv_json_table_simple/csv_json_table_simple.csv b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/csv/csv_json_table_simple/csv_json_table_simple.csv new file mode 100644 index 00000000000000..7e17354e1610e7 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/csv/csv_json_table_simple/csv_json_table_simple.csv @@ -0,0 +1,10 @@ +"1001","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""1""}}" +"1002","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""20""}}" +"1003","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""1""}}" +"1004","{""code"": ""200"", ""message"": ""query failed"", ""data"": {}}" +"1005","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""20""}}" +"1006","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""1""}}" +"1007","{""code"": ""300"", ""message"": ""invalid request"", ""data"": null}" +"1008","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""0""}}" +"1009","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""1""}}" +"1010","{""code"": ""100"", ""message"": ""query success"", ""data"": {""status"": ""20""}}" diff --git a/regression-test/data/external_table_p0/hive/test_hive_serde_prop.out b/regression-test/data/external_table_p0/hive/test_hive_serde_prop.out index fd8e7bb4313ea6..c2415c058f14f1 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_serde_prop.out +++ b/regression-test/data/external_table_p0/hive/test_hive_serde_prop.out @@ -41,7 +41,7 @@ b 2.2 -- !test_open_csv_default_prop -- 1 John Doe 28 50000.75 TRUE 2022-01-15 2023-10-21 14:30:00 4.5 Senior Developer -2 Jane,Smith 2020-05-20 ""Project Manager"" +2 Jane,Smith 2020-05-20 "Project Manager" -- !test_open_csv_standard_prop -- 1 John Doe 28 50000.75 TRUE 2022-01-15 2023-10-21 14:30:00 4.5 Senior Developer @@ -93,7 +93,7 @@ b 2.2 -- !test_open_csv_default_prop -- 1 John Doe 28 50000.75 TRUE 2022-01-15 2023-10-21 14:30:00 4.5 Senior Developer -2 Jane,Smith FALSE 2020-05-20 ""Project Manager"" +2 Jane,Smith FALSE 2020-05-20 "Project Manager" -- !test_open_csv_standard_prop -- 1 John Doe 28 50000.75 TRUE 2022-01-15 2023-10-21 14:30:00 4.5 Senior Developer diff --git a/regression-test/data/external_table_p0/hive/test_open_csv_serde.out b/regression-test/data/external_table_p0/hive/test_open_csv_serde.out new file mode 100644 index 00000000000000..a071f731f45f0b --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_open_csv_serde.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !csv_escape_quote_in_enclose -- +1001 {"code": "100", "message": "query success", "data": {"status": "1"}} +1002 {"code": "100", "message": "query success", "data": {"status": "20"}} +1003 {"code": "100", "message": "query success", "data": {"status": "1"}} +1004 {"code": "200", "message": "query failed", "data": {}} +1005 {"code": "100", "message": "query success", "data": {"status": "20"}} +1006 {"code": "100", "message": "query success", "data": {"status": "1"}} +1007 {"code": "300", "message": "invalid request", "data": null} +1008 {"code": "100", "message": "query success", "data": {"status": "0"}} +1009 {"code": "100", "message": "query success", "data": {"status": "1"}} +1010 {"code": "100", "message": "query success", "data": {"status": "20"}} + +-- !csv_escape_quote_in_enclose -- +1001 {"code": "100", "message": "query success", "data": {"status": "1"}} +1002 {"code": "100", "message": "query success", "data": {"status": "20"}} +1003 {"code": "100", "message": "query success", "data": {"status": "1"}} +1004 {"code": "200", "message": "query failed", "data": {}} +1005 {"code": "100", "message": "query success", "data": {"status": "20"}} +1006 {"code": "100", "message": "query success", "data": {"status": "1"}} +1007 {"code": "300", "message": "invalid request", "data": null} +1008 {"code": "100", "message": "query success", "data": {"status": "0"}} +1009 {"code": "100", "message": "query success", "data": {"status": "1"}} +1010 {"code": "100", "message": "query success", "data": {"status": "20"}} + diff --git a/regression-test/data/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.out b/regression-test/data/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.out index 0c1450e35fe1c0..8d4444ac4188d8 100644 --- a/regression-test/data/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.out +++ b/regression-test/data/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.out @@ -10,4 +10,5 @@ 6 ab"c 2023-07-20 d"ef" 2023-07-20:05:48:31 "g"hi 7 aaa 2023-07-20 2023-07-20:05:48:31 8 aaa"bbb"ccc 2023-07-20 "aa"bb 2023-07-20:05:48:31 aa"bb" -9 aa,"bbb cc" 2023-07-20 ""aa"bb ,2023-07-20:05:48:31,"aa"bb" +9 aa,"bbb cc" 2023-07-20 "aa"bb ,2023-07-20:05:48:31,"aa"bb" + diff --git a/regression-test/data/load_p0/stream_load/enclose_with_escape_doublequote.csv b/regression-test/data/load_p0/stream_load/enclose_with_escape_doublequote.csv new file mode 100644 index 00000000000000..41a0d65cf26a7b --- /dev/null +++ b/regression-test/data/load_p0/stream_load/enclose_with_escape_doublequote.csv @@ -0,0 +1 @@ +5,"abc""de,fg""h","2023-07-17","""abc""def""","2023-07-22:07:00:00","test ""escape"" in enclose" diff --git a/regression-test/data/load_p0/stream_load/enclose_with_escape_quote.csv b/regression-test/data/load_p0/stream_load/enclose_with_escape_quote.csv new file mode 100644 index 00000000000000..0ade6f869001b0 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/enclose_with_escape_quote.csv @@ -0,0 +1 @@ +4,"abc\"de,fg\"h","2023-07-16","\"abc\"def\"","2023-07-21:06:00:00","test \"escape\" in enclose" diff --git a/regression-test/data/load_p0/stream_load/test_csv_with_enclose_and_escape.out b/regression-test/data/load_p0/stream_load/test_csv_with_enclose_and_escape.out index 5646d96230f4a9..d4509165cf4fd9 100644 --- a/regression-test/data/load_p0/stream_load/test_csv_with_enclose_and_escape.out +++ b/regression-test/data/load_p0/stream_load/test_csv_with_enclose_and_escape.out @@ -3,6 +3,8 @@ 1 abc,d\nef, 2023-07-15 ,,\nghi,j 2023-07-20T05:48:31 kl,mn,\n 2 abc 2023-07-15 def 2023-07-20T05:48:31 ghi 3 abc"de,fg"h 2023-07-15 i\nj,k\n" 2023-07-20T05:48:31 ghi +4 abc"de,fg"h 2023-07-16 "abc"def" 2023-07-21T06:00 test "escape" in enclose +5 abc"de,fg"h 2023-07-17 "abc"def" 2023-07-22T07:00 test "escape" in enclose 6 ab"c 2023-07-20 d"ef" 2023-07-20T05:48:31 "g"hi 7 aaa 2023-07-20 2023-07-20T05:48:31 8 aaa"bbb"ccc 2023-07-20 "aa"bb 2023-07-20T05:48:31 aa"bb" diff --git a/regression-test/suites/external_table_p0/hive/test_open_csv_serde.groovy b/regression-test/suites/external_table_p0/hive/test_open_csv_serde.groovy new file mode 100644 index 00000000000000..c3b5794323d35c --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_open_csv_serde.groovy @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_open_csv_serde","p0,external,tvf,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + for (String hivePrefix : ["hive2","hive3"]) { + + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${hivePrefix}_test_open_csv_serde" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def hdfsUserName = "doris" + String hdfs_port = context.config.otherConfigs.get(hivePrefix + "HdfsPort") + def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" + + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + sql """use `${catalog_name}`.`multi_catalog`""" + + qt_csv_escape_quote_in_enclose """select * from csv_json_table_simple order by id;""" + // TODO: add more case after refactor csv_reader and text_reader + } +} \ No newline at end of file diff --git a/regression-test/suites/load_p0/stream_load/test_csv_with_enclose_and_escape.groovy b/regression-test/suites/load_p0/stream_load/test_csv_with_enclose_and_escape.groovy index 1562fa35cfd99d..5625a7e6de6d52 100644 --- a/regression-test/suites/load_p0/stream_load/test_csv_with_enclose_and_escape.groovy +++ b/regression-test/suites/load_p0/stream_load/test_csv_with_enclose_and_escape.groovy @@ -1,4 +1,3 @@ - // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -38,6 +37,8 @@ suite("test_csv_with_enclose_and_escape", "p0") { def normalCases = [ 'enclose_normal.csv', 'enclose_with_escape.csv', + 'enclose_with_escape_quote.csv', + 'enclose_with_escape_doublequote.csv', 'enclose_wrong_position.csv', 'enclose_empty_values.csv' ] From 1934a1523f5a91420d2229678667ebfd675a77eb Mon Sep 17 00:00:00 2001 From: 924060929 Date: Sat, 28 Jun 2025 17:29:34 +0800 Subject: [PATCH 105/572] [chore](test) remove tests of nereids distribute planner (#52379) ### What problem does this PR solve? remove tests of nereids distribute planner because we will not use distribute planner in 3.0 --- .../hint_tpcds/shape/query24.out | 4 +- .../hint_tpcds/shape/query72.out | 6 +- .../new_shapes_p0/hint_tpch/shape/q13.out | 2 +- .../data/new_shapes_p0/hint_tpch/shape/q3.out | 2 +- .../data/new_shapes_p0/hint_tpch/shape/q5.out | 2 +- .../data/new_shapes_p0/hint_tpch/shape/q7.out | 2 +- .../data/new_shapes_p0/hint_tpch/shape/q9.out | 4 +- .../tpcds_sf100/noStatsRfPrune/query14.out | 26 ++- .../tpcds_sf100/noStatsRfPrune/query30.out | 2 +- .../tpcds_sf100/noStatsRfPrune/query35.out | 2 +- .../tpcds_sf100/noStatsRfPrune/query37.out | 14 +- .../tpcds_sf100/noStatsRfPrune/query46.out | 2 +- .../tpcds_sf100/noStatsRfPrune/query6.out | 2 +- .../tpcds_sf100/noStatsRfPrune/query64.out | 90 ++++----- .../tpcds_sf100/noStatsRfPrune/query68.out | 2 +- .../tpcds_sf100/noStatsRfPrune/query81.out | 2 +- .../tpcds_sf100/noStatsRfPrune/query82.out | 14 +- .../tpcds_sf100/no_stats_shape/query14.out | 26 ++- .../tpcds_sf100/no_stats_shape/query30.out | 2 +- .../tpcds_sf100/no_stats_shape/query35.out | 2 +- .../tpcds_sf100/no_stats_shape/query37.out | 14 +- .../tpcds_sf100/no_stats_shape/query46.out | 2 +- .../tpcds_sf100/no_stats_shape/query6.out | 2 +- .../tpcds_sf100/no_stats_shape/query64.out | 92 ++++----- .../tpcds_sf100/no_stats_shape/query68.out | 2 +- .../tpcds_sf100/no_stats_shape/query81.out | 2 +- .../tpcds_sf100/no_stats_shape/query82.out | 14 +- .../tpcds_sf100/rf_prune/query14.out | 46 +++-- .../tpcds_sf100/rf_prune/query15.out | 10 +- .../tpcds_sf100/rf_prune/query18.out | 2 +- .../tpcds_sf100/rf_prune/query24.out | 12 +- .../tpcds_sf100/rf_prune/query25.out | 2 +- .../tpcds_sf100/rf_prune/query26.out | 6 +- .../tpcds_sf100/rf_prune/query27.out | 2 +- .../tpcds_sf100/rf_prune/query29.out | 6 +- .../tpcds_sf100/rf_prune/query30.out | 6 +- .../tpcds_sf100/rf_prune/query33.out | 4 +- .../tpcds_sf100/rf_prune/query35.out | 2 +- .../tpcds_sf100/rf_prune/query45.out | 10 +- .../tpcds_sf100/rf_prune/query46.out | 4 +- .../tpcds_sf100/rf_prune/query48.out | 10 +- .../tpcds_sf100/rf_prune/query56.out | 2 +- .../tpcds_sf100/rf_prune/query58.out | 10 +- .../tpcds_sf100/rf_prune/query60.out | 12 +- .../tpcds_sf100/rf_prune/query64.out | 6 +- .../tpcds_sf100/rf_prune/query7.out | 2 +- .../tpcds_sf100/rf_prune/query72.out | 2 +- .../tpcds_sf100/rf_prune/query8.out | 2 +- .../tpcds_sf100/rf_prune/query81.out | 2 +- .../tpcds_sf100/rf_prune/query83.out | 8 +- .../tpcds_sf100/rf_prune/query85.out | 10 +- .../tpcds_sf100/shape/query14.out | 46 +++-- .../tpcds_sf100/shape/query15.out | 6 +- .../tpcds_sf100/shape/query18.out | 2 +- .../tpcds_sf100/shape/query24.out | 6 +- .../tpcds_sf100/shape/query25.out | 2 +- .../tpcds_sf100/shape/query26.out | 8 +- .../tpcds_sf100/shape/query27.out | 2 +- .../tpcds_sf100/shape/query29.out | 10 +- .../tpcds_sf100/shape/query30.out | 8 +- .../tpcds_sf100/shape/query33.out | 4 +- .../tpcds_sf100/shape/query35.out | 2 +- .../tpcds_sf100/shape/query45.out | 6 +- .../tpcds_sf100/shape/query46.out | 4 +- .../tpcds_sf100/shape/query48.out | 10 +- .../tpcds_sf100/shape/query56.out | 2 +- .../tpcds_sf100/shape/query58.out | 12 +- .../tpcds_sf100/shape/query60.out | 12 +- .../tpcds_sf100/shape/query64.out | 8 +- .../tpcds_sf100/shape/query7.out | 2 +- .../tpcds_sf100/shape/query72.out | 2 +- .../tpcds_sf100/shape/query8.out | 2 +- .../tpcds_sf100/shape/query81.out | 2 +- .../tpcds_sf100/shape/query83.out | 8 +- .../tpcds_sf100/shape/query85.out | 10 +- .../bs_downgrade_shape/query45.out | 2 +- .../bs_downgrade_shape/query56.out | 4 +- .../bs_downgrade_shape/query8.out | 2 +- .../tpcds_sf1000/shape/query15.out | 2 +- .../tpcds_sf1000/shape/query24.out | 6 +- .../tpcds_sf1000/shape/query27.out | 2 +- .../tpcds_sf1000/shape/query30.out | 2 +- .../tpcds_sf1000/shape/query33.out | 2 +- .../tpcds_sf1000/shape/query35.out | 2 +- .../tpcds_sf1000/shape/query45.out | 2 +- .../tpcds_sf1000/shape/query46.out | 16 +- .../tpcds_sf1000/shape/query48.out | 10 +- .../tpcds_sf1000/shape/query56.out | 4 +- .../tpcds_sf1000/shape/query60.out | 2 +- .../tpcds_sf1000/shape/query64.out | 20 +- .../tpcds_sf1000/shape/query72.out | 2 +- .../tpcds_sf1000/shape/query8.out | 2 +- .../tpcds_sf1000/shape/query81.out | 2 +- .../tpcds_sf1000/shape/query83.out | 4 +- .../tpch_sf1000/nostats_rf_prune/q10.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q13.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q14.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q15.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q17.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q18.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q19.out | 2 +- .../nostats_rf_prune/q20-rewrite.out | 4 +- .../tpch_sf1000/nostats_rf_prune/q20.out | 4 +- .../tpch_sf1000/nostats_rf_prune/q22.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q3.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q5.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q7.out | 2 +- .../tpch_sf1000/nostats_rf_prune/q8.out | 4 +- .../tpch_sf1000/nostats_rf_prune/q9.out | 12 +- .../tpch_sf1000/rf_prune/q13.out | 2 +- .../new_shapes_p0/tpch_sf1000/rf_prune/q2.out | 24 +-- .../tpch_sf1000/rf_prune/q20-rewrite.out | 4 +- .../tpch_sf1000/rf_prune/q20.out | 4 +- .../tpch_sf1000/rf_prune/q22.out | 20 +- .../new_shapes_p0/tpch_sf1000/rf_prune/q5.out | 6 +- .../new_shapes_p0/tpch_sf1000/rf_prune/q8.out | 28 +-- .../new_shapes_p0/tpch_sf1000/rf_prune/q9.out | 4 +- .../new_shapes_p0/tpch_sf1000/shape/q13.out | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q2.out | 24 +-- .../tpch_sf1000/shape/q20-rewrite.out | 4 +- .../new_shapes_p0/tpch_sf1000/shape/q20.out | 4 +- .../new_shapes_p0/tpch_sf1000/shape/q22.out | 20 +- .../new_shapes_p0/tpch_sf1000/shape/q5.out | 12 +- .../new_shapes_p0/tpch_sf1000/shape/q8.out | 28 +-- .../new_shapes_p0/tpch_sf1000/shape/q9.out | 4 +- .../tpch_sf1000/shape_no_stats/q10.out | 2 +- .../tpch_sf1000/shape_no_stats/q13.out | 2 +- .../tpch_sf1000/shape_no_stats/q14.out | 2 +- .../tpch_sf1000/shape_no_stats/q15.out | 2 +- .../tpch_sf1000/shape_no_stats/q17.out | 2 +- .../tpch_sf1000/shape_no_stats/q18.out | 2 +- .../tpch_sf1000/shape_no_stats/q19.out | 2 +- .../shape_no_stats/q20-rewrite.out | 4 +- .../tpch_sf1000/shape_no_stats/q20.out | 4 +- .../tpch_sf1000/shape_no_stats/q22.out | 2 +- .../tpch_sf1000/shape_no_stats/q3.out | 2 +- .../tpch_sf1000/shape_no_stats/q5.out | 2 +- .../tpch_sf1000/shape_no_stats/q7.out | 2 +- .../tpch_sf1000/shape_no_stats/q8.out | 4 +- .../tpch_sf1000/shape_no_stats/q9.out | 10 +- .../distribute/colocate_agg_join.groovy | 84 -------- .../distribute/colocate_union_numbers.groovy | 72 ------- .../distribute/distribution_expr.groovy | 130 ------------ .../distribute/local_shuffle.groovy | 187 ------------------ ...une_bucket_with_bucket_shuffle_join.groovy | 100 ---------- .../distribute/query_constant.groovy | 39 ---- .../distribute/read_metadata.groovy | 28 --- .../distribute/shuffle.groovy | 34 ---- .../distribute/shuffle_left_join.groovy | 117 ----------- .../new_shapes_p0/clickbench/query1.groovy | 2 +- .../new_shapes_p0/clickbench/query10.groovy | 2 +- .../new_shapes_p0/clickbench/query11.groovy | 2 +- .../new_shapes_p0/clickbench/query12.groovy | 2 +- .../new_shapes_p0/clickbench/query13.groovy | 2 +- .../new_shapes_p0/clickbench/query14.groovy | 2 +- .../new_shapes_p0/clickbench/query15.groovy | 2 +- .../new_shapes_p0/clickbench/query16.groovy | 2 +- .../new_shapes_p0/clickbench/query17.groovy | 2 +- .../new_shapes_p0/clickbench/query18.groovy | 2 +- .../new_shapes_p0/clickbench/query19.groovy | 2 +- .../new_shapes_p0/clickbench/query2.groovy | 2 +- .../new_shapes_p0/clickbench/query20.groovy | 2 +- .../new_shapes_p0/clickbench/query21.groovy | 2 +- .../new_shapes_p0/clickbench/query22.groovy | 2 +- .../new_shapes_p0/clickbench/query23.groovy | 2 +- .../new_shapes_p0/clickbench/query24.groovy | 2 +- .../new_shapes_p0/clickbench/query25.groovy | 2 +- .../new_shapes_p0/clickbench/query26.groovy | 2 +- .../new_shapes_p0/clickbench/query27.groovy | 2 +- .../new_shapes_p0/clickbench/query28.groovy | 2 +- .../new_shapes_p0/clickbench/query29.groovy | 2 +- .../new_shapes_p0/clickbench/query3.groovy | 2 +- .../new_shapes_p0/clickbench/query30.groovy | 2 +- .../new_shapes_p0/clickbench/query31.groovy | 2 +- .../new_shapes_p0/clickbench/query32.groovy | 2 +- .../new_shapes_p0/clickbench/query33.groovy | 2 +- .../new_shapes_p0/clickbench/query34.groovy | 2 +- .../new_shapes_p0/clickbench/query35.groovy | 2 +- .../new_shapes_p0/clickbench/query36.groovy | 2 +- .../new_shapes_p0/clickbench/query37.groovy | 2 +- .../new_shapes_p0/clickbench/query38.groovy | 2 +- .../new_shapes_p0/clickbench/query39.groovy | 2 +- .../new_shapes_p0/clickbench/query4.groovy | 2 +- .../new_shapes_p0/clickbench/query40.groovy | 2 +- .../new_shapes_p0/clickbench/query41.groovy | 2 +- .../new_shapes_p0/clickbench/query42.groovy | 2 +- .../new_shapes_p0/clickbench/query43.groovy | 2 +- .../new_shapes_p0/clickbench/query5.groovy | 2 +- .../new_shapes_p0/clickbench/query6.groovy | 2 +- .../new_shapes_p0/clickbench/query7.groovy | 2 +- .../new_shapes_p0/clickbench/query8.groovy | 2 +- .../new_shapes_p0/clickbench/query9.groovy | 2 +- .../new_shapes_p0/hint_tpcds/ddl/shape.tmpl | 2 +- .../hint_tpcds/shape/query1.groovy | 2 +- .../hint_tpcds/shape/query24.groovy | 2 +- .../hint_tpcds/shape/query64.groovy | 2 +- .../hint_tpcds/shape/query67.groovy | 2 +- .../hint_tpcds/shape/query72.groovy | 2 +- .../hint_tpcds/shape/query78.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q10.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q11.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q12.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q13.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q14.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q15.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q17.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q19.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q3.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q4.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q5.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q7.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q8.groovy | 2 +- .../new_shapes_p0/hint_tpch/shape/q9.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/flat.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q1.1.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q1.2.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q1.3.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q2.1.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q2.2.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q2.3.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q3.1.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q3.2.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q3.3.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q3.4.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q4.1.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q4.2.groovy | 2 +- .../new_shapes_p0/ssb_sf100/shape/q4.3.groovy | 2 +- .../tpcds_sf100/constraints/query23.groovy | 2 +- .../tpcds_sf100/ddl/rf_prune.tmpl | 2 +- .../new_shapes_p0/tpcds_sf100/ddl/shape.tmpl | 2 +- .../tpcds_sf100/noStatsRfPrune/query1.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query10.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query11.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query12.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query13.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query14.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query15.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query16.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query17.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query18.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query19.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query2.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query20.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query21.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query22.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query23.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query24.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query25.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query26.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query27.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query28.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query29.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query3.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query30.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query31.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query32.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query33.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query34.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query35.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query36.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query37.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query38.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query39.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query4.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query40.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query41.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query42.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query43.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query44.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query45.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query46.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query47.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query48.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query49.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query5.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query50.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query51.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query52.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query53.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query54.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query55.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query56.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query57.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query58.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query59.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query6.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query60.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query61.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query62.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query63.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query64.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query65.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query66.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query67.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query68.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query69.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query7.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query70.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query71.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query72.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query73.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query74.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query75.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query76.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query77.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query78.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query79.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query8.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query80.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query81.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query82.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query83.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query84.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query85.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query86.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query87.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query88.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query89.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query9.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query90.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query91.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query92.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query93.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query94.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query95.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query96.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query97.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query98.groovy | 2 +- .../tpcds_sf100/noStatsRfPrune/query99.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query1.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query10.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query11.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query12.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query13.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query14.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query15.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query16.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query17.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query18.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query19.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query2.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query20.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query21.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query22.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query23.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query24.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query25.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query26.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query27.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query28.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query29.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query3.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query30.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query31.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query32.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query33.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query34.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query35.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query36.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query37.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query38.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query39.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query4.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query40.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query41.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query42.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query43.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query44.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query45.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query46.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query47.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query48.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query49.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query5.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query50.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query51.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query52.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query53.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query54.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query55.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query56.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query57.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query58.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query59.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query6.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query60.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query61.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query62.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query63.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query64.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query65.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query66.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query67.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query68.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query69.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query7.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query70.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query71.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query72.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query73.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query74.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query75.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query76.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query77.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query78.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query79.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query8.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query80.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query81.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query82.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query83.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query84.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query85.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query86.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query87.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query88.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query89.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query9.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query90.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query91.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query92.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query93.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query94.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query95.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query96.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query97.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query98.groovy | 2 +- .../tpcds_sf100/no_stats_shape/query99.groovy | 2 +- .../tpcds_sf100/rf_prune/query1.groovy | 2 +- .../tpcds_sf100/rf_prune/query10.groovy | 2 +- .../tpcds_sf100/rf_prune/query11.groovy | 2 +- .../tpcds_sf100/rf_prune/query12.groovy | 2 +- .../tpcds_sf100/rf_prune/query13.groovy | 2 +- .../tpcds_sf100/rf_prune/query14.groovy | 2 +- .../tpcds_sf100/rf_prune/query15.groovy | 2 +- .../tpcds_sf100/rf_prune/query16.groovy | 2 +- .../tpcds_sf100/rf_prune/query17.groovy | 2 +- .../tpcds_sf100/rf_prune/query18.groovy | 2 +- .../tpcds_sf100/rf_prune/query19.groovy | 2 +- .../tpcds_sf100/rf_prune/query2.groovy | 2 +- .../tpcds_sf100/rf_prune/query20.groovy | 2 +- .../tpcds_sf100/rf_prune/query21.groovy | 2 +- .../tpcds_sf100/rf_prune/query22.groovy | 2 +- .../tpcds_sf100/rf_prune/query23.groovy | 2 +- .../tpcds_sf100/rf_prune/query24.groovy | 2 +- .../tpcds_sf100/rf_prune/query25.groovy | 2 +- .../tpcds_sf100/rf_prune/query26.groovy | 2 +- .../tpcds_sf100/rf_prune/query27.groovy | 2 +- .../tpcds_sf100/rf_prune/query28.groovy | 2 +- .../tpcds_sf100/rf_prune/query29.groovy | 2 +- .../tpcds_sf100/rf_prune/query3.groovy | 2 +- .../tpcds_sf100/rf_prune/query30.groovy | 2 +- .../tpcds_sf100/rf_prune/query31.groovy | 2 +- .../tpcds_sf100/rf_prune/query32.groovy | 2 +- .../tpcds_sf100/rf_prune/query33.groovy | 2 +- .../tpcds_sf100/rf_prune/query34.groovy | 2 +- .../tpcds_sf100/rf_prune/query35.groovy | 2 +- .../tpcds_sf100/rf_prune/query36.groovy | 2 +- .../tpcds_sf100/rf_prune/query37.groovy | 2 +- .../tpcds_sf100/rf_prune/query38.groovy | 2 +- .../tpcds_sf100/rf_prune/query39.groovy | 2 +- .../tpcds_sf100/rf_prune/query4.groovy | 2 +- .../tpcds_sf100/rf_prune/query40.groovy | 2 +- .../tpcds_sf100/rf_prune/query41.groovy | 2 +- .../tpcds_sf100/rf_prune/query42.groovy | 2 +- .../tpcds_sf100/rf_prune/query43.groovy | 2 +- .../tpcds_sf100/rf_prune/query44.groovy | 2 +- .../tpcds_sf100/rf_prune/query45.groovy | 2 +- .../tpcds_sf100/rf_prune/query46.groovy | 2 +- .../tpcds_sf100/rf_prune/query47.groovy | 2 +- .../tpcds_sf100/rf_prune/query48.groovy | 2 +- .../tpcds_sf100/rf_prune/query49.groovy | 2 +- .../tpcds_sf100/rf_prune/query5.groovy | 2 +- .../tpcds_sf100/rf_prune/query50.groovy | 2 +- .../tpcds_sf100/rf_prune/query51.groovy | 2 +- .../tpcds_sf100/rf_prune/query52.groovy | 2 +- .../tpcds_sf100/rf_prune/query53.groovy | 2 +- .../tpcds_sf100/rf_prune/query54.groovy | 2 +- .../tpcds_sf100/rf_prune/query55.groovy | 2 +- .../tpcds_sf100/rf_prune/query56.groovy | 2 +- .../tpcds_sf100/rf_prune/query57.groovy | 2 +- .../tpcds_sf100/rf_prune/query58.groovy | 2 +- .../tpcds_sf100/rf_prune/query59.groovy | 2 +- .../tpcds_sf100/rf_prune/query6.groovy | 2 +- .../tpcds_sf100/rf_prune/query60.groovy | 2 +- .../tpcds_sf100/rf_prune/query61.groovy | 2 +- .../tpcds_sf100/rf_prune/query62.groovy | 2 +- .../tpcds_sf100/rf_prune/query63.groovy | 2 +- .../tpcds_sf100/rf_prune/query64.groovy | 2 +- .../tpcds_sf100/rf_prune/query65.groovy | 2 +- .../tpcds_sf100/rf_prune/query66.groovy | 2 +- .../tpcds_sf100/rf_prune/query67.groovy | 2 +- .../tpcds_sf100/rf_prune/query68.groovy | 2 +- .../tpcds_sf100/rf_prune/query69.groovy | 2 +- .../tpcds_sf100/rf_prune/query7.groovy | 2 +- .../tpcds_sf100/rf_prune/query70.groovy | 2 +- .../tpcds_sf100/rf_prune/query71.groovy | 2 +- .../tpcds_sf100/rf_prune/query72.groovy | 2 +- .../tpcds_sf100/rf_prune/query73.groovy | 2 +- .../tpcds_sf100/rf_prune/query74.groovy | 2 +- .../tpcds_sf100/rf_prune/query75.groovy | 2 +- .../tpcds_sf100/rf_prune/query76.groovy | 2 +- .../tpcds_sf100/rf_prune/query77.groovy | 2 +- .../tpcds_sf100/rf_prune/query78.groovy | 2 +- .../tpcds_sf100/rf_prune/query79.groovy | 2 +- .../tpcds_sf100/rf_prune/query8.groovy | 2 +- .../tpcds_sf100/rf_prune/query80.groovy | 2 +- .../tpcds_sf100/rf_prune/query81.groovy | 2 +- .../tpcds_sf100/rf_prune/query82.groovy | 2 +- .../tpcds_sf100/rf_prune/query83.groovy | 2 +- .../tpcds_sf100/rf_prune/query84.groovy | 2 +- .../tpcds_sf100/rf_prune/query85.groovy | 2 +- .../tpcds_sf100/rf_prune/query86.groovy | 2 +- .../tpcds_sf100/rf_prune/query87.groovy | 2 +- .../tpcds_sf100/rf_prune/query88.groovy | 2 +- .../tpcds_sf100/rf_prune/query89.groovy | 2 +- .../tpcds_sf100/rf_prune/query9.groovy | 2 +- .../tpcds_sf100/rf_prune/query90.groovy | 2 +- .../tpcds_sf100/rf_prune/query91.groovy | 2 +- .../tpcds_sf100/rf_prune/query92.groovy | 2 +- .../tpcds_sf100/rf_prune/query93.groovy | 2 +- .../tpcds_sf100/rf_prune/query94.groovy | 2 +- .../tpcds_sf100/rf_prune/query95.groovy | 2 +- .../tpcds_sf100/rf_prune/query96.groovy | 2 +- .../tpcds_sf100/rf_prune/query97.groovy | 2 +- .../tpcds_sf100/rf_prune/query98.groovy | 2 +- .../tpcds_sf100/rf_prune/query99.groovy | 2 +- .../tpcds_sf100/shape/query1.groovy | 2 +- .../tpcds_sf100/shape/query10.groovy | 2 +- .../tpcds_sf100/shape/query11.groovy | 2 +- .../tpcds_sf100/shape/query12.groovy | 2 +- .../tpcds_sf100/shape/query13.groovy | 2 +- .../tpcds_sf100/shape/query14.groovy | 2 +- .../tpcds_sf100/shape/query15.groovy | 2 +- .../tpcds_sf100/shape/query16.groovy | 2 +- .../tpcds_sf100/shape/query17.groovy | 2 +- .../tpcds_sf100/shape/query18.groovy | 2 +- .../tpcds_sf100/shape/query19.groovy | 2 +- .../tpcds_sf100/shape/query2.groovy | 2 +- .../tpcds_sf100/shape/query20.groovy | 2 +- .../tpcds_sf100/shape/query21.groovy | 2 +- .../tpcds_sf100/shape/query22.groovy | 2 +- .../tpcds_sf100/shape/query23.groovy | 2 +- .../tpcds_sf100/shape/query24.groovy | 2 +- .../tpcds_sf100/shape/query25.groovy | 2 +- .../tpcds_sf100/shape/query26.groovy | 2 +- .../tpcds_sf100/shape/query27.groovy | 2 +- .../tpcds_sf100/shape/query28.groovy | 2 +- .../tpcds_sf100/shape/query29.groovy | 2 +- .../tpcds_sf100/shape/query3.groovy | 2 +- .../tpcds_sf100/shape/query30.groovy | 2 +- .../tpcds_sf100/shape/query31.groovy | 2 +- .../tpcds_sf100/shape/query32.groovy | 2 +- .../tpcds_sf100/shape/query33.groovy | 2 +- .../tpcds_sf100/shape/query34.groovy | 2 +- .../tpcds_sf100/shape/query35.groovy | 2 +- .../tpcds_sf100/shape/query36.groovy | 2 +- .../tpcds_sf100/shape/query37.groovy | 2 +- .../tpcds_sf100/shape/query38.groovy | 2 +- .../tpcds_sf100/shape/query39.groovy | 2 +- .../tpcds_sf100/shape/query4.groovy | 2 +- .../tpcds_sf100/shape/query40.groovy | 2 +- .../tpcds_sf100/shape/query41.groovy | 2 +- .../tpcds_sf100/shape/query42.groovy | 2 +- .../tpcds_sf100/shape/query43.groovy | 2 +- .../tpcds_sf100/shape/query44.groovy | 2 +- .../tpcds_sf100/shape/query45.groovy | 2 +- .../tpcds_sf100/shape/query46.groovy | 2 +- .../tpcds_sf100/shape/query47.groovy | 2 +- .../tpcds_sf100/shape/query48.groovy | 2 +- .../tpcds_sf100/shape/query49.groovy | 2 +- .../tpcds_sf100/shape/query5.groovy | 2 +- .../tpcds_sf100/shape/query50.groovy | 2 +- .../tpcds_sf100/shape/query51.groovy | 2 +- .../tpcds_sf100/shape/query52.groovy | 2 +- .../tpcds_sf100/shape/query53.groovy | 2 +- .../tpcds_sf100/shape/query54.groovy | 2 +- .../tpcds_sf100/shape/query55.groovy | 2 +- .../tpcds_sf100/shape/query56.groovy | 2 +- .../tpcds_sf100/shape/query57.groovy | 2 +- .../tpcds_sf100/shape/query58.groovy | 2 +- .../tpcds_sf100/shape/query59.groovy | 2 +- .../tpcds_sf100/shape/query6.groovy | 2 +- .../tpcds_sf100/shape/query60.groovy | 2 +- .../tpcds_sf100/shape/query61.groovy | 2 +- .../tpcds_sf100/shape/query62.groovy | 2 +- .../tpcds_sf100/shape/query63.groovy | 2 +- .../tpcds_sf100/shape/query64.groovy | 2 +- .../tpcds_sf100/shape/query65.groovy | 2 +- .../tpcds_sf100/shape/query66.groovy | 2 +- .../tpcds_sf100/shape/query67.groovy | 2 +- .../tpcds_sf100/shape/query68.groovy | 2 +- .../tpcds_sf100/shape/query69.groovy | 2 +- .../tpcds_sf100/shape/query7.groovy | 2 +- .../tpcds_sf100/shape/query70.groovy | 2 +- .../tpcds_sf100/shape/query71.groovy | 2 +- .../tpcds_sf100/shape/query72.groovy | 2 +- .../tpcds_sf100/shape/query73.groovy | 2 +- .../tpcds_sf100/shape/query74.groovy | 2 +- .../tpcds_sf100/shape/query75.groovy | 2 +- .../tpcds_sf100/shape/query76.groovy | 2 +- .../tpcds_sf100/shape/query77.groovy | 2 +- .../tpcds_sf100/shape/query78.groovy | 2 +- .../tpcds_sf100/shape/query79.groovy | 2 +- .../tpcds_sf100/shape/query8.groovy | 2 +- .../tpcds_sf100/shape/query80.groovy | 2 +- .../tpcds_sf100/shape/query81.groovy | 2 +- .../tpcds_sf100/shape/query82.groovy | 2 +- .../tpcds_sf100/shape/query83.groovy | 2 +- .../tpcds_sf100/shape/query84.groovy | 2 +- .../tpcds_sf100/shape/query85.groovy | 2 +- .../tpcds_sf100/shape/query86.groovy | 2 +- .../tpcds_sf100/shape/query87.groovy | 2 +- .../tpcds_sf100/shape/query88.groovy | 2 +- .../tpcds_sf100/shape/query89.groovy | 2 +- .../tpcds_sf100/shape/query9.groovy | 2 +- .../tpcds_sf100/shape/query90.groovy | 2 +- .../tpcds_sf100/shape/query91.groovy | 2 +- .../tpcds_sf100/shape/query92.groovy | 2 +- .../tpcds_sf100/shape/query93.groovy | 2 +- .../tpcds_sf100/shape/query94.groovy | 2 +- .../tpcds_sf100/shape/query95.groovy | 2 +- .../tpcds_sf100/shape/query96.groovy | 2 +- .../tpcds_sf100/shape/query97.groovy | 2 +- .../tpcds_sf100/shape/query98.groovy | 2 +- .../tpcds_sf100/shape/query99.groovy | 2 +- .../bs_downgrade_shape/query13.groovy | 2 +- .../bs_downgrade_shape/query19.groovy | 2 +- .../bs_downgrade_shape/query44.groovy | 2 +- .../bs_downgrade_shape/query45.groovy | 2 +- .../bs_downgrade_shape/query54.groovy | 2 +- .../bs_downgrade_shape/query56.groovy | 2 +- .../bs_downgrade_shape/query6.groovy | 2 +- .../bs_downgrade_shape/query61.groovy | 2 +- .../bs_downgrade_shape/query68.groovy | 2 +- .../bs_downgrade_shape/query8.groovy | 2 +- .../bs_downgrade_shape/query91.groovy | 2 +- .../bs_downgrade_shape/query95.groovy | 2 +- .../new_shapes_p0/tpcds_sf1000/ddl/shape.tmpl | 2 +- .../eliminate_empty/query10_empty.groovy | 2 +- .../tpcds_sf1000/shape/query1.groovy | 2 +- .../tpcds_sf1000/shape/query10.groovy | 2 +- .../tpcds_sf1000/shape/query11.groovy | 2 +- .../tpcds_sf1000/shape/query12.groovy | 2 +- .../tpcds_sf1000/shape/query13.groovy | 2 +- .../tpcds_sf1000/shape/query14.groovy | 2 +- .../tpcds_sf1000/shape/query15.groovy | 2 +- .../tpcds_sf1000/shape/query16.groovy | 2 +- .../tpcds_sf1000/shape/query17.groovy | 2 +- .../tpcds_sf1000/shape/query18.groovy | 2 +- .../tpcds_sf1000/shape/query19.groovy | 2 +- .../tpcds_sf1000/shape/query2.groovy | 2 +- .../tpcds_sf1000/shape/query20.groovy | 2 +- .../tpcds_sf1000/shape/query21.groovy | 2 +- .../tpcds_sf1000/shape/query22.groovy | 2 +- .../tpcds_sf1000/shape/query23.groovy | 2 +- .../tpcds_sf1000/shape/query24.groovy | 2 +- .../tpcds_sf1000/shape/query25.groovy | 2 +- .../tpcds_sf1000/shape/query26.groovy | 2 +- .../tpcds_sf1000/shape/query27.groovy | 2 +- .../tpcds_sf1000/shape/query28.groovy | 2 +- .../tpcds_sf1000/shape/query29.groovy | 2 +- .../tpcds_sf1000/shape/query3.groovy | 2 +- .../tpcds_sf1000/shape/query30.groovy | 2 +- .../tpcds_sf1000/shape/query31.groovy | 2 +- .../tpcds_sf1000/shape/query32.groovy | 2 +- .../tpcds_sf1000/shape/query33.groovy | 2 +- .../tpcds_sf1000/shape/query34.groovy | 2 +- .../tpcds_sf1000/shape/query35.groovy | 2 +- .../tpcds_sf1000/shape/query36.groovy | 2 +- .../tpcds_sf1000/shape/query37.groovy | 2 +- .../tpcds_sf1000/shape/query38.groovy | 2 +- .../tpcds_sf1000/shape/query39.groovy | 2 +- .../tpcds_sf1000/shape/query4.groovy | 2 +- .../tpcds_sf1000/shape/query40.groovy | 2 +- .../tpcds_sf1000/shape/query41.groovy | 2 +- .../tpcds_sf1000/shape/query42.groovy | 2 +- .../tpcds_sf1000/shape/query43.groovy | 2 +- .../tpcds_sf1000/shape/query44.groovy | 2 +- .../tpcds_sf1000/shape/query45.groovy | 2 +- .../tpcds_sf1000/shape/query46.groovy | 2 +- .../tpcds_sf1000/shape/query47.groovy | 2 +- .../tpcds_sf1000/shape/query48.groovy | 2 +- .../tpcds_sf1000/shape/query49.groovy | 2 +- .../tpcds_sf1000/shape/query5.groovy | 2 +- .../tpcds_sf1000/shape/query50.groovy | 2 +- .../tpcds_sf1000/shape/query51.groovy | 2 +- .../tpcds_sf1000/shape/query52.groovy | 2 +- .../tpcds_sf1000/shape/query53.groovy | 2 +- .../tpcds_sf1000/shape/query54.groovy | 2 +- .../tpcds_sf1000/shape/query55.groovy | 2 +- .../tpcds_sf1000/shape/query56.groovy | 2 +- .../tpcds_sf1000/shape/query57.groovy | 2 +- .../tpcds_sf1000/shape/query58.groovy | 2 +- .../tpcds_sf1000/shape/query59.groovy | 2 +- .../tpcds_sf1000/shape/query6.groovy | 2 +- .../tpcds_sf1000/shape/query60.groovy | 2 +- .../tpcds_sf1000/shape/query61.groovy | 2 +- .../tpcds_sf1000/shape/query62.groovy | 2 +- .../tpcds_sf1000/shape/query63.groovy | 2 +- .../tpcds_sf1000/shape/query64.groovy | 2 +- .../tpcds_sf1000/shape/query65.groovy | 2 +- .../tpcds_sf1000/shape/query66.groovy | 2 +- .../tpcds_sf1000/shape/query67.groovy | 2 +- .../tpcds_sf1000/shape/query68.groovy | 2 +- .../tpcds_sf1000/shape/query69.groovy | 2 +- .../tpcds_sf1000/shape/query7.groovy | 2 +- .../tpcds_sf1000/shape/query70.groovy | 2 +- .../tpcds_sf1000/shape/query71.groovy | 2 +- .../tpcds_sf1000/shape/query72.groovy | 2 +- .../tpcds_sf1000/shape/query73.groovy | 2 +- .../tpcds_sf1000/shape/query74.groovy | 2 +- .../tpcds_sf1000/shape/query75.groovy | 2 +- .../tpcds_sf1000/shape/query76.groovy | 2 +- .../tpcds_sf1000/shape/query77.groovy | 2 +- .../tpcds_sf1000/shape/query78.groovy | 2 +- .../tpcds_sf1000/shape/query79.groovy | 2 +- .../tpcds_sf1000/shape/query8.groovy | 2 +- .../tpcds_sf1000/shape/query80.groovy | 2 +- .../tpcds_sf1000/shape/query81.groovy | 2 +- .../tpcds_sf1000/shape/query82.groovy | 2 +- .../tpcds_sf1000/shape/query83.groovy | 2 +- .../tpcds_sf1000/shape/query84.groovy | 2 +- .../tpcds_sf1000/shape/query85.groovy | 2 +- .../tpcds_sf1000/shape/query86.groovy | 2 +- .../tpcds_sf1000/shape/query87.groovy | 2 +- .../tpcds_sf1000/shape/query88.groovy | 2 +- .../tpcds_sf1000/shape/query89.groovy | 2 +- .../tpcds_sf1000/shape/query9.groovy | 2 +- .../tpcds_sf1000/shape/query90.groovy | 2 +- .../tpcds_sf1000/shape/query91.groovy | 2 +- .../tpcds_sf1000/shape/query92.groovy | 2 +- .../tpcds_sf1000/shape/query93.groovy | 2 +- .../tpcds_sf1000/shape/query94.groovy | 2 +- .../tpcds_sf1000/shape/query95.groovy | 2 +- .../tpcds_sf1000/shape/query96.groovy | 2 +- .../tpcds_sf1000/shape/query97.groovy | 2 +- .../tpcds_sf1000/shape/query98.groovy | 2 +- .../tpcds_sf1000/shape/query99.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q1.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q10.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q11.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q12.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q13.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q14.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q15.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q16.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q17.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q18.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q19.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q2.groovy | 2 +- .../nostats_rf_prune/q20-rewrite.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q20.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q21.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q22.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q3.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q4.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q5.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q6.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q7.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q8.groovy | 2 +- .../tpch_sf1000/nostats_rf_prune/q9.groovy | 2 +- .../tpch_sf1000/rf_prune/q1.groovy | 2 +- .../tpch_sf1000/rf_prune/q10.groovy | 2 +- .../tpch_sf1000/rf_prune/q11.groovy | 2 +- .../tpch_sf1000/rf_prune/q12.groovy | 2 +- .../tpch_sf1000/rf_prune/q13.groovy | 2 +- .../tpch_sf1000/rf_prune/q14.groovy | 2 +- .../tpch_sf1000/rf_prune/q15.groovy | 2 +- .../tpch_sf1000/rf_prune/q16.groovy | 2 +- .../tpch_sf1000/rf_prune/q17.groovy | 2 +- .../tpch_sf1000/rf_prune/q18.groovy | 2 +- .../tpch_sf1000/rf_prune/q19.groovy | 2 +- .../tpch_sf1000/rf_prune/q2.groovy | 2 +- .../tpch_sf1000/rf_prune/q20-rewrite.groovy | 2 +- .../tpch_sf1000/rf_prune/q20.groovy | 2 +- .../tpch_sf1000/rf_prune/q21.groovy | 2 +- .../tpch_sf1000/rf_prune/q22.groovy | 2 +- .../tpch_sf1000/rf_prune/q3.groovy | 2 +- .../tpch_sf1000/rf_prune/q4.groovy | 2 +- .../tpch_sf1000/rf_prune/q5.groovy | 2 +- .../tpch_sf1000/rf_prune/q6.groovy | 2 +- .../tpch_sf1000/rf_prune/q7.groovy | 2 +- .../tpch_sf1000/rf_prune/q8.groovy | 2 +- .../tpch_sf1000/rf_prune/q9.groovy | 2 +- .../runtime_filter/test_pushdown_setop.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q1.groovy | 2 +- .../tpch_sf1000/shape/q10.groovy | 2 +- .../tpch_sf1000/shape/q11.groovy | 2 +- .../tpch_sf1000/shape/q12.groovy | 2 +- .../tpch_sf1000/shape/q13.groovy | 2 +- .../tpch_sf1000/shape/q14.groovy | 2 +- .../tpch_sf1000/shape/q15.groovy | 2 +- .../tpch_sf1000/shape/q16.groovy | 2 +- .../tpch_sf1000/shape/q17.groovy | 2 +- .../tpch_sf1000/shape/q18.groovy | 2 +- .../tpch_sf1000/shape/q19.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q2.groovy | 2 +- .../tpch_sf1000/shape/q20-rewrite.groovy | 2 +- .../tpch_sf1000/shape/q20.groovy | 2 +- .../tpch_sf1000/shape/q21.groovy | 2 +- .../tpch_sf1000/shape/q22.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q3.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q4.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q5.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q6.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q7.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q8.groovy | 2 +- .../new_shapes_p0/tpch_sf1000/shape/q9.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q1.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q10.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q11.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q12.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q13.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q14.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q15.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q16.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q17.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q18.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q19.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q2.groovy | 2 +- .../shape_no_stats/q20-rewrite.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q20.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q21.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q22.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q3.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q4.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q5.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q6.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q7.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q8.groovy | 2 +- .../tpch_sf1000/shape_no_stats/q9.groovy | 2 +- 832 files changed, 1210 insertions(+), 2005 deletions(-) delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/colocate_agg_join.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/distribution_expr.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/read_metadata.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy delete mode 100644 regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy diff --git a/regression-test/data/new_shapes_p0/hint_tpcds/shape/query24.out b/regression-test/data/new_shapes_p0/hint_tpcds/shape/query24.out index 11fb1e7c9be3e6..8d7436a61327cf 100644 --- a/regression-test/data/new_shapes_p0/hint_tpcds/shape/query24.out +++ b/regression-test/data/new_shapes_p0/hint_tpcds/shape/query24.out @@ -9,7 +9,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF5 sr_ticket_number->[ss_ticket_number];RF6 sr_item_sk->[i_item_sk,ss_item_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[ss_item_sk] +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[ss_item_sk] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk];RF3 ca_zip->[s_zip] ------------------------PhysicalProject @@ -20,7 +20,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 5)) --------------------------------PhysicalOlapScan[store] apply RFs: RF3 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->[c_current_addr_sk] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/hint_tpcds/shape/query72.out b/regression-test/data/new_shapes_p0/hint_tpcds/shape/query72.out index 33ba178690ab03..685a1763ddbbed 100644 --- a/regression-test/data/new_shapes_p0/hint_tpcds/shape/query72.out +++ b/regression-test/data/new_shapes_p0/hint_tpcds/shape/query72.out @@ -15,11 +15,11 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d1.d_week_seq = d2.d_week_seq)) otherCondition=() build RFs:RF7 d_week_seq->[d_week_seq] --------------------------PhysicalProject -----------------------------hashJoin[RIGHT_OUTER_JOIN bucketShuffle] hashCondition=((catalog_returns.cr_item_sk = catalog_sales.cs_item_sk) and (catalog_returns.cr_order_number = catalog_sales.cs_order_number)) otherCondition=() build RFs:RF5 cs_order_number->[cr_order_number];RF6 cs_item_sk->[cr_item_sk] +----------------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((catalog_returns.cr_item_sk = catalog_sales.cs_item_sk) and (catalog_returns.cr_order_number = catalog_sales.cs_order_number)) otherCondition=() build RFs:RF5 cs_order_number->[cr_order_number];RF6 cs_item_sk->[cr_item_sk] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF5 RF6 ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk] +--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk] ----------------------------------PhysicalProject ------------------------------------hashJoin[LEFT_OUTER_JOIN broadcast] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=() --------------------------------------PhysicalProject @@ -27,7 +27,7 @@ PhysicalResultSink ------------------------------------------PhysicalProject --------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[cs_bill_hdemo_sk] ----------------------------------------------PhysicalProject -------------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk) and (catalog_sales.cs_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_ship_date_sk];RF1 d_date_sk->[cs_sold_date_sk] +------------------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk) and (catalog_sales.cs_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_ship_date_sk];RF1 d_date_sk->[cs_sold_date_sk] --------------------------------------------------PhysicalProject ----------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 RF3 RF4 --------------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q13.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q13.out index 99b5297e0d6a0d..e2c3de59271bad 100644 --- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q13.out +++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q13.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[RIGHT_OUTER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +--------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q3.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q3.out index b284ef9355077d..a58bf8720464f4 100644 --- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q3.out +++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q3.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------filter((lineitem.l_shipdate > '1995-03-15')) ------------------PhysicalOlapScan[lineitem] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ------------------PhysicalProject --------------------filter((orders.o_orderdate < '1995-03-15')) ----------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q5.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q5.out index 1c001e63b89cfe..924789b8f7e9a2 100644 --- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q5.out +++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q5.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey) and (customer.c_nationkey = supplier.s_nationkey)) otherCondition=() +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey) and (customer.c_nationkey = supplier.s_nationkey)) otherCondition=() ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_suppkey = supplier.s_suppkey)) otherCondition=() ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out index 74cca15a749172..9cffd25181dbe0 100644 --- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out +++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q7.out @@ -22,7 +22,7 @@ PhysicalResultSink ----------------------------filter(n_name IN ('FRANCE', 'GERMANY')) ------------------------------PhysicalOlapScan[nation] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ----------------------PhysicalProject ------------------------PhysicalOlapScan[orders] ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/hint_tpch/shape/q9.out b/regression-test/data/new_shapes_p0/hint_tpch/shape/q9.out index 42e3f4eb2072cd..336e07627bf97c 100644 --- a/regression-test/data/new_shapes_p0/hint_tpch/shape/q9.out +++ b/regression-test/data/new_shapes_p0/hint_tpch/shape/q9.out @@ -10,13 +10,13 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() --------------------------PhysicalProject ----------------------------PhysicalOlapScan[orders] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.out index 10192bf86cb782..d817c6f0053791 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.out @@ -112,14 +112,13 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] ----------------------------------------PhysicalProject -------------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() +--------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF15 +----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF15 -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[item] ---------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------------------------PhysicalOlapScan[item] ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) --------------------------------------------PhysicalOlapScan[date_dim] @@ -136,14 +135,13 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] ----------------------------------------PhysicalProject -------------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +--------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF18 +----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF18 -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[item] ---------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------------------------PhysicalOlapScan[item] ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) --------------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.out index 6f1b848ed00034..e956468874dbb5 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.out @@ -9,7 +9,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[wr_returned_date_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() --------------------PhysicalProject ----------------------PhysicalOlapScan[web_returns] apply RFs: RF1 --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out index a3f752e2ce0e3d..9f3bb8dab0f9e3 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() ------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=() --------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.out index 2dba5f8dad05c5..756dbd1de4efea 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.out @@ -8,20 +8,20 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[cs_item_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[cs_item_sk] ------------------PhysicalProject --------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk] +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[inv_item_sk] --------------------------PhysicalProject ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) -------------------------------PhysicalOlapScan[date_dim] +----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) +------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) ---------------------------PhysicalOlapScan[item] +------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) +--------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out index 495863c5dc9ef9..be021ce25b68ae 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.out index 55144c3f93417e..fe17bd5d3c7235 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.out @@ -22,7 +22,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 --------------------------------------PhysicalProject -----------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((a.ca_address_sk = c.c_current_addr_sk)) otherCondition=() +----------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((a.ca_address_sk = c.c_current_addr_sk)) otherCondition=() ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[customer] ------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.out index ac5d0d6d739e29..2cff316c5a22aa 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.out @@ -3,88 +3,90 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --PhysicalCteProducer ( cteId=CTEId#1 ) ----PhysicalProject -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 i_item_sk->[cr_item_sk,cs_item_sk,sr_item_sk,ss_item_sk] +------hashAgg[GLOBAL] +--------PhysicalDistribute[DistributionSpecHash] +----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 i_item_sk->[cr_item_sk,cs_item_sk,sr_item_sk,ss_item_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = hd1.hd_demo_sk)) otherCondition=() +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = hd1.hd_demo_sk)) otherCondition=() ----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ss_sold_date_sk] +--------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() ----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() +------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ss_sold_date_sk] --------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cs_ui.cs_item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[sr_item_sk,ss_item_sk] +----------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() ------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() +--------------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cs_ui.cs_item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[sr_item_sk,ss_item_sk] ----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF6 RF19 +------------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() +--------------------------------------------------------------------------------PhysicalProject +----------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF6 RF19 +--------------------------------------------------------------------------------PhysicalProject +----------------------------------------------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF4 RF19 ----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF4 RF19 -------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter((sale > (2 * refund))) -----------------------------------------------------------------------------hashAgg[GLOBAL] -------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------------------------------------------------------hashAgg[LOCAL] -----------------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((catalog_sales.cs_item_sk = catalog_returns.cr_item_sk) and (catalog_sales.cs_order_number = catalog_returns.cr_order_number)) otherCondition=() ---------------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF19 +------------------------------------------------------------------------------filter((sale > (2 * refund))) +--------------------------------------------------------------------------------hashAgg[GLOBAL] +----------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF19 +----------------------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((catalog_sales.cs_item_sk = catalog_returns.cr_item_sk) and (catalog_sales.cs_order_number = catalog_returns.cr_order_number)) otherCondition=() +------------------------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF19 +------------------------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF19 +------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------PhysicalOlapScan[customer] --------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------PhysicalOlapScan[customer] +----------------------------------------------------------------------filter(d_year IN (2001, 2002)) +------------------------------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------filter(d_year IN (2001, 2002)) ---------------------------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------------------------------PhysicalOlapScan[store] ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[store] +--------------------------------------------------------------PhysicalOlapScan[date_dim] --------------------------------------------------------PhysicalProject ----------------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------------------PhysicalProject --------------------------------------------------PhysicalOlapScan[customer_demographics] --------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[customer_demographics] +----------------------------------------------PhysicalOlapScan[promotion] ----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[promotion] +------------------------------------------PhysicalOlapScan[household_demographics] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[household_demographics] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[household_demographics] +----------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject ---------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalOlapScan[income_band] --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band] ----------------PhysicalProject -------------------PhysicalOlapScan[income_band] -------------PhysicalProject ---------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) -----------------PhysicalOlapScan[item] +------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) +--------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out index 046c5866f5196c..52f4671d103618 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.out index 8e795ffc2e0eda..08951b2ec22771 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DOUBLE) > cast((avg(cast(ctr_total_return as DECIMALV3(38, 4))) * 1.2) as DOUBLE))) --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ctr_customer_sk] ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.out index 7c415fed511e6e..4d01fe025918e7 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.out @@ -8,20 +8,20 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ------------------PhysicalProject --------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk] +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[inv_item_sk] --------------------------PhysicalProject ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) -------------------------------PhysicalOlapScan[date_dim] +----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) +------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) ---------------------------PhysicalOlapScan[item] +------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) +--------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.out index 966f8701126465..e73d45b0732736 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.out @@ -112,14 +112,13 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF15 d_date_sk->[cs_sold_date_sk] ----------------------------------------PhysicalProject -------------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk,i_item_sk] +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk,ss_item_sk] +--------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF13 ss_item_sk->[cs_item_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF14 --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF13 i_item_sk->[cs_item_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[item] apply RFs: RF14 ---------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------------------------PhysicalOlapScan[item] ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) --------------------------------------------PhysicalOlapScan[date_dim] @@ -136,14 +135,13 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF18 d_date_sk->[ws_sold_date_sk] ----------------------------------------PhysicalProject -------------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[i_item_sk,ws_item_sk] +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ss_item_sk,ws_item_sk] +--------------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF16 ss_item_sk->[ws_item_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +----------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF17 --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF16 i_item_sk->[ws_item_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[item] apply RFs: RF17 ---------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +----------------------------------------------PhysicalOlapScan[item] ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) --------------------------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.out index 6671347af5cc6c..748165ced2fb2a 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.out @@ -9,7 +9,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------PhysicalProject --------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[wr_returned_date_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[wr_returning_addr_sk] +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[wr_returning_addr_sk] --------------------PhysicalProject ----------------------PhysicalOlapScan[web_returns] apply RFs: RF0 RF1 --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out index 14b71f576d97e0..f5f9508f9b6cfb 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk] ------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((c.c_customer_sk = web_sales.ws_bill_customer_sk)) otherCondition=() --------------------------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((c.c_customer_sk = store_sales.ss_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk] ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.out index 2dba5f8dad05c5..756dbd1de4efea 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.out @@ -8,20 +8,20 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[cs_item_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[cs_item_sk] ------------------PhysicalProject --------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2 ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk] +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[inv_item_sk] --------------------------PhysicalProject ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) -------------------------------PhysicalOlapScan[date_dim] +----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) +------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856)) ---------------------------PhysicalOlapScan[item] +------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21')) +--------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out index ae7ea5a5a310fe..71f3746264834f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.out index 5f53b66408a500..81aba1038a542a 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.out @@ -22,7 +22,7 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3 --------------------------------------PhysicalProject -----------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((a.ca_address_sk = c.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +----------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((a.ca_address_sk = c.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] ------------------------------------------PhysicalProject --------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.out index b7cf8115b6db75..d1d49f78ba895b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.out @@ -3,88 +3,90 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --PhysicalCteProducer ( cteId=CTEId#1 ) ----PhysicalProject -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 i_item_sk->[cr_item_sk,cs_item_sk,sr_item_sk,ss_item_sk] +------hashAgg[GLOBAL] +--------PhysicalDistribute[DistributionSpecHash] +----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() build RFs:RF18 ib_income_band_sk->[hd_income_band_sk] +--------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF19 i_item_sk->[cr_item_sk,cs_item_sk,sr_item_sk,ss_item_sk] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() build RFs:RF17 ib_income_band_sk->[hd_income_band_sk] +------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() build RFs:RF18 ib_income_band_sk->[hd_income_band_sk] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 ca_address_sk->[c_current_addr_sk] +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd1.hd_income_band_sk = ib1.ib_income_band_sk)) otherCondition=() build RFs:RF17 ib_income_band_sk->[hd_income_band_sk] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() build RFs:RF15 ca_address_sk->[ss_addr_sk] +--------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 ca_address_sk->[c_current_addr_sk] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() build RFs:RF14 hd_demo_sk->[c_current_hdemo_sk] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() build RFs:RF15 ca_address_sk->[ss_addr_sk] --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = hd1.hd_demo_sk)) otherCondition=() build RFs:RF13 hd_demo_sk->[ss_hdemo_sk] +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() build RFs:RF14 hd_demo_sk->[c_current_hdemo_sk] ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF12 p_promo_sk->[ss_promo_sk] +--------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = hd1.hd_demo_sk)) otherCondition=() build RFs:RF13 hd_demo_sk->[ss_hdemo_sk] ----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF11 cd_demo_sk->[c_current_cdemo_sk] +------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF12 p_promo_sk->[ss_promo_sk] --------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF10 cd_demo_sk->[ss_cdemo_sk] +----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF11 cd_demo_sk->[c_current_cdemo_sk] ------------------------------------------------PhysicalProject ---------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[c_first_shipto_date_sk] +--------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=() build RFs:RF10 cd_demo_sk->[ss_cdemo_sk] ----------------------------------------------------PhysicalProject -------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[c_first_sales_date_sk] +------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_shipto_date_sk = d3.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[c_first_shipto_date_sk] --------------------------------------------------------PhysicalProject -----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF7 s_store_sk->[ss_store_sk] +----------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[c_first_sales_date_sk] ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ss_sold_date_sk] +--------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF7 s_store_sk->[ss_store_sk] ----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk] +------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ss_sold_date_sk] --------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cs_ui.cs_item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[sr_item_sk,ss_item_sk] +----------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk] ------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_item_sk->[ss_item_sk];RF3 sr_ticket_number->[ss_ticket_number] +--------------------------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = cs_ui.cs_item_sk)) otherCondition=() build RFs:RF4 cs_item_sk->[sr_item_sk,ss_item_sk] ----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF4 RF5 RF6 RF7 RF10 RF12 RF13 RF15 RF19 +------------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_item_sk->[ss_item_sk];RF3 sr_ticket_number->[ss_ticket_number] +--------------------------------------------------------------------------------PhysicalProject +----------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3 RF4 RF5 RF6 RF7 RF10 RF12 RF13 RF15 RF19 +--------------------------------------------------------------------------------PhysicalProject +----------------------------------------------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF4 RF19 ----------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF4 RF19 -------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------filter((sale > (2 * refund))) -----------------------------------------------------------------------------hashAgg[GLOBAL] -------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------------------------------------------------------------hashAgg[LOCAL] -----------------------------------------------------------------------------------PhysicalProject -------------------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((catalog_sales.cs_item_sk = catalog_returns.cr_item_sk) and (catalog_sales.cs_order_number = catalog_returns.cr_order_number)) otherCondition=() build RFs:RF0 cr_item_sk->[cs_item_sk];RF1 cr_order_number->[cs_order_number] ---------------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF19 +------------------------------------------------------------------------------filter((sale > (2 * refund))) +--------------------------------------------------------------------------------hashAgg[GLOBAL] +----------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------------------------------------------hashAgg[LOCAL] --------------------------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF19 +----------------------------------------------------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((catalog_sales.cs_item_sk = catalog_returns.cr_item_sk) and (catalog_sales.cs_order_number = catalog_returns.cr_order_number)) otherCondition=() build RFs:RF0 cr_item_sk->[cs_item_sk];RF1 cr_order_number->[cs_order_number] +------------------------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF19 +------------------------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------------------------PhysicalOlapScan[catalog_returns] apply RFs: RF19 +------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 RF11 RF14 RF16 --------------------------------------------------------------------PhysicalProject -----------------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF8 RF9 RF11 RF14 RF16 +----------------------------------------------------------------------filter(d_year IN (2001, 2002)) +------------------------------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------------------------------PhysicalProject -------------------------------------------------------------------filter(d_year IN (2001, 2002)) ---------------------------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------------------------------PhysicalOlapScan[store] ------------------------------------------------------------PhysicalProject ---------------------------------------------------------------PhysicalOlapScan[store] +--------------------------------------------------------------PhysicalOlapScan[date_dim] --------------------------------------------------------PhysicalProject ----------------------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------------------------PhysicalProject -------------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------------------PhysicalOlapScan[customer_demographics] ------------------------------------------------PhysicalProject --------------------------------------------------PhysicalOlapScan[customer_demographics] --------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[customer_demographics] +----------------------------------------------PhysicalOlapScan[promotion] ----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[promotion] +------------------------------------------PhysicalOlapScan[household_demographics] apply RFs: RF17 ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[household_demographics] apply RFs: RF17 +--------------------------------------PhysicalOlapScan[household_demographics] apply RFs: RF18 --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[household_demographics] apply RFs: RF18 +----------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject ---------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalOlapScan[income_band] --------------------PhysicalProject ----------------------PhysicalOlapScan[income_band] ----------------PhysicalProject -------------------PhysicalOlapScan[income_band] -------------PhysicalProject ---------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) -----------------PhysicalOlapScan[item] +------------------filter((item.i_current_price <= 33.00) and (item.i_current_price >= 24.00) and i_color IN ('blanched', 'brown', 'burlywood', 'chocolate', 'drab', 'medium')) +--------------------PhysicalOlapScan[item] --PhysicalResultSink ----PhysicalQuickSort[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out index 70945beb33b729..04dab40ade6bb2 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalProject ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.out index 8006799a3cc5fd..c06a48a1f5f922 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.out @@ -24,7 +24,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------PhysicalProject ------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((ctr1.ctr_state = ctr2.ctr_state)) otherCondition=((cast(ctr_total_return as DOUBLE) > cast((avg(cast(ctr_total_return as DECIMALV3(38, 4))) * 1.2) as DOUBLE))) build RFs:RF4 ctr_state->[ctr_state] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN shuffle] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ctr_customer_sk] ----------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF2 RF4 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.out index 7c415fed511e6e..4d01fe025918e7 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.out @@ -8,20 +8,20 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ------------------PhysicalProject --------------------PhysicalOlapScan[store_sales] apply RFs: RF2 ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk] +--------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk] +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[inv_item_sk] --------------------------PhysicalProject ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100)) ------------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 --------------------------PhysicalProject -----------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) -------------------------------PhysicalOlapScan[date_dim] +----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) +------------------------------PhysicalOlapScan[item] ----------------------PhysicalProject -------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639)) ---------------------------PhysicalOlapScan[item] +------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09')) +--------------------------PhysicalOlapScan[date_dim] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query14.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query14.out index 2a29746e37ef07..48ac240d961d98 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query14.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query14.out @@ -110,19 +110,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] ------------------------------------PhysicalProject ---------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() +----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 -------------------------------------------------PhysicalProject ---------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -----------------------------------------------------PhysicalOlapScan[date_dim] ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------PhysicalAssertNumRows --------------------------------PhysicalDistribute[DistributionSpecGather] @@ -134,19 +133,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] ------------------------------------PhysicalProject ---------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) ----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 -------------------------------------------------PhysicalProject ---------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -----------------------------------------------------PhysicalOlapScan[date_dim] ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] -----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------PhysicalAssertNumRows --------------------------------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out index 93b2d0cee9c5d7..ed2ea026a241c9 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query15.out @@ -8,18 +8,18 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((ca_state IN ('CA', 'GA', 'WA') OR substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274')) OR (catalog_sales.cs_sales_price > 500.00))) +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_customer_sk = customer.c_customer_sk)) otherCondition=(((ca_state IN ('CA', 'GA', 'WA') OR substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274')) OR (catalog_sales.cs_sales_price > 500.00))) build RFs:RF2 c_customer_sk->[cs_bill_customer_sk] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk] ----------------------PhysicalProject -------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 +------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF1 RF2 ----------------------PhysicalProject ------------------------filter((date_dim.d_qoy = 1) and (date_dim.d_year = 2001)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() +--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------PhysicalProject -------------------------PhysicalOlapScan[customer] +------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------PhysicalProject -------------------------PhysicalOlapScan[customer_address] +------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query18.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query18.out index 57183675eb5fc2..b1490c33c43896 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query18.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query18.out @@ -29,7 +29,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF1 ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] --------------------------------------PhysicalProject ----------------------------------------filter(c_birth_month IN (1, 10, 2, 4, 7, 8)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query24.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query24.out index 3d2e80e2381a00..2c6c329803d85f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query24.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query24.out @@ -11,20 +11,20 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 ca_zip->[s_zip];RF3 c_customer_sk->[ss_customer_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[ss_store_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 +------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF3 ----------------------------PhysicalProject ------------------------------filter((store.s_market_id = 8)) ---------------------------------PhysicalOlapScan[store] +--------------------------------PhysicalOlapScan[store] apply RFs: RF2 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +------------------------------PhysicalOlapScan[customer] --------------------PhysicalProject ----------------------PhysicalOlapScan[item] ----------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query25.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query25.out index cbc48e3165ade9..f4a203e70a0ca2 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query25.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query25.out @@ -19,7 +19,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_customer_sk->[ss_customer_sk];RF3 sr_item_sk->[ss_item_sk];RF4 sr_ticket_number->[ss_ticket_number] ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out index c1cc05ec8ee03f..2ee30b92ac642f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query26.out @@ -8,7 +8,9 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 cs_item_sk->[i_item_sk] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] apply RFs: RF3 ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF2 p_promo_sk->[cs_promo_sk] ----------------------PhysicalProject @@ -26,6 +28,4 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N'))) --------------------------PhysicalOlapScan[promotion] -------------------PhysicalProject ---------------------PhysicalOlapScan[item] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query27.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query27.out index 9b311ff91423bf..2dc8f171dee0d8 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query27.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query27.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query29.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query29.out index 4e04f8042e2e31..e4d5bd8050a855 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query29.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query29.out @@ -16,7 +16,9 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() +----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 ss_item_sk->[i_item_sk] +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[item] apply RFs: RF5 ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_customer_sk->[ss_customer_sk];RF3 sr_item_sk->[ss_item_sk];RF4 sr_ticket_number->[ss_ticket_number] ----------------------------------PhysicalProject @@ -33,8 +35,6 @@ PhysicalResultSink --------------------------------------PhysicalProject ----------------------------------------filter((d2.d_moy <= 7) and (d2.d_moy >= 4) and (d2.d_year = 1999)) ------------------------------------------PhysicalOlapScan[date_dim] -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[item] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] ------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query30.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query30.out index 99f54520a25e12..9c11294b4ec73f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query30.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query30.out @@ -7,7 +7,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 wr_returning_addr_sk->[ca_address_sk] +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] apply RFs: RF1 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[wr_returned_date_sk] --------------------PhysicalProject @@ -15,8 +17,6 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2002)) ------------------------PhysicalOlapScan[date_dim] -----------------PhysicalProject -------------------PhysicalOlapScan[customer_address] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query33.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query33.out index 3cc7c048f5784a..3cafa5359ba070 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query33.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query33.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk] ----------------------------------PhysicalProject @@ -38,7 +38,7 @@ PhysicalResultSink ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[cs_bill_addr_sk] ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out index 4da981f140aa0e..49d318314dee3a 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query35.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() +--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer] apply RFs: RF5 ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out index 83f6b9ca5df1f0..6f437edc5cb1a9 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query45.out @@ -11,20 +11,20 @@ PhysicalResultSink ----------------filter((substring(ca_zip, 1, 5) IN ('80348', '81792', '83405', '85392', '85460', '85669', '86197', '86475', '88274') OR $c$1)) ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ws_bill_customer_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_sold_date_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF3 +------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3 ----------------------------PhysicalProject ------------------------------filter((date_dim.d_qoy = 2) and (date_dim.d_year = 2000)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +------------------------------PhysicalOlapScan[customer] --------------------PhysicalProject ----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=() ------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out index 2895fa954ff4eb..23864cbc2a5e14 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query46.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalProject --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ------------------------PhysicalProject @@ -30,7 +30,7 @@ PhysicalResultSink --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=() +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=() ----------------PhysicalProject ------------------PhysicalOlapScan[customer] ----------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out index 89ce4eb76395d6..b25d7ab6b754a2 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query48.out @@ -7,23 +7,23 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ss_addr_sk->[ca_address_sk] +----------------PhysicalProject +------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) +--------------------PhysicalOlapScan[customer_address] apply RFs: RF2 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk] ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) -----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 +----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 ------------------------PhysicalProject --------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')))) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 1999)) ------------------------PhysicalOlapScan[date_dim] -----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) ---------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------PhysicalOlapScan[store] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query56.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query56.out index 97c4f27b14edb6..710ae87d9a4a7b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query56.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query56.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] --------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query58.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query58.out index 8b1c60c8c19a1b..154cbf3229240b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query58.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query58.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[cs_item_sk] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[cs_item_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF11 d_date_sk->[cs_sold_date_sk] ----------------------------PhysicalProject @@ -38,7 +38,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[ss_item_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[ss_item_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ss_sold_date_sk] --------------------------------PhysicalProject @@ -63,7 +63,9 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 ws_item_sk->[i_item_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[item] apply RFs: RF3 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] --------------------------------PhysicalProject @@ -81,6 +83,4 @@ PhysicalResultSink --------------------------------------------PhysicalProject ----------------------------------------------filter((date_dim.d_date = '2001-03-24')) ------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query60.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query60.out index f3678f64aece45..a4eb83e16299bd 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query60.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query60.out @@ -37,7 +37,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] --------------------------------PhysicalProject @@ -61,13 +61,16 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ca_address_sk->[ws_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ws_bill_addr_sk->[ca_address_sk] +----------------------------PhysicalProject +------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +--------------------------------PhysicalOlapScan[customer_address] apply RFs: RF11 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk] --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[ws_sold_date_sk] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF9 RF10 RF11 +--------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF9 RF10 ------------------------------------PhysicalProject --------------------------------------filter((date_dim.d_moy = 8) and (date_dim.d_year = 2000)) ----------------------------------------PhysicalOlapScan[date_dim] @@ -77,7 +80,4 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------filter((item.i_category = 'Children')) --------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalProject -------------------------------filter((customer_address.ca_gmt_offset = -7.00)) ---------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query64.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query64.out index 6aa36d179b1db1..4c71286f010ec6 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query64.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query64.out @@ -13,7 +13,9 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF17 ss_customer_sk->[c_customer_sk] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 c_current_addr_sk->[ca_address_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF16 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=() --------------------------------PhysicalProject @@ -28,8 +30,6 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------PhysicalOlapScan[income_band] --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[customer_demographics] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF11 ss_item_sk->[sr_item_sk];RF12 ss_ticket_number->[sr_ticket_number] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out index 3d8133e9159fea..911daeee06f6a4 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query7.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF2 p_promo_sk->[ss_promo_sk] ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query72.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query72.out index 0964cf6998d4ec..c433b1f9f01200 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query72.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query72.out @@ -19,7 +19,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d1.d_week_seq = d2.d_week_seq)) otherCondition=() ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() +------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() --------------------------------------PhysicalProject ----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, INTERVAL 5 DAY))) ------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query8.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query8.out index fd4ec64bb81546..52bfb51f9c2c70 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query8.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query8.out @@ -29,7 +29,7 @@ PhysicalResultSink ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------hashAgg[LOCAL] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] --------------------------------------PhysicalProject ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query81.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query81.out index fdc3edc4efb31f..809170997c3eed 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query81.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query81.out @@ -7,7 +7,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_returns.cr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_returns.cr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_returns.cr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cr_returned_date_sk] --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query83.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query83.out index c6d7033d8cecea..dc18aa5ff8e1a4 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query83.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query83.out @@ -36,11 +36,13 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_returns.sr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[sr_item_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_returns.sr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 sr_item_sk->[i_item_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[item] apply RFs: RF7 RF8 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_returns.sr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[sr_returned_date_sk] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store_returns] apply RFs: RF6 RF7 +----------------------------------PhysicalOlapScan[store_returns] apply RFs: RF6 --------------------------------PhysicalProject ----------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((date_dim.d_date = date_dim.d_date)) otherCondition=() build RFs:RF5 d_date->[d_date] ------------------------------------PhysicalProject @@ -52,8 +54,6 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) --------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out index a2183ccaa646ca..74ace999b87a48 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query85.out @@ -22,11 +22,14 @@ PhysicalResultSink ----------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')))) ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk] +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk] +------------------------------------PhysicalProject +--------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) +----------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number] ----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 RF3 +------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk] --------------------------------------------PhysicalProject @@ -35,9 +38,6 @@ PhysicalResultSink --------------------------------------------PhysicalProject ----------------------------------------------filter((date_dim.d_year = 2000)) ------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) -----------------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_page] --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query14.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query14.out index 196a98b5a2f51d..5aad6142d951f9 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query14.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query14.out @@ -110,19 +110,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] ------------------------------------PhysicalProject ---------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF15 ss_item_sk->[cs_item_sk,i_item_sk] +--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF15 i_item_sk->[cs_item_sk,ss_item_sk] +----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF14 ss_item_sk->[cs_item_sk] +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF15 ----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF14 i_item_sk->[cs_item_sk] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF13 d_date_sk->[cs_sold_date_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF13 RF14 RF15 -------------------------------------------------PhysicalProject ---------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -----------------------------------------------------PhysicalOlapScan[date_dim] ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] apply RFs: RF15 -----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------PhysicalAssertNumRows --------------------------------PhysicalDistribute[DistributionSpecGather] @@ -134,19 +133,18 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] ------------------------------------PhysicalProject ---------------------------------------hashJoin[LEFT_SEMI_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF18 ss_item_sk->[i_item_sk,ws_item_sk] +--------------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF18 i_item_sk->[ss_item_sk,ws_item_sk] +----------------------------------------hashJoin[LEFT_SEMI_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = cross_items.ss_item_sk)) otherCondition=() build RFs:RF17 ss_item_sk->[ws_item_sk] +------------------------------------------PhysicalProject +--------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] +----------------------------------------------PhysicalProject +------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 +----------------------------------------------PhysicalProject +------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) +--------------------------------------------------PhysicalOlapScan[date_dim] +------------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF18 ----------------------------------------PhysicalProject -------------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF17 i_item_sk->[ws_item_sk] ---------------------------------------------PhysicalProject -----------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF16 d_date_sk->[ws_sold_date_sk] -------------------------------------------------PhysicalProject ---------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF16 RF17 RF18 -------------------------------------------------PhysicalProject ---------------------------------------------------filter((date_dim.d_moy = 11) and (date_dim.d_year = 2002)) -----------------------------------------------------PhysicalOlapScan[date_dim] ---------------------------------------------PhysicalProject -----------------------------------------------PhysicalOlapScan[item] apply RFs: RF18 -----------------------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) +------------------------------------------PhysicalOlapScan[item] ----------------------------PhysicalProject ------------------------------PhysicalAssertNumRows --------------------------------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out index f617754de74f6b..ed2ea026a241c9 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query15.out @@ -17,9 +17,9 @@ PhysicalResultSink ------------------------filter((date_dim.d_qoy = 1) and (date_dim.d_year = 2001)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------PhysicalProject -------------------------PhysicalOlapScan[customer] apply RFs: RF0 +------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------PhysicalProject -------------------------PhysicalOlapScan[customer_address] +------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query18.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query18.out index 57183675eb5fc2..b1490c33c43896 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query18.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query18.out @@ -29,7 +29,7 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF1 ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] --------------------------------------PhysicalProject ----------------------------------------filter(c_birth_month IN (1, 10, 2, 4, 7, 8)) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query24.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query24.out index 0dc2c851744de2..14dbf92e98e5fd 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query24.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query24.out @@ -20,11 +20,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 8)) --------------------------------PhysicalOlapScan[store] apply RFs: RF2 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] apply RFs: RF0 +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +------------------------------PhysicalOlapScan[customer] --------------------PhysicalProject ----------------------PhysicalOlapScan[item] apply RFs: RF6 ----------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query25.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query25.out index e4e15d711aeb70..a9f0a2a72e7793 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query25.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query25.out @@ -19,7 +19,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[sr_item_sk,ss_item_sk] +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[sr_item_sk,ss_item_sk] --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_customer_sk->[ss_customer_sk];RF3 sr_item_sk->[ss_item_sk];RF4 sr_ticket_number->[ss_ticket_number] ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out index d5fbe1d1762fb9..2ee30b92ac642f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query26.out @@ -8,7 +8,9 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[cs_item_sk] +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 cs_item_sk->[i_item_sk] +------------------PhysicalProject +--------------------PhysicalOlapScan[item] apply RFs: RF3 ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF2 p_promo_sk->[cs_promo_sk] ----------------------PhysicalProject @@ -16,7 +18,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[cs_bill_cdemo_sk] ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 RF3 +--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 ------------------------------PhysicalProject --------------------------------filter((customer_demographics.cd_education_status = 'Unknown') and (customer_demographics.cd_gender = 'M') and (customer_demographics.cd_marital_status = 'S')) ----------------------------------PhysicalOlapScan[customer_demographics] @@ -26,6 +28,4 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------filter(((promotion.p_channel_email = 'N') OR (promotion.p_channel_event = 'N'))) --------------------------PhysicalOlapScan[promotion] -------------------PhysicalProject ---------------------PhysicalOlapScan[item] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query27.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query27.out index c6137e774ae1b9..3ddc47823586a0 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query27.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query27.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query29.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query29.out index 0ad4d176c10d08..f62f9918450dde 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query29.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query29.out @@ -16,25 +16,25 @@ PhysicalResultSink ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[sr_item_sk,ss_item_sk] +----------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 ss_item_sk->[i_item_sk] +------------------------------PhysicalProject +--------------------------------PhysicalOlapScan[item] apply RFs: RF5 ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_customer_sk = store_returns.sr_customer_sk) and (store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF2 sr_customer_sk->[ss_customer_sk];RF3 sr_item_sk->[ss_item_sk];RF4 sr_ticket_number->[ss_ticket_number] ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d1.d_date_sk = store_sales.ss_sold_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] --------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3 RF4 RF5 RF6 +----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3 RF4 RF6 --------------------------------------PhysicalProject ----------------------------------------filter((d1.d_moy = 4) and (d1.d_year = 1999)) ------------------------------------------PhysicalOlapScan[date_dim] ----------------------------------PhysicalProject ------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_returns.sr_returned_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[sr_returned_date_sk] --------------------------------------PhysicalProject -----------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 RF5 +----------------------------------------PhysicalOlapScan[store_returns] apply RFs: RF0 --------------------------------------PhysicalProject ----------------------------------------filter((d2.d_moy <= 7) and (d2.d_moy >= 4) and (d2.d_year = 1999)) ------------------------------------------PhysicalOlapScan[date_dim] -------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[item] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[store] ------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query30.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query30.out index 1fd2b5a1688c12..b60e62f5e7bade 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query30.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query30.out @@ -7,16 +7,16 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[wr_returning_addr_sk] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 wr_returning_addr_sk->[ca_address_sk] +----------------PhysicalProject +------------------PhysicalOlapScan[customer_address] apply RFs: RF1 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[wr_returned_date_sk] --------------------PhysicalProject -----------------------PhysicalOlapScan[web_returns] apply RFs: RF0 RF1 +----------------------PhysicalOlapScan[web_returns] apply RFs: RF0 --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2002)) ------------------------PhysicalOlapScan[date_dim] -----------------PhysicalProject -------------------PhysicalOlapScan[customer_address] --PhysicalResultSink ----PhysicalTopN[MERGE_SORT] ------PhysicalDistribute[DistributionSpecGather] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query33.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query33.out index 3cc7c048f5784a..3cafa5359ba070 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query33.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query33.out @@ -14,7 +14,7 @@ PhysicalResultSink ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk] ----------------------------------PhysicalProject @@ -38,7 +38,7 @@ PhysicalResultSink ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[cs_bill_addr_sk] ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out index 93fdd630b3352d..dcc86d24029059 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query35.out @@ -16,7 +16,7 @@ PhysicalResultSink --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_demographics.cd_demo_sk = c.c_current_cdemo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] +--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[c_current_addr_sk] ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4 RF5 ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out index e05c3f0537a669..6f437edc5cb1a9 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query45.out @@ -20,11 +20,11 @@ PhysicalResultSink ------------------------------filter((date_dim.d_qoy = 2) and (date_dim.d_year = 2000)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] apply RFs: RF0 +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +------------------------------PhysicalOlapScan[customer] --------------------PhysicalProject ----------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((item.i_item_id = item.i_item_id)) otherCondition=() ------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out index 0a6a5e279d3f80..8279b1c53b5c39 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query46.out @@ -9,7 +9,7 @@ PhysicalResultSink ------------PhysicalProject --------------hashAgg[LOCAL] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[ss_addr_sk] +------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[ss_addr_sk] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ------------------------PhysicalProject @@ -30,7 +30,7 @@ PhysicalResultSink --------------------PhysicalProject ----------------------PhysicalOlapScan[customer_address] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] ----------------PhysicalProject ------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out index 8bb4cfa24cd5f9..0bb694c50c5abe 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query48.out @@ -7,23 +7,23 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IA', 'MD', 'MN') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('IL', 'TX', 'VA') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('IN', 'MI', 'WI') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ss_addr_sk->[ca_address_sk] +----------------PhysicalProject +------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) +--------------------PhysicalOlapScan[customer_address] apply RFs: RF2 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk] ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) -----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 +----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF3 ------------------------PhysicalProject --------------------------filter(((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) OR ((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')))) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 1999)) ------------------------PhysicalOlapScan[date_dim] -----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('IA', 'IL', 'IN', 'MD', 'MI', 'MN', 'TX', 'VA', 'WI')) ---------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------PhysicalOlapScan[store] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query56.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query56.out index 97c4f27b14edb6..710ae87d9a4a7b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query56.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query56.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] --------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query58.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query58.out index 62cf69cc400980..154cbf3229240b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query58.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query58.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[cs_item_sk] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF12 i_item_sk->[cs_item_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF11 d_date_sk->[cs_sold_date_sk] ----------------------------PhysicalProject @@ -38,7 +38,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[ss_item_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[ss_item_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ss_sold_date_sk] --------------------------------PhysicalProject @@ -63,11 +63,13 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 ws_item_sk->[i_item_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[item] apply RFs: RF3 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3 +----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 --------------------------------PhysicalProject ----------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((date_dim.d_date = date_dim.d_date)) otherCondition=() build RFs:RF1 d_date->[d_date] ------------------------------------PhysicalProject @@ -81,6 +83,4 @@ PhysicalResultSink --------------------------------------------PhysicalProject ----------------------------------------------filter((date_dim.d_date = '2001-03-24')) ------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query60.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query60.out index f3678f64aece45..a4eb83e16299bd 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query60.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query60.out @@ -37,7 +37,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] --------------------------------PhysicalProject @@ -61,13 +61,16 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ca_address_sk->[ws_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ws_bill_addr_sk->[ca_address_sk] +----------------------------PhysicalProject +------------------------------filter((customer_address.ca_gmt_offset = -7.00)) +--------------------------------PhysicalOlapScan[customer_address] apply RFs: RF11 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk] --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[ws_sold_date_sk] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF9 RF10 RF11 +--------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF9 RF10 ------------------------------------PhysicalProject --------------------------------------filter((date_dim.d_moy = 8) and (date_dim.d_year = 2000)) ----------------------------------------PhysicalOlapScan[date_dim] @@ -77,7 +80,4 @@ PhysicalResultSink ----------------------------------PhysicalProject ------------------------------------filter((item.i_category = 'Children')) --------------------------------------PhysicalOlapScan[item] -----------------------------PhysicalProject -------------------------------filter((customer_address.ca_gmt_offset = -7.00)) ---------------------------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query64.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query64.out index 2ef8d6701ee4c5..68698c4fbb3894 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query64.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query64.out @@ -13,13 +13,15 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF17 ss_customer_sk->[c_customer_sk] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF16 c_current_addr_sk->[ca_address_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF16 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=() build RFs:RF15 cd_demo_sk->[c_current_cdemo_sk] --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() build RFs:RF14 hd_demo_sk->[c_current_hdemo_sk] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[customer] apply RFs: RF14 RF15 RF16 RF17 RF18 RF19 +--------------------------------------PhysicalOlapScan[customer] apply RFs: RF14 RF15 RF17 RF18 RF19 ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() build RFs:RF13 ib_income_band_sk->[hd_income_band_sk] ----------------------------------------PhysicalProject @@ -28,8 +30,6 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------PhysicalOlapScan[income_band] --------------------------------PhysicalProject ----------------------------------PhysicalOlapScan[customer_demographics] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF11 ss_item_sk->[sr_item_sk];RF12 ss_ticket_number->[sr_ticket_number] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out index 03f95005ce9cbc..516105a1c2b73f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query7.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF2 p_promo_sk->[ss_promo_sk] ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query72.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query72.out index e0d8acf21bdec5..a463f23cb489af 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query72.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query72.out @@ -19,7 +19,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d1.d_week_seq = d2.d_week_seq)) otherCondition=() build RFs:RF5 d_week_seq->[d_week_seq] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk] +------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk] --------------------------------------PhysicalProject ----------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, INTERVAL 5 DAY))) build RFs:RF3 d_date_sk->[cs_ship_date_sk] ------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query8.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query8.out index a254eeae049f91..00f211d12927a2 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query8.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query8.out @@ -29,7 +29,7 @@ PhysicalResultSink ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------hashAgg[LOCAL] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] --------------------------------------PhysicalProject ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query81.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query81.out index e6aef6266d392b..27fc8430ff3e83 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query81.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query81.out @@ -7,7 +7,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_returns.cr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[cr_returning_addr_sk] +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_returns.cr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[cr_returning_addr_sk] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_returns.cr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cr_returned_date_sk] --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query83.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query83.out index c6d7033d8cecea..dc18aa5ff8e1a4 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query83.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query83.out @@ -36,11 +36,13 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_returns.sr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[sr_item_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_returns.sr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 sr_item_sk->[i_item_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[item] apply RFs: RF7 RF8 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_returns.sr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[sr_returned_date_sk] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[store_returns] apply RFs: RF6 RF7 +----------------------------------PhysicalOlapScan[store_returns] apply RFs: RF6 --------------------------------PhysicalProject ----------------------------------hashJoin[LEFT_SEMI_JOIN broadcast] hashCondition=((date_dim.d_date = date_dim.d_date)) otherCondition=() build RFs:RF5 d_date->[d_date] ------------------------------------PhysicalProject @@ -52,8 +54,6 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter(d_date IN ('2001-06-06', '2001-09-02', '2001-11-11')) --------------------------------------------PhysicalOlapScan[date_dim] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[item] apply RFs: RF8 ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out index f922c1edf94241..da46d54e2f8be5 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query85.out @@ -22,11 +22,14 @@ PhysicalResultSink ----------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')))) ------------------------------------PhysicalOlapScan[customer_demographics] apply RFs: RF4 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 ca_address_sk->[wr_refunded_addr_sk] +----------------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer_address.ca_address_sk = web_returns.wr_refunded_addr_sk)) otherCondition=((((ca_state IN ('DE', 'FL', 'TX') AND ((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00))) OR (ca_state IN ('ID', 'IN', 'ND') AND ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00)))) OR (ca_state IN ('IL', 'MT', 'OH') AND ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))))) build RFs:RF3 wr_refunded_addr_sk->[ca_address_sk] +------------------------------------PhysicalProject +--------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) +----------------------------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF1 ws_item_sk->[wr_item_sk];RF2 ws_order_number->[wr_order_number] ----------------------------------------PhysicalProject -------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 RF3 RF9 +------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 RF9 ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk] --------------------------------------------PhysicalProject @@ -35,9 +38,6 @@ PhysicalResultSink --------------------------------------------PhysicalProject ----------------------------------------------filter((date_dim.d_year = 2000)) ------------------------------------------------PhysicalOlapScan[date_dim] -------------------------------------PhysicalProject ---------------------------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('DE', 'FL', 'ID', 'IL', 'IN', 'MT', 'ND', 'OH', 'TX')) -----------------------------------------PhysicalOlapScan[customer_address] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[web_page] --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out index 6ac3b85090aceb..b65fa9047c0c87 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------filter((date_dim.d_qoy = 1) and (date_dim.d_year = 2000)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.out index d3ec5b9b1b897e..193bf4c8b221bb 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] --------------------------------PhysicalProject @@ -37,7 +37,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] --------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.out index a254eeae049f91..00f211d12927a2 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.out @@ -29,7 +29,7 @@ PhysicalResultSink ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------hashAgg[LOCAL] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] --------------------------------------PhysicalProject ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out index 3e0b283a513dd2..d65941c949e46b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query15.out @@ -17,7 +17,7 @@ PhysicalResultSink ------------------------filter((date_dim.d_qoy = 2) and (date_dim.d_year = 2001)) --------------------------PhysicalOlapScan[date_dim] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] ----------------------PhysicalProject ------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query24.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query24.out index d93b73ade16ecb..b1e5bf29828394 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query24.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query24.out @@ -20,11 +20,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) ------------------------------filter((store.s_market_id = 5)) --------------------------------PhysicalOlapScan[store] apply RFs: RF2 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=(( not (c_birth_country = upper(ca_country)))) build RFs:RF0 c_current_addr_sk->[ca_address_sk] ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer] apply RFs: RF0 +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF0 ----------------------------PhysicalProject -------------------------------PhysicalOlapScan[customer_address] +------------------------------PhysicalOlapScan[customer] --------------------PhysicalProject ----------------------PhysicalOlapScan[item] apply RFs: RF6 ----------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query27.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query27.out index 0fa387fb0d6bb9..47ceeb712c2a8c 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query27.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query27.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------hashAgg[LOCAL] ----------------PhysicalRepeat ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk] --------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query30.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query30.out index c894fcceff19a5..8bea4f656a6132 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query30.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query30.out @@ -7,7 +7,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[wr_returning_addr_sk] +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[wr_returning_addr_sk] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[wr_returned_date_sk] --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query33.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query33.out index 721bd289919ecf..1f4f083cdcf52f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query33.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query33.out @@ -62,7 +62,7 @@ PhysicalResultSink ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk] ------------------------------PhysicalProject --------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF9 ca_address_sk->[ws_bill_addr_sk] ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out index a7a8d54bd3e135..a69bca459222b5 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query35.out @@ -23,7 +23,7 @@ PhysicalResultSink ------------------------------------filter((date_dim.d_qoy < 4) and (date_dim.d_year = 1999)) --------------------------------------PhysicalOlapScan[date_dim] ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk] +--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((c.c_current_addr_sk = ca.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[c_current_addr_sk] ----------------------------------PhysicalProject ------------------------------------PhysicalOlapScan[customer] apply RFs: RF2 RF5 ----------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out index 6ac3b85090aceb..b65fa9047c0c87 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query45.out @@ -20,7 +20,7 @@ PhysicalResultSink ------------------------------filter((date_dim.d_qoy = 1) and (date_dim.d_year = 2000)) --------------------------------PhysicalOlapScan[date_dim] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_current_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer] apply RFs: RF0 ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out index b80a9affecc0d6..e84100fe65fac8 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query46.out @@ -5,15 +5,19 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 ca_address_sk->[c_current_addr_sk] +----------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = current_addr.ca_address_sk)) otherCondition=(( not (ca_city = bought_city))) build RFs:RF5 c_current_addr_sk->[ca_address_sk] +------------PhysicalProject +--------------PhysicalOlapScan[customer_address] apply RFs: RF5 ------------PhysicalProject --------------hashJoin[INNER_JOIN shuffle] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 ss_customer_sk->[c_customer_sk] ----------------PhysicalProject -------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5 +------------------PhysicalOlapScan[customer] apply RFs: RF4 ----------------PhysicalProject ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ss_addr_sk->[ca_address_sk] +------------------------PhysicalProject +--------------------------PhysicalOlapScan[customer_address] apply RFs: RF3 ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF2 hd_demo_sk->[ss_hdemo_sk] ----------------------------PhysicalProject @@ -21,7 +25,7 @@ PhysicalResultSink --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF0 s_store_sk->[ss_store_sk] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 +--------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 ------------------------------------PhysicalProject --------------------------------------filter(s_city IN ('Fairview', 'Midway')) ----------------------------------------PhysicalOlapScan[store] @@ -31,8 +35,4 @@ PhysicalResultSink ----------------------------PhysicalProject ------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = 0))) --------------------------------PhysicalOlapScan[household_demographics] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[customer_address] -------------PhysicalProject ---------------PhysicalOlapScan[customer_address] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out index 086a41be4265ed..c20cd07f178f50 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query48.out @@ -7,23 +7,23 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('ND', 'NY', 'SD') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('GA', 'KS', 'MD') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('CO', 'MN', 'NC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ca_address_sk->[ss_addr_sk] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('ND', 'NY', 'SD') AND ((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00))) OR (ca_state IN ('GA', 'KS', 'MD') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00)))) OR (ca_state IN ('CO', 'MN', 'NC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))))) build RFs:RF2 ss_addr_sk->[ca_address_sk] +----------------PhysicalProject +------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) +--------------------PhysicalOlapScan[customer_address] apply RFs: RF2 ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk] --------------------PhysicalProject ----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk] ------------------------PhysicalProject --------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00)) -----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 +----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF3 ------------------------PhysicalProject --------------------------filter(((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) OR ((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree'))) OR ((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree')))) ----------------------------PhysicalOlapScan[customer_demographics] --------------------PhysicalProject ----------------------filter((date_dim.d_year = 2001)) ------------------------PhysicalOlapScan[date_dim] -----------------PhysicalProject -------------------filter((customer_address.ca_country = 'United States') and ca_state IN ('CO', 'GA', 'KS', 'MD', 'MN', 'NC', 'ND', 'NY', 'SD')) ---------------------PhysicalOlapScan[customer_address] ------------PhysicalProject --------------PhysicalOlapScan[store] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query56.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query56.out index d3ec5b9b1b897e..193bf4c8b221bb 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query56.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query56.out @@ -13,7 +13,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk] --------------------------------PhysicalProject @@ -37,7 +37,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF7 ca_address_sk->[cs_bill_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk] --------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query60.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query60.out index e56a60c6eebef7..f024124050e0f8 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query60.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query60.out @@ -61,7 +61,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ca_address_sk->[ws_bill_addr_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF11 ca_address_sk->[ws_bill_addr_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk] --------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query64.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query64.out index 8155e898243e42..ebc2519e119218 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query64.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query64.out @@ -11,9 +11,11 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------PhysicalProject ------------------hashJoin[INNER_JOIN shuffle] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF18 c_customer_sk->[ss_customer_sk] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() build RFs:RF17 ca_address_sk->[ss_addr_sk] +----------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF17 p_promo_sk->[ss_promo_sk] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_promo_sk = promotion.p_promo_sk)) otherCondition=() build RFs:RF16 p_promo_sk->[ss_promo_sk] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((store_sales.ss_addr_sk = ad1.ca_address_sk)) otherCondition=() build RFs:RF16 ss_addr_sk->[ca_address_sk] +----------------------------PhysicalProject +------------------------------PhysicalOlapScan[customer_address] apply RFs: RF16 ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF14 ss_item_sk->[sr_item_sk];RF15 ss_ticket_number->[sr_ticket_number] --------------------------------PhysicalProject @@ -32,7 +34,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------------------PhysicalProject --------------------------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((store_sales.ss_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[ss_sold_date_sk] ----------------------------------------------------------PhysicalProject -------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF7 RF8 RF9 RF11 RF12 RF13 RF16 RF17 RF18 +------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF7 RF8 RF9 RF11 RF12 RF13 RF17 RF18 ----------------------------------------------------------PhysicalProject ------------------------------------------------------------filter(d_year IN (1999, 2000)) --------------------------------------------------------------PhysicalOlapScan[date_dim] @@ -58,12 +60,12 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ------------------------------------------PhysicalOlapScan[item] ------------------------------------PhysicalProject --------------------------------------PhysicalOlapScan[customer_demographics] -----------------------------PhysicalProject -------------------------------PhysicalOlapScan[promotion] ------------------------PhysicalProject ---------------------------PhysicalOlapScan[customer_address] +--------------------------PhysicalOlapScan[promotion] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF4 ca_address_sk->[c_current_addr_sk] +----------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_current_addr_sk = ad2.ca_address_sk)) otherCondition=() build RFs:RF4 c_current_addr_sk->[ca_address_sk] +------------------------PhysicalProject +--------------------------PhysicalOlapScan[customer_address] apply RFs: RF4 ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=() build RFs:RF3 cd_demo_sk->[c_current_cdemo_sk] ----------------------------PhysicalProject @@ -71,7 +73,7 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_current_hdemo_sk = hd2.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[c_current_hdemo_sk] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[customer] apply RFs: RF1 RF2 RF3 RF4 RF19 +--------------------------------------PhysicalOlapScan[customer] apply RFs: RF1 RF2 RF3 RF19 ------------------------------------PhysicalProject --------------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((hd2.hd_income_band_sk = ib2.ib_income_band_sk)) otherCondition=() build RFs:RF0 ib_income_band_sk->[hd_income_band_sk] ----------------------------------------PhysicalProject @@ -82,8 +84,6 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) ----------------------------------PhysicalOlapScan[date_dim] ----------------------------PhysicalProject ------------------------------PhysicalOlapScan[customer_demographics] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[customer_address] ----------------PhysicalProject ------------------PhysicalOlapScan[date_dim] --PhysicalResultSink diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query72.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query72.out index 06bd3cd70cf867..e0e1603e128cc4 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query72.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query72.out @@ -12,7 +12,7 @@ PhysicalResultSink ------------------PhysicalProject --------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((catalog_sales.cs_item_sk = inventory.inv_item_sk) and (inventory.inv_date_sk = d2.d_date_sk)) otherCondition=((inventory.inv_quantity_on_hand < catalog_sales.cs_quantity)) build RFs:RF8 inv_date_sk->[d_date_sk];RF9 inv_item_sk->[cs_item_sk,i_item_sk] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cs_item_sk] +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cs_item_sk] --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((d1.d_week_seq = d2.d_week_seq)) otherCondition=() build RFs:RF6 d_week_seq->[d_week_seq] ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query8.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query8.out index a254eeae049f91..00f211d12927a2 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query8.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query8.out @@ -29,7 +29,7 @@ PhysicalResultSink ------------------------------PhysicalDistribute[DistributionSpecHash] --------------------------------hashAgg[LOCAL] ----------------------------------PhysicalProject -------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] +------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer_address.ca_address_sk = customer.c_current_addr_sk)) otherCondition=() build RFs:RF0 ca_address_sk->[c_current_addr_sk] --------------------------------------PhysicalProject ----------------------------------------filter((customer.c_preferred_cust_flag = 'Y')) ------------------------------------------PhysicalOlapScan[customer] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query81.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query81.out index b3b6627dd0716d..97ae3085a37ff7 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query81.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query81.out @@ -7,7 +7,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) --------PhysicalDistribute[DistributionSpecHash] ----------hashAgg[LOCAL] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_returns.cr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[cr_returning_addr_sk] +--------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_returns.cr_returning_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[cr_returning_addr_sk] ----------------PhysicalProject ------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_returns.cr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cr_returned_date_sk] --------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query83.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query83.out index 24a4ca5fa16b24..bb6216b2f45c7e 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query83.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query83.out @@ -36,7 +36,7 @@ PhysicalResultSink --------------------PhysicalDistribute[DistributionSpecHash] ----------------------hashAgg[LOCAL] ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((catalog_returns.cr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cr_item_sk] +--------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((catalog_returns.cr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cr_item_sk] ----------------------------PhysicalProject ------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((catalog_returns.cr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[cr_returned_date_sk] --------------------------------PhysicalProject @@ -59,7 +59,7 @@ PhysicalResultSink ----------------PhysicalDistribute[DistributionSpecHash] ------------------hashAgg[LOCAL] --------------------PhysicalProject -----------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((web_returns.wr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[wr_item_sk] +----------------------hashJoin[INNER_JOIN shuffle] hashCondition=((web_returns.wr_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[wr_item_sk] ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[wr_returned_date_sk] ----------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.out index a532a3f74c5395..bf421bf4350674 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------filter((lineitem.l_returnflag = 'R')) --------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() --------------------------PhysicalProject ----------------------------filter((orders.o_orderdate < '1994-01-01') and (orders.o_orderdate >= '1993-10-01')) ------------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.out index 19361de35a19df..9aefb642529297 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[RIGHT_OUTER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +--------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.out index 53fedeb6ad34c5..be755b04f33939 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((lineitem.l_partkey = part.p_partkey)) otherCondition=() +------------hashJoin[INNER_JOIN shuffle] hashCondition=((lineitem.l_partkey = part.p_partkey)) otherCondition=() --------------PhysicalProject ----------------filter((lineitem.l_shipdate < '1995-10-01') and (lineitem.l_shipdate >= '1995-09-01')) ------------------PhysicalOlapScan[lineitem] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.out index 519a64b38aa1ce..9e6b383230a34f 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((revenue0.total_revenue = max(total_revenue))) otherCondition=() ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = revenue0.supplier_no)) otherCondition=() +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((supplier.s_suppkey = revenue0.supplier_no)) otherCondition=() ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.out index 92cc8c93708400..5574a3599beb7c 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] ----------------------PhysicalProject ------------------------PhysicalOlapScan[lineitem] apply RFs: RF0 ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.out index 79164f3a7abb0d..f2dd337eadfe86 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------PhysicalOlapScan[lineitem] apply RFs: RF2 --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ------------------hashJoin[LEFT_SEMI_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF0 l_orderkey->[o_orderkey] --------------------PhysicalProject ----------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out index 8f425073388280..60e7b8446bd1a0 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey] +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey] ------------PhysicalProject --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.out index f1ec59e40d8b98..91208a71c2c13d 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF4 n_nationkey->[s_nationkey] ------------PhysicalProject ---------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] +--------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] --------------------PhysicalProject ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.out index bcf875640f9a7d..e5d22d11cfa918 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF4 n_nationkey->[s_nationkey] ------------PhysicalProject ---------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] +--------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.out index 63d82280b35b16..176b64caf7833f 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_ANTI_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +----------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.out index 48d4e37ec466e3..2fa14f8a3abf28 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------filter((lineitem.l_shipdate > '1995-03-15')) ------------------PhysicalOlapScan[lineitem] apply RFs: RF1 --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ------------------PhysicalProject --------------------filter((orders.o_orderdate < '1995-03-15')) ----------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.out index 55a5eab9536c60..e375e65d7c3815 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.out @@ -18,7 +18,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 RF2 ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ----------------------------------PhysicalProject ------------------------------------filter((orders.o_orderdate < '1995-01-01') and (orders.o_orderdate >= '1994-01-01')) --------------------------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out index f434b57fc9f511..9c2ccaeb016726 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n2.n_nationkey) and (supplier.s_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF3 n_nationkey->[c_nationkey];RF4 n_nationkey->[s_nationkey] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF2 c_custkey->[o_custkey] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF2 c_custkey->[o_custkey] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF1 l_orderkey->[o_orderkey] --------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.out index 36e6e6b1ee6c61..eedc274096c789 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.out @@ -15,11 +15,11 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF4 n_nationkey->[c_nationkey] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF3 c_custkey->[o_custkey] +------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF3 c_custkey->[o_custkey] --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] +--------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF0 o_orderkey->[l_orderkey] --------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.out b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.out index b60760ec115acb..f2155663071dfa 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.out @@ -10,20 +10,20 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN colocated] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF4 p_partkey->[l_partkey,ps_partkey] +--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF2 p_partkey->[l_partkey] --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[lineitem] apply RFs: RF4 +--------------------------------PhysicalOlapScan[lineitem] apply RFs: RF2 ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[orders] --------------------------PhysicalProject -----------------------------PhysicalOlapScan[partsupp] apply RFs: RF4 +----------------------------filter((p_name like '%green%')) +------------------------------PhysicalOlapScan[part] ----------------------PhysicalProject -------------------------filter((p_name like '%green%')) ---------------------------PhysicalOlapScan[part] +------------------------PhysicalOlapScan[partsupp] ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q13.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q13.out index 19361de35a19df..9aefb642529297 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q13.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q13.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[RIGHT_OUTER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() +--------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q2.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q2.out index fc7b45224906e4..6fdccbcadb8073 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q2.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q2.out @@ -10,22 +10,22 @@ PhysicalResultSink --------------PhysicalQuickSort[LOCAL_SORT] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[ps_suppkey] +--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 ps_suppkey->[s_suppkey] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN colocated] hashCondition=((part.p_partkey = partsupp.ps_partkey)) otherCondition=() build RFs:RF2 p_partkey->[ps_partkey] +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF2 n_nationkey->[s_nationkey] +--------------------------PhysicalOlapScan[supplier] apply RFs: RF2 RF3 --------------------------PhysicalProject -----------------------------PhysicalOlapScan[partsupp] apply RFs: RF2 RF3 ---------------------------PhysicalProject -----------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) -------------------------------PhysicalOlapScan[part] -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF1 n_nationkey->[s_nationkey] ---------------------------PhysicalOlapScan[supplier] apply RFs: RF1 ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((nation.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF0 r_regionkey->[n_regionkey] +----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((nation.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF1 r_regionkey->[n_regionkey] ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[nation] apply RFs: RF0 +--------------------------------PhysicalOlapScan[nation] apply RFs: RF1 ------------------------------PhysicalProject --------------------------------filter((region.r_name = 'EUROPE')) ----------------------------------PhysicalOlapScan[region] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN colocated] hashCondition=((part.p_partkey = partsupp.ps_partkey)) otherCondition=() build RFs:RF0 p_partkey->[ps_partkey] +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 +--------------------------PhysicalProject +----------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +------------------------------PhysicalOlapScan[part] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.out index 80f86bf96e455e..3c5d16708be512 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.out @@ -5,9 +5,9 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalProject -----------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] +----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20.out index aca1634e470c2f..38ecc268fdf8a0 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q20.out @@ -5,9 +5,9 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalProject -----------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] +----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q22.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q22.out index 63d82280b35b16..207538326c7118 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q22.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q22.out @@ -8,18 +8,18 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_ANTI_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(cast(c_acctbal as DECIMALV3(17, 4)))) ------------------PhysicalProject ---------------------PhysicalOlapScan[orders] apply RFs: RF0 -------------------PhysicalProject ---------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(cast(c_acctbal as DECIMALV3(17, 4)))) +--------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() ----------------------PhysicalProject ------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------hashAgg[LOCAL] -----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) ---------------------------------PhysicalOlapScan[customer] +----------------------PhysicalProject +------------------------PhysicalOlapScan[orders] +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecGather] +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +----------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q5.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q5.out index 2f45b1e87b401d..9d05d167d12ad6 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q5.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q5.out @@ -8,7 +8,9 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey) and (customer.c_nationkey = supplier.s_nationkey)) otherCondition=() +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_custkey = orders.o_custkey) and (customer.c_nationkey = supplier.s_nationkey)) otherCondition=() build RFs:RF4 s_nationkey->[c_nationkey];RF5 o_custkey->[c_custkey] +------------------PhysicalProject +--------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5 ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_suppkey = supplier.s_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey] ----------------------PhysicalProject @@ -29,6 +31,4 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------filter((region.r_name = 'ASIA')) ----------------------------------PhysicalOlapScan[region] -------------------PhysicalProject ---------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q8.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q8.out index fc46b49054b927..f6f7e8c16222cb 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q8.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q8.out @@ -15,30 +15,30 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[supplier] apply RFs: RF5 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF4 c_custkey->[o_custkey] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF4 o_custkey->[c_custkey] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF3 l_orderkey->[o_orderkey] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF3 n_nationkey->[c_nationkey] --------------------------------PhysicalProject -----------------------------------filter((orders.o_orderdate <= '1996-12-31') and (orders.o_orderdate >= '1995-01-01')) -------------------------------------PhysicalOlapScan[orders] apply RFs: RF3 RF4 +----------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF2 p_partkey->[l_partkey] +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((n1.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF2 r_regionkey->[n_regionkey] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[lineitem] apply RFs: RF2 +--------------------------------------PhysicalOlapScan[nation] apply RFs: RF2 ------------------------------------PhysicalProject ---------------------------------------filter((part.p_type = 'ECONOMY ANODIZED STEEL')) -----------------------------------------PhysicalOlapScan[part] +--------------------------------------filter((region.r_name = 'AMERICA')) +----------------------------------------PhysicalOlapScan[region] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF1 n_nationkey->[c_nationkey] +------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF1 l_orderkey->[o_orderkey] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer] apply RFs: RF1 +----------------------------------filter((orders.o_orderdate <= '1996-12-31') and (orders.o_orderdate >= '1995-01-01')) +------------------------------------PhysicalOlapScan[orders] apply RFs: RF1 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((n1.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF0 r_regionkey->[n_regionkey] +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[nation] apply RFs: RF0 +--------------------------------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------------------------------PhysicalProject ---------------------------------------filter((region.r_name = 'AMERICA')) -----------------------------------------PhysicalOlapScan[region] +--------------------------------------filter((part.p_type = 'ECONOMY ANODIZED STEEL')) +----------------------------------------PhysicalOlapScan[part] --------------------PhysicalProject ----------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q9.out b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q9.out index d70d5886607341..21fb9db6430bca 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q9.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/rf_prune/q9.out @@ -10,13 +10,13 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF2 l_orderkey->[o_orderkey] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[orders] apply RFs: RF2 --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q13.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q13.out index 5d94f6d22b76d8..8da72977f78cf2 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q13.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q13.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[RIGHT_OUTER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +--------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q2.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q2.out index fc7b45224906e4..6fdccbcadb8073 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q2.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q2.out @@ -10,22 +10,22 @@ PhysicalResultSink --------------PhysicalQuickSort[LOCAL_SORT] ----------------PhysicalDistribute[DistributionSpecHash] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[ps_suppkey] +--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 ps_suppkey->[s_suppkey] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN colocated] hashCondition=((part.p_partkey = partsupp.ps_partkey)) otherCondition=() build RFs:RF2 p_partkey->[ps_partkey] +------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF2 n_nationkey->[s_nationkey] +--------------------------PhysicalOlapScan[supplier] apply RFs: RF2 RF3 --------------------------PhysicalProject -----------------------------PhysicalOlapScan[partsupp] apply RFs: RF2 RF3 ---------------------------PhysicalProject -----------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) -------------------------------PhysicalOlapScan[part] -----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF1 n_nationkey->[s_nationkey] ---------------------------PhysicalOlapScan[supplier] apply RFs: RF1 ---------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((nation.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF0 r_regionkey->[n_regionkey] +----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((nation.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF1 r_regionkey->[n_regionkey] ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[nation] apply RFs: RF0 +--------------------------------PhysicalOlapScan[nation] apply RFs: RF1 ------------------------------PhysicalProject --------------------------------filter((region.r_name = 'EUROPE')) ----------------------------------PhysicalOlapScan[region] +----------------------PhysicalProject +------------------------hashJoin[INNER_JOIN colocated] hashCondition=((part.p_partkey = partsupp.ps_partkey)) otherCondition=() build RFs:RF0 p_partkey->[ps_partkey] +--------------------------PhysicalProject +----------------------------PhysicalOlapScan[partsupp] apply RFs: RF0 +--------------------------PhysicalProject +----------------------------filter((p_type like '%BRASS') and (part.p_size = 15)) +------------------------------PhysicalOlapScan[part] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.out index 80f86bf96e455e..3c5d16708be512 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.out @@ -5,9 +5,9 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalProject -----------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] +----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20.out index aca1634e470c2f..38ecc268fdf8a0 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q20.out @@ -5,9 +5,9 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalProject -----------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] +----------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF4 s_suppkey->[l_suppkey,ps_suppkey] ------------PhysicalProject ---------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF2 ps_partkey->[l_partkey];RF3 ps_suppkey->[l_suppkey] ----------------hashAgg[GLOBAL] ------------------PhysicalDistribute[DistributionSpecHash] --------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q22.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q22.out index 63d82280b35b16..207538326c7118 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q22.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q22.out @@ -8,18 +8,18 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_ANTI_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +----------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(cast(c_acctbal as DECIMALV3(17, 4)))) ------------------PhysicalProject ---------------------PhysicalOlapScan[orders] apply RFs: RF0 -------------------PhysicalProject ---------------------NestedLoopJoin[INNER_JOIN](cast(c_acctbal as DECIMALV3(38, 4)) > avg(cast(c_acctbal as DECIMALV3(17, 4)))) +--------------------hashJoin[LEFT_ANTI_JOIN bucketShuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() ----------------------PhysicalProject ------------------------filter(substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) --------------------------PhysicalOlapScan[customer] -----------------------hashAgg[GLOBAL] -------------------------PhysicalDistribute[DistributionSpecGather] ---------------------------hashAgg[LOCAL] -----------------------------PhysicalProject -------------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) ---------------------------------PhysicalOlapScan[customer] +----------------------PhysicalProject +------------------------PhysicalOlapScan[orders] +------------------hashAgg[GLOBAL] +--------------------PhysicalDistribute[DistributionSpecGather] +----------------------hashAgg[LOCAL] +------------------------PhysicalProject +--------------------------filter((customer.c_acctbal > 0.00) and substring(c_phone, 1, 2) IN ('13', '17', '18', '23', '29', '30', '31')) +----------------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q5.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q5.out index f2ebd3c8b31b86..9d05d167d12ad6 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q5.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q5.out @@ -8,7 +8,9 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey) and (customer.c_nationkey = supplier.s_nationkey)) otherCondition=() build RFs:RF4 c_nationkey->[n_nationkey,s_nationkey];RF5 c_custkey->[o_custkey] +----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((customer.c_custkey = orders.o_custkey) and (customer.c_nationkey = supplier.s_nationkey)) otherCondition=() build RFs:RF4 s_nationkey->[c_nationkey];RF5 o_custkey->[c_custkey] +------------------PhysicalProject +--------------------PhysicalOlapScan[customer] apply RFs: RF4 RF5 ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((lineitem.l_suppkey = supplier.s_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey] ----------------------PhysicalProject @@ -17,18 +19,16 @@ PhysicalResultSink ----------------------------PhysicalOlapScan[lineitem] apply RFs: RF2 RF3 --------------------------PhysicalProject ----------------------------filter((orders.o_orderdate < '1995-01-01') and (orders.o_orderdate >= '1994-01-01')) -------------------------------PhysicalOlapScan[orders] apply RFs: RF5 +------------------------------PhysicalOlapScan[orders] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF1 n_nationkey->[s_nationkey] --------------------------PhysicalProject -----------------------------PhysicalOlapScan[supplier] apply RFs: RF1 RF4 +----------------------------PhysicalOlapScan[supplier] apply RFs: RF1 --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((nation.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF0 r_regionkey->[n_regionkey] ------------------------------PhysicalProject ---------------------------------PhysicalOlapScan[nation] apply RFs: RF0 RF4 +--------------------------------PhysicalOlapScan[nation] apply RFs: RF0 ------------------------------PhysicalProject --------------------------------filter((region.r_name = 'ASIA')) ----------------------------------PhysicalOlapScan[region] -------------------PhysicalProject ---------------------PhysicalOlapScan[customer] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q8.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q8.out index ef2b3523bbbe98..32eda956cf1b02 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q8.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q8.out @@ -15,30 +15,30 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------PhysicalOlapScan[supplier] apply RFs: RF5 RF6 ------------------------PhysicalProject ---------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF4 c_custkey->[o_custkey] +--------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF4 o_custkey->[c_custkey] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF3 l_orderkey->[o_orderkey] +------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF3 n_nationkey->[c_nationkey] --------------------------------PhysicalProject -----------------------------------filter((orders.o_orderdate <= '1996-12-31') and (orders.o_orderdate >= '1995-01-01')) -------------------------------------PhysicalOlapScan[orders] apply RFs: RF3 RF4 +----------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF2 p_partkey->[l_partkey] +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((n1.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF2 r_regionkey->[n_regionkey] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[lineitem] apply RFs: RF2 +--------------------------------------PhysicalOlapScan[nation] apply RFs: RF2 ------------------------------------PhysicalProject ---------------------------------------filter((part.p_type = 'ECONOMY ANODIZED STEEL')) -----------------------------------------PhysicalOlapScan[part] +--------------------------------------filter((region.r_name = 'AMERICA')) +----------------------------------------PhysicalOlapScan[region] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF1 n_nationkey->[c_nationkey] +------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF1 l_orderkey->[o_orderkey] --------------------------------PhysicalProject -----------------------------------PhysicalOlapScan[customer] apply RFs: RF1 +----------------------------------filter((orders.o_orderdate <= '1996-12-31') and (orders.o_orderdate >= '1995-01-01')) +------------------------------------PhysicalOlapScan[orders] apply RFs: RF1 --------------------------------PhysicalProject -----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((n1.n_regionkey = region.r_regionkey)) otherCondition=() build RFs:RF0 r_regionkey->[n_regionkey] +----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] ------------------------------------PhysicalProject ---------------------------------------PhysicalOlapScan[nation] apply RFs: RF0 +--------------------------------------PhysicalOlapScan[lineitem] apply RFs: RF0 ------------------------------------PhysicalProject ---------------------------------------filter((region.r_name = 'AMERICA')) -----------------------------------------PhysicalOlapScan[region] +--------------------------------------filter((part.p_type = 'ECONOMY ANODIZED STEEL')) +----------------------------------------PhysicalOlapScan[part] --------------------PhysicalProject ----------------------PhysicalOlapScan[nation] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q9.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q9.out index cd77e18b02c916..bee3d7f4b52394 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q9.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape/q9.out @@ -10,13 +10,13 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF4 ps_suppkey->[l_suppkey,s_suppkey];RF5 ps_partkey->[l_partkey,p_partkey] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF2 l_orderkey->[o_orderkey] --------------------------PhysicalProject ----------------------------PhysicalOlapScan[orders] apply RFs: RF2 --------------------------PhysicalProject -----------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] +----------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 RF3 RF4 RF5 ------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.out index 0820ef5c2c6526..b089c525af8864 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.out @@ -15,7 +15,7 @@ PhysicalResultSink ------------------------filter((lineitem.l_returnflag = 'R')) --------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] --------------------------PhysicalProject ----------------------------filter((orders.o_orderdate < '1994-01-01') and (orders.o_orderdate >= '1993-10-01')) ------------------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.out index 5d94f6d22b76d8..8da72977f78cf2 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashAgg[LOCAL] ------------------PhysicalProject ---------------------hashJoin[RIGHT_OUTER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +--------------------hashJoin[RIGHT_OUTER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ----------------------PhysicalProject ------------------------filter(( not (o_comment like '%special%requests%'))) --------------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.out index d6457aadcccc10..fd7cd9d438a5ed 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.out @@ -6,7 +6,7 @@ PhysicalResultSink ------PhysicalDistribute[DistributionSpecGather] --------hashAgg[LOCAL] ----------PhysicalProject -------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((lineitem.l_partkey = part.p_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] +------------hashJoin[INNER_JOIN shuffle] hashCondition=((lineitem.l_partkey = part.p_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] --------------PhysicalProject ----------------filter((lineitem.l_shipdate < '1995-10-01') and (lineitem.l_shipdate >= '1995-09-01')) ------------------PhysicalOlapScan[lineitem] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.out index f29b77317c3bfb..e9b45b5888ce54 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.out @@ -7,7 +7,7 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((revenue0.total_revenue = max(total_revenue))) otherCondition=() ------------PhysicalProject ---------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = revenue0.supplier_no)) otherCondition=() build RFs:RF0 s_suppkey->[l_suppkey] +--------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((supplier.s_suppkey = revenue0.supplier_no)) otherCondition=() build RFs:RF0 s_suppkey->[l_suppkey] ----------------PhysicalProject ------------------hashAgg[GLOBAL] --------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.out index 92cc8c93708400..5574a3599beb7c 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalWindow ----------------PhysicalQuickSort[LOCAL_SORT] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF0 p_partkey->[l_partkey] ----------------------PhysicalProject ------------------------PhysicalOlapScan[lineitem] apply RFs: RF0 ----------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.out index 44c12faa6d47d5..580d66d2c717c5 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------PhysicalOlapScan[lineitem] apply RFs: RF2 --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF1 c_custkey->[o_custkey] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF1 c_custkey->[o_custkey] ------------------hashJoin[LEFT_SEMI_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF0 l_orderkey->[o_orderkey] --------------------PhysicalProject ----------------------PhysicalOlapScan[orders] apply RFs: RF0 RF1 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out index 8f425073388280..60e7b8446bd1a0 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.out @@ -5,7 +5,7 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] --------PhysicalProject -----------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey] +----------hashJoin[INNER_JOIN broadcast] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey] ------------PhysicalProject --------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR')) ----------------PhysicalOlapScan[lineitem] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.out index f1ec59e40d8b98..91208a71c2c13d 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF4 n_nationkey->[s_nationkey] ------------PhysicalProject ---------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] +--------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = t3.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((t2.l_partkey = t1.ps_partkey) and (t2.l_suppkey = t1.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > t2.l_q)) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] --------------------PhysicalProject ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecHash] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.out index bcf875640f9a7d..e5d22d11cfa918 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.out @@ -7,9 +7,9 @@ PhysicalResultSink --------PhysicalProject ----------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF4 n_nationkey->[s_nationkey] ------------PhysicalProject ---------------hashJoin[RIGHT_SEMI_JOIN shuffleBucket] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] +--------------hashJoin[RIGHT_SEMI_JOIN shuffle] hashCondition=((supplier.s_suppkey = partsupp.ps_suppkey)) otherCondition=() build RFs:RF3 s_suppkey->[l_suppkey,ps_suppkey] ----------------PhysicalProject -------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] +------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((lineitem.l_partkey = partsupp.ps_partkey) and (lineitem.l_suppkey = partsupp.ps_suppkey)) otherCondition=((cast(ps_availqty as DECIMALV3(38, 3)) > (0.5 * sum(l_quantity)))) build RFs:RF1 ps_partkey->[l_partkey];RF2 ps_suppkey->[l_suppkey] --------------------hashAgg[GLOBAL] ----------------------PhysicalDistribute[DistributionSpecHash] ------------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.out index 63d82280b35b16..176b64caf7833f 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.out @@ -8,7 +8,7 @@ PhysicalResultSink ----------PhysicalDistribute[DistributionSpecHash] ------------hashAgg[LOCAL] --------------PhysicalProject -----------------hashJoin[RIGHT_ANTI_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +----------------hashJoin[RIGHT_ANTI_JOIN shuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ------------------PhysicalProject --------------------PhysicalOlapScan[orders] apply RFs: RF0 ------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.out index 48d4e37ec466e3..2fa14f8a3abf28 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.out @@ -11,7 +11,7 @@ PhysicalResultSink ----------------filter((lineitem.l_shipdate > '1995-03-15')) ------------------PhysicalOlapScan[lineitem] apply RFs: RF1 --------------PhysicalProject -----------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +----------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ------------------PhysicalProject --------------------filter((orders.o_orderdate < '1995-03-15')) ----------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.out index 640dabeb70f0bb..8d817e1c7b7e68 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.out @@ -18,7 +18,7 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[lineitem] apply RFs: RF1 RF2 ------------------------------PhysicalProject ---------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] +--------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF0 c_custkey->[o_custkey] ----------------------------------PhysicalProject ------------------------------------filter((orders.o_orderdate < '1995-01-01') and (orders.o_orderdate >= '1994-01-01')) --------------------------------------PhysicalOlapScan[orders] apply RFs: RF0 diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out index f434b57fc9f511..9c2ccaeb016726 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.out @@ -10,7 +10,7 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n2.n_nationkey) and (supplier.s_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF3 n_nationkey->[c_nationkey];RF4 n_nationkey->[s_nationkey] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF2 c_custkey->[o_custkey] +--------------------hashJoin[INNER_JOIN shuffle] hashCondition=((customer.c_custkey = orders.o_custkey)) otherCondition=() build RFs:RF2 c_custkey->[o_custkey] ----------------------PhysicalProject ------------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF1 l_orderkey->[o_orderkey] --------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.out index f3abaf5956c1a6..7b034a6386f270 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.out @@ -15,11 +15,11 @@ PhysicalResultSink ------------------------PhysicalProject --------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((customer.c_nationkey = n1.n_nationkey)) otherCondition=() build RFs:RF4 n_nationkey->[c_nationkey] ----------------------------PhysicalProject -------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF3 c_custkey->[o_custkey] +------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((orders.o_custkey = customer.c_custkey)) otherCondition=() build RFs:RF3 c_custkey->[o_custkey] --------------------------------PhysicalProject ----------------------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF2 s_suppkey->[l_suppkey] ------------------------------------PhysicalProject ---------------------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] +--------------------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF1 p_partkey->[l_partkey] ----------------------------------------PhysicalProject ------------------------------------------hashJoin[INNER_JOIN colocated] hashCondition=((lineitem.l_orderkey = orders.o_orderkey)) otherCondition=() build RFs:RF0 o_orderkey->[l_orderkey] --------------------------------------------PhysicalProject diff --git a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.out b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.out index ab51e5595023a9..ae79c7950f230b 100644 --- a/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.out +++ b/regression-test/data/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.out @@ -10,9 +10,9 @@ PhysicalResultSink --------------PhysicalProject ----------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF5 s_suppkey->[l_suppkey,ps_suppkey] ------------------PhysicalProject ---------------------hashJoin[INNER_JOIN colocated] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF4 p_partkey->[l_partkey,ps_partkey] +--------------------hashJoin[INNER_JOIN bucketShuffle] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF3 ps_suppkey->[l_suppkey];RF4 ps_partkey->[l_partkey,p_partkey] ----------------------PhysicalProject -------------------------hashJoin[INNER_JOIN shuffleBucket] hashCondition=((partsupp.ps_partkey = lineitem.l_partkey) and (partsupp.ps_suppkey = lineitem.l_suppkey)) otherCondition=() build RFs:RF2 ps_suppkey->[l_suppkey];RF3 ps_partkey->[l_partkey] +------------------------hashJoin[INNER_JOIN shuffle] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=() build RFs:RF2 p_partkey->[l_partkey] --------------------------PhysicalProject ----------------------------hashJoin[INNER_JOIN colocated] hashCondition=((orders.o_orderkey = lineitem.l_orderkey)) otherCondition=() build RFs:RF1 o_orderkey->[l_orderkey] ------------------------------PhysicalProject @@ -20,10 +20,10 @@ PhysicalResultSink ------------------------------PhysicalProject --------------------------------PhysicalOlapScan[orders] --------------------------PhysicalProject -----------------------------PhysicalOlapScan[partsupp] apply RFs: RF4 RF5 +----------------------------filter((p_name like '%green%')) +------------------------------PhysicalOlapScan[part] apply RFs: RF4 ----------------------PhysicalProject -------------------------filter((p_name like '%green%')) ---------------------------PhysicalOlapScan[part] +------------------------PhysicalOlapScan[partsupp] apply RFs: RF5 ------------------PhysicalProject --------------------hashJoin[INNER_JOIN broadcast] hashCondition=((supplier.s_nationkey = nation.n_nationkey)) otherCondition=() build RFs:RF0 n_nationkey->[s_nationkey] ----------------------PhysicalProject diff --git a/regression-test/suites/nereids_syntax_p0/distribute/colocate_agg_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/colocate_agg_join.groovy deleted file mode 100644 index d7d88b89cd8881..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/colocate_agg_join.groovy +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("colocate_agg_join") { - multi_sql """ - drop table if exists colocate_agg_join1; - drop table if exists colocate_agg_join2; - - create table colocate_agg_join1 ( - col_int_undef_signed_not_null int not null , - col_date_undef_signed_not_null date not null , - pk int, - col_int_undef_signed int null , - col_date_undef_signed date null , - col_varchar_10__undef_signed varchar(10) null , - col_varchar_10__undef_signed_not_null varchar(10) not null , - col_varchar_1024__undef_signed varchar(1024) null , - col_varchar_1024__undef_signed_not_null varchar(1024) not null - ) engine=olap - DUPLICATE KEY(col_int_undef_signed_not_null, col_date_undef_signed_not_null, pk) - PARTITION BY RANGE(col_int_undef_signed_not_null, col_date_undef_signed_not_null) (PARTITION p0 VALUES [('-10000', '2023-12-01'), ('3', '2023-12-10')), PARTITION p1 VALUES [('3', '2023-12-10'), ('6', '2023-12-15')), PARTITION p2 VALUES [('6', '2023-12-15'), ('10000', '2023-12-21'))) - distributed by hash(pk) buckets 10 - properties("replication_num" = "1"); - insert into colocate_agg_join1(pk,col_int_undef_signed,col_int_undef_signed_not_null,col_date_undef_signed,col_date_undef_signed_not_null,col_varchar_10__undef_signed,col_varchar_10__undef_signed_not_null,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_not_null) values (0,null,6,'2023-12-18','2023-12-20','b','g','were','as'),(1,4,0,'2023-12-16','2023-12-16','have','l','be','do'),(2,null,3,'2023-12-14','2023-12-14','there','why','were','s'),(3,null,2,'2023-12-20','2023-12-16','i','y','at','v'),(4,null,1,'2023-12-20','2023-12-09','x','some','why','e'),(5,null,0,'2023-12-13','2023-12-13','n','didn''t','get','z'),(6,2,7,'2023-12-11','2023-12-12','want','h','you','m'),(7,6,6,'2023-12-18','2023-12-19','your','look','e','i'),(8,8,1,'2023-12-11','2023-12-19','j','a','when','i'),(9,2,8,'2023-12-13','2023-12-16','v','his','v','had'),(10,null,2,'2023-12-17','2023-12-14','who','z','were','d'),(11,5,6,'2023-12-12','2023-12-15','z','been','z','he'),(12,2,0,'2023-12-16','2023-12-13','no','a','not','been'),(13,4,7,'2023-12-16','2023-12-18','o','about','z','it'),(14,6,2,'2023-12-10','2023-12-09','he','ok','n','about'),(15,2,1,'2023-12-15','2023-12-19','y','e','l','his'),(16,3,0,'2023-12-16','2023-12-18','b','that','for','yes'),(17,3,5,'2023-12-15','2023-12-15','e','I''m','h','could'),(18,null,9,'2023-12-15','2023-12-16','j','n','about','on'),(19,1,3,'2023-12-15','2023-12-15','had','would','in','no'); - - create table colocate_agg_join2 ( - pk int, - col_int_undef_signed int null , - col_int_undef_signed_not_null int not null , - col_date_undef_signed date null , - col_date_undef_signed_not_null date not null , - col_varchar_10__undef_signed varchar(10) null , - col_varchar_10__undef_signed_not_null varchar(10) not null , - col_varchar_1024__undef_signed varchar(1024) null , - col_varchar_1024__undef_signed_not_null varchar(1024) not null - ) engine=olap - DUPLICATE KEY(pk) - distributed by hash(pk) buckets 10 - properties("replication_num" = "1"); - insert into colocate_agg_join2(pk,col_int_undef_signed,col_int_undef_signed_not_null,col_date_undef_signed,col_date_undef_signed_not_null,col_varchar_10__undef_signed,col_varchar_10__undef_signed_not_null,col_varchar_1024__undef_signed,col_varchar_1024__undef_signed_not_null) values (0,4,0,'2023-12-18','2023-12-11','to','see','he''s','r'),(1,4,5,'2023-12-10','2023-12-09','him','s','didn''t','k'); - - set disable_join_reorder=true; - set enable_local_shuffle=true; - set runtime_filter_mode=off; - - set disable_nereids_rules='ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI,ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT,TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI,TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT,TWO_PHASE_AGGREGATE_WITHOUT_DISTINCT,FOUR_PHASE_AGGREGATE_WITH_DISTINCT,FOUR_PHASE_AGGREGATE_WITH_DISTINCT_WITH_FULL_DISTRIBUTE'; - """ - - for (def i in 1..10) { - sql "set parallel_pipeline_task_num=${i}" - test { - sql """ - select avg_pk - from ( - select t1.* - from colocate_agg_join1 AS alias1 - right anti join ( - select pk, avg(distinct pk) avg_pk - from colocate_agg_join2 - group by pk - ) t1 - ON t1.`pk` = alias1.`pk` - )a - order by pk - limit 1000 - """ - rowNum 0 - } - } -} \ No newline at end of file diff --git a/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy b/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy deleted file mode 100644 index 3119b2ddb2c5dd..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/colocate_union_numbers.groovy +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("colocate_union_numbers") { - multi_sql """ - set enable_nereids_distribute_planner=false; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=false; - set force_to_local_shuffle=false; - """ - - def extractFragment = { String sqlStr, String containsString, Closure checkExchangeNum -> - retry(120, 1000) { - explain { - sql sqlStr - check { result -> - log.info("Explain result:\n${result}") - - assertTrue(result.contains(containsString)) - - def fragmentContainsJoin = result.split("PLAN FRAGMENT") - .toList() - .stream() - .filter { it.contains(containsString) } - .findFirst() - .get() - - log.info("Fragment:\n${fragmentContainsJoin}") - - checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE")) - } - } - } - } - - def sqlStr = """ - select * from numbers('number'='3')a - union all - select * from numbers('number'='4')b - """ - - extractFragment(sqlStr, "VUNION") { exchangeNum -> - assertTrue(exchangeNum == 2) - } - - multi_sql """ - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - """ - - extractFragment(sqlStr, "VUNION") { exchangeNum -> - assertTrue(exchangeNum == 0) - } - - order_qt_union_all sqlStr -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/distribution_expr.groovy b/regression-test/suites/nereids_syntax_p0/distribute/distribution_expr.groovy deleted file mode 100644 index c2ca8a40134d3b..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/distribution_expr.groovy +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("distribution_expr") { - multi_sql """ - drop table if exists table_100_undef_partitions2_keys3_properties4_distributed_by52; - drop table if exists table_6_undef_partitions2_keys3_properties4_distributed_by53; - drop table if exists table_7_undef_partitions2_keys3_properties4_distributed_by5; - drop table if exists table_8_undef_partitions2_keys3_properties4_distributed_by5; - - create table table_100_undef_partitions2_keys3_properties4_distributed_by52 ( - `pk` int, - `col_int_undef_signed` int , - `col_varchar_10__undef_signed` varchar(10) , - `col_varchar_1024__undef_signed` varchar(1024) MAX - ) engine=olap - AGGREGATE KEY(pk, col_int_undef_signed, col_varchar_10__undef_signed) - distributed by hash(pk) buckets 10 - properties("replication_num" = "1"); - insert into table_100_undef_partitions2_keys3_properties4_distributed_by52(pk,col_int_undef_signed,col_varchar_10__undef_signed,col_varchar_1024__undef_signed) values (0,null,'k',"mean"),(1,null,"I'll","who"),(2,null,"i",'v'),(3,null,"really",'w'),(4,null,"when",'e'),(5,9,"what","who"),(6,5,"been","like"),(7,null,"and","ok"),(8,3,'t',"then"),(9,null,'y',"up"),(10,8,'b',"think"),(11,0,"if",'l'),(12,null,"there",'q'),(13,null,"out",'q'),(14,3,'c','h'),(15,2,"out","yes"),(16,null,'t',"think"),(17,2,'a',"did"),(18,null,'j','a'),(19,7,'p',"with"),(20,null,'x','v'),(21,null,"I'll",'f'),(22,9,'i',"this"),(23,4,'l',"or"),(24,3,"oh","have"),(25,null,"go",'g'),(26,null,"did","been"),(27,null,'x',"what"),(28,2,"see",'b'),(29,null,'c','d'),(30,null,'b',"some"),(31,null,"didn't","out"),(32,6,"that's","did"),(33,7,"will","got"),(34,null,'w','h'),(35,null,"the",'d'),(36,4,'k',"good"),(37,null,'u','m'),(38,null,"for",'c'),(39,8,"good","on"),(40,1,'d',"will"),(41,null,"ok",'t'),(42,null,"see",'a'),(43,null,"mean","something"),(44,4,"did","be"),(45,3,'k',"been"),(46,4,'t',"yes"),(47,null,"but","think"),(48,null,'b',"some"),(49,null,'o',"like"),(50,null,"on","there"),(51,0,'q','u'),(52,0,"a",'s'),(53,6,'d',"yes"),(54,7,"that's",'e'),(55,3,"been",'f'),(56,null,"tell",'y'),(57,null,'m','v'),(58,null,"i","get"),(59,2,"why",'v'),(60,3,'g',"for"),(61,null,'x',"if"),(62,null,"can","did"),(63,null,"i",'t'),(64,3,"who","I'll"),(65,1,'x',"if"),(66,9,"he","a"),(67,0,"get",'h'),(68,0,"don't","some"),(69,null,'r',"with"),(70,3,'i','j'),(71,null,"can't",'v'),(72,2,"ok",'j'),(73,null,'e',"what"),(74,null,'w',"in"),(75,8,"well","mean"),(76,null,'z','s'),(77,9,'d','z'),(78,9,"oh","you"),(79,null,'k','c'),(80,2,"know","I'll"),(81,null,"say","had"),(82,null,'x',"about"),(83,9,"a","me"),(84,1,"be",'a'),(85,7,"the",'t'),(86,null,'t',"been"),(87,null,"not","are"),(88,null,"how",'m'),(89,2,'w',"will"),(90,null,"what","i"),(91,1,"will","we"),(92,null,'l','o'),(93,null,"all",'o'),(94,null,'i',"me"),(95,null,'e','l'),(96,6,'q',"you're"),(97,9,"your",'g'),(98,null,"okay",'o'),(99,7,"my",'v'); - - - create table table_6_undef_partitions2_keys3_properties4_distributed_by53 ( - `pk` int, - `col_varchar_10__undef_signed` varchar(10) , - `col_int_undef_signed` int , - `col_varchar_1024__undef_signed` varchar(1024) - ) engine=olap - DUPLICATE KEY(pk, col_varchar_10__undef_signed) - distributed by hash(pk) buckets 10 - properties("replication_num" = "1"); - insert into table_6_undef_partitions2_keys3_properties4_distributed_by53(pk,col_int_undef_signed,col_varchar_10__undef_signed,col_varchar_1024__undef_signed) values (0,0,"think","she"),(1,null,"was",'r'),(2,8,'g',"i"),(3,9,'s',"he's"),(4,4,"they",'n'),(5,null,"time","really"); - - create table table_7_undef_partitions2_keys3_properties4_distributed_by5 ( - `col_int_undef_signed` int/*agg_type_placeholder*/ , - `col_varchar_10__undef_signed` varchar(10)/*agg_type_placeholder*/ , - `col_varchar_1024__undef_signed` varchar(1024)/*agg_type_placeholder*/ , - `pk` int/*agg_type_placeholder*/ - ) engine=olap - distributed by hash(pk) buckets 10 - properties("replication_num" = "1"); - insert into table_7_undef_partitions2_keys3_properties4_distributed_by5(pk,col_int_undef_signed,col_varchar_10__undef_signed,col_varchar_1024__undef_signed) values (0,7,'y','g'),(1,null,'t',"can"),(2,8,"would",'l'),(3,null,"will","he"),(4,null,'k',"I'll"),(5,null,'m',"ok"),(6,null,'s',"that"); - - create table table_8_undef_partitions2_keys3_properties4_distributed_by5 ( - `col_int_undef_signed` int/*agg_type_placeholder*/ , - `col_varchar_10__undef_signed` varchar(10)/*agg_type_placeholder*/ , - `col_varchar_1024__undef_signed` varchar(1024)/*agg_type_placeholder*/ , - `pk` int/*agg_type_placeholder*/ - ) engine=olap - distributed by hash(pk) buckets 10 - properties("replication_num" = "1"); - insert into table_8_undef_partitions2_keys3_properties4_distributed_by5(pk,col_int_undef_signed,col_varchar_10__undef_signed,col_varchar_1024__undef_signed) values (0,4,'d',"she"),(1,3,"okay",'e'),(2,null,'s',"as"),(3,null,"you","up"),(4,3,'f','q'),(5,null,'f','s'),(6,null,'h',"time"),(7,8,'o',"i"); - - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=true; - set force_to_local_shuffle=true; - """ - - explain { - sql """ - SELECT * - FROM - (SELECT alias3.`pk` from table_100_undef_partitions2_keys3_properties4_distributed_by52 AS alias4 INNER JOIN - table_6_undef_partitions2_keys3_properties4_distributed_by53 AS alias3 - ON alias3.`pk` = alias4.`pk` - WHERE (alias3.`pk` < alias4.`pk` OR alias3.`pk` <= 4 ) - ) tmp2 - INNER JOIN[shuffle] - (select alias1.pk from table_7_undef_partitions2_keys3_properties4_distributed_by5 AS alias1 - LEFT JOIN table_8_undef_partitions2_keys3_properties4_distributed_by5 AS alias2 - ON alias1.`col_varchar_10__undef_signed` = alias2.`col_varchar_1024__undef_signed`) tmp1 - ON tmp1 . `pk` = tmp2 . `pk`; - """ - contains "BUCKET_SHUFFLE" - contains "distribute expr lists: pk[#27]" - contains "distribute expr lists: pk[#14]" - } - - multi_sql """ - drop table if exists baseall; - drop table if exists test; - CREATE TABLE IF NOT EXISTS `baseall` ( - `k1` tinyint(4) null comment "" - ) engine=olap - DISTRIBUTED BY HASH(`k1`) BUCKETS 3 properties("replication_num" = "1"); - - CREATE TABLE IF NOT EXISTS `test` ( - `k1` tinyint(4) null comment "" - ) engine=olap - DISTRIBUTED BY HASH(`k1`) BUCKETS 3 properties("replication_num" = "1"); - - insert into baseall values (1); - insert into baseall values (2); - insert into baseall values (3); - insert into test values (1); - insert into test values (2); - insert into test values (3); - - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=true; - set force_to_local_shuffle=true; - """ - - explain { - sql """ - select tmp.k1 from baseall d join (select a.k1 as k1 from baseall b join test a on (a.k1=b.k1)) tmp on tmp.k1 = d.k1; - """ - contains "COLOCATE" - contains "distribute expr lists: k1[#6]" - contains "distribute expr lists: k1[#5]" - } -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy b/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy deleted file mode 100644 index 950f42f428c2a2..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/local_shuffle.groovy +++ /dev/null @@ -1,187 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("local_shuffle") { - multi_sql """ - drop table if exists test_local_shuffle1; - drop table if exists test_local_shuffle2; - - CREATE TABLE `test_local_shuffle1` ( - id int, - id2 int - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "colocate_with" = "test_local_shuffle_with_colocate" - ); - - CREATE TABLE `test_local_shuffle2` ( - id int, - id2 int - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "colocate_with" = "test_local_shuffle_with_colocate" - ); - - insert into test_local_shuffle1 values (1, 1), (2, 2); - insert into test_local_shuffle2 values (2, 2), (3, 3); - - -- set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=true; - set force_to_local_shuffle=true; - """ - - order_qt_read_single_olap_table "select * from test_local_shuffle1 order by id, id2" - - order_qt_broadcast_join """ - select * - from test_local_shuffle1 - join [broadcast] - test_local_shuffle2 - on test_local_shuffle1.id=test_local_shuffle2.id - """ - - order_qt_shuffle_join """ - select * - from test_local_shuffle1 - join [shuffle] - test_local_shuffle2 - on test_local_shuffle1.id2=test_local_shuffle2.id2 - """ - - order_qt_bucket_shuffle_join """ - select * - from test_local_shuffle1 - join [shuffle] - test_local_shuffle2 - on test_local_shuffle1.id2=test_local_shuffle2.id2 - """ - - order_qt_colocate_join """ - select * - from test_local_shuffle1 - join [shuffle] - test_local_shuffle2 - on test_local_shuffle1.id=test_local_shuffle2.id - """ - - order_qt_bucket_shuffle_with_prune_tablets """ - select * - from - ( - select * - from test_local_shuffle1 - where id=1 - ) a - right outer join [shuffle] - test_local_shuffle2 - on a.id=test_local_shuffle2.id2 order by test_local_shuffle2.id, test_local_shuffle2.id2 - """ - - order_qt_bucket_shuffle_with_prune_tablets2 """ - select * - from - test_local_shuffle2 - left outer join [shuffle] - ( - select * - from test_local_shuffle1 - where id=1 - ) a - on a.id=test_local_shuffle2.id2 order by test_local_shuffle2.id, test_local_shuffle2.id2 - """ - - order_qt_bucket_shuffle_with_prune_tablets3 """ - select * - from - ( - select * - from test_local_shuffle1 - where id=1 - ) a - left outer join [shuffle] - test_local_shuffle2 - on a.id=test_local_shuffle2.id2 - """ - - multi_sql """ - drop table if exists test_local_shuffle3; - CREATE TABLE IF NOT EXISTS test_local_shuffle3 ( - c0 int - ) - DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1"); - - drop table if exists test_local_shuffle4; - CREATE TABLE IF NOT EXISTS test_local_shuffle4 ( - c0 int - ) - DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1"); - INSERT INTO test_local_shuffle3 (c0) VALUES (1), (3); - INSERT INTO test_local_shuffle4 (c0) VALUES (2), (3); - - sync; - - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=true; - set force_to_local_shuffle=true; - """ - - order_qt_fillup_bucket """ - SELECT cast(a.c0 as int), cast(b.c0 as int) res FROM - (select * from test_local_shuffle3 where c0 =1)a - RIGHT OUTER JOIN - (select * from test_local_shuffle4)b - ON a.c0 = b.c0 order by res - """ - - multi_sql """ - drop table if exists test_shuffle_left_with_local_shuffle; - CREATE TABLE `test_shuffle_left_with_local_shuffle` ( - id int, - id2 int - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - - insert into test_shuffle_left_with_local_shuffle values (1, 1), (2, 2), (3, 4); - """ - - order_qt_shuffle_left """ - select * - from - ( - select id2 - from test_shuffle_left_with_local_shuffle - group by id2 - ) a - inner join [shuffle] - test_shuffle_left_with_local_shuffle b - on a.id2=b.id order by a.id2; - """ -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy deleted file mode 100644 index 7006b8c226e170..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/prune_bucket_with_bucket_shuffle_join.groovy +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("prune_bucket_with_bucket_shuffle_join") { - multi_sql """ - drop table if exists test_outer_join1; - CREATE TABLE IF NOT EXISTS test_outer_join1 ( - c0 DECIMALV3(8,3) - ) - DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1"); - - drop table if exists test_outer_join2; - CREATE TABLE IF NOT EXISTS test_outer_join2 ( - c0 DECIMALV3(8,3) - ) - DISTRIBUTED BY HASH (c0) BUCKETS 10 PROPERTIES ("replication_num" = "1"); - INSERT INTO test_outer_join1 (c0) VALUES (1), (3); - INSERT INTO test_outer_join2 (c0) VALUES (2), (3); - - sync; - - set enable_nereids_distribute_planner=false; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=false; - set force_to_local_shuffle=false; - """ - - def extractFragment = { String sqlStr, String containsString, Closure checkExchangeNum -> - retry(120, 1000) { - explain { - sql sqlStr - check { result -> - log.info("Explain result:\n${result}") - - assertTrue(result.contains(containsString)) - - def fragmentContainsJoin = result.split("PLAN FRAGMENT") - .toList() - .stream() - .filter { it.contains(containsString) } - .findFirst() - .get() - - log.info("Fragment:\n${fragmentContainsJoin}") - - checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE")) - } - } - } - } - - String sqlStr = """ - SELECT * FROM - (select * from test_outer_join1 where c0 =1)a - RIGHT OUTER JOIN - (select * from test_outer_join2)b - ON a.c0 = b.c0 - """ - - extractFragment(sqlStr, "RIGHT OUTER JOIN(PARTITIONED)") { exchangeNum -> - assertTrue(exchangeNum > 1) - } - - multi_sql """ - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - """ - - extractFragment(sqlStr, "RIGHT OUTER JOIN(BUCKET_SHUFFLE)") { exchangeNum -> - assertTrue(exchangeNum == 1) - } - - explain { - sql "distributed plan ${sqlStr}" - check { explainStr -> - log.info("Distributed plan:\n${explainStr}") - - // some tablets of left table are pruned - assertTrue(explainStr.count("tablet ") < 20) - } - } - - order_qt_fillup_bucket sqlStr -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy b/regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy deleted file mode 100644 index 19521294975934..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/query_constant.groovy +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("query_constant") { - multi_sql """ - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set enable_local_shuffle=false; - set force_to_local_shuffle=false; - """ - - order_qt_query_one_row "select 100 id, 'abc' name" - - order_qt_union_all """ - select 100 id, 'hello' name - union all - select 200 id, 'world' name - """ - - order_qt_union """ - select 100 id, 'hello' name - union - select 200 id, 'world' name - """ -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/read_metadata.groovy b/regression-test/suites/nereids_syntax_p0/distribute/read_metadata.groovy deleted file mode 100644 index 48d8c20e1c213a..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/read_metadata.groovy +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("shuffle") { - - multi_sql """ - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set enable_local_shuffle=false; - set force_to_local_shuffle=false; - """ - - sql "select * from internal.information_schema.tables" -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy b/regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy deleted file mode 100644 index 42ec0df05459a8..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/shuffle.groovy +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("shuffle") { - createTestTable "test_shuffle" - - multi_sql """ - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set enable_local_shuffle=false; - set force_to_local_shuffle=false; - """ - - order_qt_4_phase_agg """ - select /*+SET_VAR(disable_nereids_rules='TWO_PHASE_AGGREGATE_WITH_MULTI_DISTINCT,TWO_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI,THREE_PHASE_AGGREGATE_WITH_COUNT_DISTINCT_MULTI,THREE_PHASE_AGGREGATE_WITH_DISTINCT,FOUR_PHASE_AGGREGATE_WITH_DISTINCT')*/ - id, count(distinct value) - from test_shuffle - group by id - """ -} diff --git a/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy b/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy deleted file mode 100644 index 8c56c257b0e2d9..00000000000000 --- a/regression-test/suites/nereids_syntax_p0/distribute/shuffle_left_join.groovy +++ /dev/null @@ -1,117 +0,0 @@ -import java.util.stream.Collectors - -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("shuffle_left_join") { - multi_sql """ - drop table if exists test_shuffle_left; - - CREATE TABLE `test_shuffle_left` ( - id int, - id2 int - ) ENGINE=OLAP - DUPLICATE KEY(`id`) - DISTRIBUTED BY HASH(`id`) BUCKETS 10 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1" - ); - - insert into test_shuffle_left values (1, 1), (2, 2), (3, 4); - - sync; - - set enable_nereids_distribute_planner=false; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - set enable_local_shuffle=false; - set force_to_local_shuffle=false; - """ - - def extractFragment = { String sqlStr, String containsString, Closure checkExchangeNum -> - explain { - sql sqlStr - check { result -> - log.info("Explain result:\n${result}") - - assertTrue(result.contains(containsString)) - - def fragmentContainsJoin = result.split("PLAN FRAGMENT") - .toList() - .stream() - .filter { it.contains(containsString) } - .findFirst() - .get() - - log.info("Fragment:\n${fragmentContainsJoin}") - - checkExchangeNum(fragmentContainsJoin.count("VEXCHANGE")) - } - } - } - - def sqlStr = """ - select * - from - ( - select id2 - from test_shuffle_left - group by id2 - ) a - inner join [shuffle] - test_shuffle_left b - on a.id2=b.id; - """ - - extractFragment(sqlStr, "INNER JOIN(PARTITIONED)") { exchangeNum -> - assertTrue(exchangeNum == 2) - } - - order_qt_shuffle_left_and_right sqlStr - - multi_sql """ - set enable_nereids_distribute_planner=true; - set enable_pipeline_x_engine=true; - set disable_join_reorder=true; - """ - - def variables = sql "show variables" - def variableString = variables.stream() - .map { it.toString() } - .collect(Collectors.joining("\n")) - logger.info("Variables:\n${variableString}") - - extractFragment(sqlStr, "INNER JOIN(BUCKET_SHUFFLE)") { exchangeNum -> - assertTrue(exchangeNum == 1) - } - - explain { - sql "plan $sqlStr" - check { explainStr -> - log.info("explain plan:\n${explainStr}") - } - } - - def rows = sql sqlStr - def rowsString = rows.stream() - .map { it.toString() } - .collect(Collectors.joining("\n")) - logger.info("Rows:\n${rowsString}") - - - order_qt_shuffle_left sqlStr -} diff --git a/regression-test/suites/new_shapes_p0/clickbench/query1.groovy b/regression-test/suites/new_shapes_p0/clickbench/query1.groovy index e1294c1e2563fd..51db86f351cd42 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query1.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query1.groovy @@ -22,7 +22,7 @@ suite("query1") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set topn_opt_limit_threshold = 1024' diff --git a/regression-test/suites/new_shapes_p0/clickbench/query10.groovy b/regression-test/suites/new_shapes_p0/clickbench/query10.groovy index 027b06ed7919f5..69590185ac5c82 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query10.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query10.groovy @@ -22,7 +22,7 @@ suite("query10") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query11.groovy b/regression-test/suites/new_shapes_p0/clickbench/query11.groovy index c87be65cb43346..35d24b09ee48ff 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query11.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query11.groovy @@ -23,7 +23,7 @@ suite("query11") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query12.groovy b/regression-test/suites/new_shapes_p0/clickbench/query12.groovy index e58d4d3ff0a4dd..233c27d62820a8 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query12.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query12.groovy @@ -23,7 +23,7 @@ suite("query12") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query13.groovy b/regression-test/suites/new_shapes_p0/clickbench/query13.groovy index e156e51bc4cc6d..a5c4d84b348f00 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query13.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query13.groovy @@ -23,7 +23,7 @@ suite("query13") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query14.groovy b/regression-test/suites/new_shapes_p0/clickbench/query14.groovy index 2bb1aa260fe958..8ae1bddc188dab 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query14.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query14.groovy @@ -23,7 +23,7 @@ suite("query14") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query15.groovy b/regression-test/suites/new_shapes_p0/clickbench/query15.groovy index f8becb72a625d4..93dd8b31176b40 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query15.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query15.groovy @@ -23,7 +23,7 @@ suite("query15") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query16.groovy b/regression-test/suites/new_shapes_p0/clickbench/query16.groovy index 4d742899e6b704..9abe0b2a766112 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query16.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query16.groovy @@ -23,7 +23,7 @@ suite("query16") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query17.groovy b/regression-test/suites/new_shapes_p0/clickbench/query17.groovy index 76307e1167b8ec..6487f25c602e98 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query17.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query17.groovy @@ -23,7 +23,7 @@ suite("query17") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query18.groovy b/regression-test/suites/new_shapes_p0/clickbench/query18.groovy index 8a9d8f19b619da..c2681db7f879a0 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query18.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query18.groovy @@ -23,7 +23,7 @@ suite("query18") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query19.groovy b/regression-test/suites/new_shapes_p0/clickbench/query19.groovy index da080ef696aab9..836f5dd5386b63 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query19.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query19.groovy @@ -23,7 +23,7 @@ suite("query19") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query2.groovy b/regression-test/suites/new_shapes_p0/clickbench/query2.groovy index 0deae902dbd810..7e310f7e8dc78a 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query2.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query2.groovy @@ -22,7 +22,7 @@ suite("query2") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set topn_opt_limit_threshold = 1024' diff --git a/regression-test/suites/new_shapes_p0/clickbench/query20.groovy b/regression-test/suites/new_shapes_p0/clickbench/query20.groovy index 2d7fc829508e6e..038b7d114de220 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query20.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query20.groovy @@ -23,7 +23,7 @@ suite("query20") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql "set enable_parallel_result_sink=false;" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query21.groovy b/regression-test/suites/new_shapes_p0/clickbench/query21.groovy index 870316ee6ea8c6..a7fe4cca7332df 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query21.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query21.groovy @@ -23,7 +23,7 @@ suite("query21") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query22.groovy b/regression-test/suites/new_shapes_p0/clickbench/query22.groovy index c4e27d56c2b817..94bdb4778ad2a0 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query22.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query22.groovy @@ -23,7 +23,7 @@ suite("query22") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query23.groovy b/regression-test/suites/new_shapes_p0/clickbench/query23.groovy index ff19e0ec4ea667..03732c40da9872 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query23.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query23.groovy @@ -23,7 +23,7 @@ suite("query23") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query24.groovy b/regression-test/suites/new_shapes_p0/clickbench/query24.groovy index 7ea248701a9b6b..7d304dd057690a 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query24.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query24.groovy @@ -23,7 +23,7 @@ suite("query24") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query25.groovy b/regression-test/suites/new_shapes_p0/clickbench/query25.groovy index d61c7475d137c5..9499c83ff887b4 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query25.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query25.groovy @@ -23,7 +23,7 @@ suite("query25") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query26.groovy b/regression-test/suites/new_shapes_p0/clickbench/query26.groovy index b5cf08c9482011..03bcec9b3e544c 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query26.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query26.groovy @@ -23,7 +23,7 @@ suite("query26") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query27.groovy b/regression-test/suites/new_shapes_p0/clickbench/query27.groovy index e72528e97db520..92298fa70631a6 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query27.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query27.groovy @@ -23,7 +23,7 @@ suite("query27") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query28.groovy b/regression-test/suites/new_shapes_p0/clickbench/query28.groovy index 08c2fa6b2505d5..03ade0817cd723 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query28.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query28.groovy @@ -23,7 +23,7 @@ suite("query28") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query29.groovy b/regression-test/suites/new_shapes_p0/clickbench/query29.groovy index e855ad7ff7bda3..2e3c0f5ecf07e3 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query29.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query29.groovy @@ -23,7 +23,7 @@ suite("query29") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query3.groovy b/regression-test/suites/new_shapes_p0/clickbench/query3.groovy index ac376f98ff2d2a..ba4fca2d0471d5 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query3.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query3.groovy @@ -22,7 +22,7 @@ suite("query3") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set topn_opt_limit_threshold = 1024' diff --git a/regression-test/suites/new_shapes_p0/clickbench/query30.groovy b/regression-test/suites/new_shapes_p0/clickbench/query30.groovy index 1a9b9f914d4147..eae6d2db039514 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query30.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query30.groovy @@ -23,7 +23,7 @@ suite("query30") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query31.groovy b/regression-test/suites/new_shapes_p0/clickbench/query31.groovy index 4e0dab037fb90e..6493dd08c073a3 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query31.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query31.groovy @@ -23,7 +23,7 @@ suite("query31") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query32.groovy b/regression-test/suites/new_shapes_p0/clickbench/query32.groovy index a4fc0f66d2d59c..8a0fdcd85686f3 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query32.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query32.groovy @@ -23,7 +23,7 @@ suite("query32") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query33.groovy b/regression-test/suites/new_shapes_p0/clickbench/query33.groovy index 31d89c03410dcb..e368c1364c918e 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query33.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query33.groovy @@ -23,7 +23,7 @@ suite("query33") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query34.groovy b/regression-test/suites/new_shapes_p0/clickbench/query34.groovy index 144dbd8579ef35..100794fadf1997 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query34.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query34.groovy @@ -23,7 +23,7 @@ suite("query34") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query35.groovy b/regression-test/suites/new_shapes_p0/clickbench/query35.groovy index 14b91739e89a5a..967104a9f8a7af 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query35.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query35.groovy @@ -23,7 +23,7 @@ suite("query35") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query36.groovy b/regression-test/suites/new_shapes_p0/clickbench/query36.groovy index 4067c45ff16b75..99e224ab0d5dc8 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query36.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query36.groovy @@ -23,7 +23,7 @@ suite("query36") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query37.groovy b/regression-test/suites/new_shapes_p0/clickbench/query37.groovy index c1b3882e96eb00..12d582c6780d6e 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query37.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query37.groovy @@ -23,7 +23,7 @@ suite("query37") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query38.groovy b/regression-test/suites/new_shapes_p0/clickbench/query38.groovy index da44d88d84595a..97d6a4fb54f58d 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query38.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query38.groovy @@ -23,7 +23,7 @@ suite("query38") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query39.groovy b/regression-test/suites/new_shapes_p0/clickbench/query39.groovy index 9aaf41f0303988..db267d3696e9f7 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query39.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query39.groovy @@ -23,7 +23,7 @@ suite("query39") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query4.groovy b/regression-test/suites/new_shapes_p0/clickbench/query4.groovy index 5f2727592daee5..985b0f29b3a7f1 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query4.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query4.groovy @@ -22,7 +22,7 @@ suite("query4") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set topn_opt_limit_threshold = 1024' diff --git a/regression-test/suites/new_shapes_p0/clickbench/query40.groovy b/regression-test/suites/new_shapes_p0/clickbench/query40.groovy index a256bd9d40dbea..06f52e94c93c7e 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query40.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query40.groovy @@ -23,7 +23,7 @@ suite("query40") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query41.groovy b/regression-test/suites/new_shapes_p0/clickbench/query41.groovy index e0617d1e5d6a32..f8ac07e8cb07a6 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query41.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query41.groovy @@ -23,7 +23,7 @@ suite("query41") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query42.groovy b/regression-test/suites/new_shapes_p0/clickbench/query42.groovy index cd53a2a7d4d030..f950310259f79d 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query42.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query42.groovy @@ -23,7 +23,7 @@ suite("query42") { } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query43.groovy b/regression-test/suites/new_shapes_p0/clickbench/query43.groovy index 8de968093369a9..8b71ca4510a503 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query43.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query43.groovy @@ -22,7 +22,7 @@ suite("query43") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query5.groovy b/regression-test/suites/new_shapes_p0/clickbench/query5.groovy index cf727f94101dee..928f1774ad1602 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query5.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query5.groovy @@ -22,7 +22,7 @@ suite("query5") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set topn_opt_limit_threshold = 1024' diff --git a/regression-test/suites/new_shapes_p0/clickbench/query6.groovy b/regression-test/suites/new_shapes_p0/clickbench/query6.groovy index 3ec9bbbe25b2d3..65e561ebe2766c 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query6.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query6.groovy @@ -22,7 +22,7 @@ suite("query6") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set topn_opt_limit_threshold = 1024' diff --git a/regression-test/suites/new_shapes_p0/clickbench/query7.groovy b/regression-test/suites/new_shapes_p0/clickbench/query7.groovy index 1c9d80a33709f8..249f78e8bee359 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query7.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query7.groovy @@ -22,7 +22,7 @@ suite("query7") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query8.groovy b/regression-test/suites/new_shapes_p0/clickbench/query8.groovy index 36c1322ff5a7ef..a86dfbd0336201 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query8.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query8.groovy @@ -22,7 +22,7 @@ suite("query8") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/clickbench/query9.groovy b/regression-test/suites/new_shapes_p0/clickbench/query9.groovy index 91a8184de49c97..506684fb78cd8a 100644 --- a/regression-test/suites/new_shapes_p0/clickbench/query9.groovy +++ b/regression-test/suites/new_shapes_p0/clickbench/query9.groovy @@ -22,7 +22,7 @@ suite("query9") { return } sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/ddl/shape.tmpl b/regression-test/suites/new_shapes_p0/hint_tpcds/ddl/shape.tmpl index c25fd3f36b03f4..ffe47491a10337 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/ddl/shape.tmpl +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/ddl/shape.tmpl @@ -24,7 +24,7 @@ suite("query{--}") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query1.groovy b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query1.groovy index d43c8dfb357846..fbca9893ed480c 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query1.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query1.groovy @@ -24,7 +24,7 @@ suite("query1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query24.groovy b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query24.groovy index 01b4beedbdf434..75a2cede8f1366 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query24.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query24.groovy @@ -24,7 +24,7 @@ suite("query24") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query64.groovy b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query64.groovy index 950e9416d2c47e..3131572f952c40 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query64.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query64.groovy @@ -24,7 +24,7 @@ suite("query64") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query67.groovy b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query67.groovy index ce258f814d8047..41f3616243ca2a 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query67.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query67.groovy @@ -24,7 +24,7 @@ suite("query67") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query72.groovy b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query72.groovy index 76efe58c0ad620..2d6a5747280c8e 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query72.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query72.groovy @@ -24,7 +24,7 @@ suite("query72") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query78.groovy b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query78.groovy index a96d26a95caf78..1c30657793e0dc 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query78.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpcds/shape/query78.groovy @@ -24,7 +24,7 @@ suite("query78") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q10.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q10.groovy index 9322e2485a0221..ed04ca74b3f3f5 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q10.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q10.groovy @@ -24,7 +24,7 @@ suite("q10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q11.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q11.groovy index 1581c654ecdc73..b00780474fdc25 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q11.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q11.groovy @@ -24,7 +24,7 @@ suite("q11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q12.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q12.groovy index c55966a2bfb546..f2a37bb2a8d524 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q12.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q12.groovy @@ -24,7 +24,7 @@ suite("q12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q13.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q13.groovy index 54a83a15265010..2da54efbd71c51 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q13.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q13.groovy @@ -24,7 +24,7 @@ suite("q13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q14.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q14.groovy index d1a05921a7ddd7..21871ba78c1721 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q14.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q14.groovy @@ -24,7 +24,7 @@ suite("q14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q15.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q15.groovy index 70bc802e3220a7..f4b55b35fd9bfd 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q15.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q15.groovy @@ -24,7 +24,7 @@ suite("q15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q17.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q17.groovy index aa595d59bce9ed..cfd08d12d4c949 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q17.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q17.groovy @@ -24,7 +24,7 @@ suite("q17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q19.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q19.groovy index ce166235d63322..68dfd245d6a94a 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q19.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q19.groovy @@ -25,7 +25,7 @@ suite("q19") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q3.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q3.groovy index 543193d069821d..a302bab46b2620 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q3.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q3.groovy @@ -25,7 +25,7 @@ suite("q3") { // db = "tpch" sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q4.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q4.groovy index fd004fe6981d4a..0aed8a4a71357c 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q4.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q4.groovy @@ -24,7 +24,7 @@ suite("q4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q5.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q5.groovy index 0e1527e34a44ad..a473e5c03615ed 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q5.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q5.groovy @@ -24,7 +24,7 @@ suite("q5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q7.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q7.groovy index 0f33f61c395502..1929f59f483b66 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q7.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q7.groovy @@ -24,7 +24,7 @@ suite("q7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q8.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q8.groovy index e70333dd1b051b..144a668d2295ca 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q8.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q8.groovy @@ -24,7 +24,7 @@ suite("q8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q9.groovy b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q9.groovy index d1fd50b47c56b8..bdb32c7065dca6 100644 --- a/regression-test/suites/new_shapes_p0/hint_tpch/shape/q9.groovy +++ b/regression-test/suites/new_shapes_p0/hint_tpch/shape/q9.groovy @@ -24,7 +24,7 @@ suite("q9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/flat.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/flat.groovy index 1ba75597477e43..d0d9beb169d654 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/flat.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/flat.groovy @@ -24,7 +24,7 @@ suite("q1.1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.1.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.1.groovy index 42b34b901e9c0e..3544b0fe311aec 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.1.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.1.groovy @@ -24,7 +24,7 @@ suite("q1.1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.2.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.2.groovy index 5ce77e665c070d..dacc313bc698a6 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.2.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.2.groovy @@ -24,7 +24,7 @@ suite("q1.2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.3.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.3.groovy index 61155a26f647af..eff9503af2c444 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.3.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q1.3.groovy @@ -24,7 +24,7 @@ suite("q1.3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.1.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.1.groovy index 8d3232ec783355..eb295ddd973841 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.1.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.1.groovy @@ -24,7 +24,7 @@ suite("q2.1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.2.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.2.groovy index dd614fecc779b5..cf34f685680c98 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.2.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.2.groovy @@ -24,7 +24,7 @@ suite("q2.2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.3.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.3.groovy index cf39d3d1047e8d..812a9ab75c1f27 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.3.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q2.3.groovy @@ -24,7 +24,7 @@ suite("q2.3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.1.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.1.groovy index a5a7eadb3735e2..ce4446ca9d8745 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.1.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.1.groovy @@ -24,7 +24,7 @@ suite("q3.1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.2.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.2.groovy index 4755927b522171..faa94b24ccf303 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.2.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.2.groovy @@ -24,7 +24,7 @@ suite("q3.2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.3.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.3.groovy index 7f349a8f0841fc..b8d6782b6328d0 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.3.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.3.groovy @@ -24,7 +24,7 @@ suite("q3.3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.4.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.4.groovy index ff34697a8521aa..51154c2d003a39 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.4.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q3.4.groovy @@ -24,7 +24,7 @@ suite("q3.4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.1.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.1.groovy index 91d4bf499a1c39..705f6e0c0140af 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.1.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.1.groovy @@ -24,7 +24,7 @@ suite("q4.1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.2.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.2.groovy index da8b425a810a4b..28e693b506e9b1 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.2.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.2.groovy @@ -24,7 +24,7 @@ suite("q4.2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.3.groovy b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.3.groovy index bb3f5de73837bd..5b65a7f46b9c88 100644 --- a/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.3.groovy +++ b/regression-test/suites/new_shapes_p0/ssb_sf100/shape/q4.3.groovy @@ -24,7 +24,7 @@ suite("q4.3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy index b17e9c6d3d0e7f..780c16b62cb8c5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/constraints/query23.groovy @@ -25,7 +25,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/rf_prune.tmpl b/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/rf_prune.tmpl index b29318db8e806a..a1a2635806bda1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/rf_prune.tmpl +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/rf_prune.tmpl @@ -24,7 +24,7 @@ suite("query{--}") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/shape.tmpl b/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/shape.tmpl index c74e55d30c8965..2e0fb06b1cc042 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/shape.tmpl +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/ddl/shape.tmpl @@ -24,7 +24,7 @@ suite("query{--}") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query1.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query1.groovy index ac9bfd0d8cb6b4..ad6362ddc6b49c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query1.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query1.groovy @@ -24,7 +24,7 @@ suite("query1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.groovy index 903d035d22e918..f50d48530809fd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query10.groovy @@ -24,7 +24,7 @@ suite("query10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.groovy index e3712ab6cee089..7a6fc6be94f1ac 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query11.groovy @@ -24,7 +24,7 @@ suite("query11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query12.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query12.groovy index 04255c01f57aa1..c45847b7006b38 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query12.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query12.groovy @@ -24,7 +24,7 @@ suite("query12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.groovy index f1d8338c174e11..ff94cc3a981661 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query13.groovy @@ -24,7 +24,7 @@ suite("query13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.groovy index d9f561d7fd313f..273ad3624e5aa3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query14.groovy @@ -24,7 +24,7 @@ suite("query14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.groovy index 7e5f7c5ff17bd2..b64ccc2bdd2378 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query15.groovy @@ -24,7 +24,7 @@ suite("query15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query16.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query16.groovy index 81886a513f9a0e..b3ff0c66cb8442 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query16.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query16.groovy @@ -24,7 +24,7 @@ suite("query16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query17.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query17.groovy index 7f53dfaa3a0c16..f3ea294c28b419 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query17.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query17.groovy @@ -24,7 +24,7 @@ suite("query17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query18.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query18.groovy index fec13a57cf6313..d85bb07df1fb14 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query18.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query18.groovy @@ -24,7 +24,7 @@ suite("query18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query19.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query19.groovy index beb66b47ff57f5..c2a5c9399e295f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query19.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query19.groovy @@ -24,7 +24,7 @@ suite("query19") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query2.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query2.groovy index 2fb61ba9fbd3da..1f2b3c1d577459 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query2.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query2.groovy @@ -24,7 +24,7 @@ suite("query2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query20.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query20.groovy index aead630449670f..4aec20af735fc2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query20.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query20.groovy @@ -24,7 +24,7 @@ suite("query20") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query21.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query21.groovy index 8d7a39c97354b0..01c54a31454395 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query21.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query21.groovy @@ -24,7 +24,7 @@ suite("query21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query22.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query22.groovy index c31efd8b73852d..11b2709eb8f8df 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query22.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query22.groovy @@ -24,7 +24,7 @@ suite("query22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.groovy index 91494ee79761b8..d7a2c2b1f9caac 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query23.groovy @@ -25,7 +25,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query24.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query24.groovy index bfbd00add04fb4..e33a809c06fad5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query24.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query24.groovy @@ -24,7 +24,7 @@ suite("query24") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query25.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query25.groovy index 0bb08f70c81758..06c80cfd586825 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query25.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query25.groovy @@ -24,7 +24,7 @@ suite("query25") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.groovy index 01f80b4596d062..a727e33015d135 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query26.groovy @@ -24,7 +24,7 @@ suite("query26") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query27.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query27.groovy index 407f2ab9ea9af7..6729825f644877 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query27.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query27.groovy @@ -24,7 +24,7 @@ suite("query27") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.groovy index aacdcc03a4a946..0c74c6973ba889 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query28.groovy @@ -24,7 +24,7 @@ suite("query28") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query29.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query29.groovy index 98f4c2ffd57a4d..b5f6eb6cef316f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query29.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query29.groovy @@ -24,7 +24,7 @@ suite("query29") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query3.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query3.groovy index d91dc9bdbc63f3..b9734bafa3a5ed 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query3.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query3.groovy @@ -24,7 +24,7 @@ suite("query3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.groovy index f3c4f793cb206e..e796be74ca67ee 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query30.groovy @@ -24,7 +24,7 @@ suite("query30") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query31.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query31.groovy index 35a6d672ddc2dd..a6c21ae749d754 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query31.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query31.groovy @@ -24,7 +24,7 @@ suite("query31") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.groovy index 0e712c8804d29c..087dd39fe51771 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query32.groovy @@ -25,7 +25,7 @@ suite("query32") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query33.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query33.groovy index 61fe0c580384d0..b88809a55d2fea 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query33.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query33.groovy @@ -24,7 +24,7 @@ suite("query33") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.groovy index 7b0a9cb22fffbc..e6c0f37819aa0c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query34.groovy @@ -24,7 +24,7 @@ suite("query34") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.groovy index 715fb701b86a60..56af4d00349358 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query35.groovy @@ -24,7 +24,7 @@ suite("query35") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query36.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query36.groovy index b67ecc699fec84..0d15d91e6b4710 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query36.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query36.groovy @@ -24,7 +24,7 @@ suite("query36") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.groovy index ee3b6b8ebf2224..749cc9dd4c46b0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query37.groovy @@ -24,7 +24,7 @@ suite("query37") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.groovy index 2e1a2d1a5a12fa..3c5dc7fa100928 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query38.groovy @@ -25,7 +25,7 @@ suite("query38") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query39.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query39.groovy index f041f52ac11295..915ed85252cd2d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query39.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query39.groovy @@ -24,7 +24,7 @@ suite("query39") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.groovy index 1af702e366764a..1799bfcd312704 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query4.groovy @@ -24,7 +24,7 @@ suite("query4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query40.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query40.groovy index b943f2471b568b..d4f06aaf925fa8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query40.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query40.groovy @@ -24,7 +24,7 @@ suite("query40") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.groovy index c14fdbb8a94abb..899534a510c93d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query41.groovy @@ -24,7 +24,7 @@ suite("query41") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query42.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query42.groovy index e533209b57668f..ec70d134a7314b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query42.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query42.groovy @@ -24,7 +24,7 @@ suite("query42") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query43.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query43.groovy index cc53ae91ec0368..c40260d0c8f24c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query43.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query43.groovy @@ -24,7 +24,7 @@ suite("query43") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query44.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query44.groovy index cc655c04360072..f5769ab66cb896 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query44.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query44.groovy @@ -24,7 +24,7 @@ suite("query44") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.groovy index f60e79ad7c85ff..b0144c807757c7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query45.groovy @@ -24,7 +24,7 @@ suite("query45") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.groovy index 8a0afcea593227..f3c2a0904d40ad 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query46.groovy @@ -24,7 +24,7 @@ suite("query46") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.groovy index 4845dfd8a32c6f..f1c934bf648a80 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query47.groovy @@ -24,7 +24,7 @@ suite("query47") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.groovy index 9f6f638c5fd373..a76c0970faab79 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query48.groovy @@ -24,7 +24,7 @@ suite("query48") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.groovy index b13177064ac384..5255d1e7e42b3f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query49.groovy @@ -24,7 +24,7 @@ suite("query49") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query5.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query5.groovy index 222b4ae79923b6..ddc14fd886362c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query5.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query5.groovy @@ -24,7 +24,7 @@ suite("query5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query50.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query50.groovy index e97fd67f20d883..67e634bc4e1e7f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query50.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query50.groovy @@ -24,7 +24,7 @@ suite("query50") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.groovy index e80a42a2a9e8c4..2c0396c55949dc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.groovy @@ -24,7 +24,7 @@ suite("query51") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query52.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query52.groovy index b827f1c2c0dc39..78b0b8ee6de88c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query52.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query52.groovy @@ -24,7 +24,7 @@ suite("query52") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.groovy index fb8fe661c8b401..a1b0b161cd8b4b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query53.groovy @@ -24,7 +24,7 @@ suite("query53") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query54.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query54.groovy index a919601deab68b..e2d15bd0dc6795 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query54.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query54.groovy @@ -24,7 +24,7 @@ suite("query54") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query55.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query55.groovy index a1760bd16d2a23..6fcceb2ccb22ab 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query55.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query55.groovy @@ -24,7 +24,7 @@ suite("query55") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query56.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query56.groovy index 6e611882852e61..5f2b06ca9e4ae2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query56.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query56.groovy @@ -24,7 +24,7 @@ suite("query56") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.groovy index 5be0c3041c2082..b9e143922d8cf8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query57.groovy @@ -24,7 +24,7 @@ suite("query57") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query58.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query58.groovy index a34ff64eee162c..22dc53f9739410 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query58.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query58.groovy @@ -24,7 +24,7 @@ suite("query58") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query59.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query59.groovy index be96d9b5ffe71c..4ab86289f67bd3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query59.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query59.groovy @@ -24,7 +24,7 @@ suite("query59") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.groovy index bbd815ee463cb0..544956a7859b99 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query6.groovy @@ -24,7 +24,7 @@ suite("query6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query60.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query60.groovy index 6c60d4a4b7ca22..dfc01f14e199b6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query60.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query60.groovy @@ -24,7 +24,7 @@ suite("query60") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.groovy index 11dc3db78d1f35..5fe0b026dc2e9c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query61.groovy @@ -24,7 +24,7 @@ suite("query61") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query62.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query62.groovy index 6eac332278c1b4..a073dfc97d6a78 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query62.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query62.groovy @@ -24,7 +24,7 @@ suite("query62") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.groovy index 600981b9cebedc..6eb7e26eb8f0a7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query63.groovy @@ -24,7 +24,7 @@ suite("query63") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.groovy index 9c06fec135a9a9..d789dad3c3f5c2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query64.groovy @@ -24,7 +24,7 @@ suite("query64") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query65.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query65.groovy index bec1515c2ee85e..ee126b83f877f9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query65.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query65.groovy @@ -24,7 +24,7 @@ suite("query65") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query66.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query66.groovy index 83804dfb545594..fae748c8273ee9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query66.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query66.groovy @@ -24,7 +24,7 @@ suite("query66") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query67.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query67.groovy index e6a5a63192f5a8..5a1d9076aa3c1e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query67.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query67.groovy @@ -24,7 +24,7 @@ suite("query67") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.groovy index e9dd6bb74416c0..181138d4d2e64a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query68.groovy @@ -24,7 +24,7 @@ suite("query68") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query69.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query69.groovy index b80eec4fc048d6..8684da34bd7e01 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query69.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query69.groovy @@ -24,7 +24,7 @@ suite("query69") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.groovy index 547f1c0ac3afeb..836cf86985417f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query7.groovy @@ -24,7 +24,7 @@ suite("query7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query70.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query70.groovy index ec131a31f78102..af4399eca98db4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query70.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query70.groovy @@ -24,7 +24,7 @@ suite("query70") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query71.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query71.groovy index 34836552508d38..4ab6f3f56947ff 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query71.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query71.groovy @@ -24,7 +24,7 @@ suite("query71") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query72.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query72.groovy index 30d399176fa3a4..406e2c20220850 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query72.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query72.groovy @@ -24,7 +24,7 @@ suite("query72") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query73.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query73.groovy index 906dce3697abad..c710757e5a9b8e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query73.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query73.groovy @@ -24,7 +24,7 @@ suite("query73") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query74.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query74.groovy index cd7c8028b6d47b..6e6d40c1110254 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query74.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query74.groovy @@ -24,7 +24,7 @@ suite("query74") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query75.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query75.groovy index 1845cf2c2acd21..641e9f7ebdd258 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query75.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query75.groovy @@ -24,7 +24,7 @@ suite("query75") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query76.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query76.groovy index cf5fa8f8c232e5..e9f2bc9ae11281 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query76.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query76.groovy @@ -24,7 +24,7 @@ suite("query76") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query77.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query77.groovy index fb196585a8b692..52584ace286143 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query77.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query77.groovy @@ -24,7 +24,7 @@ suite("query77") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.groovy index 70587543e584fd..6fa8fce4fd3cef 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query78.groovy @@ -24,7 +24,7 @@ suite("query78") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.groovy index c5eeb99d6d7603..2609cda906a6f0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query79.groovy @@ -24,7 +24,7 @@ suite("query79") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query8.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query8.groovy index b01eab837b63ac..b01d281b04870a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query8.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query8.groovy @@ -24,7 +24,7 @@ suite("query8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query80.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query80.groovy index c3973871b98c1d..9e4850d5172084 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query80.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query80.groovy @@ -24,7 +24,7 @@ suite("query80") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.groovy index b86d7bb4dc575d..10ed17e9f043cd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query81.groovy @@ -24,7 +24,7 @@ suite("query81") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.groovy index c6ea139fcb164b..7b6e24b764002c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query82.groovy @@ -24,7 +24,7 @@ suite("query82") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query83.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query83.groovy index c363bd4037969c..3c4faaa0609fee 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query83.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query83.groovy @@ -24,7 +24,7 @@ suite("query83") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query84.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query84.groovy index a030a4c1ae132f..6563ddae9d0bca 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query84.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query84.groovy @@ -24,7 +24,7 @@ suite("query84") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.groovy index c8931084b4a0e1..d34b6b67f45ac4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query85.groovy @@ -24,7 +24,7 @@ suite("query85") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query86.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query86.groovy index 669e3fae1d357d..ef43ae335a9d01 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query86.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query86.groovy @@ -24,7 +24,7 @@ suite("query86") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query87.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query87.groovy index baf43511edeed0..8423a7204a8f7b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query87.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query87.groovy @@ -24,7 +24,7 @@ suite("query87") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.groovy index cc1dc227786134..bcb9664e5822c6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query88.groovy @@ -24,7 +24,7 @@ suite("query88") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.groovy index 4ca9b8f69f923d..300323c20a49bb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query89.groovy @@ -24,7 +24,7 @@ suite("query89") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query9.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query9.groovy index 72321f8a522999..e95875a1e9b656 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query9.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query9.groovy @@ -24,7 +24,7 @@ suite("query9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query90.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query90.groovy index 25e457398299db..0c61499277e493 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query90.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query90.groovy @@ -24,7 +24,7 @@ suite("query90") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.groovy index 655387b20a885c..b349517f28f9bb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query91.groovy @@ -24,7 +24,7 @@ suite("query91") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query92.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query92.groovy index 1e6bc33766f29f..a5bd427ede3b2e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query92.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query92.groovy @@ -24,7 +24,7 @@ suite("query92") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query93.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query93.groovy index 067576f5ac8318..5397ea9e973f20 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query93.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query93.groovy @@ -24,7 +24,7 @@ suite("query93") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query94.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query94.groovy index 5832e550cdcdc8..a0eb5d4d588d7d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query94.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query94.groovy @@ -24,7 +24,7 @@ suite("query94") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query95.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query95.groovy index 10a0394d19d56b..55425ece72a9aa 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query95.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query95.groovy @@ -24,7 +24,7 @@ suite("query95") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query96.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query96.groovy index ee419513883b8b..09f945721e01d1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query96.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query96.groovy @@ -24,7 +24,7 @@ suite("query96") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.groovy index 253ea22b361497..e9d6e7c6146aac 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query97.groovy @@ -25,7 +25,7 @@ suite("query97") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query98.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query98.groovy index f48270d50ad8d6..88b0525d73afef 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query98.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query98.groovy @@ -24,7 +24,7 @@ suite("query98") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query99.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query99.groovy index a2bb765ae6bd3b..e04edeedc45b37 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query99.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query99.groovy @@ -24,7 +24,7 @@ suite("query99") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query1.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query1.groovy index 6fe68f53233903..4ceae6e7f42d12 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query1.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query1.groovy @@ -24,7 +24,7 @@ suite("query1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.groovy index fc895a2048160c..d3d257b9ea4702 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query10.groovy @@ -24,7 +24,7 @@ suite("query10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.groovy index e63b9d8f685006..6068724f03f898 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query11.groovy @@ -24,7 +24,7 @@ suite("query11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query12.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query12.groovy index 47ac46798f9264..ae97a7d53c4a02 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query12.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query12.groovy @@ -24,7 +24,7 @@ suite("query12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.groovy index 9aa9d301878f38..1d309166c0e594 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query13.groovy @@ -24,7 +24,7 @@ suite("query13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.groovy index a491062d05363e..ad58cdabf0946b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query14.groovy @@ -24,7 +24,7 @@ suite("query14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.groovy index c42ff735202802..992534f4bd1bad 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query15.groovy @@ -24,7 +24,7 @@ suite("query15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query16.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query16.groovy index 6516e0d007dc92..85fe68bee81774 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query16.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query16.groovy @@ -24,7 +24,7 @@ suite("query16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query17.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query17.groovy index 367fee559d579b..03de884f49a342 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query17.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query17.groovy @@ -24,7 +24,7 @@ suite("query17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query18.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query18.groovy index beb7e30d525844..676068731a9b22 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query18.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query18.groovy @@ -24,7 +24,7 @@ suite("query18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query19.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query19.groovy index b845f9d6a08212..39c85ed4583b15 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query19.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query19.groovy @@ -24,7 +24,7 @@ suite("query19") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query2.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query2.groovy index 89c63655fc5c3e..561f0625eb9883 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query2.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query2.groovy @@ -24,7 +24,7 @@ suite("query2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query20.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query20.groovy index 2262f12f7e2a77..750d69a7b39cc8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query20.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query20.groovy @@ -24,7 +24,7 @@ suite("query20") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query21.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query21.groovy index 4d9d0f7b8f5639..9a8bf4270f387e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query21.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query21.groovy @@ -24,7 +24,7 @@ suite("query21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query22.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query22.groovy index 0feaf26850b36f..33f1e7e483a97e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query22.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query22.groovy @@ -24,7 +24,7 @@ suite("query22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.groovy index 3d8aee1907c112..8c6cfe68684e50 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query23.groovy @@ -25,7 +25,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query24.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query24.groovy index 8ce1a2c29f8b7f..bbec6ac8fe9645 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query24.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query24.groovy @@ -24,7 +24,7 @@ suite("query24") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query25.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query25.groovy index ca4fea0419c96e..1a6fcd34a67534 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query25.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query25.groovy @@ -24,7 +24,7 @@ suite("query25") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.groovy index a5e0a760a24706..c93d191b2137a4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query26.groovy @@ -24,7 +24,7 @@ suite("query26") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query27.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query27.groovy index e0888d00f3785a..92bfa6557cf8d7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query27.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query27.groovy @@ -24,7 +24,7 @@ suite("query27") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.groovy index c2cd90ef2d1dc6..e1a660e437748b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query28.groovy @@ -24,7 +24,7 @@ suite("query28") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query29.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query29.groovy index 238f0e434d9772..92b80b97b5fd56 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query29.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query29.groovy @@ -24,7 +24,7 @@ suite("query29") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query3.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query3.groovy index de3b2bbbece903..5674abe3712dcb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query3.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query3.groovy @@ -24,7 +24,7 @@ suite("query3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.groovy index a39c25ea9bd016..f856cb44d9c9bd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query30.groovy @@ -24,7 +24,7 @@ suite("query30") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query31.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query31.groovy index 12fb0f766c7385..b53ee705921e25 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query31.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query31.groovy @@ -24,7 +24,7 @@ suite("query31") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.groovy index 1132ffd6bea511..890fc083cfd62f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query32.groovy @@ -25,7 +25,7 @@ suite("query32") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query33.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query33.groovy index 78587586a51685..858e04d231ed42 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query33.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query33.groovy @@ -24,7 +24,7 @@ suite("query33") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.groovy index 379b29ccdf7c59..e0d3c0070f3b8b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query34.groovy @@ -24,7 +24,7 @@ suite("query34") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.groovy index 755ea40079011e..272da51d8558d7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query35.groovy @@ -24,7 +24,7 @@ suite("query35") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query36.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query36.groovy index def05c702aba48..db43de88525686 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query36.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query36.groovy @@ -24,7 +24,7 @@ suite("query36") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.groovy index 82ce00de92bd42..754b6b572a1204 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query37.groovy @@ -24,7 +24,7 @@ suite("query37") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.groovy index d2175cb65cc79a..8a3fe5efac82a1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query38.groovy @@ -25,7 +25,7 @@ suite("query38") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query39.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query39.groovy index 9dfcaf127d58d0..ed78f237be81d3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query39.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query39.groovy @@ -24,7 +24,7 @@ suite("query39") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.groovy index 0470f27267ebc9..a98f41ae5ca4f1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query4.groovy @@ -24,7 +24,7 @@ suite("query4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query40.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query40.groovy index 55fdce027dc4a7..88fe8e53affb76 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query40.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query40.groovy @@ -24,7 +24,7 @@ suite("query40") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.groovy index 3604cfa49f66ac..5c3b52ea042e47 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query41.groovy @@ -24,7 +24,7 @@ suite("query41") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query42.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query42.groovy index 1b2aad016d79eb..6bf9621f4b97bd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query42.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query42.groovy @@ -24,7 +24,7 @@ suite("query42") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query43.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query43.groovy index f75ac2dae53e27..51249fb35bda18 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query43.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query43.groovy @@ -24,7 +24,7 @@ suite("query43") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query44.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query44.groovy index bccd8304610749..e5d5ff8d53e859 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query44.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query44.groovy @@ -24,7 +24,7 @@ suite("query44") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.groovy index 54bdeaa5cd2e5e..3568daa5784c31 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query45.groovy @@ -24,7 +24,7 @@ suite("query45") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.groovy index 05edaa7a4dcb79..41ccd4ce87e4e9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query46.groovy @@ -24,7 +24,7 @@ suite("query46") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.groovy index 6bf7243f0e2cea..ee90d2651fcf3b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query47.groovy @@ -24,7 +24,7 @@ suite("query47") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.groovy index 0ccf5809bc6d42..739dfa35695c77 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query48.groovy @@ -24,7 +24,7 @@ suite("query48") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.groovy index 4a9ab3f4a2ba4f..05410bb8b40945 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query49.groovy @@ -24,7 +24,7 @@ suite("query49") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query5.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query5.groovy index 7f2e92f5fe7d8f..4711ed3ac0c867 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query5.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query5.groovy @@ -24,7 +24,7 @@ suite("query5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query50.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query50.groovy index 1cc8aa811d2dc0..a2d7feb1466de9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query50.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query50.groovy @@ -24,7 +24,7 @@ suite("query50") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.groovy index 3fb187cdc3ec0e..0a9185d5d3a44d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.groovy @@ -24,7 +24,7 @@ suite("query51") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query52.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query52.groovy index 00f0d1cdce76b9..afd98314db8e24 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query52.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query52.groovy @@ -24,7 +24,7 @@ suite("query52") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.groovy index 0bbb9f8a7aa7aa..8c1d97f570516d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query53.groovy @@ -24,7 +24,7 @@ suite("query53") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query54.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query54.groovy index cf2a0806fb7a76..a39c21b0be2c74 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query54.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query54.groovy @@ -24,7 +24,7 @@ suite("query54") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query55.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query55.groovy index 440cc0a53a375b..9905875d8499ca 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query55.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query55.groovy @@ -24,7 +24,7 @@ suite("query55") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query56.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query56.groovy index bd8405821ee155..cc1c480b7bf3f7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query56.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query56.groovy @@ -24,7 +24,7 @@ suite("query56") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.groovy index 4e7d37d147ebed..837fb564e0266a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query57.groovy @@ -24,7 +24,7 @@ suite("query57") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query58.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query58.groovy index 29451458a6f266..adb51074084439 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query58.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query58.groovy @@ -24,7 +24,7 @@ suite("query58") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query59.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query59.groovy index c8b985a5290cdf..f39499bb8bc7c5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query59.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query59.groovy @@ -24,7 +24,7 @@ suite("query59") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.groovy index 8455347f43c105..96c3b6e43c515e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query6.groovy @@ -24,7 +24,7 @@ suite("query6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query60.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query60.groovy index 02e6dedaa3c90f..88deb42f487758 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query60.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query60.groovy @@ -24,7 +24,7 @@ suite("query60") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.groovy index 035daafadec3da..ea293daf2686bf 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query61.groovy @@ -24,7 +24,7 @@ suite("query61") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query62.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query62.groovy index 820da22663fc33..0a2830fe2a5977 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query62.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query62.groovy @@ -24,7 +24,7 @@ suite("query62") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.groovy index 5873e1c2753d3d..24efcd18055531 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query63.groovy @@ -24,7 +24,7 @@ suite("query63") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.groovy index b4566e90f9575c..83e01f83e55439 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query64.groovy @@ -24,7 +24,7 @@ suite("query64") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query65.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query65.groovy index 0fa05938bcf3d7..fb25732f5f27ff 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query65.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query65.groovy @@ -24,7 +24,7 @@ suite("query65") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query66.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query66.groovy index 407bb39f6117ff..b20e2ee5a56ea6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query66.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query66.groovy @@ -24,7 +24,7 @@ suite("query66") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query67.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query67.groovy index d337474611f5cb..31d4a0dd9f7957 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query67.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query67.groovy @@ -24,7 +24,7 @@ suite("query67") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.groovy index b234cbee914806..70f5522be8e3de 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query68.groovy @@ -24,7 +24,7 @@ suite("query68") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query69.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query69.groovy index f1b09b13229f3b..5074f63828fcfe 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query69.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query69.groovy @@ -24,7 +24,7 @@ suite("query69") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.groovy index 3e2f47b9a374d1..264f11ef1e2323 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query7.groovy @@ -24,7 +24,7 @@ suite("query7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query70.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query70.groovy index 5e6196cec5ec14..1cf5e69748fc57 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query70.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query70.groovy @@ -24,7 +24,7 @@ suite("query70") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query71.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query71.groovy index dedf696d632767..217b670896442d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query71.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query71.groovy @@ -24,7 +24,7 @@ suite("query71") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query72.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query72.groovy index 8029b93fe637cd..ded25606d24d41 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query72.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query72.groovy @@ -24,7 +24,7 @@ suite("query72") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query73.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query73.groovy index 4dba313ee28c5d..0c66890cfae40c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query73.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query73.groovy @@ -24,7 +24,7 @@ suite("query73") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query74.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query74.groovy index eebd2286ded0a8..6d7d5cf6f5856a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query74.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query74.groovy @@ -24,7 +24,7 @@ suite("query74") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query75.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query75.groovy index f1556bbc51f91f..078c860f23829f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query75.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query75.groovy @@ -24,7 +24,7 @@ suite("query75") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query76.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query76.groovy index 9410eb8c8341c2..ff59837463e0c8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query76.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query76.groovy @@ -24,7 +24,7 @@ suite("query76") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query77.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query77.groovy index 54b615fd67f852..7c5ecefa2da501 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query77.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query77.groovy @@ -24,7 +24,7 @@ suite("query77") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.groovy index 6ad5e1a8be9e44..f28c908d1e90fa 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query78.groovy @@ -24,7 +24,7 @@ suite("query78") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.groovy index fb04d65650dd31..7dc7c5314e9a0a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query79.groovy @@ -24,7 +24,7 @@ suite("query79") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query8.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query8.groovy index 821ffb2cf236a8..4d75553573ca6a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query8.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query8.groovy @@ -24,7 +24,7 @@ suite("query8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query80.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query80.groovy index 8ab44b3653b5bf..abcdc83d06ae8a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query80.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query80.groovy @@ -24,7 +24,7 @@ suite("query80") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.groovy index 5a2fa3ae7b547d..a5624b04f52042 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query81.groovy @@ -24,7 +24,7 @@ suite("query81") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.groovy index 39d87006a1d80f..b2ace1d573fcb3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query82.groovy @@ -24,7 +24,7 @@ suite("query82") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query83.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query83.groovy index 119fe440d9a756..1c38236e6d16d9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query83.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query83.groovy @@ -24,7 +24,7 @@ suite("query83") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query84.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query84.groovy index fb0804cececcaa..7042ea72566aa2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query84.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query84.groovy @@ -24,7 +24,7 @@ suite("query84") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.groovy index 517301a858d109..ec609def91b65a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query85.groovy @@ -24,7 +24,7 @@ suite("query85") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query86.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query86.groovy index c2594db8e9e045..3d52f646dbecb0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query86.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query86.groovy @@ -24,7 +24,7 @@ suite("query86") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query87.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query87.groovy index 51b50c928aa507..7eac982452f721 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query87.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query87.groovy @@ -24,7 +24,7 @@ suite("query87") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.groovy index 3e23dc65efdc5a..bbe85f32ccd8ed 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query88.groovy @@ -24,7 +24,7 @@ suite("query88") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.groovy index 070a9145a60f00..72115de19a8417 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query89.groovy @@ -24,7 +24,7 @@ suite("query89") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query9.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query9.groovy index 8afa6c0b913a4c..a13ce3af104203 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query9.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query9.groovy @@ -24,7 +24,7 @@ suite("query9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query90.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query90.groovy index 359090a313e9cd..95a96aeb1a3e05 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query90.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query90.groovy @@ -24,7 +24,7 @@ suite("query90") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.groovy index a8d4b4895bf616..67f013d1148439 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query91.groovy @@ -24,7 +24,7 @@ suite("query91") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query92.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query92.groovy index b314dfb25374bf..4f2de708c32d95 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query92.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query92.groovy @@ -24,7 +24,7 @@ suite("query92") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query93.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query93.groovy index d504874817236c..9e98a1a9b6df05 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query93.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query93.groovy @@ -24,7 +24,7 @@ suite("query93") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query94.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query94.groovy index dd66d44f3efb67..9622e82a078895 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query94.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query94.groovy @@ -24,7 +24,7 @@ suite("query94") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query95.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query95.groovy index 9472af9d687cdc..6ceb72a912c22a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query95.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query95.groovy @@ -24,7 +24,7 @@ suite("query95") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query96.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query96.groovy index 9973173945983f..e0f14d594e9d8a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query96.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query96.groovy @@ -24,7 +24,7 @@ suite("query96") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.groovy index 7038a8179b8f87..bdf1916bab6e06 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query97.groovy @@ -25,7 +25,7 @@ suite("query97") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query98.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query98.groovy index cb803a7032b4d8..cf33f1dd5698dd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query98.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query98.groovy @@ -24,7 +24,7 @@ suite("query98") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query99.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query99.groovy index 0806ca2a7bdecd..4e7660bd9c6b9b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query99.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/no_stats_shape/query99.groovy @@ -24,7 +24,7 @@ suite("query99") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query1.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query1.groovy index 7b93b538c6a947..64467b33b06d79 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query1.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query1.groovy @@ -24,7 +24,7 @@ suite("query1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query10.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query10.groovy index 8ebc3d34f8c92b..6c420c7bac7d52 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query10.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query10.groovy @@ -24,7 +24,7 @@ suite("query10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query11.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query11.groovy index 1eba17bdfa959d..b03bddc6d35a04 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query11.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query11.groovy @@ -24,7 +24,7 @@ suite("query11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query12.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query12.groovy index 3de0eddfeacefb..5e480a1d2a2177 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query12.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query12.groovy @@ -24,7 +24,7 @@ suite("query12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query13.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query13.groovy index 4bf64c23d264ea..da4bad001956fc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query13.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query13.groovy @@ -24,7 +24,7 @@ suite("query13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query14.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query14.groovy index 29d8d419dd1536..ff8158ec66e331 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query14.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query14.groovy @@ -24,7 +24,7 @@ suite("query14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query15.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query15.groovy index d2d371da6c95c3..677c6e3d0fc0fb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query15.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query15.groovy @@ -24,7 +24,7 @@ suite("query15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query16.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query16.groovy index ba5e1073fccd59..07cbde4da88ce8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query16.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query16.groovy @@ -24,7 +24,7 @@ suite("query16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query17.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query17.groovy index 3d6fe36e5b0f89..5a85070fc95d2d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query17.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query17.groovy @@ -24,7 +24,7 @@ suite("query17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query18.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query18.groovy index ee8f64d80ea204..4b5f9c505b23c0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query18.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query18.groovy @@ -24,7 +24,7 @@ suite("query18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query19.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query19.groovy index 8461069c183e93..f21330afd7e94a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query19.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query19.groovy @@ -24,7 +24,7 @@ suite("query19") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query2.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query2.groovy index 61574ba16f58fa..82f7a23330e9cc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query2.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query2.groovy @@ -24,7 +24,7 @@ suite("query2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query20.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query20.groovy index 321e1d913dc48f..fc359c96bf646f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query20.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query20.groovy @@ -24,7 +24,7 @@ suite("query20") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query21.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query21.groovy index feff18960306b0..3d79919edb3f39 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query21.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query21.groovy @@ -24,7 +24,7 @@ suite("query21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query22.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query22.groovy index bfb87e1402b41d..a00b779e65824f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query22.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query22.groovy @@ -24,7 +24,7 @@ suite("query22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query23.groovy index c97e2fb303e1f2..56082b128c45ed 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query23.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query23.groovy @@ -25,7 +25,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query24.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query24.groovy index cfc582e615208f..457afcc6829ece 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query24.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query24.groovy @@ -24,7 +24,7 @@ suite("query24") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query25.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query25.groovy index 1227c775eeb559..48c32ce1040f30 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query25.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query25.groovy @@ -24,7 +24,7 @@ suite("query25") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query26.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query26.groovy index e80f67d935c302..a76f20dd157ff7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query26.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query26.groovy @@ -24,7 +24,7 @@ suite("query26") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query27.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query27.groovy index ac99f5918c6b3f..54e16f63420796 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query27.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query27.groovy @@ -24,7 +24,7 @@ suite("query27") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query28.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query28.groovy index 6d72e2f2e6142e..c1f182ff3159b3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query28.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query28.groovy @@ -24,7 +24,7 @@ suite("query28") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query29.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query29.groovy index 76d0424a831ed2..dd4d880e1b89d3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query29.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query29.groovy @@ -24,7 +24,7 @@ suite("query29") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query3.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query3.groovy index fdaa33c8b8f68b..30e2dc7d9ef6d4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query3.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query3.groovy @@ -24,7 +24,7 @@ suite("query3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query30.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query30.groovy index a5a28b942da603..b14479ffc54d0b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query30.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query30.groovy @@ -24,7 +24,7 @@ suite("query30") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query31.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query31.groovy index a9e8e54f2e03cb..f97e74740b8f91 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query31.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query31.groovy @@ -24,7 +24,7 @@ suite("query31") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query32.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query32.groovy index ff511671303007..d122dc1df3b01c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query32.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query32.groovy @@ -25,7 +25,7 @@ suite("query32") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query33.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query33.groovy index 139221711c9092..47826999b184c7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query33.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query33.groovy @@ -24,7 +24,7 @@ suite("query33") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query34.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query34.groovy index 2dfb1a3fab0432..3df13fde1af615 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query34.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query34.groovy @@ -24,7 +24,7 @@ suite("query34") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query35.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query35.groovy index 7c579674b5bdc2..c40ef46e1aa254 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query35.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query35.groovy @@ -24,7 +24,7 @@ suite("query35") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query36.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query36.groovy index b18d481ad47615..679016f72ebeb9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query36.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query36.groovy @@ -24,7 +24,7 @@ suite("query36") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query37.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query37.groovy index af1b5737791539..0cc173917f3023 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query37.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query37.groovy @@ -24,7 +24,7 @@ suite("query37") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query38.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query38.groovy index 861df1065d10a2..3e2a7773509f6d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query38.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query38.groovy @@ -25,7 +25,7 @@ suite("query38") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query39.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query39.groovy index dde7b5e771220b..fe56912b97c2f7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query39.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query39.groovy @@ -24,7 +24,7 @@ suite("query39") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query4.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query4.groovy index 950cc47c083427..4f98c9a5f7d471 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query4.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query4.groovy @@ -24,7 +24,7 @@ suite("query4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query40.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query40.groovy index ee2016698d4fc6..f8f3a0e3c12fc8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query40.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query40.groovy @@ -24,7 +24,7 @@ suite("query40") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query41.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query41.groovy index 9e042b7b73e7ad..fc4eb7bb5ab12b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query41.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query41.groovy @@ -24,7 +24,7 @@ suite("query41") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query42.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query42.groovy index a801f21595be87..1e9065587ea160 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query42.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query42.groovy @@ -24,7 +24,7 @@ suite("query42") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query43.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query43.groovy index 65eb573d9f663f..000c5dbacfc644 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query43.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query43.groovy @@ -24,7 +24,7 @@ suite("query43") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query44.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query44.groovy index afa0b2691d9fe0..55fb2eb596c70f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query44.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query44.groovy @@ -24,7 +24,7 @@ suite("query44") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query45.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query45.groovy index 0655abe5962517..d5ac2d4303c2cd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query45.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query45.groovy @@ -24,7 +24,7 @@ suite("query45") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query46.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query46.groovy index 56432e72023569..c3abfeab74f905 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query46.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query46.groovy @@ -24,7 +24,7 @@ suite("query46") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query47.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query47.groovy index 3cef5880701fc7..76b328568b92ee 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query47.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query47.groovy @@ -24,7 +24,7 @@ suite("query47") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query48.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query48.groovy index 788a32a3d9af4f..87f643413ba4d3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query48.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query48.groovy @@ -24,7 +24,7 @@ suite("query48") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query49.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query49.groovy index 08c2e111116002..773ff9bc58cf7e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query49.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query49.groovy @@ -24,7 +24,7 @@ suite("query49") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query5.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query5.groovy index d2bb54dfc468e3..ad164cd365c99f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query5.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query5.groovy @@ -24,7 +24,7 @@ suite("query5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query50.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query50.groovy index b13bfe4e95837e..aed58bc6763af1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query50.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query50.groovy @@ -24,7 +24,7 @@ suite("query50") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query51.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query51.groovy index 3db2f3a5abb815..f8ece9df2cfbb0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query51.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query51.groovy @@ -24,7 +24,7 @@ suite("query51") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query52.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query52.groovy index 87393c0825fb46..4cfe6a471eb62d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query52.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query52.groovy @@ -24,7 +24,7 @@ suite("query52") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query53.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query53.groovy index 8c249ff3044e8e..af41198d6a223d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query53.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query53.groovy @@ -24,7 +24,7 @@ suite("query53") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query54.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query54.groovy index 083e4d648d299b..aa9d2342ca201d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query54.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query54.groovy @@ -24,7 +24,7 @@ suite("query54") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query55.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query55.groovy index dab3d5e7cde7f4..78b37165b87a56 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query55.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query55.groovy @@ -24,7 +24,7 @@ suite("query55") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query56.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query56.groovy index 2143dcb2dd4795..afe60dd0f70e50 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query56.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query56.groovy @@ -24,7 +24,7 @@ suite("query56") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query57.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query57.groovy index e3e2a1ad57e5b9..1752611f41d926 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query57.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query57.groovy @@ -24,7 +24,7 @@ suite("query57") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query58.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query58.groovy index c160d0ada1ba91..d82cb72bd2c9f1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query58.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query58.groovy @@ -24,7 +24,7 @@ suite("query58") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query59.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query59.groovy index 79d68ee081ff36..0a0596994524d2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query59.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query59.groovy @@ -24,7 +24,7 @@ suite("query59") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query6.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query6.groovy index 9a18350057d29c..b34540a89c6ec6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query6.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query6.groovy @@ -24,7 +24,7 @@ suite("query6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query60.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query60.groovy index a778c869418df6..44943800cbfb58 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query60.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query60.groovy @@ -24,7 +24,7 @@ suite("query60") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query61.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query61.groovy index 3efb6f2051c734..d74eddc78d717c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query61.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query61.groovy @@ -24,7 +24,7 @@ suite("query61") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query62.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query62.groovy index d9e8e6a5638e42..5a596f80933cb7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query62.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query62.groovy @@ -24,7 +24,7 @@ suite("query62") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query63.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query63.groovy index ef7bfcb661924c..bf14ee2576b2f1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query63.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query63.groovy @@ -24,7 +24,7 @@ suite("query63") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query64.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query64.groovy index d0c4ada0d677ae..15a82d00a35844 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query64.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query64.groovy @@ -24,7 +24,7 @@ suite("query64") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query65.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query65.groovy index 13ff66a236f908..cc5da20228fe93 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query65.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query65.groovy @@ -24,7 +24,7 @@ suite("query65") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query66.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query66.groovy index 3ad2fbecfa9cb7..d8993ec4c729af 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query66.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query66.groovy @@ -24,7 +24,7 @@ suite("query66") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query67.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query67.groovy index 60725af0b19fce..030f4c31ab3950 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query67.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query67.groovy @@ -24,7 +24,7 @@ suite("query67") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query68.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query68.groovy index c2d21e2d218559..08195b7400c437 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query68.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query68.groovy @@ -24,7 +24,7 @@ suite("query68") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query69.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query69.groovy index 4ce0907dd769b2..c9a7a7a38bc4c4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query69.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query69.groovy @@ -24,7 +24,7 @@ suite("query69") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query7.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query7.groovy index 015f3504c6f0a5..0e9a56a7439164 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query7.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query7.groovy @@ -24,7 +24,7 @@ suite("query7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query70.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query70.groovy index 34f11394e63dcf..6eb3824254cb65 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query70.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query70.groovy @@ -24,7 +24,7 @@ suite("query70") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query71.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query71.groovy index a6bd9c8884d29b..02a7c0d5e89286 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query71.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query71.groovy @@ -24,7 +24,7 @@ suite("query71") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query72.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query72.groovy index 6e7360ed7ddfd3..22a75fbb501614 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query72.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query72.groovy @@ -24,7 +24,7 @@ suite("query72") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query73.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query73.groovy index 5b02c9a994842e..00844e448dc4b3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query73.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query73.groovy @@ -24,7 +24,7 @@ suite("query73") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query74.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query74.groovy index 06cd139cb4e834..95ee2623c4f667 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query74.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query74.groovy @@ -24,7 +24,7 @@ suite("query74") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query75.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query75.groovy index 9397c42c127d46..0bac0118afaa46 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query75.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query75.groovy @@ -24,7 +24,7 @@ suite("query75") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query76.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query76.groovy index 9b0c5ca496c1b1..00366a557bc148 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query76.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query76.groovy @@ -24,7 +24,7 @@ suite("query76") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query77.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query77.groovy index 72f7a552e13b3d..ef7996391c445b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query77.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query77.groovy @@ -24,7 +24,7 @@ suite("query77") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query78.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query78.groovy index aa17385358d0d7..2fa085e9df80a8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query78.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query78.groovy @@ -24,7 +24,7 @@ suite("query78") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query79.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query79.groovy index 34f613e0974752..14eb66d85fd4aa 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query79.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query79.groovy @@ -24,7 +24,7 @@ suite("query79") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query8.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query8.groovy index 5fbbac6e3d11fd..e4ec57a65d25e5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query8.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query8.groovy @@ -24,7 +24,7 @@ suite("query8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query80.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query80.groovy index 6fadaafdb42a4a..e3414403747a85 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query80.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query80.groovy @@ -24,7 +24,7 @@ suite("query80") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query81.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query81.groovy index c7233532657d5a..9b470ac2d2af40 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query81.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query81.groovy @@ -24,7 +24,7 @@ suite("query81") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query82.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query82.groovy index 94ddfc873b1053..7b20dcf9639db9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query82.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query82.groovy @@ -24,7 +24,7 @@ suite("query82") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query83.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query83.groovy index d4696ed1a2350e..63fb5aa06f1039 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query83.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query83.groovy @@ -24,7 +24,7 @@ suite("query83") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query84.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query84.groovy index 1cea65929d1ea0..4ff3a5a5b38080 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query84.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query84.groovy @@ -24,7 +24,7 @@ suite("query84") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query85.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query85.groovy index 5455c49f9d813c..07e1241705488b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query85.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query85.groovy @@ -24,7 +24,7 @@ suite("query85") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query86.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query86.groovy index 9ba88449ca98f7..f893ea7eb219ac 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query86.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query86.groovy @@ -24,7 +24,7 @@ suite("query86") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query87.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query87.groovy index 391627f33adaac..152c63a009236b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query87.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query87.groovy @@ -24,7 +24,7 @@ suite("query87") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query88.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query88.groovy index f7a330a6963eea..e360dfbbdff0c3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query88.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query88.groovy @@ -24,7 +24,7 @@ suite("query88") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query89.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query89.groovy index b0f23ecd8a66b1..2ba59fa077532e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query89.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query89.groovy @@ -24,7 +24,7 @@ suite("query89") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query9.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query9.groovy index 76f5a7ed84c454..4aeee68747a7a8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query9.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query9.groovy @@ -24,7 +24,7 @@ suite("query9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query90.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query90.groovy index 5023a13e1f25dc..a49bcb21f2c55e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query90.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query90.groovy @@ -24,7 +24,7 @@ suite("query90") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query91.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query91.groovy index 199e374915a32b..13266112ccba2a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query91.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query91.groovy @@ -24,7 +24,7 @@ suite("query91") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query92.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query92.groovy index a8d060e557c4cd..34e17dcbb0aaa9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query92.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query92.groovy @@ -24,7 +24,7 @@ suite("query92") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query93.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query93.groovy index fcbb8872980f5d..ad155e3d94e0c3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query93.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query93.groovy @@ -24,7 +24,7 @@ suite("query93") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query94.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query94.groovy index 81e63de5b22b05..3fd4331681bdc1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query94.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query94.groovy @@ -24,7 +24,7 @@ suite("query94") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query95.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query95.groovy index be8b36d1c9c253..5df3fe1c59e657 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query95.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query95.groovy @@ -24,7 +24,7 @@ suite("query95") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query96.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query96.groovy index c3d1721650f207..682bb32f1d5907 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query96.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query96.groovy @@ -24,7 +24,7 @@ suite("query96") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query97.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query97.groovy index 5ea649b1046797..af942bda85113e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query97.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query97.groovy @@ -25,7 +25,7 @@ suite("query97") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query98.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query98.groovy index ac78b3adcf5da7..bd60f4ac00d8d0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query98.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query98.groovy @@ -24,7 +24,7 @@ suite("query98") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query99.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query99.groovy index 1e867ca4987f0f..833fe51d449b80 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query99.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/rf_prune/query99.groovy @@ -24,7 +24,7 @@ suite("query99") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query1.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query1.groovy index cdaa360eafaacd..42918f697fb961 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query1.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query1.groovy @@ -24,7 +24,7 @@ suite("query1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query10.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query10.groovy index 7e492e1ec8cec8..1f48b6d036fd89 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query10.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query10.groovy @@ -24,7 +24,7 @@ suite("query10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query11.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query11.groovy index ab874a3853f8da..06c5cb274c7712 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query11.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query11.groovy @@ -24,7 +24,7 @@ suite("query11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query12.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query12.groovy index 6fc3a9149599d4..e8e6ee842bd7fc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query12.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query12.groovy @@ -24,7 +24,7 @@ suite("query12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query13.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query13.groovy index 152c4132629c63..357cbebfb28f47 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query13.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query13.groovy @@ -24,7 +24,7 @@ suite("query13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query14.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query14.groovy index c621b72b18c4e4..13fe837cda9a62 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query14.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query14.groovy @@ -24,7 +24,7 @@ suite("query14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query15.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query15.groovy index 48dc7923ebaa3f..1c665dc59872f0 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query15.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query15.groovy @@ -24,7 +24,7 @@ suite("query15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query16.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query16.groovy index 16103efe13173f..cc23568bacff0a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query16.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query16.groovy @@ -24,7 +24,7 @@ suite("query16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query17.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query17.groovy index 5cafc500be9674..30163c87b9a887 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query17.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query17.groovy @@ -24,7 +24,7 @@ suite("query17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query18.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query18.groovy index 42bc9b85e8a581..4deba13860803f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query18.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query18.groovy @@ -24,7 +24,7 @@ suite("query18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query19.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query19.groovy index 102142a9975b04..c79a3ee2ec35d4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query19.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query19.groovy @@ -24,7 +24,7 @@ suite("query19") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query2.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query2.groovy index c8afcaca91ae45..8902787b572395 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query2.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query2.groovy @@ -24,7 +24,7 @@ suite("query2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query20.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query20.groovy index e64fd0a4d64957..363a4ad7a72b93 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query20.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query20.groovy @@ -24,7 +24,7 @@ suite("query20") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query21.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query21.groovy index 8c0d46200b8da3..44cd7866862d4b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query21.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query21.groovy @@ -24,7 +24,7 @@ suite("query21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query22.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query22.groovy index ab0afa82a7f8b1..ed1cbd44da0bdc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query22.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query22.groovy @@ -24,7 +24,7 @@ suite("query22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query23.groovy index b09ccf2079a525..2781c6a97fd06b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query23.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query23.groovy @@ -25,7 +25,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query24.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query24.groovy index 7f8d9159cd3ab2..5dd5dc90bd9951 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query24.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query24.groovy @@ -24,7 +24,7 @@ suite("query24") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query25.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query25.groovy index 7d6b7930cd82f2..6c82937b56a16d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query25.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query25.groovy @@ -24,7 +24,7 @@ suite("query25") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query26.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query26.groovy index a269a64d6600b9..af437adc11d7ba 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query26.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query26.groovy @@ -24,7 +24,7 @@ suite("query26") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query27.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query27.groovy index e02f0e1baf82f2..2b81670e436d23 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query27.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query27.groovy @@ -24,7 +24,7 @@ suite("query27") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query28.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query28.groovy index cbd3938230d209..ba98ce97443d6c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query28.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query28.groovy @@ -24,7 +24,7 @@ suite("query28") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query29.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query29.groovy index a8c46eb13c7b70..4e9cabccc6747d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query29.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query29.groovy @@ -24,7 +24,7 @@ suite("query29") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query3.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query3.groovy index 2030616e958181..959647f77f2c1e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query3.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query3.groovy @@ -24,7 +24,7 @@ suite("query3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query30.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query30.groovy index 301ac521b84480..0325d199e9473d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query30.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query30.groovy @@ -24,7 +24,7 @@ suite("query30") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query31.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query31.groovy index 607dc93051d195..7ecc40546c2a9e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query31.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query31.groovy @@ -24,7 +24,7 @@ suite("query31") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query32.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query32.groovy index e90fa68b73443a..d8d436025a6425 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query32.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query32.groovy @@ -25,7 +25,7 @@ suite("query32") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query33.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query33.groovy index b09523f54b7d46..0e88a050ddf1aa 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query33.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query33.groovy @@ -24,7 +24,7 @@ suite("query33") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query34.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query34.groovy index 6e61686781f9ff..23fd7dbddc1760 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query34.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query34.groovy @@ -24,7 +24,7 @@ suite("query34") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query35.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query35.groovy index 0359da335facba..ac36ae96e02e1d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query35.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query35.groovy @@ -24,7 +24,7 @@ suite("query35") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query36.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query36.groovy index 80a114afc269a8..9450865fb8a25a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query36.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query36.groovy @@ -24,7 +24,7 @@ suite("query36") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query37.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query37.groovy index e5f353ab44a134..7cc68b08a155e1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query37.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query37.groovy @@ -24,7 +24,7 @@ suite("query37") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query38.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query38.groovy index 7a7c71813664ed..70c0488051c282 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query38.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query38.groovy @@ -25,7 +25,7 @@ suite("query38") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query39.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query39.groovy index 55c7135fca9e8d..bf8ad503715c21 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query39.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query39.groovy @@ -24,7 +24,7 @@ suite("query39") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query4.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query4.groovy index 969981b25f7cac..5ead0a54f5ad70 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query4.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query4.groovy @@ -24,7 +24,7 @@ suite("query4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query40.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query40.groovy index 8d261843553c84..efa451619499c7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query40.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query40.groovy @@ -24,7 +24,7 @@ suite("query40") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query41.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query41.groovy index 68ea4200724d18..72a54cb955762a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query41.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query41.groovy @@ -24,7 +24,7 @@ suite("query41") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query42.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query42.groovy index df1baef595771e..638f40df576f5b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query42.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query42.groovy @@ -24,7 +24,7 @@ suite("query42") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query43.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query43.groovy index fdc21823808f4f..59671767cbb226 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query43.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query43.groovy @@ -24,7 +24,7 @@ suite("query43") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query44.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query44.groovy index 085ef7323dc884..b7a1c7e28d8d89 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query44.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query44.groovy @@ -24,7 +24,7 @@ suite("query44") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query45.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query45.groovy index a8d4bc2bff2d7f..f8732e5b95b813 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query45.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query45.groovy @@ -24,7 +24,7 @@ suite("query45") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query46.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query46.groovy index 13edc5bad59a57..b660531c7ce87f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query46.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query46.groovy @@ -24,7 +24,7 @@ suite("query46") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query47.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query47.groovy index 63b6ee07cbee57..b5c65fb4bbbafc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query47.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query47.groovy @@ -24,7 +24,7 @@ suite("query47") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query48.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query48.groovy index c8594b7fd10cf7..7fb530551d6ebb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query48.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query48.groovy @@ -24,7 +24,7 @@ suite("query48") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query49.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query49.groovy index 08a1e2ca819b96..6c1fd26a205e2a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query49.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query49.groovy @@ -24,7 +24,7 @@ suite("query49") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query5.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query5.groovy index 43524c4c6e8a28..b301852275a140 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query5.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query5.groovy @@ -24,7 +24,7 @@ suite("query5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query50.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query50.groovy index fa5bd0908d6201..85fcc4fe7cc5e8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query50.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query50.groovy @@ -24,7 +24,7 @@ suite("query50") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query51.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query51.groovy index 7a00b2b7a52de4..fd4be5f78bd98b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query51.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query51.groovy @@ -24,7 +24,7 @@ suite("query51") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query52.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query52.groovy index aca150ca790e5d..383ca211cd0979 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query52.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query52.groovy @@ -24,7 +24,7 @@ suite("query52") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query53.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query53.groovy index 62240282e24579..8500921341c9c2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query53.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query53.groovy @@ -24,7 +24,7 @@ suite("query53") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query54.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query54.groovy index 0d7fb531304d95..ade0d3aaf2cad5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query54.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query54.groovy @@ -24,7 +24,7 @@ suite("query54") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query55.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query55.groovy index 37af78fb24bf3c..364f28f623ee69 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query55.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query55.groovy @@ -24,7 +24,7 @@ suite("query55") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query56.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query56.groovy index b96a95094fe147..5d814d5e254599 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query56.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query56.groovy @@ -24,7 +24,7 @@ suite("query56") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query57.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query57.groovy index a258a524731224..2e3184a554034e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query57.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query57.groovy @@ -24,7 +24,7 @@ suite("query57") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query58.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query58.groovy index 0ebb6288536ae8..66623e34e8ab42 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query58.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query58.groovy @@ -24,7 +24,7 @@ suite("query58") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query59.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query59.groovy index 15005ea7f1638a..40678db0699e76 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query59.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query59.groovy @@ -24,7 +24,7 @@ suite("query59") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query6.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query6.groovy index 54b0e9c129fd72..977dd3669fba3d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query6.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query6.groovy @@ -24,7 +24,7 @@ suite("query6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query60.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query60.groovy index d980e0cbc3a1b8..59371d705e2463 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query60.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query60.groovy @@ -24,7 +24,7 @@ suite("query60") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query61.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query61.groovy index 0ea2596fdf01be..0f8b50313087a8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query61.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query61.groovy @@ -24,7 +24,7 @@ suite("query61") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query62.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query62.groovy index 95fe33f4c608c0..3743a81d632c6b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query62.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query62.groovy @@ -24,7 +24,7 @@ suite("query62") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query63.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query63.groovy index 367ee302c10a70..d23cf913dd604d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query63.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query63.groovy @@ -24,7 +24,7 @@ suite("query63") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query64.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query64.groovy index 5644db2d92c867..2d5f644d877092 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query64.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query64.groovy @@ -24,7 +24,7 @@ suite("query64") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query65.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query65.groovy index 9f5108267560a3..f71151d151dcbf 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query65.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query65.groovy @@ -24,7 +24,7 @@ suite("query65") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query66.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query66.groovy index e9b0f8c7e89f05..e3b73014b641d4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query66.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query66.groovy @@ -24,7 +24,7 @@ suite("query66") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query67.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query67.groovy index bba4022c805c07..a3d364e582f12b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query67.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query67.groovy @@ -24,7 +24,7 @@ suite("query67") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query68.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query68.groovy index 08bcc07785d10d..06ea58ae212cc7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query68.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query68.groovy @@ -24,7 +24,7 @@ suite("query68") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query69.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query69.groovy index 1122aa1716cd53..28ad62bf4b10ec 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query69.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query69.groovy @@ -24,7 +24,7 @@ suite("query69") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query7.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query7.groovy index 0031a3088da690..185c02bdc57461 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query7.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query7.groovy @@ -24,7 +24,7 @@ suite("query7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query70.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query70.groovy index d79b36bec668a6..57b850cf9c4955 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query70.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query70.groovy @@ -24,7 +24,7 @@ suite("query70") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query71.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query71.groovy index 480c77aaffce75..5cf28843997747 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query71.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query71.groovy @@ -24,7 +24,7 @@ suite("query71") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query72.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query72.groovy index 73b574b9fdd0dc..7b77dcfd49b186 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query72.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query72.groovy @@ -24,7 +24,7 @@ suite("query72") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query73.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query73.groovy index 19d399ee7ea84b..0a94e5bcc42286 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query73.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query73.groovy @@ -24,7 +24,7 @@ suite("query73") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query74.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query74.groovy index 65c65d66bcf294..626865961c44f1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query74.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query74.groovy @@ -24,7 +24,7 @@ suite("query74") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query75.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query75.groovy index 0afdc93abf33f8..f5a9aab3b5142a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query75.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query75.groovy @@ -24,7 +24,7 @@ suite("query75") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query76.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query76.groovy index 52b75d1628bfd7..67e3e8be04985b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query76.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query76.groovy @@ -24,7 +24,7 @@ suite("query76") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query77.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query77.groovy index 22483589531b2d..ef7441613d76af 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query77.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query77.groovy @@ -24,7 +24,7 @@ suite("query77") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query78.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query78.groovy index eee40a4fc03dfc..012f5525a326ce 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query78.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query78.groovy @@ -24,7 +24,7 @@ suite("query78") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query79.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query79.groovy index 45bd5452c078c6..4564fdc22cec8f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query79.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query79.groovy @@ -24,7 +24,7 @@ suite("query79") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query8.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query8.groovy index e44e47a69e9fdd..ec4a0ee05b40d3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query8.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query8.groovy @@ -24,7 +24,7 @@ suite("query8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query80.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query80.groovy index a8924221229cbf..5741e2a42e5cbf 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query80.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query80.groovy @@ -24,7 +24,7 @@ suite("query80") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query81.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query81.groovy index 85088a4553fa7c..4ede380afe35ee 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query81.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query81.groovy @@ -24,7 +24,7 @@ suite("query81") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query82.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query82.groovy index 1663dcdb3ee7b9..eea00f6d71e086 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query82.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query82.groovy @@ -24,7 +24,7 @@ suite("query82") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query83.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query83.groovy index a8fd8561e20a09..a517c451fc4c14 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query83.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query83.groovy @@ -24,7 +24,7 @@ suite("query83") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query84.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query84.groovy index 01433c96e96718..9157caa6f9219f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query84.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query84.groovy @@ -24,7 +24,7 @@ suite("query84") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query85.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query85.groovy index 858a94cd81d36b..adf857726914ef 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query85.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query85.groovy @@ -24,7 +24,7 @@ suite("query85") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query86.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query86.groovy index 71e726382da63e..5a0fab661cd2e6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query86.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query86.groovy @@ -24,7 +24,7 @@ suite("query86") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query87.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query87.groovy index 52e6ab0b62425c..85ff05f3a44438 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query87.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query87.groovy @@ -24,7 +24,7 @@ suite("query87") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query88.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query88.groovy index dcdeb6a1ac4386..fdb7b5d6aa5b52 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query88.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query88.groovy @@ -24,7 +24,7 @@ suite("query88") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query89.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query89.groovy index 9ddbe3cf68ce57..7c73edce5c797f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query89.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query89.groovy @@ -24,7 +24,7 @@ suite("query89") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query9.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query9.groovy index df25c356c66e21..ef266b835093ce 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query9.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query9.groovy @@ -24,7 +24,7 @@ suite("query9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query90.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query90.groovy index 0f5cde4e25e30a..058cd5452a33ef 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query90.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query90.groovy @@ -24,7 +24,7 @@ suite("query90") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query91.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query91.groovy index 3c706057a02f5d..e7d34747805a28 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query91.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query91.groovy @@ -24,7 +24,7 @@ suite("query91") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query92.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query92.groovy index 1013519e91975e..b29f2bf91241ac 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query92.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query92.groovy @@ -24,7 +24,7 @@ suite("query92") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query93.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query93.groovy index 53d1d63184839e..3404c7cc9c8a94 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query93.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query93.groovy @@ -24,7 +24,7 @@ suite("query93") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query94.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query94.groovy index 4888bb535c6dc0..ef8677741898b6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query94.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query94.groovy @@ -24,7 +24,7 @@ suite("query94") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query95.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query95.groovy index 4f1712c93fd90b..6d67e79c8936ce 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query95.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query95.groovy @@ -24,7 +24,7 @@ suite("query95") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query96.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query96.groovy index 724e1a1f171702..730ade3e4d5c91 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query96.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query96.groovy @@ -24,7 +24,7 @@ suite("query96") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query97.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query97.groovy index 0a4c7627524edb..bdc2fe5ed575a6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query97.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query97.groovy @@ -25,7 +25,7 @@ suite("query97") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query98.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query98.groovy index 9a401b0184eb3e..b8f68770034a29 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query98.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query98.groovy @@ -24,7 +24,7 @@ suite("query98") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query99.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query99.groovy index 6c16d02330fb8e..bba82591c186f8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query99.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf100/shape/query99.groovy @@ -24,7 +24,7 @@ suite("query99") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.groovy index f7acdd26eb8690..75849873362331 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query13.groovy @@ -24,7 +24,7 @@ suite("query13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query19.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query19.groovy index 17466ecacf346e..cf67e74241fe11 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query19.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query19.groovy @@ -24,7 +24,7 @@ suite("query19") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query44.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query44.groovy index d42411e0c011d2..65cab366fd2277 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query44.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query44.groovy @@ -24,7 +24,7 @@ suite("query44") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.groovy index 9a0eb67545c428..298ec8932ff1b1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query45.groovy @@ -24,7 +24,7 @@ suite("query45") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query54.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query54.groovy index 198b39b8583407..9934e3759778ce 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query54.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query54.groovy @@ -24,7 +24,7 @@ suite("query54") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.groovy index c1dd7cbd87be1b..f627f8d4b56f77 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query56.groovy @@ -24,7 +24,7 @@ suite("query56") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query6.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query6.groovy index 80639ff76d3720..8c5e0615326deb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query6.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query6.groovy @@ -24,7 +24,7 @@ suite("query6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.groovy index d34a2c3fb719c5..10cd099329eea6 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query61.groovy @@ -24,7 +24,7 @@ suite("query61") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.groovy index 2477c21b67ce2d..1a5bda76b35b38 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query68.groovy @@ -24,7 +24,7 @@ suite("query68") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.groovy index 33e361246ba77c..578c2e20313705 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query8.groovy @@ -24,7 +24,7 @@ suite("query8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.groovy index 0502b8403fb321..a36e3d19ce3f61 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query91.groovy @@ -24,7 +24,7 @@ suite("query91") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query95.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query95.groovy index b4e6f86e2c50bd..5047f3ed89da31 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query95.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/bs_downgrade_shape/query95.groovy @@ -24,7 +24,7 @@ suite("query95") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/ddl/shape.tmpl b/regression-test/suites/new_shapes_p0/tpcds_sf1000/ddl/shape.tmpl index c25fd3f36b03f4..ffe47491a10337 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/ddl/shape.tmpl +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/ddl/shape.tmpl @@ -24,7 +24,7 @@ suite("query{--}") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.groovy index aada0585602b1d..d7f50d8f1bd301 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/eliminate_empty/query10_empty.groovy @@ -25,7 +25,7 @@ suite("query10") { sql "use ${db}" multi_sql """ set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query1.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query1.groovy index 52a88c9c294b1b..2b6a204c62d31b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query1.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query1.groovy @@ -24,7 +24,7 @@ suite("query1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query10.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query10.groovy index edff37bb8d673a..6b471dfe1290ac 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query10.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query10.groovy @@ -24,7 +24,7 @@ suite("query10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query11.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query11.groovy index 5b659ebd3d24ff..d54fcc90c3f3a4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query11.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query11.groovy @@ -24,7 +24,7 @@ suite("query11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query12.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query12.groovy index 90f49e49c24c7e..b8b8ba19c50c75 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query12.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query12.groovy @@ -24,7 +24,7 @@ suite("query12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query13.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query13.groovy index 64da9d0bcd9de2..ea58b41d9dd58b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query13.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query13.groovy @@ -24,7 +24,7 @@ suite("query13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query14.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query14.groovy index 07ef2b99688004..15eb273065e0d7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query14.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query14.groovy @@ -24,7 +24,7 @@ suite("query14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query15.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query15.groovy index 05d8d44bcdaddc..e236f57bd7c1ad 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query15.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query15.groovy @@ -24,7 +24,7 @@ suite("query15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query16.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query16.groovy index 23e94ad465c6b8..2818bfcc7069aa 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query16.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query16.groovy @@ -24,7 +24,7 @@ suite("query16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query17.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query17.groovy index 6bde5e8c30e2d2..13ba576a342dec 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query17.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query17.groovy @@ -24,7 +24,7 @@ suite("query17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query18.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query18.groovy index 1b3ee9b374b321..ad631e67c87835 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query18.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query18.groovy @@ -24,7 +24,7 @@ suite("query18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query19.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query19.groovy index 43a9dbdd37a710..a48df38afd5fc5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query19.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query19.groovy @@ -24,7 +24,7 @@ suite("query19") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query2.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query2.groovy index e6a85af6b5c792..e2d262a91384f5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query2.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query2.groovy @@ -24,7 +24,7 @@ suite("query2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query20.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query20.groovy index e132b59a3806b7..3c7a62289afc00 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query20.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query20.groovy @@ -24,7 +24,7 @@ suite("query20") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query21.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query21.groovy index 7f079f8806f79c..b0da36f96ddefc 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query21.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query21.groovy @@ -24,7 +24,7 @@ suite("query21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'SET enable_fold_constant_by_be = false' //plan shape will be different sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query22.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query22.groovy index 3eea3092249ead..9e02e17afd94e8 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query22.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query22.groovy @@ -24,7 +24,7 @@ suite("query22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query23.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query23.groovy index 28268f8fcbcdf9..89fe2bbb9ec0c2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query23.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query23.groovy @@ -25,7 +25,7 @@ suite("query23") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query24.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query24.groovy index b64674097d7b86..23f748fb13dfff 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query24.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query24.groovy @@ -24,7 +24,7 @@ suite("query24") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query25.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query25.groovy index e206d6f27dd536..4a074d6820d883 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query25.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query25.groovy @@ -24,7 +24,7 @@ suite("query25") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query26.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query26.groovy index 2f2a1f1ca16a4f..65db6b5100a210 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query26.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query26.groovy @@ -24,7 +24,7 @@ suite("query26") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query27.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query27.groovy index 6bdbc27bc295c0..45fa85d291e03b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query27.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query27.groovy @@ -24,7 +24,7 @@ suite("query27") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query28.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query28.groovy index e7f8ae691933b6..8c212b0f919d81 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query28.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query28.groovy @@ -24,7 +24,7 @@ suite("query28") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query29.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query29.groovy index bc0a11be1f4227..1736054fb75e8c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query29.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query29.groovy @@ -24,7 +24,7 @@ suite("query29") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query3.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query3.groovy index c396e81a258154..46f2f5d6b9d8b4 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query3.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query3.groovy @@ -24,7 +24,7 @@ suite("query3") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query30.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query30.groovy index 3677b56c442266..f76ee9e65a0dff 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query30.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query30.groovy @@ -24,7 +24,7 @@ suite("query30") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query31.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query31.groovy index faca7bb52750e0..b8e476b1a3706d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query31.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query31.groovy @@ -24,7 +24,7 @@ suite("query31") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query32.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query32.groovy index 7f6ed9a0c5a721..e57f2c32e0a9bd 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query32.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query32.groovy @@ -25,7 +25,7 @@ suite("query32") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query33.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query33.groovy index 8e1ab9f66da27e..cf4ed712e7664a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query33.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query33.groovy @@ -24,7 +24,7 @@ suite("query33") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query34.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query34.groovy index 98a2d27c001b41..6239ae8ec5a5ed 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query34.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query34.groovy @@ -24,7 +24,7 @@ suite("query34") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query35.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query35.groovy index 3d68f9a85ac51c..f53eb6c3f10f33 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query35.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query35.groovy @@ -24,7 +24,7 @@ suite("query35") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query36.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query36.groovy index 7a681f2906089a..7bd518b61609e9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query36.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query36.groovy @@ -24,7 +24,7 @@ suite("query36") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query37.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query37.groovy index 5d41320124388e..b3aa26dc0cce0f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query37.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query37.groovy @@ -24,7 +24,7 @@ suite("query37") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query38.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query38.groovy index d758d7a43561a7..2abdb0f7693fd2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query38.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query38.groovy @@ -25,7 +25,7 @@ suite("query38") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query39.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query39.groovy index 5392d2b82f8c21..9d8c7112dc5272 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query39.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query39.groovy @@ -24,7 +24,7 @@ suite("query39") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query4.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query4.groovy index 24689dda916653..0f6339163403e2 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query4.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query4.groovy @@ -24,7 +24,7 @@ suite("query4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query40.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query40.groovy index ac7a71d3233856..af3342c98b8112 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query40.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query40.groovy @@ -24,7 +24,7 @@ suite("query40") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query41.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query41.groovy index 0754878e39e7a2..e89c335660c565 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query41.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query41.groovy @@ -24,7 +24,7 @@ suite("query41") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query42.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query42.groovy index 44f5c9f0a6fff7..f0e2461263d004 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query42.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query42.groovy @@ -24,7 +24,7 @@ suite("query42") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query43.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query43.groovy index abb3e84a2dc74f..2fecc1ebf409a1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query43.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query43.groovy @@ -24,7 +24,7 @@ suite("query43") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query44.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query44.groovy index 46c52ced591560..5b78dbb02a3617 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query44.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query44.groovy @@ -24,7 +24,7 @@ suite("query44") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query45.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query45.groovy index 9c27eb60e16869..83090aba4ed3ee 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query45.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query45.groovy @@ -24,7 +24,7 @@ suite("query45") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query46.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query46.groovy index 44f48ecde635c3..a28db59088d2c3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query46.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query46.groovy @@ -24,7 +24,7 @@ suite("query46") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query47.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query47.groovy index ee5856493b5f98..f9fa489afc621c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query47.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query47.groovy @@ -24,7 +24,7 @@ suite("query47") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query48.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query48.groovy index fa89e8b5976f70..48a3d276c86400 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query48.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query48.groovy @@ -24,7 +24,7 @@ suite("query48") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query49.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query49.groovy index 5883d3522930e5..cdc9e367ce7b0e 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query49.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query49.groovy @@ -24,7 +24,7 @@ suite("query49") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query5.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query5.groovy index f1db7b7207b66d..bb2dd30f4b722a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query5.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query5.groovy @@ -24,7 +24,7 @@ suite("query5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query50.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query50.groovy index 3a0a41bad7a6d9..9fbfc1cb810338 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query50.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query50.groovy @@ -24,7 +24,7 @@ suite("query50") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query51.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query51.groovy index d7fe227ff00e3e..9c9f6aeff5ea05 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query51.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query51.groovy @@ -26,7 +26,7 @@ suite("query51") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query52.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query52.groovy index 327bf50ae0ed32..452f46ed5b9846 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query52.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query52.groovy @@ -24,7 +24,7 @@ suite("query52") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query53.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query53.groovy index b23a25424e4a6f..5343a957b43971 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query53.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query53.groovy @@ -24,7 +24,7 @@ suite("query53") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query54.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query54.groovy index 1ee7c497f8f1c1..e7cd9e02ba70d1 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query54.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query54.groovy @@ -24,7 +24,7 @@ suite("query54") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query55.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query55.groovy index c2d4c1db731a68..6748d587fdab49 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query55.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query55.groovy @@ -24,7 +24,7 @@ suite("query55") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query56.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query56.groovy index f9a99301af1af6..8669e8cab8ef4a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query56.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query56.groovy @@ -24,7 +24,7 @@ suite("query56") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query57.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query57.groovy index b1c5ec41918363..9c25e1e93e8bdb 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query57.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query57.groovy @@ -24,7 +24,7 @@ suite("query57") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query58.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query58.groovy index 5d618277923ebb..a193e181c25b94 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query58.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query58.groovy @@ -24,7 +24,7 @@ suite("query58") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query59.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query59.groovy index 9bda7dd0b108ce..8728c2aa7c580a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query59.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query59.groovy @@ -24,7 +24,7 @@ suite("query59") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query6.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query6.groovy index 5267c9b608cf70..d0335b21772c69 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query6.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query6.groovy @@ -24,7 +24,7 @@ suite("query6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query60.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query60.groovy index 362298991deb80..8d0cc7cf4e1032 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query60.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query60.groovy @@ -24,7 +24,7 @@ suite("query60") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query61.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query61.groovy index d2e1029fa5c139..e0ba666b441507 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query61.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query61.groovy @@ -24,7 +24,7 @@ suite("query61") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query62.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query62.groovy index 686760596adddd..8547fd75f03e9d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query62.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query62.groovy @@ -24,7 +24,7 @@ suite("query62") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query63.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query63.groovy index 1b0cd11db4190b..7921ddd0ce3421 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query63.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query63.groovy @@ -24,7 +24,7 @@ suite("query63") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query64.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query64.groovy index ae63cbc2dab186..255104a6461f18 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query64.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query64.groovy @@ -24,7 +24,7 @@ suite("query64") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query65.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query65.groovy index 25955c1ce93d77..7952361781ecf9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query65.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query65.groovy @@ -24,7 +24,7 @@ suite("query65") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query66.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query66.groovy index 19cd030b878127..94f6f715b4f673 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query66.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query66.groovy @@ -24,7 +24,7 @@ suite("query66") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query67.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query67.groovy index 881919b24ab14d..cdc51fe9f86c50 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query67.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query67.groovy @@ -24,7 +24,7 @@ suite("query67") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query68.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query68.groovy index 30e37515cbe938..584cd3324e6b62 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query68.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query68.groovy @@ -24,7 +24,7 @@ suite("query68") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query69.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query69.groovy index 09cbd0185b150d..711ac55545bf89 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query69.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query69.groovy @@ -24,7 +24,7 @@ suite("query69") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query7.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query7.groovy index b018fd1ff86677..1a9e1d60339c84 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query7.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query7.groovy @@ -24,7 +24,7 @@ suite("query7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query70.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query70.groovy index 764f9bd2483034..de0f8b8dd6ec7d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query70.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query70.groovy @@ -24,7 +24,7 @@ suite("query70") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query71.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query71.groovy index 409be785e48011..6a20534ff50100 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query71.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query71.groovy @@ -24,7 +24,7 @@ suite("query71") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query72.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query72.groovy index 5880246f558a34..1291c220d8188a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query72.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query72.groovy @@ -24,7 +24,7 @@ suite("query72") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query73.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query73.groovy index fbd8df6bec621e..9fc96b6d63afbe 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query73.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query73.groovy @@ -24,7 +24,7 @@ suite("query73") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query74.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query74.groovy index 90580c2e6b756b..0c87e782b31054 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query74.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query74.groovy @@ -24,7 +24,7 @@ suite("query74") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query75.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query75.groovy index 775798604b2ee8..aff5b004f36422 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query75.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query75.groovy @@ -24,7 +24,7 @@ suite("query75") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query76.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query76.groovy index 4937f060c5e736..8ef3cd7bea3f5f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query76.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query76.groovy @@ -24,7 +24,7 @@ suite("query76") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query77.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query77.groovy index b188b1982d560c..9451e54aac257a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query77.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query77.groovy @@ -24,7 +24,7 @@ suite("query77") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query78.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query78.groovy index b96778997d3d48..8f390284bfb02b 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query78.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query78.groovy @@ -24,7 +24,7 @@ suite("query78") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query79.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query79.groovy index dda5a1cfd58448..7c24454e39c51c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query79.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query79.groovy @@ -24,7 +24,7 @@ suite("query79") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query8.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query8.groovy index 11f5a76e10d94e..c88174aa71eb1c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query8.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query8.groovy @@ -24,7 +24,7 @@ suite("query8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query80.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query80.groovy index f9e62e637081f6..18701194bcdfb7 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query80.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query80.groovy @@ -24,7 +24,7 @@ suite("query80") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query81.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query81.groovy index 0f39073baf4b47..2918b83b596b06 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query81.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query81.groovy @@ -24,7 +24,7 @@ suite("query81") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query82.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query82.groovy index 6f3260b90f1206..e1061469079a2c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query82.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query82.groovy @@ -24,7 +24,7 @@ suite("query82") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query83.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query83.groovy index da47ab88acbd69..26373896bf05fe 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query83.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query83.groovy @@ -24,7 +24,7 @@ suite("query83") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query84.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query84.groovy index 98b9cea582ad5c..8446f051984793 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query84.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query84.groovy @@ -24,7 +24,7 @@ suite("query84") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query85.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query85.groovy index 8add0348dae1fd..96cab231cb491f 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query85.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query85.groovy @@ -24,7 +24,7 @@ suite("query85") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query86.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query86.groovy index e5247041ec8897..15b12c720675ca 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query86.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query86.groovy @@ -24,7 +24,7 @@ suite("query86") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query87.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query87.groovy index b082d2bdbc1299..9cd479e8ef3bb3 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query87.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query87.groovy @@ -24,7 +24,7 @@ suite("query87") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query88.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query88.groovy index 0b9c4c7c7d2de6..4845d53a42e957 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query88.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query88.groovy @@ -24,7 +24,7 @@ suite("query88") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query89.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query89.groovy index dd5752b43fa4f0..8a82601ca3f764 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query89.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query89.groovy @@ -24,7 +24,7 @@ suite("query89") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query9.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query9.groovy index d72b880214bcc9..6d69c5eb38189a 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query9.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query9.groovy @@ -24,7 +24,7 @@ suite("query9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query90.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query90.groovy index 9bce112a5c2d07..2115df945ce527 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query90.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query90.groovy @@ -24,7 +24,7 @@ suite("query90") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query91.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query91.groovy index e250435336edfa..48a30d985bc0d9 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query91.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query91.groovy @@ -24,7 +24,7 @@ suite("query91") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query92.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query92.groovy index 5ec81ae0610cae..76d84812da91cf 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query92.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query92.groovy @@ -24,7 +24,7 @@ suite("query92") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query93.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query93.groovy index f0f52194f2d4f6..f93d2f01a27925 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query93.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query93.groovy @@ -24,7 +24,7 @@ suite("query93") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query94.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query94.groovy index cee27ca765ab57..574bc2fb2a2b35 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query94.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query94.groovy @@ -24,7 +24,7 @@ suite("query94") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query95.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query95.groovy index 8a19be8d039e4d..821aefaeaa7c9c 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query95.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query95.groovy @@ -24,7 +24,7 @@ suite("query95") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query96.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query96.groovy index 14645aee61163f..e79a089ba5eee5 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query96.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query96.groovy @@ -24,7 +24,7 @@ suite("query96") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query97.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query97.groovy index e30c1ec14a7857..5677adb7eab27d 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query97.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query97.groovy @@ -25,7 +25,7 @@ suite("query97") { multi_sql """ use ${db}; set enable_nereids_planner=true; - set enable_nereids_distribute_planner=true; + set enable_nereids_distribute_planner=false; set enable_fallback_to_original_planner=false; set exec_mem_limit=21G; set be_number_for_test=3; diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query98.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query98.groovy index dedcf82dae4d04..e05bf840bf3a75 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query98.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query98.groovy @@ -24,7 +24,7 @@ suite("query98") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query99.groovy b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query99.groovy index 6f9e47f2140a90..7d680ad8d8dc80 100644 --- a/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query99.groovy +++ b/regression-test/suites/new_shapes_p0/tpcds_sf1000/shape/query99.groovy @@ -24,7 +24,7 @@ suite("query99") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q1.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q1.groovy index ae7072c55ed913..66a099fbfc1a2c 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q1.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q1.groovy @@ -24,7 +24,7 @@ suite("q1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.groovy index 2ea2891b4a2c6e..9815e7ba946b55 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q10.groovy @@ -24,7 +24,7 @@ suite("q10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q11.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q11.groovy index 997aa07898b61e..46c2e914a95331 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q11.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q11.groovy @@ -24,7 +24,7 @@ suite("q11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q12.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q12.groovy index b78ad09e0e6bee..e3b3624245bcb5 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q12.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q12.groovy @@ -24,7 +24,7 @@ suite("q12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.groovy index 52ce1a04b62a5d..fa4dd199f8e38b 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q13.groovy @@ -24,7 +24,7 @@ suite("q13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.groovy index 28d7599970a0e0..aef032a16084bf 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q14.groovy @@ -24,7 +24,7 @@ suite("q14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.groovy index 00897a7819000d..6f3452c20f368a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q15.groovy @@ -24,7 +24,7 @@ suite("q15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q16.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q16.groovy index 8d682c8edb9004..aaaa8f71c6a285 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q16.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q16.groovy @@ -24,7 +24,7 @@ suite("q16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.groovy index 52937ee96b7b84..5d465132333e7d 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q17.groovy @@ -24,7 +24,7 @@ suite("q17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.groovy index 2e4245d06ee338..2943cd4030cf0c 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q18.groovy @@ -24,7 +24,7 @@ suite("q18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.groovy index c2fffd575f7e3a..79c460ce29b68c 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q19.groovy @@ -25,7 +25,7 @@ suite("q19") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q2.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q2.groovy index cabce2fc86697b..21cb95c7a145f8 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q2.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q2.groovy @@ -24,7 +24,7 @@ suite("q2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.groovy index 32efcdde07b9a0..85f8262d7c2471 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20-rewrite.groovy @@ -25,7 +25,7 @@ suite("q20-rewrite") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.groovy index ea9819fd3cfed5..3b63309ea060bf 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q20.groovy @@ -25,7 +25,7 @@ suite("q20") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q21.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q21.groovy index 28b5d0f7648fef..d03c7bb1519e44 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q21.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q21.groovy @@ -24,7 +24,7 @@ suite("q21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.groovy index d363362a329540..3eefeabf759acc 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q22.groovy @@ -24,7 +24,7 @@ suite("q22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.groovy index cc0f648b264a5f..23db17eb185458 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q3.groovy @@ -25,7 +25,7 @@ suite("q3") { // db = "tpch" sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q4.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q4.groovy index 3454d5bb6fef9d..510f914a9278f6 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q4.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q4.groovy @@ -24,7 +24,7 @@ suite("q4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.groovy index f49ff1f561aef6..fdbc7871d4addd 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q5.groovy @@ -24,7 +24,7 @@ suite("q5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q6.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q6.groovy index 43da4d59e6ba78..f4b4f087ec80d5 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q6.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q6.groovy @@ -24,7 +24,7 @@ suite("q6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.groovy index bac6387a3cb048..b6c1d4e70bab75 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q7.groovy @@ -24,7 +24,7 @@ suite("q7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.groovy index 44199ff620ebdf..1bea5cbcf598fb 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q8.groovy @@ -24,7 +24,7 @@ suite("q8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.groovy index 263f4f6f16d914..e7a95b00413974 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/nostats_rf_prune/q9.groovy @@ -24,7 +24,7 @@ suite("q9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q1.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q1.groovy index f90bc9891cb428..37f4509ccf184e 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q1.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q1.groovy @@ -24,7 +24,7 @@ suite("q1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q10.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q10.groovy index 7079db86ee7c77..119e35a69be087 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q10.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q10.groovy @@ -24,7 +24,7 @@ suite("q10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q11.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q11.groovy index 6c3ad7913a6958..2625e5ab50ea09 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q11.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q11.groovy @@ -24,7 +24,7 @@ suite("q11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q12.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q12.groovy index 75846854982e85..8c38b4efa74520 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q12.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q12.groovy @@ -24,7 +24,7 @@ suite("q12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q13.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q13.groovy index bb49201e823132..50bf99bf78517b 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q13.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q13.groovy @@ -24,7 +24,7 @@ suite("q13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q14.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q14.groovy index 04f031f141ff23..c9d8069d599609 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q14.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q14.groovy @@ -24,7 +24,7 @@ suite("q14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q15.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q15.groovy index 4c4288133d3e70..c483cc2904df82 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q15.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q15.groovy @@ -24,7 +24,7 @@ suite("q15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q16.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q16.groovy index 4927b90c071a7f..eb8bc241664868 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q16.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q16.groovy @@ -24,7 +24,7 @@ suite("q16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q17.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q17.groovy index 748d16fc9d537d..2b95304f937f98 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q17.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q17.groovy @@ -24,7 +24,7 @@ suite("q17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q18.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q18.groovy index 9ad92d40f2e8ed..72cf02aa2bf0ca 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q18.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q18.groovy @@ -24,7 +24,7 @@ suite("q18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q19.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q19.groovy index 86b4ea66b18a6c..b3e25f3eae820c 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q19.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q19.groovy @@ -25,7 +25,7 @@ suite("q19") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q2.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q2.groovy index bde54070a72c3e..24cb9e8f27bb8e 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q2.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q2.groovy @@ -24,7 +24,7 @@ suite("q2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.groovy index 0f618516b8be62..1d9d5a60d7717c 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20-rewrite.groovy @@ -25,7 +25,7 @@ suite("q20-rewrite") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20.groovy index 50315e7a95aafd..dc9ca4b2c7b84a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q20.groovy @@ -25,7 +25,7 @@ suite("q20") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q21.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q21.groovy index ee7e8f9be4cd98..15a4aab2cc966f 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q21.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q21.groovy @@ -24,7 +24,7 @@ suite("q21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q22.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q22.groovy index c14fdbe4a4c172..ad024685825992 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q22.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q22.groovy @@ -24,7 +24,7 @@ suite("q22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q3.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q3.groovy index 836a30172eb0ec..2a10f46901f868 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q3.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q3.groovy @@ -25,7 +25,7 @@ suite("q3") { // db = "tpch" sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q4.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q4.groovy index de22ca1cafda73..646185008ad4be 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q4.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q4.groovy @@ -24,7 +24,7 @@ suite("q4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q5.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q5.groovy index ba5abe1ed4719e..5a5a7d751c29b6 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q5.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q5.groovy @@ -24,7 +24,7 @@ suite("q5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q6.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q6.groovy index 5a03ad454af74d..c75ff18edf27b2 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q6.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q6.groovy @@ -24,7 +24,7 @@ suite("q6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q7.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q7.groovy index 08c7532ba994bc..83fa3b38d8b3af 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q7.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q7.groovy @@ -24,7 +24,7 @@ suite("q7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q8.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q8.groovy index 56455092954780..8c28d22842e2b7 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q8.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q8.groovy @@ -24,7 +24,7 @@ suite("q8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q9.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q9.groovy index 8b5b664928891c..c9b9adc31d9eef 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q9.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/rf_prune/q9.groovy @@ -24,7 +24,7 @@ suite("q9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy index 3d9b2012693ce3..ca8ba0f6ebdaa8 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/runtime_filter/test_pushdown_setop.groovy @@ -24,7 +24,7 @@ suite("test_pushdown_setop") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'set be_number_for_test=3' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q1.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q1.groovy index 76bffda65f8c4a..5025d9b727e109 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q1.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q1.groovy @@ -24,7 +24,7 @@ suite("q1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q10.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q10.groovy index 89ca6dc051fd3f..49b54e182ba4a5 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q10.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q10.groovy @@ -24,7 +24,7 @@ suite("q10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q11.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q11.groovy index b21ba41bf4187b..8bce5af288c16a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q11.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q11.groovy @@ -24,7 +24,7 @@ suite("q11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q12.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q12.groovy index ced1db1a561937..9b52ed8654a7b3 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q12.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q12.groovy @@ -24,7 +24,7 @@ suite("q12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q13.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q13.groovy index b7eaca41ac5d36..8db08c47cd5faa 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q13.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q13.groovy @@ -24,7 +24,7 @@ suite("q13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q14.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q14.groovy index 5617134c86680c..bafb1178563b24 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q14.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q14.groovy @@ -24,7 +24,7 @@ suite("q14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q15.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q15.groovy index 02ad529f9b314d..951b36ead3cba0 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q15.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q15.groovy @@ -24,7 +24,7 @@ suite("q15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q16.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q16.groovy index 333b6d837cac19..3ce400937f5e98 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q16.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q16.groovy @@ -24,7 +24,7 @@ suite("q16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q17.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q17.groovy index 75ee0508980951..e0b7ebc8287faa 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q17.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q17.groovy @@ -24,7 +24,7 @@ suite("q17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q18.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q18.groovy index 425d7da6119e43..0e2f33f282e20a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q18.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q18.groovy @@ -24,7 +24,7 @@ suite("q18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q19.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q19.groovy index f5a3aa43a8dd29..8437a3c22d499c 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q19.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q19.groovy @@ -25,7 +25,7 @@ suite("q19") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q2.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q2.groovy index ae39c1116b805d..8ece3ee4833baf 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q2.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q2.groovy @@ -24,7 +24,7 @@ suite("q2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.groovy index 97e6a2272c60e4..a152f41ad646fd 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20-rewrite.groovy @@ -25,7 +25,7 @@ suite("q20-rewrite") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20.groovy index f24004cec88626..d505ffa6ce9a46 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q20.groovy @@ -25,7 +25,7 @@ suite("q20") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q21.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q21.groovy index 6bc96c09407a3e..b70e58f5ec8736 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q21.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q21.groovy @@ -24,7 +24,7 @@ suite("q21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q22.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q22.groovy index e4c5d7ceaa2411..562016209c6210 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q22.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q22.groovy @@ -24,7 +24,7 @@ suite("q22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q3.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q3.groovy index 3f4693ae74591c..5e94ea5fde8166 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q3.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q3.groovy @@ -25,7 +25,7 @@ suite("q3") { // db = "tpch" sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q4.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q4.groovy index 6a68d84009ddf7..df7cfcd83cfc9a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q4.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q4.groovy @@ -24,7 +24,7 @@ suite("q4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q5.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q5.groovy index b334dbbbf3bec7..c9b1ba4ec2d9ab 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q5.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q5.groovy @@ -24,7 +24,7 @@ suite("q5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q6.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q6.groovy index 5f0e58c5d0d421..40a1a85e28510a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q6.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q6.groovy @@ -24,7 +24,7 @@ suite("q6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q7.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q7.groovy index 36eacb5f6fd120..6a84a4dad9165a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q7.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q7.groovy @@ -24,7 +24,7 @@ suite("q7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q8.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q8.groovy index de9e89b2261f42..4f7e666347e99e 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q8.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q8.groovy @@ -24,7 +24,7 @@ suite("q8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q9.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q9.groovy index 7ee7e2ec186f7b..8f2d4ec03422d5 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q9.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape/q9.groovy @@ -24,7 +24,7 @@ suite("q9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q1.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q1.groovy index dfb5a6a0d2e83b..2f71665b476a44 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q1.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q1.groovy @@ -24,7 +24,7 @@ suite("q1") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql 'set exec_mem_limit=21G' sql 'SET enable_pipeline_engine = true' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.groovy index 30cc6abdbdef4c..11e487170fb3d3 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q10.groovy @@ -24,7 +24,7 @@ suite("q10") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q11.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q11.groovy index fd49b25488a426..10820bb7bde145 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q11.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q11.groovy @@ -24,7 +24,7 @@ suite("q11") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q12.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q12.groovy index 3b2b4fe1f536a3..e12d58d0512b36 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q12.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q12.groovy @@ -24,7 +24,7 @@ suite("q12") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.groovy index 72252a9779675b..b64ca543b18cab 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q13.groovy @@ -24,7 +24,7 @@ suite("q13") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.groovy index f9a0eeefd33fa0..e62d341e9e7cb1 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q14.groovy @@ -24,7 +24,7 @@ suite("q14") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.groovy index b79e8d3855e80e..fdb11b4a7b7710 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q15.groovy @@ -24,7 +24,7 @@ suite("q15") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q16.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q16.groovy index 2ebca626bbdf19..475bbf649f4680 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q16.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q16.groovy @@ -24,7 +24,7 @@ suite("q16") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.groovy index 06aa3299d5022f..70103ffaf94974 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q17.groovy @@ -24,7 +24,7 @@ suite("q17") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.groovy index 4f5a07cfbc1371..d8ff31ec4e3810 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q18.groovy @@ -24,7 +24,7 @@ suite("q18") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.groovy index c800d256d5c66f..fb6e97a63620ef 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q19.groovy @@ -25,7 +25,7 @@ suite("q19") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q2.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q2.groovy index ee920ca6e50d78..eefb3a7a467311 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q2.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q2.groovy @@ -24,7 +24,7 @@ suite("q2") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.groovy index c30a636d3ff73a..516966dd02c450 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20-rewrite.groovy @@ -25,7 +25,7 @@ suite("q20-rewrite") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set parallel_pipeline_task_num=8' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.groovy index 4007273901e112..a8831ab89bf8d7 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q20.groovy @@ -25,7 +25,7 @@ suite("q20") { sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q21.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q21.groovy index 89216bd88a2b39..216025826adc53 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q21.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q21.groovy @@ -24,7 +24,7 @@ suite("q21") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.groovy index e2f7e1096b1786..e31d0a2119c6f5 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q22.groovy @@ -24,7 +24,7 @@ suite("q22") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.groovy index 01c926f1efc768..e5051e34529c02 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q3.groovy @@ -25,7 +25,7 @@ suite("q3") { // db = "tpch" sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q4.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q4.groovy index 55ff31bb4577df..2fb70eca0f091b 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q4.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q4.groovy @@ -24,7 +24,7 @@ suite("q4") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" sql 'set exec_mem_limit=21G' diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.groovy index bdc6d994c27245..7932e0a90d92eb 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q5.groovy @@ -24,7 +24,7 @@ suite("q5") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q6.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q6.groovy index f1474a164444af..f2cb9d010f088a 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q6.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q6.groovy @@ -24,7 +24,7 @@ suite("q6") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.groovy index 2adc348b633426..e497df53000556 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q7.groovy @@ -24,7 +24,7 @@ suite("q7") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.groovy index f99324ba64cb5b..07effb1e2ba20b 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q8.groovy @@ -24,7 +24,7 @@ suite("q8") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" diff --git a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.groovy b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.groovy index 692afad084f535..5e3288966f0144 100644 --- a/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.groovy +++ b/regression-test/suites/new_shapes_p0/tpch_sf1000/shape_no_stats/q9.groovy @@ -24,7 +24,7 @@ suite("q9") { String db = context.config.getDbNameByFile(new File(context.file.parent)) sql "use ${db}" sql 'set enable_nereids_planner=true' - sql 'set enable_nereids_distribute_planner=true' + sql 'set enable_nereids_distribute_planner=false' sql 'set enable_fallback_to_original_planner=false' sql "set runtime_filter_mode='GLOBAL'" From 34d83429949b5b1866450ef3200a44b8a8cae18b Mon Sep 17 00:00:00 2001 From: koarz Date: Sat, 28 Jun 2025 17:30:33 +0800 Subject: [PATCH 106/572] branch-3.0: [Enhancement] add information_schema backend_tablets table (#52195) https://github.com/apache/doris/pull/51282 --- be/src/cloud/cloud_schema_change_job.cpp | 9 +- be/src/cloud/cloud_tablet.cpp | 4 + be/src/cloud/cloud_tablet.h | 2 + be/src/cloud/cloud_tablet_mgr.cpp | 7 + be/src/cloud/cloud_tablet_mgr.h | 2 + be/src/exec/schema_scanner.cpp | 3 + .../schema_scanner/schema_scanner_helper.cpp | 43 +++- .../schema_scanner/schema_scanner_helper.h | 5 +- .../schema_scanner/schema_tablets_scanner.cpp | 227 ++++++++++++++++++ .../schema_scanner/schema_tablets_scanner.h | 56 +++++ be/src/olap/base_tablet.h | 12 + be/src/olap/tablet.h | 17 +- .../doris/analysis/SchemaTableType.java | 1 + .../org/apache/doris/catalog/SchemaTable.java | 23 +- .../BackendPartitionedSchemaScanNode.java | 1 + gensrc/thrift/Descriptors.thrift | 3 +- .../info_schema_db/backend_tablets.out | 4 + .../info_schema_db/backend_tablets.groovy | 126 ++++++++++ 18 files changed, 516 insertions(+), 29 deletions(-) create mode 100644 be/src/exec/schema_scanner/schema_tablets_scanner.cpp create mode 100644 be/src/exec/schema_scanner/schema_tablets_scanner.h create mode 100644 regression-test/data/external_table_p0/info_schema_db/backend_tablets.out create mode 100644 regression-test/suites/external_table_p0/info_schema_db/backend_tablets.groovy diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index 05d29383af4213..a5707e51bb64a6 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -80,6 +80,13 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque std::unique_lock schema_change_lock(_base_tablet->get_schema_change_lock(), std::try_to_lock); + _new_tablet->set_alter_failed(false); + Defer defer([this] { + // if tablet state is not TABLET_RUNNING when return, indicates that alter has failed. + if (_new_tablet->tablet_state() != TABLET_RUNNING) { + _new_tablet->set_alter_failed(true); + } + }); if (!schema_change_lock.owns_lock()) { LOG(WARNING) << "Failed to obtain schema change lock. base_tablet=" << request.base_tablet_id; @@ -134,7 +141,7 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque RETURN_IF_ERROR(_base_tablet->capture_rs_readers({2, start_resp.alter_version()}, &rs_splits, false)); } - Defer defer {[&]() { + Defer defer2 {[&]() { _new_tablet->set_alter_version(-1); _base_tablet->set_alter_version(-1); }}; diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index a26ce0dd2c7172..582a4b771265d2 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -86,6 +86,10 @@ Status CloudTablet::capture_consistent_rowsets_unlocked( return _capture_consistent_rowsets_unlocked(version_path, rowsets); } +std::string CloudTablet::tablet_path() const { + return ""; +} + Status CloudTablet::capture_rs_readers(const Version& spec_version, std::vector* rs_splits, bool skip_missing_version) { diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index f3af8b09b27869..2dd1d3c4425a3a 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -62,6 +62,8 @@ class CloudTablet final : public BaseTablet { return _approximate_data_size.load(std::memory_order_relaxed); } + std::string tablet_path() const override; + // clang-format off int64_t fetch_add_approximate_num_rowsets (int64_t x) { return _approximate_num_rowsets .fetch_add(x, std::memory_order_relaxed); } int64_t fetch_add_approximate_num_segments(int64_t x) { return _approximate_num_segments.fetch_add(x, std::memory_order_relaxed); } diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index efcfdde73616f8..cbf9a29ee907ff 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -508,6 +508,13 @@ void CloudTabletMgr::get_topn_tablet_delete_bitmap_score( << max_base_rowset_delete_bitmap_score_tablet_id << ",tablets=[" << ss.str() << "]"; } +std::vector> CloudTabletMgr::get_all_tablet() { + std::vector> tablets; + tablets.reserve(_tablet_map->size()); + _tablet_map->traverse([&tablets](auto& t) { tablets.push_back(t); }); + return tablets; +} + void CloudTabletMgr::put_tablet_for_UT(std::shared_ptr tablet) { _tablet_map->put(tablet); } diff --git a/be/src/cloud/cloud_tablet_mgr.h b/be/src/cloud/cloud_tablet_mgr.h index c8ff133df821fb..a1ce6d2b8cfb1b 100644 --- a/be/src/cloud/cloud_tablet_mgr.h +++ b/be/src/cloud/cloud_tablet_mgr.h @@ -91,6 +91,8 @@ class CloudTabletMgr { // **ATTN: JUST FOR UT** void put_tablet_for_UT(std::shared_ptr tablet); + std::vector> get_all_tablet(); + private: CloudStorageEngine& _engine; diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp index e9d068baca6102..5f60ffb7bc6c2a 100644 --- a/be/src/exec/schema_scanner.cpp +++ b/be/src/exec/schema_scanner.cpp @@ -49,6 +49,7 @@ #include "exec/schema_scanner/schema_table_privileges_scanner.h" #include "exec/schema_scanner/schema_table_properties_scanner.h" #include "exec/schema_scanner/schema_tables_scanner.h" +#include "exec/schema_scanner/schema_tablets_scanner.h" #include "exec/schema_scanner/schema_user_privileges_scanner.h" #include "exec/schema_scanner/schema_user_scanner.h" #include "exec/schema_scanner/schema_variables_scanner.h" @@ -247,6 +248,8 @@ std::unique_ptr SchemaScanner::create(TSchemaTableType::type type return SchemaCatalogMetaCacheStatsScanner::create_unique(); case TSchemaTableType::SCH_ROUTINE_LOAD_JOBS: return SchemaRoutineLoadJobScanner::create_unique(); + case TSchemaTableType::SCH_BACKEND_TABLETS: + return SchemaTabletsScanner::create_unique(); default: return SchemaDummyScanner::create_unique(); break; diff --git a/be/src/exec/schema_scanner/schema_scanner_helper.cpp b/be/src/exec/schema_scanner/schema_scanner_helper.cpp index 0fea9d8c39f328..5f4af06c52d3ab 100644 --- a/be/src/exec/schema_scanner/schema_scanner_helper.cpp +++ b/be/src/exec/schema_scanner/schema_scanner_helper.cpp @@ -30,29 +30,54 @@ namespace doris { void SchemaScannerHelper::insert_string_value(int col_index, std::string str_val, vectorized::Block* block) { vectorized::MutableColumnPtr mutable_col_ptr; - mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + mutable_col_ptr = block->get_by_position(col_index).column->assume_mutable(); auto* nullable_column = assert_cast(mutable_col_ptr.get()); vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); assert_cast(col_ptr)->insert_data(str_val.data(), str_val.size()); nullable_column->get_null_map_data().emplace_back(0); } -void SchemaScannerHelper::insert_datetime_value(int col_index, const std::vector& datas, - vectorized::Block* block) { +void SchemaScannerHelper::insert_datetime_value(int col_index, int64_t timestamp, + const std::string& ctz, vectorized::Block* block) { vectorized::MutableColumnPtr mutable_col_ptr; - mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + mutable_col_ptr = block->get_by_position(col_index).column->assume_mutable(); auto* nullable_column = assert_cast(mutable_col_ptr.get()); vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); - auto data = datas[0]; - assert_cast*>(col_ptr)->insert_data( - reinterpret_cast(data), 0); + + std::vector datas(1); + VecDateTimeValue src[1]; + src[0].from_unixtime(timestamp, ctz); + datas[0] = src; + auto* data = datas[0]; + assert_cast(col_ptr)->insert_data(reinterpret_cast(data), + 0); + nullable_column->get_null_map_data().emplace_back(0); +} + +void SchemaScannerHelper::insert_bool_value(int col_index, bool bool_val, + vectorized::Block* block) { + vectorized::MutableColumnPtr mutable_col_ptr; + mutable_col_ptr = block->get_by_position(col_index).column->assume_mutable(); + auto* nullable_column = assert_cast(mutable_col_ptr.get()); + vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); + assert_cast(col_ptr)->insert_value(bool_val); + nullable_column->get_null_map_data().emplace_back(0); +} + +void SchemaScannerHelper::insert_int32_value(int col_index, int32_t int_val, + vectorized::Block* block) { + vectorized::MutableColumnPtr mutable_col_ptr; + mutable_col_ptr = block->get_by_position(col_index).column->assume_mutable(); + auto* nullable_column = assert_cast(mutable_col_ptr.get()); + vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); + assert_cast(col_ptr)->insert_value(int_val); nullable_column->get_null_map_data().emplace_back(0); } void SchemaScannerHelper::insert_int64_value(int col_index, int64_t int_val, vectorized::Block* block) { vectorized::MutableColumnPtr mutable_col_ptr; - mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + mutable_col_ptr = block->get_by_position(col_index).column->assume_mutable(); auto* nullable_column = assert_cast(mutable_col_ptr.get()); vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); assert_cast*>(col_ptr)->insert_value(int_val); @@ -62,7 +87,7 @@ void SchemaScannerHelper::insert_int64_value(int col_index, int64_t int_val, void SchemaScannerHelper::insert_double_value(int col_index, double double_val, vectorized::Block* block) { vectorized::MutableColumnPtr mutable_col_ptr; - mutable_col_ptr = std::move(*block->get_by_position(col_index).column).assume_mutable(); + mutable_col_ptr = block->get_by_position(col_index).column->assume_mutable(); auto* nullable_column = assert_cast(mutable_col_ptr.get()); vectorized::IColumn* col_ptr = &nullable_column->get_nested_column(); assert_cast*>(col_ptr)->insert_value(double_val); diff --git a/be/src/exec/schema_scanner/schema_scanner_helper.h b/be/src/exec/schema_scanner/schema_scanner_helper.h index f7b47ede91bb5d..aed14677363965 100644 --- a/be/src/exec/schema_scanner/schema_scanner_helper.h +++ b/be/src/exec/schema_scanner/schema_scanner_helper.h @@ -32,9 +32,12 @@ class Block; class SchemaScannerHelper { public: static void insert_string_value(int col_index, std::string str_val, vectorized::Block* block); - static void insert_datetime_value(int col_index, const std::vector& datas, + static void insert_datetime_value(int col_index, int64_t timestamp, const std::string& ctz, vectorized::Block* block); + static void insert_bool_value(int col_index, bool bool_val, vectorized::Block* block); + + static void insert_int32_value(int col_index, int32_t int_val, vectorized::Block* block); static void insert_int64_value(int col_index, int64_t int_val, vectorized::Block* block); static void insert_double_value(int col_index, double double_val, vectorized::Block* block); }; diff --git a/be/src/exec/schema_scanner/schema_tablets_scanner.cpp b/be/src/exec/schema_scanner/schema_tablets_scanner.cpp new file mode 100644 index 00000000000000..972492e9151b08 --- /dev/null +++ b/be/src/exec/schema_scanner/schema_tablets_scanner.cpp @@ -0,0 +1,227 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/schema_scanner/schema_tablets_scanner.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" +#include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" +#include "common/status.h" +#include "exec/schema_scanner.h" +#include "exec/schema_scanner/schema_scanner_helper.h" +#include "olap/storage_engine.h" +#include "olap/tablet_fwd.h" +#include "olap/tablet_manager.h" +#include "runtime/define_primitive_type.h" +#include "runtime/exec_env.h" +#include "runtime/runtime_state.h" +#include "vec/common/string_ref.h" + +namespace doris { +namespace vectorized { +class Block; +} // namespace vectorized + +#include "common/compile_check_begin.h" + +std::vector SchemaTabletsScanner::_s_tbls_columns = { + // name, type, size, is_null + {"BE_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"TABLET_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"REPLICA_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"PARTITION_ID", TYPE_BIGINT, sizeof(int64_t), true}, + {"TABLET_PATH", TYPE_STRING, sizeof(StringRef), true}, + {"TABLET_LOCAL_SIZE", TYPE_BIGINT, sizeof(int64_t), true}, + {"TABLET_REMOTE_SIZE", TYPE_BIGINT, sizeof(int64_t), true}, + {"VERSION_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"SEGMENT_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"NUM_COLUMNS", TYPE_BIGINT, sizeof(int64_t), true}, + {"ROW_SIZE", TYPE_BIGINT, sizeof(int64_t), true}, + {"COMPACTION_SCORE", TYPE_INT, sizeof(int32_t), true}, + {"COMPRESS_KIND", TYPE_STRING, sizeof(StringRef), true}, + {"IS_USED", TYPE_BOOLEAN, sizeof(bool), true}, + {"IS_ALTER_FAILED", TYPE_BOOLEAN, sizeof(bool), true}, + {"CREATE_TIME", TYPE_DATETIME, sizeof(int64_t), true}, + {"UPDATE_TIME", TYPE_DATETIME, sizeof(int64_t), true}, + {"IS_OVERLAP", TYPE_BOOLEAN, sizeof(bool), true}, +}; + +SchemaTabletsScanner::SchemaTabletsScanner() + : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_BACKEND_TABLETS) {}; + +Status SchemaTabletsScanner::start(RuntimeState* state) { + if (!_is_init) { + return Status::InternalError("used before initialized."); + } + _backend_id = state->backend_id(); + RETURN_IF_ERROR(_get_all_tablets()); + return Status::OK(); +} + +Status SchemaTabletsScanner::_get_all_tablets() { + if (config::is_cloud_mode()) { + auto tablets = + ExecEnv::GetInstance()->storage_engine().to_cloud().tablet_mgr().get_all_tablet(); + std::ranges::for_each(tablets, [&](auto& tablet) { + _tablets.push_back(std::static_pointer_cast(tablet)); + }); + } else { + auto tablets = ExecEnv::GetInstance() + ->storage_engine() + .to_local() + .tablet_manager() + ->get_all_tablet(); + std::ranges::for_each(tablets, [&](auto& tablet) { + _tablets.push_back(std::static_pointer_cast(tablet)); + }); + } + return Status::OK(); +} + +Status SchemaTabletsScanner::get_next_block_internal(vectorized::Block* block, bool* eos) { + if (!_is_init) { + return Status::InternalError("Used before initialized."); + } + if (nullptr == block || nullptr == eos) { + return Status::InternalError("input pointer is nullptr."); + } + *eos = true; + return _fill_block_impl(block); +} + +Status SchemaTabletsScanner::_fill_block_impl(vectorized::Block* block) { + SCOPED_TIMER(_fill_block_timer); + + size_t row_num = _tablets.size(); + if (row_num == 0) { + return Status::OK(); + } + + size_t fill_tablets_num = _tablets.size(); + std::vector datas(fill_tablets_num); + + for (int i = 0; i < _tablets.size(); i++) { + BaseTabletSPtr tablet = _tablets[i]; + // BE_ID + SchemaScannerHelper::insert_int64_value(0, _backend_id, block); + + // TABLET_ID + SchemaScannerHelper::insert_int64_value(1, tablet->tablet_meta()->tablet_id(), block); + + // REPLICA_ID + SchemaScannerHelper::insert_int64_value(2, tablet->tablet_meta()->replica_id(), block); + + // PARTITION_ID + SchemaScannerHelper::insert_int64_value(3, tablet->tablet_meta()->partition_id(), block); + + // TABLET_PATH + SchemaScannerHelper::insert_string_value(4, tablet->tablet_path(), block); + + // TABLET_LOCAL_SIZE + SchemaScannerHelper::insert_int64_value(5, tablet->tablet_meta()->tablet_local_size(), + block); + + // TABLET_REMOTE_SIZE + SchemaScannerHelper::insert_int64_value(6, tablet->tablet_meta()->tablet_remote_size(), + block); + + // VERSION_COUNT + SchemaScannerHelper::insert_int64_value( + 7, static_cast(tablet->tablet_meta()->version_count()), block); + + // SEGMENT_COUNT + SchemaScannerHelper::insert_int64_value( + 8, + [&tablet]() { + auto rs_metas = tablet->tablet_meta()->all_rs_metas(); + return std::accumulate(rs_metas.begin(), rs_metas.end(), 0, + [](int64_t val, RowsetMetaSharedPtr& rs_meta) { + return val + rs_meta->num_segments(); + }); + }(), + block); + + // NUM_COLUMNS + SchemaScannerHelper::insert_int64_value(9, tablet->tablet_meta()->tablet_columns_num(), + block); + + // ROW_SIZE + SchemaScannerHelper::insert_int64_value(10, static_cast(tablet->row_size()), + block); + + // COMPACTION_SCORE + SchemaScannerHelper::insert_int32_value(11, tablet->get_real_compaction_score(), block); + + // COMPRESS_KIND + SchemaScannerHelper::insert_string_value(12, CompressKind_Name(tablet->compress_kind()), + block); + + // IS_USED + SchemaScannerHelper::insert_bool_value( + 13, + [&tablet]() { + if (config::is_cloud_mode()) { + return true; + } + return std::static_pointer_cast(tablet)->is_used(); + }(), + block); + + // IS_ALTER_FAILED + SchemaScannerHelper::insert_bool_value(14, tablet->is_alter_failed(), block); + + // CREATE_TIME + SchemaScannerHelper::insert_datetime_value(15, tablet->tablet_meta()->creation_time(), + TimezoneUtils::default_time_zone, block); + + // UPDATE_TIME + SchemaScannerHelper::insert_datetime_value( + 16, + [&tablet]() { + auto rowset = tablet->get_rowset_with_max_version(); + return rowset == nullptr ? 0 : rowset->newest_write_timestamp(); + }(), + TimezoneUtils::default_time_zone, block); + + // IS_OVERLAP + SchemaScannerHelper::insert_bool_value( + 17, + [&tablet]() { + const auto& rs_metas = tablet->tablet_meta()->all_rs_metas(); + return std::any_of(rs_metas.begin(), rs_metas.end(), + [](const RowsetMetaSharedPtr& rs_meta) { + return rs_meta->is_segments_overlapping(); + }); + }(), + block); + } + + return Status::OK(); +} +} // namespace doris diff --git a/be/src/exec/schema_scanner/schema_tablets_scanner.h b/be/src/exec/schema_scanner/schema_tablets_scanner.h new file mode 100644 index 00000000000000..dc8b77067244be --- /dev/null +++ b/be/src/exec/schema_scanner/schema_tablets_scanner.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "exec/schema_scanner.h" +#include "olap/tablet.h" + +namespace doris { +class RuntimeState; + +namespace vectorized { +class Block; +} // namespace vectorized + +class SchemaTabletsScanner : public SchemaScanner { + ENABLE_FACTORY_CREATOR(SchemaTabletsScanner) + +public: + SchemaTabletsScanner(); + + ~SchemaTabletsScanner() override = default; + + Status start(RuntimeState* state) override; + + Status get_next_block_internal(vectorized::Block* block, bool* eos) override; + +private: + Status _get_all_tablets(); + + Status _fill_block_impl(vectorized::Block* block); + + int64_t _backend_id {}; + std::vector _tablets; + static std::vector _s_tbls_columns; +}; +} // namespace doris diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 4df16de7eb35ef..9dd69d0bd9a20c 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -17,6 +17,8 @@ #pragma once +#include + #include #include #include @@ -63,10 +65,12 @@ class BaseTablet { int64_t partition_id() const { return _tablet_meta->partition_id(); } int64_t tablet_id() const { return _tablet_meta->tablet_id(); } int32_t schema_hash() const { return _tablet_meta->schema_hash(); } + size_t row_size() const { return _tablet_meta->tablet_schema()->row_size(); } KeysType keys_type() const { return _tablet_meta->tablet_schema()->keys_type(); } size_t num_key_columns() const { return _tablet_meta->tablet_schema()->num_key_columns(); } int64_t ttl_seconds() const { return _tablet_meta->ttl_seconds(); } std::mutex& get_schema_change_lock() { return _schema_change_lock; } + CompressKind compress_kind() const { return _tablet_meta->tablet_schema()->compress_kind(); } bool enable_unique_key_merge_on_write() const { #ifdef BE_TEST if (_tablet_meta == nullptr) { @@ -93,6 +97,11 @@ class BaseTablet { return _max_version_schema; } + void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; } + bool is_alter_failed() { return _alter_failed; } + + virtual std::string tablet_path() const = 0; + virtual bool exceed_version_limit(int32_t limit) = 0; virtual Result> create_rowset_writer(RowsetWriterContext& context, @@ -334,6 +343,9 @@ class BaseTablet { const TabletMetaSharedPtr _tablet_meta; TabletSchemaSPtr _max_version_schema; + // `_alter_failed` is used to indicate whether the tablet failed to perform a schema change + std::atomic _alter_failed = false; + // metrics of this tablet std::shared_ptr _metric_entity; diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 35993116bf68c1..467fc51f98517c 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -116,7 +116,7 @@ class Tablet final : public BaseTablet { DataDir* data_dir() const { return _data_dir; } int64_t replica_id() const { return _tablet_meta->replica_id(); } - const std::string& tablet_path() const { return _tablet_path; } + std::string tablet_path() const override { return _tablet_path; } bool set_tablet_schema_into_rowset_meta(); Status init(); @@ -160,10 +160,8 @@ class Tablet final : public BaseTablet { size_t num_null_columns() const; size_t num_short_key_columns() const; size_t num_rows_per_row_block() const; - CompressKind compress_kind() const; double bloom_filter_fpp() const; size_t next_unique_id() const; - size_t row_size() const; int64_t avg_rs_meta_serialize_size() const; // operation in rowsets @@ -484,9 +482,6 @@ class Tablet final : public BaseTablet { void set_binlog_config(BinlogConfig binlog_config); - void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; } - bool is_alter_failed() { return _alter_failed; } - void set_is_full_compaction_running(bool is_full_compaction_running) { _is_full_compaction_running = is_full_compaction_running; } @@ -620,8 +615,6 @@ class Tablet final : public BaseTablet { // may delete compaction input rowsets. std::mutex _cold_compaction_lock; int64_t _last_failed_follow_cooldown_time = 0; - // `_alter_failed` is used to indicate whether the tablet failed to perform a schema change - std::atomic _alter_failed = false; int64_t _io_error_times = 0; @@ -743,10 +736,6 @@ inline size_t Tablet::num_rows_per_row_block() const { return _tablet_meta->tablet_schema()->num_rows_per_row_block(); } -inline CompressKind Tablet::compress_kind() const { - return _tablet_meta->tablet_schema()->compress_kind(); -} - inline double Tablet::bloom_filter_fpp() const { return _tablet_meta->tablet_schema()->bloom_filter_fpp(); } @@ -755,10 +744,6 @@ inline size_t Tablet::next_unique_id() const { return _tablet_meta->tablet_schema()->next_column_unique_id(); } -inline size_t Tablet::row_size() const { - return _tablet_meta->tablet_schema()->row_size(); -} - inline int64_t Tablet::avg_rs_meta_serialize_size() const { return _tablet_meta->avg_rs_meta_serialize_size(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java index 0ec8f6b851ff23..ea50de3f83f419 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SchemaTableType.java @@ -67,6 +67,7 @@ public enum SchemaTableType { SCH_CREATE_TABLE("CREATE_TABLE", "CREATE_TABLE", TSchemaTableType.SCH_CREATE_TABLE), SCH_INVALID("NULL", "NULL", TSchemaTableType.SCH_INVALID), SCH_ROWSETS("ROWSETS", "ROWSETS", TSchemaTableType.SCH_ROWSETS), + SCH_TABLETS("BACKEND_TABLETS", "BACKEND_TABLETS", TSchemaTableType.SCH_BACKEND_TABLETS), SCH_PARAMETERS("PARAMETERS", "PARAMETERS", TSchemaTableType.SCH_PARAMETERS), SCH_METADATA_NAME_IDS("METADATA_NAME_IDS", "METADATA_NAME_IDS", TSchemaTableType.SCH_METADATA_NAME_IDS), SCH_PROFILING("PROFILING", "PROFILING", TSchemaTableType.SCH_PROFILING), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index 31a80d3daf6c6d..ea8ccf8e787dae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -477,7 +477,7 @@ public class SchemaTable extends Table { .column("CURRENT_USED_MEMORY_BYTES", ScalarType.createType(PrimitiveType.BIGINT)) .column("SHUFFLE_SEND_BYTES", ScalarType.createType(PrimitiveType.BIGINT)) .column("SHUFFLE_SEND_ROWS", ScalarType.createType(PrimitiveType.BIGINT)) - .column("QUERY_TYPE", ScalarType.createVarchar(256)) + .column("QUERY_TYPE", ScalarType.createVarchar(256)) .build())) .put("active_queries", new SchemaTable(SystemIdGenerator.getNextId(), "active_queries", TableType.SCHEMA, builder().column("QUERY_ID", ScalarType.createVarchar(256)) @@ -624,6 +624,27 @@ public class SchemaTable extends Table { .column("IS_ABNORMAL_PAUSE", ScalarType.createType(PrimitiveType.BOOLEAN)) .build()) ) + .put("backend_tablets", new SchemaTable(SystemIdGenerator.getNextId(), "backend_tablets", TableType.SCHEMA, + builder().column("BE_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("TABLET_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("REPLICA_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("PARTITION_ID", ScalarType.createType(PrimitiveType.BIGINT)) + .column("TABLET_PATH", ScalarType.createStringType()) + .column("TABLET_LOCAL_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) + .column("TABLET_REMOTE_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) + .column("VERSION_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("SEGMENT_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("NUM_COLUMNS", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ROW_SIZE", ScalarType.createType(PrimitiveType.BIGINT)) + .column("COMPACTION_SCORE", ScalarType.createType(PrimitiveType.INT)) + .column("COMPRESS_KIND", ScalarType.createStringType()) + .column("IS_USED", ScalarType.createType(PrimitiveType.BOOLEAN)) + .column("IS_ALTER_FAILED", ScalarType.createType(PrimitiveType.BOOLEAN)) + .column("CREATE_TIME", ScalarType.createType(PrimitiveType.DATETIME)) + .column("UPDATE_TIME", ScalarType.createType(PrimitiveType.DATETIME)) + .column("IS_OVERLAP", ScalarType.createType(PrimitiveType.BOOLEAN)) + .build()) + ) .build(); private boolean fetchAllFe = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java index bc73cd8ce1ba58..8e96b04d3ebdd9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BackendPartitionedSchemaScanNode.java @@ -70,6 +70,7 @@ public class BackendPartitionedSchemaScanNode extends SchemaScanNode { BEACKEND_ID_COLUMN_SET.add("be_id"); BACKEND_TABLE.add("file_cache_statistics"); + BACKEND_TABLE.add("backend_tablets"); BACKEND_TABLE.add("backend_configuration"); } diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index 9c5e6f075b9006..c76e64641dc9f2 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -141,7 +141,8 @@ enum TSchemaTableType { SCH_CATALOG_META_CACHE_STATISTICS = 52; // consistent with the master SCH_ROUTINE_LOAD_JOBS = 54, - SCH_BACKEND_CONFIGURATION=55; + SCH_BACKEND_CONFIGURATION=55, + SCH_BACKEND_TABLETS = 56; } enum THdfsCompression { diff --git a/regression-test/data/external_table_p0/info_schema_db/backend_tablets.out b/regression-test/data/external_table_p0/info_schema_db/backend_tablets.out new file mode 100644 index 00000000000000..6a79ef81689cc7 --- /dev/null +++ b/regression-test/data/external_table_p0/info_schema_db/backend_tablets.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +true + diff --git a/regression-test/suites/external_table_p0/info_schema_db/backend_tablets.groovy b/regression-test/suites/external_table_p0/info_schema_db/backend_tablets.groovy new file mode 100644 index 00000000000000..cd194d958e926a --- /dev/null +++ b/regression-test/suites/external_table_p0/info_schema_db/backend_tablets.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("backend_tablets", "p0, external_table,information_schema,backend_tablets") { + if (!isCloudMode()) { + def dbName = "test_backend_tablets_db" + def tbName1 = "test_backend_tablets_1" + def tbName2 = "test_backend_tablets_2" + def tbName3 = "test_backend_tablets_3" + sql(" drop database IF EXISTS ${dbName}") + sql(" create database ${dbName}") + + sql("use ${dbName}") + sql("drop table IF EXISTS ${tbName1}") + sql("drop table IF EXISTS ${tbName2}") + sql("drop table IF EXISTS ${tbName3}") + + sql """ + CREATE TABLE IF NOT EXISTS `${tbName1}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` varchar(20) NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + CREATE TABLE IF NOT EXISTS `${tbName2}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` string NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` SMALLINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + CREATE TABLE IF NOT EXISTS `${tbName3}` ( + `aaa` varchar(170) NOT NULL COMMENT "", + `bbb` string NOT NULL COMMENT "", + `ccc` INT NULL COMMENT "", + `ddd` BIGINT NULL COMMENT "" + ) + DISTRIBUTED BY HASH(`aaa`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO `${tbName1}` (aaa, bbb, ccc, ddd) VALUES + ('value1', 'string1', 100, 10), + ('value2', 'string2', 200, 20), + ('value3', 'string3', 300, 30), + ('value4', 'string4', 400, 40), + ('value5', 'string5', 500, 50), + ('value6', 'string6', 600, 60), + ('value7', 'string7', 700, 70), + ('value8', 'string8', 800, 80), + ('value9', 'string9', 900, 90), + ('value10', 'string10', 1000, 100); + """ + + sql """ + INSERT INTO `${tbName2}` (aaa, bbb, ccc, ddd) VALUES + ('value1', 'string1', 100, 10), + ('value2', 'string2', 200, 20), + ('value3', 'string3', 300, 30), + ('value4', 'string4', 400, 40), + ('value5', 'string5', 500, 50), + ('value6', 'string6', 600, 60), + ('value7', 'string7', 700, 70), + ('value8', 'string8', 800, 80), + ('value9', 'string9', 900, 90), + ('value10', 'string10', 1000, 100); + """ + + sql """ + INSERT INTO `${tbName3}` (aaa, bbb, ccc, ddd) VALUES + ('value1', 'string1', 100, 10), + ('value2', 'string2', 200, 20), + ('value3', 'string3', 300, 30), + ('value4', 'string4', 400, 40), + ('value5', 'string5', 500, 50), + ('value6', 'string6', 600, 60), + ('value7', 'string7', 700, 70), + ('value8', 'string8', 800, 80), + ('value9', 'string9', 900, 90), + ('value10', 'string10', 1000, 100); + """ + + sql("use information_schema") + order_qt_1 """ + SELECT + CASE + WHEN tablets_count.count_result >= 3 THEN 'true' + ELSE 'false' + END AS result + FROM + (SELECT COUNT(*) AS count_result FROM backend_tablets) AS tablets_count; + """ + + } +} + From d252afd649630e6fcaacd86cbdea812ce7dd89a4 Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 30 Jun 2025 10:36:54 +0800 Subject: [PATCH 107/572] branch-3.0: [enhance](orc) Printf OrcReader SearchArgument in profile (#52438) Printf OrcReader SearchArgument in profile --- be/src/vec/exec/format/orc/vorc_reader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 891ec81e992fef..c7b4467f679ee4 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -742,6 +742,7 @@ bool OrcReader::_init_search_argument( std::unique_ptr builder = orc::SearchArgumentFactory::newBuilder(); if (build_search_argument(predicates, 0, builder)) { std::unique_ptr sargs = builder->build(); + _profile->add_info_string("OrcReader SearchArgument: ", sargs->toString()); _row_reader_options.searchArgument(std::move(sargs)); return true; } else { From e68e7cba992077ae18b548ae66e72f493126e866 Mon Sep 17 00:00:00 2001 From: seawinde Date: Mon, 30 Jun 2025 11:47:22 +0800 Subject: [PATCH 108/572] [fix](mtmv) Fix compensate union all wrongly when partition mv use partition external table (#52223) Fix compensate union all wrongly when partition mv use partition external table in test mv/external_table/part_partition_invalid.groovy order_qt_after_modify_data_and_refresh_catalog data should be as following: ``` 1 2 1 2023-10-17 2023-10-17 2 2 2 2023-10-18 2023-10-18 3 2 3 2023-10-19 2023-10-19 ``` Related PR: #49514 --- .../nereids/rules/exploration/mv/PartitionCompensator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java index 0ee88ce82af070..c8288ed6c8d890 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionCompensator.java @@ -85,8 +85,10 @@ public static Pair>, Map mvValidPartitions = cascadesContext.getStatementContext() From d6e78219d3c321f712d53052d34b1e93015a6f7e Mon Sep 17 00:00:00 2001 From: zfr95 Date: Mon, 30 Jun 2025 11:54:14 +0800 Subject: [PATCH 109/572] [test](mv)forbid create mv test case (#52348) pick from #51777 [test](mv)forbid create mv test case --- .../mv_negative/agg_negative_test.groovy | 175 ++++++++++++++++++ .../mv_negative/dup_negative_test.groovy | 151 +++++++++++++++ .../mv_negative/mor_negative_test.groovy | 156 ++++++++++++++++ .../mv_negative/mow_negative_test.groovy | 157 ++++++++++++++++ 4 files changed, 639 insertions(+) create mode 100644 regression-test/suites/mv_p0/mv_negative/agg_negative_test.groovy create mode 100644 regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy create mode 100644 regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy create mode 100644 regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy diff --git a/regression-test/suites/mv_p0/mv_negative/agg_negative_test.groovy b/regression-test/suites/mv_p0/mv_negative/agg_negative_test.groovy new file mode 100644 index 00000000000000..b85275264ba5ea --- /dev/null +++ b/regression-test/suites/mv_p0/mv_negative/agg_negative_test.groovy @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("agg_negative_mv_test", "mv_negative") { + + String db = context.config.getDbNameByFile(context.file) + def prefix_str = "mv_agg_negative" + def tb_name = prefix_str + "_tb" + + sql """set enable_agg_state=true;""" + sql """drop table if exists ${tb_name};""" + sql """ + CREATE TABLE `${tb_name}` ( + `col1` datetime NULL, + `col2` varchar(60) NULL, + `col3` int(11) NOT NULL, + `col4` boolean NULL, + `col15` ipv4 NULL, + `col5` string REPLACE NULL, + `col6` ARRAY REPLACE NULL COMMENT "", + `col7` int(11) SUM NULL DEFAULT "0", + `col8` int(11) min NULL DEFAULT "0", + `col9` int(11) max NULL DEFAULT "0", + `col10` int(11) REPLACE NULL, + `col11` bitmap BITMAP_UNION, + `col12` agg_state generic, + `col13` hll hll_union NOT NULL COMMENT "hll", + `col14` ipv4 REPLACE NULL + ) ENGINE=OLAP + AGGREGATE KEY(`col1`, `col2`, `col3`, `col4`, `col15`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`col2`, `col3`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """insert into ${tb_name} values + ("2023-08-16 22:28:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax1",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",2,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,0,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd2",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[5,4,3,2,1], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 3, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 4, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 5, 6, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(2), max_by_state(3,1), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(30,100), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(100), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'","asd",[1,2,3,4,5], 1, 1, 1, 1, to_bitmap(243), max_by_state(3,1), HLL_HASH(1), "'255.255.255.255'"), + ("2023-08-16 24:27:00","ax1",2,0,"'0.0.0.0'","asd",[5,4,3,2,1], 3, 4, 5, 6, to_bitmap(2), max_by_state(30,100), HLL_HASH(100), "'255.255.255.255'"), + ("2024-08-17 22:27:00","ax2",3,1,"'0.0.0.0'","asd3",[1,2,3,4,6], 7, 8, 9, 10, to_bitmap(3), max_by_state(6,2), HLL_HASH(1000), "'0.0.1.0'"), + ("2023-09-16 22:27:00","ax4",4,0,"'0.0.0.0'","asd2",[1,2,9,4,5], 11, 11, 11, 11, to_bitmap(4), max_by_state(3,1), HLL_HASH(1), "'0.10.0.0'");""" + + def mv_name = """${prefix_str}_mv""" + def no_mv_name = """no_${prefix_str}_mv""" + def mtmv_sql = """select col4, col1, col2, col3, col15, sum(col7) from ${tb_name} where col1 = "2023-08-16 22:27:00" group by col4, col1, col2, col3, col15 order by col4, col1, col2, col3, col15""" + create_sync_mv(db, tb_name, mv_name, mtmv_sql) + + def desc_res = sql """desc ${tb_name} all;""" + for (int i = 0; i < desc_res.size(); i++) { + if (desc_res[i][0] == mv_name) { + for (int j = i; j < i+6; j++) { + if (desc_res[j][2] != "mva_SUM__CAST(`col7` AS bigint)") { + assertTrue(desc_res[j][6] == "true") + } else { + assertTrue(desc_res[j][6] == "false") + } + } + break + } + } + def sql_hit = """select col1, col2, col3, sum(col7) from ${tb_name} where col1 = "2023-08-16 22:27:00" group by col3, col1, col2 order by col1, col2, col3""" + mv_rewrite_success_without_check_chosen(sql_hit, mv_name) + + test { + sql """create materialized view ${no_mv_name} as select col3, sum(col7) from ${tb_name} group by col3 having col3 > 1""" + exception "The having clause is not supported in add materialized view clause" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, sum(col7) from ${tb_name} group by col3 limit 1""" + exception "The limit clause is not supported in add materialized view clause" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, 1, sum(col7) from ${tb_name} group by col3""" + exception "The materialized view only support the single column or function expr" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col3, sum(col7) from ${tb_name} group by col3""" + exception "The select expr `col3` is duplicated" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, sum(col7) / 1 from ${tb_name} group by col3""" + exception "The materialized view's expr calculations cannot be included outside aggregate functions" + } + + test { + sql """create materialized view ${no_mv_name} as select sum(col7), col3 from ${tb_name} group by col3""" + exception "The aggregate column should be after the single column" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, col2, col3 from ${tb_name} order by col1, col2, col3;""" + exception """agg mv must has group by clause""" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, col2, col3, sum(col7) from ${tb_name} group by col3, col1, col2 order by col3, col1, col2""" + exception "The order of columns in order by clause must be same as the order of columns in select list" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, col2, col3, sum(col7) from ${tb_name} group by col1, col2, col3 order by col3, col1, col2""" + exception "The order of columns in order by clause must be same as the order of columns in select list" + } + + test { + sql """create materialized view ${no_mv_name} as select sum(col3) from ${tb_name}""" + exception """The materialized view must contain at least one key column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, min(col7) from ${tb_name} group by col3""" + exception """Aggregate function require same with slot aggregate type""" + } + + test { + sql """create materialized view ${no_mv_name} as select min(col8),col3 from ${tb_name} group by col3""" + exception """The aggregate column should be after the single column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, case when col2 > 1 then 1 else 2 end, sum(col7) from ${tb_name} group by 1,2,3,4,5 order by 1,2,3,4,5""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, sum(case when col2 > 1 then 1 else 2 end) from ${tb_name} group by 1,2,3,4 order by 1,2,3,4""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, sum(col7), count(col3) from ${tb_name} group by 1,2,3,4 order by 1,2,3,4""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, sum(col7), bitmap_union(to_bitmap(case when col2 > 1 then 1 else 2 end)) from ${tb_name} group by 1,2,3,4 order by 1,2,3,4""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, sum(col7), bitmap_union(to_bitmap(case when col10 > 1 then 1 else 2 end)) from ${tb_name} group by 1,2,3,4 order by 1,2,3,4""" + exception """Aggregate function require single slot argument""" + } + +} diff --git a/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy b/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy new file mode 100644 index 00000000000000..f0cac006b14a83 --- /dev/null +++ b/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("dup_negative_mv_test", "mv_negative") { + + String db = context.config.getDbNameByFile(context.file) + def prefix_str = "mv_dup_negative" + def tb_name = prefix_str + "_tb" + + sql """drop table if exists ${tb_name};""" + sql """ + CREATE TABLE `${tb_name}` ( + `col1` datetime NULL, + `col2` varchar(60) NULL, + `col3` bigint(11) NOT NULL, + `col4` boolean NULL, + `col15` ipv4 NULL, + `col8` int(11) NULL DEFAULT "0", + `col5` string NULL, + `col6` ARRAY NULL COMMENT "", + `col7` bigint(11) NOT NULL AUTO_INCREMENT, + `col9` int(11) NULL DEFAULT "0", + `col10` int(11) NULL, + `col11` bitmap NOT NULL, + `col13` hll not NULL COMMENT "hll", + `col14` ipv4 NULL + ) ENGINE=OLAP + DUPLICATE KEY(`col1`, `col2`, `col3`, `col4`, `col15`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`col2`, `col3`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """insert into ${tb_name} values + ("2023-08-16 22:28:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax1",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",2,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,0,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd2",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[5,4,3,2,1], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 3, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",4,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 5, 6, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(2), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(100), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'255.255.255.255'"), + ("2023-08-16 24:27:00","ax1",2,0,"'0.0.0.0'",4,"asd",[5,4,3,2,1], 3, 5, 6, to_bitmap(2), HLL_HASH(100), "'255.255.255.255'"), + ("2024-08-17 22:27:00","ax2",3,1,"'0.0.0.0'",8,"asd3",[1,2,3,4,6], 7, 9, 10, to_bitmap(3), HLL_HASH(1000), "'0.0.1.0'"), + ("2023-09-16 22:27:00","ax4",4,0,"'0.0.0.0'",11,"asd2",[1,2,9,4,5], 11, 11, 11, to_bitmap(4), HLL_HASH(1), "'0.10.0.0'");""" + + def mv_name = """${prefix_str}_mv""" + def no_mv_name = """no_${prefix_str}_mv""" + + + def mtmv_sql = """select col4, col1, col2, col3, col15, sum(col8) from ${tb_name} where col1 = "2023-08-16 22:27:00" group by col4, col1, col2, col3, col15 order by col4, col1, col2, col3, col15""" + create_sync_mv(db, tb_name, mv_name, mtmv_sql) + + def desc_res = sql """desc ${tb_name} all;""" + for (int i = 0; i < desc_res.size(); i++) { + if (desc_res[i][0] == mv_name) { + for (int j = i; j < i+6; j++) { + if (desc_res[j][2] != "mva_SUM__CAST(`col8` AS bigint)") { + assertTrue(desc_res[j][6] == "true") + } else { + assertTrue(desc_res[j][6] == "false") + } + } + break + } + } + def sql_hit = """select col1, col2, col3, sum(col8) from ${tb_name} where col1 = "2023-08-16 22:27:00" group by col3, col1, col2 order by col1, col2, col3""" + mv_rewrite_success_without_check_chosen(sql_hit, mv_name) + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col7 from ${tb_name} where col1 = '2023-08-16 22:27:00' order by col4, col1, col2, col3, col15, col7""" + exception "The materialized view can not involved auto increment column" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, sum(col8) from ${tb_name} group by col3 having col3 > 1""" + exception "The having clause is not supported in add materialized view clause" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, sum(col8) from ${tb_name} group by col3 limit 1""" + exception "The limit clause is not supported in add materialized view clause" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, 1, sum(col8) from ${tb_name} group by col3""" + exception "The materialized view only support the single column or function expr" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col3, sum(col8) from ${tb_name} group by col3""" + exception "The select expr `col3` is duplicated" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, sum(col8) / 1 from ${tb_name} group by col3""" + exception "The materialized view's expr calculations cannot be included outside aggregate functions" + } + + test { + sql """create materialized view ${no_mv_name} as select sum(col8), col3 from ${tb_name} group by col3""" + exception "The aggregate column should be after the single column" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, col2, col3, sum(col8) from ${tb_name} group by col3, col1, col2 order by col3, col1, col2""" + exception "The order of columns in order by clause must be same as the order of columns in select list" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, col2, col3, sum(col8) from ${tb_name} group by col1, col2, col3 order by col3, col1, col2""" + exception "The order of columns in order by clause must be same as the order of columns in select list" + } + + test { + sql """create materialized view ${no_mv_name} as select sum(col3) from ${tb_name}""" + exception """The materialized view must contain at least one key column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, case when col2 > 1 then 1 else 2 end, sum(col8) from ${tb_name} group by 1,2,3,4,5 order by 1,2,3,4,5""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select min(col8), col3 from ${tb_name} group by col3""" + exception """The aggregate column should be after the single column""" + } + + +} diff --git a/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy b/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy new file mode 100644 index 00000000000000..c027f9f17eabae --- /dev/null +++ b/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("mor_negative_mv_test", "mv_negative") { + + String db = context.config.getDbNameByFile(context.file) + def prefix_str = "mv_mor_negative" + def tb_name = prefix_str + "_tb" + + sql """drop table if exists ${tb_name};""" + sql """ + CREATE TABLE `${tb_name}` ( + `col1` datetime NULL, + `col2` varchar(60) NULL, + `col3` bigint(11) NOT NULL, + `col4` boolean NULL, + `col15` ipv4 NULL, + `col8` int(11) NULL DEFAULT "0", + `col5` string NULL, + `col6` ARRAY NULL COMMENT "", + `col7` bigint(11) NOT NULL AUTO_INCREMENT, + `col9` int(11) NULL DEFAULT "0", + `col10` int(11) NULL, + `col11` bitmap NOT NULL, + `col13` hll not NULL COMMENT "hll", + `col14` ipv4 NULL + ) ENGINE=OLAP + unique KEY(`col1`, `col2`, `col3`, `col4`, `col15`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`col2`, `col3`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "false" + ); + """ + sql """insert into ${tb_name} values + ("2023-08-16 22:28:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax1",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",2,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,0,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd2",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[5,4,3,2,1], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 3, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",4,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 5, 6, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(2), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(100), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'255.255.255.255'"), + ("2023-08-16 24:27:00","ax1",2,0,"'0.0.0.0'",4,"asd",[5,4,3,2,1], 3, 5, 6, to_bitmap(2), HLL_HASH(100), "'255.255.255.255'"), + ("2024-08-17 22:27:00","ax2",3,1,"'0.0.0.0'",8,"asd3",[1,2,3,4,6], 7, 9, 10, to_bitmap(3), HLL_HASH(1000), "'0.0.1.0'"), + ("2023-09-16 22:27:00","ax4",4,0,"'0.0.0.0'",11,"asd2",[1,2,9,4,5], 11, 11, 11, to_bitmap(4), HLL_HASH(1), "'0.10.0.0'");""" + + def mv_name = """${prefix_str}_mv""" + def no_mv_name = """no_${prefix_str}_mv""" + def mtmv_sql = """select col4, col1, col2, col3, col15 from ${tb_name} where col1 = '2023-08-16 22:27:00' order by col4, col1, col2, col3, col15""" + create_sync_mv(db, tb_name, mv_name, mtmv_sql) + def desc_res = sql """desc ${tb_name} all;""" + for (int i = 0; i < desc_res.size(); i++) { + if (desc_res[i][0] == mv_name) { + for (int j = i; j < i+5; j++) { + assertTrue(desc_res[j][6] == "true") + } + break + } + } + def sql_hit = """select col1, col2, col3, sum(col3) from ${tb_name} where col1 = "2023-08-16 22:27:00" group by col3, col1, col2 order by col1, col2, col3""" + mv_rewrite_success_without_check_chosen(sql_hit, mv_name) + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col7 from ${tb_name} where col1 = '2023-08-16 22:27:00' order by col4, col1, col2, col3, col15, col7""" + exception "The materialized view can not involved auto increment column" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col8 from ${tb_name} where col1 = '2023-08-16 22:27:00' group by col4, col1, col2, col3, col15, col8 order by col4, col1, col2, col3, col15, col8""" + exception "The materialized view of unique table must not has grouping columns" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col8, sum(col3) from ${tb_name} where col1 = '2023-08-16 22:27:00' group by col4, col1, col2, col3, col15, col8 order by col4, col1, col2, col3, col15, col8""" + exception "The materialized view of unique table must not has grouping columns" + } + + // There is a bug in the old optimizer. Please comment out this case first and remove the comment after the bug is fixed. + // test { + // sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15 from ${tb_name} having col3 > 1""" + // exception "LogicalHaving is not supported" + // } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15 from ${tb_name} limit 1""" + exception "The limit clause is not supported in add materialized view clause" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, 1 from ${tb_name}""" + exception "The materialized view only support the single column or function expr" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col3 from ${tb_name}""" + exception "The select expr `col3` is duplicated" + } + + test { + sql """create materialized view ${no_mv_name} as select col3 from ${tb_name}""" + exception "The materialized view of uniq table must contain all key columns" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15 from ${tb_name} order by col1, col2, col3, col4, col15""" + exception "The order of columns in order by clause must be same as the order of columns in select list" + } + + test { + sql """create materialized view ${no_mv_name} as select sum(col3) from ${tb_name}""" + exception """The materialized view must contain at least one key column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, min(col7) from ${tb_name} group by col3""" + exception """The materialized view can not involved auto increment column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, case when col2 > 1 then 1 else 2 end from ${tb_name} order by 1,2,3,4,5""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, bitmap_union(to_bitmap(col3)) from ${tb_name} group by col1;""" + exception "The materialized view of unique table must not has grouping columns" + } + + test { + sql """create materialized view ${no_mv_name} as select bitmap_union(col11) from ${tb_name}""" + exception "Aggregate function require same with slot aggregate type" + } + + +} diff --git a/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy b/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy new file mode 100644 index 00000000000000..b855ad926f27b4 --- /dev/null +++ b/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy @@ -0,0 +1,157 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("mow_negative_mv_test", "mv_negative") { + + String db = context.config.getDbNameByFile(context.file) + def prefix_str = "mv_mow_negative" + def tb_name = prefix_str + "_tb" + + sql """drop table if exists ${tb_name};""" + sql """ + CREATE TABLE `${tb_name}` ( + `col1` datetime NULL, + `col2` varchar(60) NULL, + `col3` bigint(11) NOT NULL, + `col4` boolean NULL, + `col15` ipv4 NULL, + `col8` int(11) NULL DEFAULT "0", + `col5` string NULL, + `col6` ARRAY NULL COMMENT "", + `col7` bigint(11) NOT NULL AUTO_INCREMENT, + `col9` int(11) NULL DEFAULT "0", + `col10` int(11) NULL, + `col11` bitmap NOT NULL, + `col13` hll not NULL COMMENT "hll", + `col14` ipv4 NULL + ) ENGINE=OLAP + unique KEY(`col1`, `col2`, `col3`, `col4`, `col15`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`col2`, `col3`) BUCKETS 2 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + sql """insert into ${tb_name} values + ("2023-08-16 22:28:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax1",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",2,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,0,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd2",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[5,4,3,2,1], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 3, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",4,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 5, 6, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(2), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(100), "'0.0.0.0'"), + ("2023-08-16 22:27:00","ax",1,1,"'0.0.0.0'",1,"asd",[1,2,3,4,5], 1, 1, 1, to_bitmap(243), HLL_HASH(1), "'255.255.255.255'"), + ("2023-08-16 24:27:00","ax1",2,0,"'0.0.0.0'",4,"asd",[5,4,3,2,1], 3, 5, 6, to_bitmap(2), HLL_HASH(100), "'255.255.255.255'"), + ("2024-08-17 22:27:00","ax2",3,1,"'0.0.0.0'",8,"asd3",[1,2,3,4,6], 7, 9, 10, to_bitmap(3), HLL_HASH(1000), "'0.0.1.0'"), + ("2023-09-16 22:27:00","ax4",4,0,"'0.0.0.0'",11,"asd2",[1,2,9,4,5], 11, 11, 11, to_bitmap(4), HLL_HASH(1), "'0.10.0.0'");""" + + def mv_name = """${prefix_str}_mv""" + def no_mv_name = """no_${prefix_str}_mv""" + def mtmv_sql = """select col4, col1, col2, col3, col15 from ${tb_name} where col1 = '2023-08-16 22:27:00' order by col4, col1, col2, col3, col15""" + create_sync_mv(db, tb_name, mv_name, mtmv_sql) + def desc_res = sql """desc ${tb_name} all;""" + for (int i = 0; i < desc_res.size(); i++) { + if (desc_res[i][0] == mv_name) { + for (int j = i; j < i+5; j++) { + assertTrue(desc_res[j][6] == "true") + } + break + } + } + def sql_hit = """select col1, col2, col3, sum(col3) from ${tb_name} where col1 = "2023-08-16 22:27:00" group by col3, col1, col2 order by col1, col2, col3""" + mv_rewrite_success_without_check_chosen(sql_hit, mv_name) + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col7 from ${tb_name} where col1 = '2023-08-16 22:27:00' order by col4, col1, col2, col3, col15, col7""" + exception "The materialized view can not involved auto increment column" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col8 from ${tb_name} where col1 = '2023-08-16 22:27:00' group by col4, col1, col2, col3, col15, col8 order by col4, col1, col2, col3, col15, col8""" + exception "The materialized view of unique table must not has grouping columns" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col8, sum(col3) from ${tb_name} where col1 = '2023-08-16 22:27:00' group by col4, col1, col2, col3, col15, col8 order by col4, col1, col2, col3, col15, col8""" + exception "The materialized view of unique table must not has grouping columns" + } + + // There is a bug in the old optimizer. Please comment out this case first and remove the comment after the bug is fixed. + // test { + // sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15 from ${tb_name} having col3 > 1""" + // exception "LogicalHaving is not supported" + // } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15 from ${tb_name} limit 1""" + exception "The limit clause is not supported in add materialized view clause" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, 1 from ${tb_name}""" + exception "The materialized view only support the single column or function expr" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15, col3 from ${tb_name}""" + exception "The select expr `col3` is duplicated" + } + + + test { + sql """create materialized view ${no_mv_name} as select col3 from ${tb_name}""" + exception "The materialized view of uniq table must contain all key columns" + } + + test { + sql """create materialized view ${no_mv_name} as select col4, col1, col2, col3, col15 from ${tb_name} order by col1, col2, col3, col4, col15""" + exception "The order of columns in order by clause must be same as the order of columns in select list" + } + + test { + sql """create materialized view ${no_mv_name} as select sum(col3) from ${tb_name}""" + exception """The materialized view must contain at least one key column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, min(col7) from ${tb_name} group by col3""" + exception """The materialized view can not involved auto increment column""" + } + + test { + sql """create materialized view ${no_mv_name} as select col3, col1, col2, col15, case when col2 > 1 then 1 else 2 end from ${tb_name} order by 1,2,3,4,5""" + exception """The order of columns in order by clause must be same as the order of columns in select list""" + } + + test { + sql """create materialized view ${no_mv_name} as select col1, bitmap_union(to_bitmap(col3)) from ${tb_name} group by col1;""" + exception "The materialized view of unique table must not has grouping columns" + } + + test { + sql """create materialized view ${no_mv_name} as select bitmap_union(col11) from ${tb_name}""" + exception "The materialized view must contain at least one key column" + } + + +} From e3ba7875bfb50c3a6284037c88600f73cfbd8298 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Mon, 30 Jun 2025 12:31:10 +0800 Subject: [PATCH 110/572] [cherry-pick](branch-30) fix array_map cause coredump as NULL (#51618) (#51740) Problem Summary: cherry-pick from https://github.com/apache/doris/pull/51618 --- .../lambda_function/varray_map_function.cpp | 34 +++++++++++++- .../test_array_map_function.out | 9 ++++ .../test_array_map_function.groovy | 44 ++++++++++++++++++- 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/be/src/vec/exprs/lambda_function/varray_map_function.cpp b/be/src/vec/exprs/lambda_function/varray_map_function.cpp index 23050b6593c5ea..5f90827a70c5b8 100644 --- a/be/src/vec/exprs/lambda_function/varray_map_function.cpp +++ b/be/src/vec/exprs/lambda_function/varray_map_function.cpp @@ -192,6 +192,39 @@ class ArrayMapFunction : public LambdaFunction { data_types.push_back(col_type.get_nested_type()); } + ColumnWithTypeAndName result_arr; + // if column_array is NULL, we know the array_data_column will not write any data, + // so the column is empty. eg : (x) -> concat('|',x + "1"). if still execute the lambda function, will cause the bolck rows are not equal + // the x column is empty, but "|" is const literal, size of column is 1, so the block rows is 1, but the x column is empty, will be coredump. + if (std::any_of(lambda_datas.begin(), lambda_datas.end(), + [](const auto& v) { return v->empty(); })) { + DataTypePtr nested_type; + bool is_nullable = result_type->is_nullable(); + if (is_nullable) { + nested_type = + assert_cast(result_type.get())->get_nested_type(); + } else { + nested_type = result_type; + } + auto empty_nested_column = assert_cast(nested_type.get()) + ->get_nested_type() + ->create_column(); + auto result_array_column = ColumnArray::create(std::move(empty_nested_column), + std::move(array_column_offset)); + + if (is_nullable) { + result_arr = {ColumnNullable::create(std::move(result_array_column), + std::move(outside_null_map)), + result_type, "Result"}; + } else { + result_arr = {std::move(result_array_column), result_type, "Result"}; + } + + block->insert(result_arr); + *result_column_id = block->columns() - 1; + return Status::OK(); + } + ColumnPtr result_col = nullptr; DataTypePtr res_type; std::string res_name; @@ -260,7 +293,6 @@ class ArrayMapFunction : public LambdaFunction { } while (args.current_row_idx < block->rows()); //4. get the result column after execution, reassemble it into a new array column, and return. - ColumnWithTypeAndName result_arr; if (result_type->is_nullable()) { if (res_type->is_nullable()) { result_arr = { diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out index 211092f3875215..0e3935d131fc4a 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_map_function.out @@ -123,3 +123,12 @@ 4 [[90, 100], [110, 120]] 5 [[130]] +-- !select_26 -- +\N + +-- !select_27 -- +\N \N + +-- !select_28 -- +[] + diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy index acf6dba0600411..fb67352a0d48d9 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_map_function.groovy @@ -103,8 +103,8 @@ suite("test_array_map_function") { } sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP TABLE IF EXISTS array_map_test" -sql "DROP TABLE IF EXISTS array_map_test" sql """ CREATE TABLE IF NOT EXISTS array_map_test ( id INT, int_array ARRAY, @@ -129,4 +129,46 @@ sql "DROP TABLE IF EXISTS array_map_test" qt_select_25 """ SELECT id, array_map(x -> array_map(y -> y * 10, x), nested_array) FROM array_map_test order by id; """ + + sql "DROP TABLE IF EXISTS db" + + sql """ CREATE TABLE `db` ( + `id` VARCHAR(255) NULL COMMENT '主键', + `QC_result_list` ARRAY NULL COMMENT '标签预刷' + ) ENGINE=OLAP + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "storage_medium" = "hdd", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); + """ + + sql """insert into db values(1,null); + """ + + qt_select_26 """ + select array_map( + (x, y, z) -> concat( + '|', + x + "1", + '|', + x + "2", + '|', + x + "3" + ), + QC_result_list, + QC_result_list, + QC_result_list + ) FROM db; + """ + + qt_select_27 """ select QC_result_list, array_map( x -> concat( '|', x + "1" ), QC_result_list ) FROM db; """ + qt_select_28 """ select array_map((x,y)->x,[],[]); """ } From ef9d890c6de65fff68823ff718b6f77c2d9b609d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:29:04 +0800 Subject: [PATCH 111/572] branch-3.0: [Enhancement](Cloud) Check auth when alter vault #51747 (#52489) Cherry-picked from #51747 Co-authored-by: Lijia Liu --- .../commands/AlterStorageVaultCommand.java | 16 ++++++++++++++++ .../alter/test_alter_vault_type.groovy | 18 ++++++++++++++++++ .../vault_p0/default/test_default_vault.groovy | 10 ++++++++++ 3 files changed, 44 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java index 7b2d99814a2707..f877ef0da14afa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java @@ -21,7 +21,11 @@ import org.apache.doris.catalog.StorageVault; import org.apache.doris.catalog.StorageVault.StorageVaultType; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.qe.ConnectContext; @@ -47,6 +51,10 @@ public AlterStorageVaultCommand(String name, final Map propertie @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + // check auth + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } StorageVault.StorageVaultType vaultType = StorageVaultType.fromString(properties.get(TYPE)); if (vaultType == StorageVault.StorageVaultType.UNKNOWN) { throw new AnalysisException("Unsupported Storage Vault type: " + type); @@ -54,6 +62,14 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { FeNameFormat.checkStorageVaultName(name); if (properties.containsKey(StorageVault.PropertyKey.VAULT_NAME)) { + String newVaultName = properties.get(StorageVault.PropertyKey.VAULT_NAME); + Pair info = Env.getCurrentEnv().getStorageVaultMgr().getDefaultStorageVault(); + if (info != null && name.equalsIgnoreCase(info.first)) { + throw new AnalysisException("Cannot rename default storage vault. Before rename it, you should execute" + + " `UNSET DEFAULT STORAGE VAULT` sql to unset default storage vault. After rename it, you can" + + " execute `SET " + newVaultName + " AS DEFAULT STORAGE VAULT` sql to set it as default" + + " storage vault."); + } String newName = properties.get(StorageVault.PropertyKey.VAULT_NAME); FeNameFormat.checkStorageVaultName(newName); Preconditions.checkArgument(!name.equalsIgnoreCase(newName), "Vault name has not been changed"); diff --git a/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy b/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy index 1a4c4c19193068..a57207712e90cf 100644 --- a/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy +++ b/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy @@ -77,4 +77,22 @@ suite("test_alter_vault_type", "nonConcurrent") { ); """ }, "is not hdfs storage vault") + + def test_user = "alter_vault_no_pri"; + def test_pass = "12345" + sql """create user ${test_user} identified by '${test_pass}'""" + + try { + def result = connect(test_user, test_pass, context.config.jdbcUrl) { + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="s3", + "VAULT_NAME" = "${s3VaultName}_rename" + ); + """ + } + } catch (Exception e) { + assertTrue(e.getMessage().contains("Access denied for user"), e.getMessage()) + } } \ No newline at end of file diff --git a/regression-test/suites/vault_p0/default/test_default_vault.groovy b/regression-test/suites/vault_p0/default/test_default_vault.groovy index 0660b47cbf82db..069c4b7daeb4da 100644 --- a/regression-test/suites/vault_p0/default/test_default_vault.groovy +++ b/regression-test/suites/vault_p0/default/test_default_vault.groovy @@ -185,6 +185,16 @@ suite("test_default_vault", "nonConcurrent") { } assertTrue(found) + expectExceptionLike({ + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="s3", + "VAULT_NAME" = "${s3VaultName}_rename" + ); + """ + }, "Cannot rename default storage vault") + sql """ CREATE TABLE ${s3TableName}_2 ( C_CUSTKEY INTEGER NOT NULL, From 0fb1f0f0170188052a2781018ec42cd51114bc50 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:51:25 +0800 Subject: [PATCH 112/572] branch-3.0: [Opt](compaction) Prune rows with delete sign=1 in full compaction #51874 (#52256) Cherry-picked from #51874 Co-authored-by: bobhan1 --- be/src/cloud/cloud_tablet.cpp | 7 +++++-- be/src/olap/tablet_reader.cpp | 3 ++- regression-test/data/compaction/test_full_compaction.out | 1 - .../data/compaction/test_full_compaction_by_table_id.out | 1 - 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 582a4b771265d2..4344b085a838dc 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -950,7 +950,9 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( if ((config::enable_missing_rows_correctness_check || config::enable_mow_compaction_correctness_check_core) && !allow_delete_in_cumu_compaction && - compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION) { + (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION || + !config::enable_prune_delete_sign_when_base_compaction)) { + // also check duplicate key for base compaction when config::enable_prune_delete_sign_when_base_compaction==false missed_rows = std::make_unique(); LOG(INFO) << "RowLocation Set inited succ for tablet:" << tablet_id(); } @@ -971,7 +973,8 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( if (missed_rows) { missed_rows_size = missed_rows->size(); if (!allow_delete_in_cumu_compaction) { - if (compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION && + if ((compaction_type == ReaderType::READER_CUMULATIVE_COMPACTION || + !config::enable_prune_delete_sign_when_base_compaction) && tablet_state() == TABLET_RUNNING) { if (merged_rows >= 0 && merged_rows != missed_rows_size) { std::string err_msg = fmt::format( diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index a0c3a9c5eda95b..a453634ca8381f 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -635,7 +635,8 @@ Status TabletReader::_init_delete_condition(const ReaderParams& read_params) { // Delete sign could not be applied when delete on cumu compaction is enabled, bucause it is meant for delete with predicates. // If delete design is applied on cumu compaction, it will lose effect when doing base compaction. // `_delete_sign_available` indicates the condition where we could apply delete signs to data. - _delete_sign_available = ((read_params.reader_type == ReaderType::READER_BASE_COMPACTION && + _delete_sign_available = (((read_params.reader_type == ReaderType::READER_BASE_COMPACTION || + read_params.reader_type == ReaderType::READER_FULL_COMPACTION) && config::enable_prune_delete_sign_when_base_compaction) || read_params.reader_type == ReaderType::READER_COLD_DATA_COMPACTION || read_params.reader_type == ReaderType::READER_CHECKSUM); diff --git a/regression-test/data/compaction/test_full_compaction.out b/regression-test/data/compaction/test_full_compaction.out index b25fdad93145cc..118191b6cc8464 100644 --- a/regression-test/data/compaction/test_full_compaction.out +++ b/regression-test/data/compaction/test_full_compaction.out @@ -39,5 +39,4 @@ -- !select_final -- 1 100 2 200 -3 0 diff --git a/regression-test/data/compaction/test_full_compaction_by_table_id.out b/regression-test/data/compaction/test_full_compaction_by_table_id.out index b25fdad93145cc..118191b6cc8464 100644 --- a/regression-test/data/compaction/test_full_compaction_by_table_id.out +++ b/regression-test/data/compaction/test_full_compaction_by_table_id.out @@ -39,5 +39,4 @@ -- !select_final -- 1 100 2 200 -3 0 From 0460f62ca5f0097e067d8fe86c4d4cf1acff2dbc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:52:13 +0800 Subject: [PATCH 113/572] branch-3.0: [Enhancement](Compaction) Make base compaction use the same tablet selection strategy as cumulative compaction #51649 (#52389) Cherry-picked from #51649 Co-authored-by: abmdocrt --- be/src/cloud/cloud_tablet_mgr.cpp | 24 +++++++++++++++++------- be/src/cloud/config.cpp | 2 +- be/src/cloud/config.h | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index cbf9a29ee907ff..25f50f7ef4bd3b 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -31,6 +31,10 @@ namespace doris { uint64_t g_tablet_report_inactive_duration_ms = 0; +bvar::Adder g_base_compaction_not_frozen_tablet_num( + "base_compaction_not_frozen_tablet_num"); +bvar::Adder g_cumu_compaction_not_frozen_tablet_num( + "cumu_compaction_not_frozen_tablet_num"); namespace { // port from @@ -353,19 +357,25 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( using namespace std::chrono; auto now = duration_cast(system_clock::now().time_since_epoch()).count(); auto skip = [now, compaction_type](CloudTablet* t) { + int32_t max_version_config = t->max_version_config(); if (compaction_type == CompactionType::BASE_COMPACTION) { - return now - t->last_base_compaction_success_time_ms < config::base_compaction_freeze_interval_s * 1000 || - now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms; + bool is_recent_failure = now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms; + bool is_frozen = (now - t->last_load_time_ms > config::compaction_load_max_freeze_interval_s * 1000 + && now - t->last_base_compaction_success_time_ms < config::base_compaction_freeze_interval_s * 1000 + && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); + g_base_compaction_not_frozen_tablet_num << !is_frozen; + return is_recent_failure || is_frozen; } + // If tablet has too many rowsets but not be compacted for a long time, compaction should be performed // regardless of whether there is a load job recently. - - int32_t max_version_config = t->max_version_config(); - return now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms || - now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms || - (now - t->last_load_time_ms > config::cu_compaction_freeze_interval_s * 1000 + bool is_recent_failure = now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms; + bool is_recent_no_suitable_version = now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms; + bool is_frozen = (now - t->last_load_time_ms > config::compaction_load_max_freeze_interval_s * 1000 && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_interval_s * 1000 && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2); + g_cumu_compaction_not_frozen_tablet_num << !is_frozen; + return is_recent_failure || is_recent_no_suitable_version || is_frozen; }; // We don't schedule tablets that are disabled for compaction auto disable = [](CloudTablet* t) { return t->tablet_meta()->tablet_schema()->disable_auto_compaction(); }; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 800df55b3d6c6b..d4ae356236b416 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -43,7 +43,7 @@ DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); DEFINE_mInt64(base_compaction_freeze_interval_s, "7200"); -DEFINE_mInt64(cu_compaction_freeze_interval_s, "1200"); +DEFINE_mInt64(compaction_load_max_freeze_interval_s, "1200"); DEFINE_mInt64(cumu_compaction_interval_s, "1800"); DEFINE_mInt32(compaction_timeout_seconds, "86400"); diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 76e37163828006..6f550ff25f86eb 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -77,7 +77,7 @@ DECLARE_mInt64(min_compaction_failure_interval_ms); DECLARE_mBool(enable_new_tablet_do_compaction); // For cloud read/write separate mode DECLARE_mInt64(base_compaction_freeze_interval_s); -DECLARE_mInt64(cu_compaction_freeze_interval_s); +DECLARE_mInt64(compaction_load_max_freeze_interval_s); DECLARE_mInt64(cumu_compaction_interval_s); DECLARE_mInt32(compaction_timeout_seconds); From 1b794ddff2c49f28eea16fa1b0ea4b5802486f1a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 15:56:29 +0800 Subject: [PATCH 114/572] branch-3.0: [fix](auth)Only treat admin@% and root@% as system users. #50904 (#52282) Cherry-picked from #50904 Co-authored-by: zhangdong --- .../apache/doris/analysis/UserIdentity.java | 4 +- .../suites/account_p0/test_system_user.groovy | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java index 6898915dab73ce..c954c16447023e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java @@ -182,11 +182,11 @@ public static UserIdentity fromString(String userIdentStr) { } public boolean isRootUser() { - return user.equals(Auth.ROOT_USER); + return this.equals(ROOT); } public boolean isAdminUser() { - return user.equals(Auth.ADMIN_USER); + return this.equals(ADMIN); } public boolean isSystemUser() { diff --git a/regression-test/suites/account_p0/test_system_user.groovy b/regression-test/suites/account_p0/test_system_user.groovy index 5993e1d238b444..d7b3ad213fbbda 100644 --- a/regression-test/suites/account_p0/test_system_user.groovy +++ b/regression-test/suites/account_p0/test_system_user.groovy @@ -62,4 +62,42 @@ suite("test_system_user","p0,auth") { revoke select_priv on *.*.* from `admin`; """ + sql """ + create user `root`@'8.8.8.8'; + """ + sql """ + grant select_priv on *.*.* to `root`@'8.8.8.8'; + """ + sql """ + revoke select_priv on *.*.* from `root`@'8.8.8.8'; + """ + test { + sql """ + grant 'operator' to `root`@'8.8.8.8'; + """ + exception "Can not grant role: operator" + } + sql """ + drop user `root`@'8.8.8.8'; + """ + + sql """ + create user `admin`@'8.8.8.8'; + """ + sql """ + grant select_priv on *.*.* to `admin`@'8.8.8.8'; + """ + sql """ + revoke select_priv on *.*.* from `admin`@'8.8.8.8'; + """ + + sql """ + grant 'admin' to `admin`@'8.8.8.8'; + """ + sql """ + revoke 'admin' from `admin`@'8.8.8.8'; + """ + sql """ + drop user `admin`@'8.8.8.8'; + """ } From f92ace30fb7c2511802a2d74e3266c6c18e5b7c4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 11:31:14 +0800 Subject: [PATCH 115/572] branch-3.0: [fix](cluster)fix fe host may be contains Scope Identifier #52076 (#52396) Cherry-picked from #52076 Co-authored-by: zhangdong --- .../apache/doris/service/FrontendOptions.java | 15 +++++- .../doris/service/FrontendOptionsTest.java | 51 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100755 fe/fe-core/src/test/java/org/apache/doris/service/FrontendOptionsTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java index 35b6d5e01e25b5..e578aea549ea3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendOptions.java @@ -183,7 +183,20 @@ public static String getLocalHostAddress() { // so we call localAddr.getCanonicalHostName() at here return localAddr.getCanonicalHostName(); } - return InetAddresses.toAddrString(localAddr); + return getIpByLocalAddr(localAddr); + } + + /** + * get ip from addr + * @param addr InetAddress + * @return ip + */ + public static String getIpByLocalAddr(InetAddress addr) { + String addrString = InetAddresses.toAddrString(addr); + if (addrString.contains("%")) { + addrString = addrString.split("%")[0]; + } + return addrString; } private static void analyzePriorityCidrs() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/FrontendOptionsTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendOptionsTest.java new file mode 100755 index 00000000000000..84a3e34702d0c2 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/service/FrontendOptionsTest.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.service; + +import org.apache.doris.common.AnalysisException; + +import com.google.common.net.InetAddresses; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.net.InetAddress; + +public class FrontendOptionsTest { + @Mocked + private InetAddresses inetAddresses; + + @Before + public void setUp() throws NoSuchMethodException, SecurityException, AnalysisException { + new Expectations() { + { + inetAddresses.toAddrString((InetAddress) any); + minTimes = 0; + result = "2408:400a:5a:ea00:2fb5:112e:39dd:9bba%eth0"; + } + }; + } + + @Test + public void testGetIpByLocalAddr() { + String ip = FrontendOptions.getIpByLocalAddr(null); + Assert.assertEquals("2408:400a:5a:ea00:2fb5:112e:39dd:9bba", ip); + } +} From d1c9b07c327ea798bb62df09aaabc6b23e378769 Mon Sep 17 00:00:00 2001 From: walter Date: Tue, 1 Jul 2025 11:32:22 +0800 Subject: [PATCH 116/572] branch-3.0: [chore](cloud) Add a defer utility `DORIS_CLOUD_DEFER` #52041 (#52371) cherry pick from #52041 --- .clang-format-ignore | 1 + cloud/src/common/defer.h | 90 ++++++++++++++++ cloud/src/common/network_util.cpp | 5 +- cloud/src/meta-service/meta_service.cpp | 9 +- cloud/src/meta-service/meta_service_helper.h | 39 +++---- cloud/src/meta-service/meta_service_job.cpp | 51 +++++---- .../meta-service/meta_service_resource.cpp | 43 ++++---- cloud/src/meta-service/meta_service_txn.cpp | 25 ++--- cloud/src/meta-service/txn_kv.cpp | 10 +- cloud/src/recycler/checker.cpp | 12 +-- cloud/src/recycler/hdfs_accessor.cpp | 5 +- cloud/src/recycler/recycler.cpp | 95 ++++++++-------- cloud/src/recycler/recycler_service.cpp | 34 +++--- cloud/src/recycler/sync_executor.h | 10 +- cloud/src/recycler/util.h | 11 +- .../src/resource-manager/resource_manager.cpp | 30 +++--- cloud/test/encryption_test.cpp | 26 +++-- cloud/test/http_encode_key_test.cpp | 6 +- cloud/test/meta_server_test.cpp | 5 +- cloud/test/meta_service_http_test.cpp | 16 +-- cloud/test/meta_service_job_test.cpp | 81 ++++++++------ cloud/test/meta_service_test.cpp | 85 ++++++++------- cloud/test/recycler_test.cpp | 102 ++++++++++-------- cloud/test/schema_kv_test.cpp | 29 ++--- cloud/test/txn_kv_test.cpp | 11 +- 25 files changed, 495 insertions(+), 336 deletions(-) create mode 100644 cloud/src/common/defer.h diff --git a/.clang-format-ignore b/.clang-format-ignore index fbbc4cb0446e14..5f10e2a221203c 100644 --- a/.clang-format-ignore +++ b/.clang-format-ignore @@ -9,3 +9,4 @@ be/src/util/sse2neon.h be/src/util/mustache/mustache.h be/src/util/mustache/mustache.cc be/src/util/utf8_check.cpp +cloud/src/common/defer.h diff --git a/cloud/src/common/defer.h b/cloud/src/common/defer.h new file mode 100644 index 00000000000000..ab087aeb2fbd29 --- /dev/null +++ b/cloud/src/common/defer.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris::cloud { + +// A simple RAII class to defer the execution of a function until the end of the +// scope. +template +requires std::is_invocable_v +class DeferFn { +public: + DeferFn(Fn &&fn) : fn_(std::move(fn)) {} + DeferFn(const DeferFn &) = delete; + DeferFn &operator=(const DeferFn &) = delete; + ~DeferFn() { fn_(); } + +private: + Fn fn_; +}; + +} // namespace doris::cloud + +// A macro to create a DeferFn object that will execute the given function +// when it goes out of scope. This is useful for cleanup tasks or finalization +// actions that should always run, regardless of how the scope is exited (e.g. +// normal return, exception thrown, etc.). +// +// Usage: +// DORIS_CLOUD_DEFER { +// // Code to execute at the end of the scope +// }; +// +#define DORIS_CLOUD_DEFER_IMPL(line, counter) \ + ::doris::cloud::DeferFn defer_fn_##line##_##counter = [&]() +#define DORIS_CLOUD_DEFER_EXPAND(line, counter) \ + DORIS_CLOUD_DEFER_IMPL(line, counter) +#define DORIS_CLOUD_DEFER DORIS_CLOUD_DEFER_EXPAND(__LINE__, __COUNTER__) + +// A macro to create a DeferFn object that will execute the given function +// with additional parameters when it goes out of scope. This is useful for +// cleanup tasks or finalization actions that should always run, regardless of +// how the scope is exited (e.g. normal return, exception thrown, etc.). +// +// Usage: +// DORIS_CLOUD_DEFER_COPY(param1, param2) { +// // Code to execute at the end of the scope, using param1 and param2 +// }; +// +// Note: The parameters are captured by copy, so they can be used safely, for +// example, +// void foo(int &a, int &b) { +// DORIS_CLOUD_DEFER_COPY(a, b) mutable { +// a += 1; +// b *= 2; +// }; +// } +// +// int x = 1, y = 2; +// foo(x, y); +// assert(x == 1 && y == 2); +// +// The captured parameters are passed by value, so they modifications inside the +// deferred function do not affect the original variables outside the scope, or +// the modifications after the definition of the defer function will not affect +// the captured values. +#define DORIS_CLOUD_DEFER_COPY_IMPL(line, counter, ...) \ + ::doris::cloud::DeferFn defer_fn_##line##_##counter = [&, __VA_ARGS__ ]() +#define DORIS_CLOUD_DEFER_COPY_EXPAND(line, counter, ...) \ + DORIS_CLOUD_DEFER_COPY_IMPL(line, counter, __VA_ARGS__) +#define DORIS_CLOUD_DEFER_COPY(...) \ + DORIS_CLOUD_DEFER_COPY_EXPAND(__LINE__, __COUNTER__, __VA_ARGS__) diff --git a/cloud/src/common/network_util.cpp b/cloud/src/common/network_util.cpp index afec16200e0573..5ac8483cc1452d 100644 --- a/cloud/src/common/network_util.cpp +++ b/cloud/src/common/network_util.cpp @@ -30,6 +30,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "common/logging.h" namespace doris::cloud { @@ -161,7 +162,7 @@ static bool get_hosts_v4(std::vector* hosts) { std::string get_local_ip(const std::string& priority_networks) { std::string localhost_str = butil::my_ip_cstr(); - std::unique_ptr> defer((int*)0x01, [&localhost_str](int*) { + DORIS_CLOUD_DEFER { // Check if ip eq 127.0.0.1, ms/recycler exit LOG(INFO) << "get the IP for ms is " << localhost_str; if (config::enable_loopback_address_for_ms || localhost_str != "127.0.0.1") return; @@ -170,7 +171,7 @@ std::string get_local_ip(const std::string& priority_networks) { << "please set priority_networks with a CIDR expression in doris_cloud.conf " << "to choose a non-loopback address accordingly"; exit(-1); - }); + }; if (priority_networks == "") { LOG(INFO) << "use butil::my_ip_cstr(), local host ip=" << localhost_str; return localhost_str; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 660a7ef7320bc4..e1b3195d50b5ed 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1544,11 +1544,10 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, std::unique_ptr it; int num_rowsets = 0; - std::unique_ptr> defer_log_range( - (int*)0x01, [key0, key1, &num_rowsets](int*) { - LOG(INFO) << "get rowset meta, num_rowsets=" << num_rowsets << " range=[" - << hex(key0) << "," << hex(key1) << "]"; - }); + DORIS_CLOUD_DEFER_COPY(key0, key1) { + LOG(INFO) << "get rowset meta, num_rowsets=" << num_rowsets << " range=[" << hex(key0) + << "," << hex(key1) << "]"; + }; std::stringstream ss; do { diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index cd9ed2f7f1d9fd..5e724eae14f619 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -28,6 +28,7 @@ #include "common/bvars.h" #include "common/config.h" +#include "common/defer.h" #include "common/logging.h" #include "common/stopwatch.h" #include "common/util.h" @@ -225,25 +226,25 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { } } -#define RPC_PREPROCESS(func_name) \ - StopWatch sw; \ - auto ctrl = static_cast(controller); \ - begin_rpc(#func_name, ctrl, request); \ - brpc::ClosureGuard closure_guard(done); \ - [[maybe_unused]] std::stringstream ss; \ - [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ - [[maybe_unused]] std::string msg; \ - [[maybe_unused]] std::string instance_id; \ - [[maybe_unused]] bool drop_request = false; \ - std::unique_ptr> defer_status((int*)0x01, [&](int*) { \ - response->mutable_status()->set_code(code); \ - response->mutable_status()->set_msg(msg); \ - finish_rpc(#func_name, ctrl, response); \ - closure_guard.reset(nullptr); \ - if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ - g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ - } \ - }); +#define RPC_PREPROCESS(func_name) \ + StopWatch sw; \ + auto ctrl = static_cast(controller); \ + begin_rpc(#func_name, ctrl, request); \ + brpc::ClosureGuard closure_guard(done); \ + [[maybe_unused]] std::stringstream ss; \ + [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ + [[maybe_unused]] std::string msg; \ + [[maybe_unused]] std::string instance_id; \ + [[maybe_unused]] bool drop_request = false; \ + DORIS_CLOUD_DEFER { \ + response->mutable_status()->set_code(code); \ + response->mutable_status()->set_msg(msg); \ + finish_rpc(#func_name, ctrl, response); \ + closure_guard.reset(nullptr); \ + if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ + g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ + } \ + }; #define RPC_RATE_LIMIT(func_name) \ if (config::enable_rate_limit && config::use_detailed_metrics && !instance_id.empty()) { \ diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index c71859decb743e..e356c198670089 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -453,17 +453,16 @@ void MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro } bool need_commit = false; - std::unique_ptr> defer_commit( - (int*)0x01, [&ss, &txn, &code, &msg, &need_commit](int*) { - if (!need_commit) return; - TxnErrorCode err = txn->commit(); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - ss << "failed to commit job kv, err=" << err; - msg = ss.str(); - return; - } - }); + DORIS_CLOUD_DEFER { + if (!need_commit) return; + TxnErrorCode err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to commit job kv, err=" << err; + msg = ss.str(); + return; + } + }; if (!request->job().compaction().empty()) { start_compaction_job(code, msg, ss, txn, request, response, instance_id, need_commit); @@ -849,11 +848,10 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string std::unique_ptr it; int num_rowsets = 0; - std::unique_ptr> defer_log_range( - (int*)0x01, [&rs_start, &rs_end, &num_rowsets, &instance_id](int*) { - INSTANCE_LOG(INFO) << "get rowset meta, num_rowsets=" << num_rowsets << " range=[" - << hex(rs_start) << "," << hex(rs_end) << "]"; - }); + DORIS_CLOUD_DEFER { + INSTANCE_LOG(INFO) << "get rowset meta, num_rowsets=" << num_rowsets << " range=[" + << hex(rs_start) << "," << hex(rs_end) << "]"; + }; auto rs_start1 = rs_start; do { @@ -1460,17 +1458,16 @@ void MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr VLOG_DEBUG << "get tablet job, tablet_id=" << tablet_id << " job=" << proto_to_json(recorded_job); - std::unique_ptr> defer_commit( - (int*)0x01, [&ss, &txn, &code, &msg, &need_commit](int*) { - if (!need_commit) return; - TxnErrorCode err = txn->commit(); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - ss << "failed to commit job kv, err=" << err; - msg = ss.str(); - return; - } - }); + DORIS_CLOUD_DEFER { + if (!need_commit) return; + TxnErrorCode err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to commit job kv, err=" << err; + msg = ss.str(); + return; + } + }; // Process compaction commit if (!request->job().compaction().empty()) { diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 8c8b0646c94cdd..44389a41e2b91f 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -1780,17 +1780,16 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller std::string msg = "OK"; [[maybe_unused]] std::stringstream ss; std::string instance_id = request->has_instance_id() ? request->instance_id() : ""; - std::unique_ptr> defer_status( - (int*)0x01, [&code, &msg, &response, &ctrl, &closure_guard, &sw, &instance_id](int*) { - response->mutable_status()->set_code(code); - response->mutable_status()->set_msg(msg); - LOG(INFO) << (code == MetaServiceCode::OK ? "succ to " : "failed to ") - << __PRETTY_FUNCTION__ << " " << ctrl->remote_side() << " " << msg; - closure_guard.reset(nullptr); - if (config::use_detailed_metrics && !instance_id.empty()) { - g_bvar_ms_alter_instance.put(instance_id, sw.elapsed_us()); - } - }); + DORIS_CLOUD_DEFER { + response->mutable_status()->set_code(code); + response->mutable_status()->set_msg(msg); + LOG(INFO) << (code == MetaServiceCode::OK ? "succ to " : "failed to ") + << __PRETTY_FUNCTION__ << " " << ctrl->remote_side() << " " << msg; + closure_guard.reset(nullptr); + if (config::use_detailed_metrics && !instance_id.empty()) { + g_bvar_ms_alter_instance.put(instance_id, sw.elapsed_us()); + } + }; std::pair ret; switch (request->op()) { @@ -3039,18 +3038,16 @@ void MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller, std::string msg = "OK"; std::string instance_id; bool drop_request = false; - std::unique_ptr> defer_status( - (int*)0x01, [&ret, &code, &msg, &response, &ctrl, &closure_guard, &sw, &instance_id, - &drop_request](int*) { - response->mutable_status()->set_code(code); - response->mutable_status()->set_msg(msg); - LOG(INFO) << (ret == 0 ? "succ to " : "failed to ") << __PRETTY_FUNCTION__ << " " - << ctrl->remote_side() << " " << msg; - closure_guard.reset(nullptr); - if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { - g_bvar_ms_drop_stage.put(instance_id, sw.elapsed_us()); - } - }); + DORIS_CLOUD_DEFER { + response->mutable_status()->set_code(code); + response->mutable_status()->set_msg(msg); + LOG(INFO) << (ret == 0 ? "succ to " : "failed to ") << __PRETTY_FUNCTION__ << " " + << ctrl->remote_side() << " " << msg; + closure_guard.reset(nullptr); + if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { + g_bvar_ms_drop_stage.put(instance_id, sw.elapsed_us()); + } + }; std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 490c7bb616bd3b..4243dc4a1b52fe 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -817,12 +817,10 @@ void scan_tmp_rowset( meta_rowset_tmp_key(rs_tmp_key_info1, &rs_tmp_key1); int num_rowsets = 0; - std::unique_ptr> defer_log_range( - (int*)0x01, [rs_tmp_key0, rs_tmp_key1, &num_rowsets, &txn_id](int*) { - LOG(INFO) << "get tmp rowset meta, txn_id=" << txn_id - << " num_rowsets=" << num_rowsets << " range=[" << hex(rs_tmp_key0) << "," - << hex(rs_tmp_key1) << ")"; - }); + DORIS_CLOUD_DEFER_COPY(rs_tmp_key0, rs_tmp_key1) { + LOG(INFO) << "get tmp rowset meta, txn_id=" << txn_id << " num_rowsets=" << num_rowsets + << " range=[" << hex(rs_tmp_key0) << "," << hex(rs_tmp_key1) << ")"; + }; std::unique_ptr it; do { @@ -1558,11 +1556,11 @@ void commit_txn_eventually( MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id, const std::vector>& tmp_rowsets_meta) { StopWatch sw; - std::unique_ptr> defer_status((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (config::use_detailed_metrics && !instance_id.empty()) { g_bvar_ms_commit_txn_eventually.put(instance_id, sw.elapsed_us()); } - }); + }; std::stringstream ss; TxnErrorCode err = TxnErrorCode::TXN_OK; @@ -2032,12 +2030,11 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* std::vector> tmp_rowsets_meta; int num_rowsets = 0; - std::unique_ptr> defer_log_range( - (int*)0x01, [rs_tmp_key0, rs_tmp_key1, &num_rowsets, &txn_id, &sub_txn_id](int*) { - LOG(INFO) << "get tmp rowset meta, txn_id=" << txn_id - << ", sub_txn_id=" << sub_txn_id << " num_rowsets=" << num_rowsets - << " range=[" << hex(rs_tmp_key0) << "," << hex(rs_tmp_key1) << ")"; - }); + DORIS_CLOUD_DEFER_COPY(rs_tmp_key_info0, rs_tmp_key_info1) { + LOG(INFO) << "get tmp rowset meta, txn_id=" << txn_id << ", sub_txn_id=" << sub_txn_id + << " num_rowsets=" << num_rowsets << " range=[" << hex(rs_tmp_key0) << "," + << hex(rs_tmp_key1) << ")"; + }; std::unique_ptr it; do { diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-service/txn_kv.cpp index a328538ed78832..ad9b62bb0c14b7 100644 --- a/cloud/src/meta-service/txn_kv.cpp +++ b/cloud/src/meta-service/txn_kv.cpp @@ -34,6 +34,7 @@ #include "common/bvars.h" #include "common/config.h" +#include "common/defer.h" #include "common/logging.h" #include "common/stopwatch.h" #include "common/util.h" @@ -416,8 +417,9 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, int limit) { StopWatch sw; approximate_bytes_ += begin.size() + end.size(); - std::unique_ptr> defer( - (int*)0x01, [&sw](int*) { g_bvar_txn_kv_range_get << sw.elapsed_us(); }); + DORIS_CLOUD_DEFER { + g_bvar_txn_kv_range_get << sw.elapsed_us(); + }; FDBFuture* fut = fdb_transaction_get_range( txn_, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)begin.data(), begin.size()), @@ -555,10 +557,10 @@ TxnErrorCode Transaction::commit() { TxnErrorCode Transaction::get_read_version(int64_t* version) { StopWatch sw; auto* fut = fdb_transaction_get_read_version(txn_); - std::unique_ptr> defer((int*)0x01, [fut, &sw](...) { + DORIS_CLOUD_DEFER { fdb_future_destroy(fut); g_bvar_txn_kv_get_read_version << sw.elapsed_us(); - }); + }; RETURN_IF_ERROR(await_future(fut)); auto err = fdb_future_get_error(fut); TEST_SYNC_POINT_CALLBACK("transaction:get_read_version:get_err", &err); diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 954a3332d7566e..3fbd862ae1a97c 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -476,7 +476,7 @@ int InstanceChecker::do_check() { long instance_volume = 0; using namespace std::chrono; auto start_time = steady_clock::now(); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { auto cost = duration(steady_clock::now() - start_time).count(); LOG(INFO) << "check instance objects finished, cost=" << cost << "s. instance_id=" << instance_id_ << " num_scanned=" << num_scanned @@ -489,7 +489,7 @@ int InstanceChecker::do_check() { g_bvar_checker_check_cost_s.put(instance_id_, static_cast(cost)); // FIXME(plat1ko): What if some list operation failed? g_bvar_checker_instance_volume.put(instance_id_, instance_volume); - }); + }; struct TabletFiles { int64_t tablet_id {0}; @@ -688,14 +688,14 @@ int InstanceChecker::do_inverted_check() { long num_file_leak = 0; using namespace std::chrono; auto start_time = steady_clock::now(); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { g_bvar_inverted_checker_num_scanned.put(instance_id_, num_scanned); g_bvar_inverted_checker_num_check_failed.put(instance_id_, num_file_leak); auto cost = duration(steady_clock::now() - start_time).count(); LOG(INFO) << "inverted check instance objects finished, cost=" << cost << "s. instance_id=" << instance_id_ << " num_scanned=" << num_scanned << " num_file_leak=" << num_file_leak; - }); + }; struct TabletRowsets { int64_t tablet_id {0}; @@ -1067,7 +1067,7 @@ int InstanceChecker::do_delete_bitmap_inverted_check() { int64_t leaked_delete_bitmaps {0}; auto start_time = std::chrono::steady_clock::now(); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { g_bvar_inverted_checker_leaked_delete_bitmaps.put(instance_id_, leaked_delete_bitmaps); g_bvar_inverted_checker_abnormal_delete_bitmaps.put(instance_id_, abnormal_delete_bitmaps); g_bvar_inverted_checker_delete_bitmaps_scanned.put(instance_id_, total_delete_bitmap_keys); @@ -1088,7 +1088,7 @@ int InstanceChecker::do_delete_bitmap_inverted_check() { "passed. cost={} ms, total_delete_bitmap_keys={}", instance_id_, cost, total_delete_bitmap_keys); } - }); + }; struct TabletsRowsetsCache { int64_t tablet_id {-1}; diff --git a/cloud/src/recycler/hdfs_accessor.cpp b/cloud/src/recycler/hdfs_accessor.cpp index 024acd0efe7852..0ff9523e073a0a 100644 --- a/cloud/src/recycler/hdfs_accessor.cpp +++ b/cloud/src/recycler/hdfs_accessor.cpp @@ -32,6 +32,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "common/logging.h" #include "common/string_util.h" #include "cpp/sync_point.h" @@ -533,12 +534,12 @@ int HdfsAccessor::put_file(const std::string& relative_path, const std::string& return -1; } - std::unique_ptr> defer((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (file) { SCOPED_BVAR_LATENCY(hdfs_close_latency); hdfsCloseFile(fs_.get(), file); } - }); + }; int64_t written_bytes = 0; { diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index c688097f83f71b..ed1028ca699f9f 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -37,6 +37,7 @@ #include #include +#include "common/defer.h" #include "common/stopwatch.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" @@ -715,12 +716,12 @@ int InstanceRecycler::recycle_deleted_instance() { int ret = 0; auto start_time = steady_clock::now(); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { auto cost = duration(steady_clock::now() - start_time).count(); LOG(WARNING) << (ret == 0 ? "successfully" : "failed to") << " recycle deleted instance, cost=" << cost << "s, instance_id=" << instance_id_; - }); + }; // delete all remote data for (auto& [_, accessor] : accessor_map_) { @@ -829,7 +830,7 @@ int InstanceRecycler::recycle_indexes() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -839,7 +840,7 @@ int InstanceRecycler::recycle_indexes() { .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) .tag("num_recycled", num_recycled); - }); + }; int64_t earlest_ts = std::numeric_limits::max(); @@ -974,8 +975,9 @@ int InstanceRecycler::recycle_indexes() { auto loop_done = [&index_keys, this]() -> int { if (index_keys.empty()) return 0; - std::unique_ptr> defer((int*)0x01, - [&](int*) { index_keys.clear(); }); + DORIS_CLOUD_DEFER { + index_keys.clear(); + }; if (0 != txn_remove(txn_kv_.get(), index_keys)) { LOG(WARNING) << "failed to delete recycle index kv, instance_id=" << instance_id_; return -1; @@ -1092,7 +1094,7 @@ int InstanceRecycler::recycle_partitions() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -1102,7 +1104,7 @@ int InstanceRecycler::recycle_partitions() { .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) .tag("num_recycled", num_recycled); - }); + }; int64_t earlest_ts = std::numeric_limits::max(); @@ -1260,10 +1262,10 @@ int InstanceRecycler::recycle_partitions() { auto loop_done = [&partition_keys, &partition_version_keys, this]() -> int { if (partition_keys.empty()) return 0; - std::unique_ptr> defer((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { partition_keys.clear(); partition_version_keys.clear(); - }); + }; std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { @@ -1299,14 +1301,14 @@ int InstanceRecycler::recycle_versions() { auto start_time = steady_clock::now(); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { auto cost = duration(steady_clock::now() - start_time).count(); metrics_context.finish_report(); LOG_WARNING("recycle table and partition versions finished, cost={}s", cost) .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_recycled", num_recycled); - }); + }; auto version_key_begin = partition_version_key({instance_id_, 0, 0, 0}); auto version_key_end = partition_version_key({instance_id_, INT64_MAX, 0, 0}); @@ -1449,7 +1451,7 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, auto start_time = steady_clock::now(); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { auto cost = duration(steady_clock::now() - start_time).count(); LOG_INFO("recycle tablets of " + tablet_belongs + " finished, cost={}s", cost) .tag("instance_id", instance_id_) @@ -1458,7 +1460,7 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, .tag("partition_id", partition_id) .tag("num_scanned", num_scanned) .tag("num_recycled", num_recycled); - }); + }; // The first string_view represents the tablet key which has been recycled // The second bool represents whether the following fdb's tablet key deletion could be done using range move or not @@ -1554,10 +1556,10 @@ int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, break; } } - std::unique_ptr> defer((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { tablet_idx_keys.clear(); init_rs_keys.clear(); - }); + }; std::unique_ptr txn; if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to delete tablet meta kv, instance_id=" << instance_id_; @@ -2041,7 +2043,7 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& int64_t min_rowset_expiration_time = INT64_MAX; int64_t max_rowset_expiration_time = 0; - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { auto cost = duration(steady_clock::now() - start_time).count(); LOG_INFO("recycle the rowsets of dropped tablet finished, cost={}s", cost) .tag("instance_id", instance_id_) @@ -2056,7 +2058,7 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& .tag("min rowset expiration time", min_rowset_expiration_time) .tag("max rowset expiration time", max_rowset_expiration_time) .tag("ret", ret); - }); + }; std::unique_ptr txn; if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { @@ -2267,7 +2269,7 @@ int InstanceRecycler::recycle_rowsets() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -2283,7 +2285,7 @@ int InstanceRecycler::recycle_rowsets() { .tag("total_rowset_meta_key_size_scanned", total_rowset_key_size) .tag("total_rowset_meta_value_size_scanned", total_rowset_value_size) .tag("expired_rowset_meta_size", expired_rowset_size); - }); + }; std::vector rowset_keys; // rowset_id -> rowset_meta @@ -2623,7 +2625,7 @@ int InstanceRecycler::recycle_tmp_rowsets() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -2636,7 +2638,7 @@ int InstanceRecycler::recycle_tmp_rowsets() { .tag("total_rowset_meta_key_size_scanned", total_rowset_key_size) .tag("total_rowset_meta_value_size_scanned", total_rowset_value_size) .tag("expired_rowset_meta_size_recycled", expired_rowset_size); - }); + }; // Elements in `tmp_rowset_keys` has the same lifetime as `it` std::vector tmp_rowset_keys; @@ -2758,10 +2760,10 @@ int InstanceRecycler::recycle_tmp_rowsets() { auto loop_done = [&tmp_rowset_keys, &tmp_rowsets, &num_recycled, &metrics_context, this]() -> int { - std::unique_ptr> defer((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { tmp_rowset_keys.clear(); tmp_rowsets.clear(); - }); + }; if (delete_rowset_data(tmp_rowsets, RowsetRecyclingState::TMP_ROWSET, metrics_context) != 0) { LOG(WARNING) << "failed to delete tmp rowset data, instance_id=" << instance_id_; @@ -2810,12 +2812,11 @@ int InstanceRecycler::scan_and_recycle( int64_t cnt = 0; int get_range_retried = 0; std::string err; - std::unique_ptr> defer_log( - (int*)0x01, [begin, end, &err, &ret, &cnt, &get_range_retried](int*) { - LOG(INFO) << "finish scan_and_recycle key_range=[" << hex(begin) << "," << hex(end) - << ") num_scanned=" << cnt << " get_range_retried=" << get_range_retried - << " ret=" << ret << " err=" << err; - }); + DORIS_CLOUD_DEFER_COPY(begin, end) { + LOG(INFO) << "finish scan_and_recycle key_range=[" << hex(begin) << "," << hex(end) + << ") num_scanned=" << cnt << " get_range_retried=" << get_range_retried + << " ret=" << ret << " err=" << err; + }; std::unique_ptr it; do { @@ -2881,7 +2882,7 @@ int InstanceRecycler::abort_timeout_txn() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -2892,7 +2893,7 @@ int InstanceRecycler::abort_timeout_txn() { .tag("num_timeout", num_timeout) .tag("num_abort", num_abort) .tag("num_advance", num_advance); - }); + }; int64_t current_time = duration_cast(system_clock::now().time_since_epoch()).count(); @@ -3066,7 +3067,7 @@ int InstanceRecycler::recycle_expired_txn_label() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -3076,7 +3077,7 @@ int InstanceRecycler::recycle_expired_txn_label() { .tag("num_scanned", num_scanned) .tag("num_expired", num_expired) .tag("num_recycled", num_recycled); - }); + }; int64_t earlest_ts = std::numeric_limits::max(); auto calc_expiration = [&earlest_ts, this](const RecycleTxnPB& recycle_txn_pb) { @@ -3215,8 +3216,9 @@ int InstanceRecycler::recycle_expired_txn_label() { }; auto loop_done = [&]() -> int { - std::unique_ptr> defer( - (int*)0x01, [&](int*) { recycle_txn_info_keys.clear(); }); + DORIS_CLOUD_DEFER { + recycle_txn_info_keys.clear(); + }; TEST_SYNC_POINT_CALLBACK( "InstanceRecycler::recycle_expired_txn_label.check_recycle_txn_info_keys", &recycle_txn_info_keys); @@ -3313,14 +3315,14 @@ struct BatchObjStoreAccessor { private: void consume() { - std::unique_ptr> defer((int*)0x01, [this](int*) { + DORIS_CLOUD_DEFER { paths_.clear(); copy_file_keys_.clear(); batch_count_++; LOG_WARNING("begin to delete {} internal stage objects in batch {}", paths_.size(), batch_count_); - }); + }; StopWatch sw; // TODO(yuejing): 在accessor的delete_objets的实现里可以考虑如果_paths数量不超过10个的话，就直接发10个delete objection operation而不是发post @@ -3386,7 +3388,7 @@ int InstanceRecycler::recycle_copy_jobs() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -3397,7 +3399,7 @@ int InstanceRecycler::recycle_copy_jobs() { .tag("num_finished", num_finished) .tag("num_expired", num_expired) .tag("num_recycled", num_recycled); - }); + }; CopyJobKeyInfo key_info0 {instance_id_, "", 0, "", 0}; CopyJobKeyInfo key_info1 {instance_id_, "\xff", 0, "", 0}; @@ -3668,7 +3670,7 @@ int InstanceRecycler::recycle_stage() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); register_recycle_task(task_name, start_time); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { unregister_recycle_task(task_name); int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; @@ -3677,7 +3679,7 @@ int InstanceRecycler::recycle_stage() { .tag("instance_id", instance_id_) .tag("num_scanned", num_scanned) .tag("num_recycled", num_recycled); - }); + }; RecycleStageKeyInfo key_info0 {instance_id_, ""}; RecycleStageKeyInfo key_info1 {instance_id_, "\xff"}; @@ -3795,8 +3797,9 @@ int InstanceRecycler::recycle_stage() { auto loop_done = [&stage_keys, this]() -> int { if (stage_keys.empty()) return 0; - std::unique_ptr> defer((int*)0x01, - [&](int*) { stage_keys.clear(); }); + DORIS_CLOUD_DEFER { + stage_keys.clear(); + }; if (0 != txn_remove(txn_kv_.get(), stage_keys)) { LOG(WARNING) << "failed to delete recycle partition kv, instance_id=" << instance_id_; return -1; @@ -3814,13 +3817,13 @@ int InstanceRecycler::recycle_expired_stage_objects() { int64_t start_time = duration_cast(steady_clock::now().time_since_epoch()).count(); RecyclerMetricsContext metrics_context(instance_id_, "recycle_expired_stage_objects"); - std::unique_ptr> defer_log_statistics((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { int64_t cost = duration_cast(steady_clock::now().time_since_epoch()).count() - start_time; metrics_context.finish_report(); LOG_WARNING("recycle expired stage objects, cost={}s", cost) .tag("instance_id", instance_id_); - }); + }; int ret = 0; // for calculate the total num or bytes of recyled objects diff --git a/cloud/src/recycler/recycler_service.cpp b/cloud/src/recycler/recycler_service.cpp index 8f3a736671cba7..c77357c764ebae 100644 --- a/cloud/src/recycler/recycler_service.cpp +++ b/cloud/src/recycler/recycler_service.cpp @@ -24,6 +24,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "common/logging.h" #include "common/util.h" #include "cpp/s3_rate_limiter.h" @@ -60,14 +61,12 @@ void RecyclerServiceImpl::recycle_instance(::google::protobuf::RpcController* co brpc::ClosureGuard closure_guard(done); MetaServiceCode code = MetaServiceCode::OK; std::string msg = "OK"; - std::unique_ptr> defer_status( - (int*)0x01, [&code, &msg, &response, &ctrl](int*) { - response->mutable_status()->set_code(code); - response->mutable_status()->set_msg(msg); - LOG(INFO) << (code == MetaServiceCode::OK ? "succ to " : "failed to ") - << "recycle_instance" - << " " << ctrl->remote_side() << " " << msg; - }); + DORIS_CLOUD_DEFER { + response->mutable_status()->set_code(code); + response->mutable_status()->set_msg(msg); + LOG(INFO) << (code == MetaServiceCode::OK ? "succ to " : "failed to ") << "recycle_instance" + << " " << ctrl->remote_side() << " " << msg; + }; std::vector instances; instances.reserve(request->instance_ids_size()); @@ -275,16 +274,15 @@ void RecyclerServiceImpl::http(::google::protobuf::RpcController* controller, std::string req; std::string response_body; std::string request_body; - std::unique_ptr> defer_status( - (int*)0x01, [&code, &msg, &status_code, &response_body, &cntl, &req](int*) { - status_code = std::get<0>(convert_ms_code_to_http_code(code)); - LOG(INFO) << (code == MetaServiceCode::OK ? "succ to " : "failed to ") << "http" - << " " << cntl->remote_side() << " request=\n" - << req << "\n ret=" << code << " msg=" << msg; - cntl->http_response().set_status_code(status_code); - cntl->response_attachment().append(response_body); - cntl->response_attachment().append("\n"); - }); + DORIS_CLOUD_DEFER { + status_code = std::get<0>(convert_ms_code_to_http_code(code)); + LOG(INFO) << (code == MetaServiceCode::OK ? "succ to " : "failed to ") << "http" + << " " << cntl->remote_side() << " request=\n" + << req << "\n ret=" << code << " msg=" << msg; + cntl->http_response().set_status_code(status_code); + cntl->response_attachment().append(response_body); + cntl->response_attachment().append("\n"); + }; // Prepare input request info auto unresolved_path = cntl->http_request().unresolved_path(); diff --git a/cloud/src/recycler/sync_executor.h b/cloud/src/recycler/sync_executor.h index 909f36a56c4c9a..95650e5316a4d5 100644 --- a/cloud/src/recycler/sync_executor.h +++ b/cloud/src/recycler/sync_executor.h @@ -27,6 +27,7 @@ #include #include +#include "common/defer.h" #include "common/simple_thread_pool.h" namespace doris::cloud { @@ -50,7 +51,9 @@ class SyncExecutor { return *this; } std::vector when_all(bool* finished) { - std::unique_ptr> defer((int*)0x01, [&](int*) { _reset(); }); + DORIS_CLOUD_DEFER { + _reset(); + }; timespec current_time; auto current_time_second = time(nullptr); current_time.tv_sec = current_time_second + 300; @@ -103,8 +106,9 @@ class SyncExecutor { _count(count), _fut(_pro.get_future()) {} void operator()(std::atomic_bool& stop_token) { - std::unique_ptr> defer((int*)0x01, - [&](int*) { _count.signal(); }); + DORIS_CLOUD_DEFER { + _count.signal(); + }; if (stop_token) { _valid = false; return; diff --git a/cloud/src/recycler/util.h b/cloud/src/recycler/util.h index 5aa929c23984b1..6efa0bec76e034 100644 --- a/cloud/src/recycler/util.h +++ b/cloud/src/recycler/util.h @@ -23,13 +23,16 @@ #include +#include "common/defer.h" + namespace doris::cloud { // The time unit is the same with BE: us -#define SCOPED_BVAR_LATENCY(bvar_item) \ - StopWatch sw; \ - std::unique_ptr> defer( \ - (int*)0x01, [&](int*) { bvar_item << sw.elapsed_us(); }); +#define SCOPED_BVAR_LATENCY(bvar_item) \ + StopWatch sw; \ + DORIS_CLOUD_DEFER { \ + bvar_item << sw.elapsed_us(); \ + }; class TxnKv; diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp index cd0381b7189aa5..99296d025cc3ce 100644 --- a/cloud/src/resource-manager/resource_manager.cpp +++ b/cloud/src/resource-manager/resource_manager.cpp @@ -53,11 +53,10 @@ int ResourceManager::init() { std::unique_ptr it; int num_instances = 0; - std::unique_ptr> defer_log_range( - (int*)0x01, [key0, key1, &num_instances](int*) { - LOG(INFO) << "get instances, num_instances=" << num_instances << " range=[" - << hex(key0) << "," << hex(key1) << "]"; - }); + DORIS_CLOUD_DEFER_COPY(key0, key1) { + LOG(INFO) << "get instances, num_instances=" << num_instances << " range=[" << hex(key0) + << "," << hex(key1) << "]"; + }; // instance_id instance std::vector> instances; @@ -312,8 +311,9 @@ std::pair ResourceManager::add_cluster(const std:: std::string msg; std::stringstream ss; - std::unique_ptr> defer( - (int*)0x01, [&msg](int*) { LOG(INFO) << "add_cluster err=" << msg; }); + DORIS_CLOUD_DEFER { + LOG(INFO) << "add_cluster err=" << msg; + }; // just check cluster_name not empty in add_cluster if (!check_cluster_params_valid(cluster.cluster, &msg, true, true)) { @@ -856,8 +856,9 @@ std::string ResourceManager::modify_nodes(const std::string& instance_id, const std::vector& to_del) { std::string msg; std::stringstream ss; - std::unique_ptr> defer( - (int*)0x01, [&msg](int*) { LOG(INFO) << "modify_nodes err=" << msg; }); + DORIS_CLOUD_DEFER { + LOG(INFO) << "modify_nodes err=" << msg; + }; if ((to_add.size() && to_del.size()) || (!to_add.size() && !to_del.size())) { msg = "to_add and to_del both empty or both not empty"; @@ -1220,12 +1221,11 @@ std::pair ResourceManager::refresh_instance( LOG(INFO) << "begin to refresh instance, instance_id=" << instance_id << " seq=" << ++seq; std::pair ret0 {MetaServiceCode::OK, "OK"}; auto& [code, msg] = ret0; - std::unique_ptr> defer_log( - (int*)0x01, [&ret0, &instance_id](int*) { - LOG(INFO) << (std::get<0>(ret0) == MetaServiceCode::OK ? "succ to " : "failed to ") - << "refresh_instance, instance_id=" << instance_id - << " code=" << std::get<0>(ret0) << " msg=" << std::get<1>(ret0); - }); + DORIS_CLOUD_DEFER { + LOG(INFO) << (std::get<0>(ret0) == MetaServiceCode::OK ? "succ to " : "failed to ") + << "refresh_instance, instance_id=" << instance_id + << " code=" << std::get<0>(ret0) << " msg=" << std::get<1>(ret0); + }; std::unique_ptr txn0; TxnErrorCode err = txn_kv_->create_txn(&txn0); diff --git a/cloud/test/encryption_test.cpp b/cloud/test/encryption_test.cpp index ed29177ac2791c..072e6a820cfd05 100644 --- a/cloud/test/encryption_test.cpp +++ b/cloud/test/encryption_test.cpp @@ -19,6 +19,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "common/encryption_util.h" #include "common/logging.h" #include "common/util.h" @@ -186,8 +187,9 @@ TEST(EncryptionTest, RootKeyTestWithKms2) { { // mock falied to generate key auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("alikms::generate_data_key", [](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = -1; @@ -208,8 +210,9 @@ TEST(EncryptionTest, RootKeyTestWithKms2) { { // mock succ to generate key auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [&](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("alikms::generate_data_key", [&](auto&& args) { auto* ciphertext = try_any_cast(args[0]); *ciphertext = mock_encoded_ciphertext; @@ -249,8 +252,9 @@ TEST(EncryptionTest, RootKeyTestWithKms2) { // mock abnormal decryption auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("alikms::decrypt", [](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = -1; @@ -267,8 +271,9 @@ TEST(EncryptionTest, RootKeyTestWithKms2) { // Decryption succeeded { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("alikms::decrypt", [&](auto&& args) { auto* output = try_any_cast(args[0]); *output = mock_encoded_plaintext; @@ -320,8 +325,9 @@ TEST(EncryptionTest, RootKeyTestWithKms3) { std::string mock_encoded_ciphertext = mock_encoded_plaintext; // mock succ to generate key auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("alikms::generate_data_key", [&](auto&& args) { auto* ciphertext = try_any_cast(args[0]); *ciphertext = mock_encoded_ciphertext; diff --git a/cloud/test/http_encode_key_test.cpp b/cloud/test/http_encode_key_test.cpp index d245682474349d..58fdb3a4c97ef1 100644 --- a/cloud/test/http_encode_key_test.cpp +++ b/cloud/test/http_encode_key_test.cpp @@ -19,6 +19,7 @@ #include #include +#include "common/defer.h" #include "common/logging.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -95,8 +96,9 @@ v v v v v // test empty body branch auto sp = doris::SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { doris::SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + doris::SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("process_http_encode_key::empty_body", [](auto&& args) { auto* body = doris::try_any_cast(args[0]); body->clear(); diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index 15fd50785ac54d..8de04af4a8a19d 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -33,6 +33,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "common/logging.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -170,12 +171,12 @@ TEST(MetaServerTest, StartAndStop) { // use structured binding for point alias (avoid multi lines of declaration) auto [meta_server_start_1, meta_server_start_2, meta_server_start_3] = sps; sp->enable_processing(); - std::unique_ptr> defer((int*)0x01, [&](...) { + DORIS_CLOUD_DEFER { for (auto& i : sps) { sp->clear_call_back(i); } // redundant sp->disable_processing(); - }); + }; auto foo = [](auto&& args) { auto* ret = try_any_cast(args[0]); diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp index 7131505ae7f70b..64a9a47515fcbe 100644 --- a/cloud/test/meta_service_http_test.cpp +++ b/cloud/test/meta_service_http_test.cpp @@ -41,6 +41,7 @@ #include "common/config.h" #include "common/configbase.h" +#include "common/defer.h" #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" @@ -1004,8 +1005,9 @@ TEST(MetaServiceHttpTest, AlterIamTest) { auto cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "alter_iam_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -1523,8 +1525,9 @@ TEST(MetaServiceHttpTest, TxnLazyCommit) { TEST(MetaServiceHttpTest, get_stage_response_sk) { auto sp = SyncPoint::get_instance(); sp->enable_processing(); - std::unique_ptr> defer((int*)0x01, - [&](...) { sp->disable_processing(); }); + DORIS_CLOUD_DEFER { + sp->disable_processing(); + }; GetStageResponse res; auto* stage = res.add_stage(); @@ -1557,8 +1560,9 @@ TEST(MetaServiceHttpTest, get_stage_response_sk) { TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) { auto sp = SyncPoint::get_instance(); sp->enable_processing(); - std::unique_ptr> defer((int*)0x01, - [&](...) { sp->disable_processing(); }); + DORIS_CLOUD_DEFER { + sp->disable_processing(); + }; GetObjStoreInfoResponse res; auto* obj_info = res.add_obj_info(); diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp index 521d01d101fbad..21ef2e41349449 100644 --- a/cloud/test/meta_service_job_test.cpp +++ b/cloud/test/meta_service_job_test.cpp @@ -30,6 +30,7 @@ #include #include +#include "common/defer.h" #include "common/util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -293,8 +294,9 @@ void finish_schema_change_job( TEST(MetaServiceJobTest, StartCompactionArguments) { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -356,8 +358,9 @@ TEST(MetaServiceJobTest, StartCompactionArguments) { TEST(MetaServiceJobTest, StartFullCompaction) { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -411,8 +414,9 @@ TEST(MetaServiceJobTest, StartFullCompaction) { TEST(MetaServiceJobTest, StartSchemaChangeArguments) { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -516,8 +520,9 @@ TEST(MetaServiceJobTest, StartSchemaChangeArguments) { TEST(MetaServiceJobTest, ProcessCompactionArguments) { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -615,8 +620,9 @@ TEST(MetaServiceJobTest, ProcessCompactionArguments) { TEST(MetaServiceJobTest, ProcessSchemaChangeArguments) { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -759,8 +765,9 @@ TEST(MetaServiceJobTest, CompactionJobTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -1100,8 +1107,9 @@ TEST(MetaServiceJobTest, CompactionJobTest) { TEST(MetaServiceJobTest, CompactionJobWithMoWTest) { auto meta_service = get_meta_service(); auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -1366,8 +1374,9 @@ TEST(MetaServiceJobTest, SchemaChangeJobTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -1590,8 +1599,9 @@ TEST(MetaServiceJobTest, RetrySchemaChangeJobTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -1755,8 +1765,9 @@ TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -1910,8 +1921,9 @@ TEST(MetaServiceJobTest, ConcurrentCompactionTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2261,8 +2273,9 @@ TEST(MetaServiceJobTest, ParallelCumuCompactionTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2380,8 +2393,9 @@ TEST(MetaServiceJobTest, SchemaChangeJobPersistTest) { auto meta_service = get_meta_service(); auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2447,8 +2461,9 @@ TEST(MetaServiceJobTest, DoCompactionWhenSC) { auto meta_service = get_meta_service(); auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2534,8 +2549,9 @@ TEST(MetaServiceJobTest, ReStartSC) { auto meta_service = get_meta_service(); auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -2579,8 +2595,9 @@ TEST(MetaServiceJobTest, CancelSC) { auto meta_service = get_meta_service(); auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index d02ea7b5011257..994086703c8854 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -351,8 +351,9 @@ TEST(MetaServiceTest, GetInstanceIdTest) { const std::string& cloud_unique_id); auto meta_service = get_meta_service(); auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id_err", [&](auto&& args) { std::string* err = try_any_cast(args[0]); *err = "can't find node from cache"; @@ -1183,8 +1184,9 @@ TEST(MetaServiceTest, BeginTxnTest) { std::condition_variable go_cv; bool go = false; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; std::atomic count_txn1 = {0}; std::atomic count_txn2 = {0}; @@ -1330,8 +1332,9 @@ TEST(MetaServiceTest, BeginTxnTest) { std::condition_variable go_cv; bool go = false; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; std::atomic count_txn1 = {0}; std::atomic count_txn2 = {0}; @@ -3453,8 +3456,9 @@ TEST(MetaServiceTest, CopyJobTest) { std::string instance_id = "copy_job_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -3931,8 +3935,9 @@ TEST(MetaServiceTest, StageTest) { auto cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "stage_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4269,8 +4274,9 @@ TEST(MetaServiceTest, GetIamTest) { auto cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "get_iam_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4352,8 +4358,9 @@ TEST(MetaServiceTest, AlterRamTest) { auto cloud_unique_id = "test_cloud_unique_id"; std::string instance_id = "alter_iam_test_instance_id"; [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4695,10 +4702,10 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsNormal) { std::string instance_id = "test_get_delete_bitmap_update_lock_normal"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [](int*) { + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); - }); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4745,10 +4752,10 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) { // the reading of tablet stats std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal1"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [](int*) { + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); - }); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4788,10 +4795,10 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) { // the reading of tablet stats std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal2"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [](int*) { + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); - }); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4831,10 +4838,10 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsError) { // 2.3 abnormal path, meeting error when reading tablets' stats std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal3"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [](int*) { + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); - }); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4871,10 +4878,10 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsError) { // this should not fail if lock is not expired std::string instance_id = "test_get_delete_bitmap_update_lock_abnormal4"; [[maybe_unused]] auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [](int*) { + DORIS_CLOUD_DEFER { SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); - }); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -6443,8 +6450,9 @@ TEST(MetaServiceTest, BatchGetVersionFallback) { constexpr size_t N = 100; size_t i = 0; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("batch_get_version_err", [&](auto&& args) { if (i++ == N / 10) { *try_any_cast(args) = TxnErrorCode::TXN_TOO_OLD; @@ -6489,8 +6497,9 @@ TEST(MetaServiceTest, IsDroppedTablet) { auto meta_service = get_meta_service(); std::string instance_id = "IsDroppedTablet"; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -6570,8 +6579,9 @@ TEST(MetaServiceTest, IndexRequest) { auto meta_service = get_meta_service(); std::string instance_id = "IndexRequest"; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -6813,8 +6823,9 @@ TEST(MetaServiceTest, PartitionRequest) { auto meta_service = get_meta_service(); std::string instance_id = "PartitionRequest"; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -8667,8 +8678,9 @@ TEST(MetaServiceTest, UpdateTmpRowsetTest) { std::string instance_id = "update_rowset_meta_test_instance_id"; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -9086,8 +9098,9 @@ TEST(MetaServiceTest, CheckJobExisted) { std::string instance_id = "check_job_existed_instance_id"; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index e70d54921826a8..bb6ba0c4b7f462 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -772,8 +772,9 @@ TEST(RecyclerTest, recycle_rowsets) { int insert_no_inverted_index = 0; int insert_inverted_index = 0; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("InvertedIndexIdCache::insert1", [&](auto&&) { ++insert_no_inverted_index; }); sp->set_call_back("InvertedIndexIdCache::insert2", [&](auto&&) { ++insert_inverted_index; }); sp->enable_processing(); @@ -849,8 +850,9 @@ TEST(RecyclerTest, bench_recycle_rowsets) { ASSERT_EQ(recycler.init(), 0); auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("memkv::Transaction::get", [](auto&& args) { auto* limit = try_any_cast(args[0]); *limit = 100; @@ -922,8 +924,9 @@ TEST(RecyclerTest, recycle_tmp_rowsets) { int insert_no_inverted_index = 0; int insert_inverted_index = 0; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("InvertedIndexIdCache::insert1", [&](auto&&) { ++insert_no_inverted_index; }); sp->set_call_back("InvertedIndexIdCache::insert2", [&](auto&&) { ++insert_inverted_index; }); sp->enable_processing(); @@ -1987,8 +1990,9 @@ TEST(RecyclerTest, recycle_copy_jobs) { TEST(RecyclerTest, recycle_batch_copy_jobs) { auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("MockAccessor::delete_files", [](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = -1; @@ -2110,8 +2114,9 @@ TEST(RecyclerTest, recycle_batch_copy_jobs) { TEST(RecyclerTest, recycle_stage) { [[maybe_unused]] auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); ASSERT_NE(txn_kv.get(), nullptr); ASSERT_EQ(txn_kv->init(), 0); @@ -2408,8 +2413,9 @@ TEST(CheckerTest, normal_inverted_check) { }, &guard); sp->enable_processing(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->disable_processing(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->disable_processing(); + }; InstanceChecker checker(txn_kv, instance_id); ASSERT_EQ(checker.init(instance), 0); @@ -2457,8 +2463,9 @@ TEST(CheckerTest, DISABLED_abnormal_inverted_check) { }, &guard); sp->enable_processing(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->disable_processing(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->disable_processing(); + }; InstanceChecker checker(txn_kv, instance_id); ASSERT_EQ(checker.init(instance), 0); @@ -2786,8 +2793,9 @@ TEST(CheckerTest, do_inspect) { { // empty job info auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("Checker:do_inspect", [](auto&& args) { auto last_ctime = *try_any_cast(args[0]); ASSERT_EQ(last_ctime, 11111); @@ -2808,8 +2816,9 @@ TEST(CheckerTest, do_inspect) { ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit()); checker.do_inspect(instance); auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("Checker:do_inspect", [](auto&& args) { ASSERT_EQ(*try_any_cast(args[0]), 11111); }); @@ -2824,8 +2833,9 @@ TEST(CheckerTest, do_inspect) { job_info.set_instance_id(instance_id); job_info.set_last_ctime_ms(12345); auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("Checker:do_inspect", [](auto&& args) { ASSERT_EQ(*try_any_cast(args[0]), 12345); }); @@ -2848,8 +2858,9 @@ TEST(CheckerTest, do_inspect) { job_info.set_instance_id(instance_id); job_info.set_last_ctime_ms(now - expiration_ms - 10); auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; bool alarm = false; sp->set_call_back("Checker:do_inspect", [&alarm](auto&&) { alarm = true; }); @@ -2944,8 +2955,9 @@ TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) { std::map>> expected_leaked_delete_bitmaps {}, real_leaked_delete_bitmaps {}; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back( "InstanceChecker::do_delete_bitmap_inverted_check.get_abnormal_delete_bitmap", [&real_abnormal_delete_bitmaps](auto&& args) { @@ -3168,8 +3180,9 @@ TEST(CheckerTest, delete_bitmap_storage_optimize_check_abnormal) { std::map> expected_abnormal_rowsets {}; std::map> real_abnormal_rowsets {}; auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("InstanceChecker::check_delete_bitmap_storage_optimize.get_abnormal_rowset", [&real_abnormal_rowsets](auto&& args) { int64_t tablet_id = *try_any_cast(args[0]); @@ -3425,11 +3438,11 @@ TEST(RecyclerTest, delete_rowset_data_without_inverted_index_storage_format) { TEST(RecyclerTest, init_vault_accessor_failed_test) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); EXPECT_EQ(txn_kv->init(), 0); @@ -3560,11 +3573,11 @@ TEST(RecyclerTest, init_vault_accessor_failed_test) { TEST(RecyclerTest, recycle_tablet_without_resource_id) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); EXPECT_EQ(txn_kv->init(), 0); @@ -3642,11 +3655,11 @@ TEST(RecyclerTest, recycle_tablet_without_resource_id) { TEST(RecyclerTest, recycle_tablet_with_wrong_resource_id) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); EXPECT_EQ(txn_kv->init(), 0); @@ -3724,11 +3737,11 @@ TEST(RecyclerTest, recycle_tablet_with_wrong_resource_id) { TEST(RecyclerTest, init_all_vault_accessors_failed_test) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); EXPECT_EQ(txn_kv->init(), 0); @@ -3770,11 +3783,11 @@ TEST(RecyclerTest, init_all_vault_accessors_failed_test) { TEST(RecyclerTest, recycler_storage_vault_white_list_test) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); EXPECT_EQ(txn_kv->init(), 0); @@ -3904,11 +3917,11 @@ TEST(RecyclerTest, recycler_storage_vault_white_list_test) { TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0); @@ -3984,11 +3997,11 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) { TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0); @@ -4063,11 +4076,11 @@ TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v2) { TEST(RecyclerTest, delete_tmp_rowset_without_resource_id) { auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer((int*)0x01, [&sp](int*) { + DORIS_CLOUD_DEFER { sp->clear_all_call_backs(); sp->clear_trace(); sp->disable_processing(); - }); + }; auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0); @@ -4463,8 +4476,9 @@ TEST(RecyclerTest, concurrent_recycle_txn_label_failure_test) { check_multiple_txn_info_kvs(txn_kv, 20000); auto* sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.check_recycle_txn_info_keys", [](auto&& args) { auto* recycle_txn_info_keys = diff --git a/cloud/test/schema_kv_test.cpp b/cloud/test/schema_kv_test.cpp index 2ab8261949e4f5..1840598b40e5da 100644 --- a/cloud/test/schema_kv_test.cpp +++ b/cloud/test/schema_kv_test.cpp @@ -24,6 +24,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" #include "meta-service/meta_service.h" @@ -114,8 +115,9 @@ TEST(DetachSchemaKVTest, TabletTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -396,8 +398,9 @@ TEST(DetachSchemaKVTest, RowsetTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -527,9 +530,9 @@ TEST(DetachSchemaKVTest, RowsetTest) { } // check get rowset response auto get_rowset_res = google::protobuf::Arena::CreateMessage(arena); - std::unique_ptr> defer((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (!arena) delete get_rowset_res; - }); + }; get_rowset(meta_service.get(), table_id, index_id, partition_id, tablet_id, *get_rowset_res); ASSERT_EQ(get_rowset_res->rowset_meta_size(), schema_versions.size()); @@ -578,8 +581,9 @@ TEST(DetachSchemaKVTest, InsertExistedRowsetTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -633,9 +637,9 @@ TEST(DetachSchemaKVTest, InsertExistedRowsetTest) { tablet_id, next_rowset_id(), 1)); auto committed_rowset = create_rowset(txn_id, tablet_id, next_rowset_id(), 2, 2); auto res = google::protobuf::Arena::CreateMessage(arena); - std::unique_ptr> defer((int*)0x01, [&](int*) { + DORIS_CLOUD_DEFER { if (!arena) delete res; - }); + }; prepare_rowset(meta_service.get(), committed_rowset, *res); ASSERT_EQ(res->status().code(), MetaServiceCode::OK); res->Clear(); @@ -671,8 +675,9 @@ TEST(SchemaKVTest, InsertExistedRowsetTest) { // meta_service->resource_mgr().reset(); // Do not use resource manager auto sp = SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; diff --git a/cloud/test/txn_kv_test.cpp b/cloud/test/txn_kv_test.cpp index 27088f3da72a4a..78c8c18da4272b 100644 --- a/cloud/test/txn_kv_test.cpp +++ b/cloud/test/txn_kv_test.cpp @@ -30,6 +30,7 @@ #include #include "common/config.h" +#include "common/defer.h" #include "common/stopwatch.h" #include "common/util.h" #include "cpp/sync_point.h" @@ -284,8 +285,9 @@ TEST(TxnKvTest, PutLargeValueTest) { auto txn_kv = std::make_shared(); auto sp = doris::SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { doris::SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + doris::SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->enable_processing(); doris::TabletSchemaCloudPB schema; @@ -573,8 +575,9 @@ TEST(TxnKvTest, FullRangeGetIterator) { encode_int64(INT64_MAX, &end); auto* sp = doris::SyncPoint::get_instance(); - std::unique_ptr> defer( - (int*)0x01, [](int*) { doris::SyncPoint::get_instance()->clear_all_call_backs(); }); + DORIS_CLOUD_DEFER { + doris::SyncPoint::get_instance()->clear_all_call_backs(); + }; sp->enable_processing(); { From 8c6809b4d3a280046d4f43f500a4e410f1a2a88b Mon Sep 17 00:00:00 2001 From: zhengyu Date: Tue, 1 Jul 2025 12:07:00 +0800 Subject: [PATCH 117/572] [optimization](filecache) speed up filecache warm up #51776 (#52556) pick #51776 pick #51776 pick #51776 this pr does the following: make file cache downloader worker pool thread num configurable make warm up job split batch size configurable split large file downloading task to smaller ones to maintain load balance between threads, thus improve concurrency use meta info to deduce size of inverted idx file size to reduce S3 HEAD ops some log print optimization in our test, this opt can improve more than 3x file cache warm up performance Signed-off-by: zhengyu --- be/src/cloud/cloud_backend_service.cpp | 4 +- be/src/cloud/cloud_tablet.cpp | 12 +- be/src/cloud/cloud_warm_up_manager.cpp | 138 +++++++++++------- be/src/cloud/cloud_warm_up_manager.h | 6 +- be/src/common/config.cpp | 3 + be/src/common/config.h | 3 + .../io/cache/block_file_cache_downloader.cpp | 4 +- be/src/io/cache/block_file_cache_downloader.h | 2 +- .../java/org/apache/doris/common/Config.java | 3 + .../doris/cloud/CacheHotspotManager.java | 2 +- 10 files changed, 116 insertions(+), 61 deletions(-) diff --git a/be/src/cloud/cloud_backend_service.cpp b/be/src/cloud/cloud_backend_service.cpp index 265e6c44aac9ab..f94807282b380c 100644 --- a/be/src/cloud/cloud_backend_service.cpp +++ b/be/src/cloud/cloud_backend_service.cpp @@ -105,7 +105,9 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response, .tag("request_type", "SET_BATCH") .tag("job_id", request.job_id) .tag("batch_id", request.batch_id) - .tag("jobs size", request.job_metas.size()); + .tag("jobs size", request.job_metas.size()) + .tag("tablet num of first meta", + request.job_metas.empty() ? 0 : request.job_metas[0].tablet_ids.size()); bool retry = false; st = manager.check_and_set_batch_id(request.job_id, request.batch_id, &retry); if (!retry && st) { diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 4344b085a838dc..78663de6ed86d6 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -279,7 +279,11 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ .expiration_time = expiration_time, .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, }, - .download_done {}, + .download_done {[](Status st) { + if (!st) { + LOG_WARNING("add rowset warm up error ").error(st); + } + }}, }); auto download_idx_file = [&](const io::Path& idx_path) { @@ -292,7 +296,11 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ .expiration_time = expiration_time, .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, }, - .download_done {}, + .download_done {[](Status st) { + if (!st) { + LOG_WARNING("add rowset warm up error ").error(st); + } + }}, }; _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); }; diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index d8bce097465dde..510c677f06f34c 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -17,7 +17,8 @@ #include "cloud/cloud_warm_up_manager.h" -#include +#include +#include #include #include @@ -34,6 +35,8 @@ namespace doris { +bvar::Adder file_cache_warm_up_failed_task_num("file_cache_warm_up", "failed_task_num"); + CloudWarmUpManager::CloudWarmUpManager(CloudStorageEngine& engine) : _engine(engine) { _download_thread = std::thread(&CloudWarmUpManager::handle_jobs, this); } @@ -59,6 +62,52 @@ std::unordered_map snapshot_rs_metas(BaseTable return id_to_rowset_meta_map; } +void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, + io::FileSystemSPtr file_system, + int64_t expiration_time, + std::shared_ptr wait) { + if (file_size < 0) { + auto st = file_system->file_size(path, &file_size); + if (!st.ok()) [[unlikely]] { + LOG(WARNING) << "get file size failed: " << path; + file_cache_warm_up_failed_task_num << 1; + return; + } + } + + const int64_t chunk_size = 10 * 1024 * 1024; // 10MB + int64_t offset = 0; + int64_t remaining_size = file_size; + + while (remaining_size > 0) { + int64_t current_chunk_size = std::min(chunk_size, remaining_size); + wait->add_count(); + + _engine.file_cache_block_downloader().submit_download_task(io::DownloadFileMeta { + .path = path, + .file_size = file_size, + .offset = offset, + .download_size = current_chunk_size, + .file_system = file_system, + .ctx = + { + .expiration_time = expiration_time, + .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + }, + .download_done = + [wait](Status st) { + if (!st) { + LOG_WARNING("Warm up error ").error(st); + } + wait->signal(); + }, + }); + + offset += current_chunk_size; + remaining_size -= current_chunk_size; + } +} + void CloudWarmUpManager::handle_jobs() { #ifndef BE_TEST constexpr int WAIT_TIME_SECONDS = 600; @@ -77,6 +126,10 @@ void CloudWarmUpManager::handle_jobs() { LOG_WARNING("Warm up job is null"); continue; } + + std::shared_ptr wait = + std::make_shared(0); + for (int64_t tablet_id : cur_job->tablet_ids) { if (_cur_job_id == 0) { // The job is canceled break; @@ -92,8 +145,7 @@ void CloudWarmUpManager::handle_jobs() { LOG_WARNING("Warm up error ").tag("tablet_id", tablet_id).error(st); continue; } - std::shared_ptr wait = - std::make_shared(0); + auto tablet_meta = tablet->tablet_meta(); auto rs_metas = snapshot_rs_metas(tablet.get()); for (auto& [_, rs] : rs_metas) { @@ -112,71 +164,51 @@ void CloudWarmUpManager::handle_jobs() { expiration_time = 0; } - wait->add_count(); - // clang-format off - _engine.file_cache_block_downloader().submit_download_task(io::DownloadFileMeta { - .path = storage_resource.value()->remote_segment_path(*rs, seg_id), - .file_size = rs->segment_file_size(seg_id), - .file_system = storage_resource.value()->fs, - .ctx = - { - .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, - }, - .download_done = - [wait](Status st) { - if (!st) { - LOG_WARNING("Warm up error ").error(st); - } - wait->signal(); - }, - }); - - auto download_idx_file = [&](const io::Path& idx_path) { - io::DownloadFileMeta meta { - .path = idx_path, - .file_size = -1, - .file_system = storage_resource.value()->fs, - .ctx = - { - .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, - }, - .download_done = - [wait](Status st) { - if (!st) { - LOG_WARNING("Warm up error ").error(st); - } - wait->signal(); - }, - }; - // clang-format on - _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); - }; + // 1st. download segment files + submit_download_tasks( + storage_resource.value()->remote_segment_path(*rs, seg_id), + rs->segment_file_size(seg_id), storage_resource.value()->fs, + expiration_time, wait); + + // 2nd. download inverted index files + int64_t file_size = -1; auto schema_ptr = rs->tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); + const auto& idx_file_info = rs->inverted_index_file_info(seg_id); if (idx_version == InvertedIndexStorageFormatPB::V1) { for (const auto& index : schema_ptr->inverted_indexes()) { - wait->add_count(); auto idx_path = storage_resource.value()->remote_idx_v1_path( *rs, seg_id, index->index_id(), index->get_index_suffix()); - download_idx_file(idx_path); + if (idx_file_info.index_info_size() > 0) { + for (const auto& idx_info : idx_file_info.index_info()) { + if (index->index_id() == idx_info.index_id() && + index->get_index_suffix() == idx_info.index_suffix()) { + file_size = idx_info.index_file_size(); + break; + } + } + } + submit_download_tasks(idx_path, file_size, storage_resource.value()->fs, + expiration_time, wait); } } else { if (schema_ptr->has_inverted_index()) { - wait->add_count(); auto idx_path = storage_resource.value()->remote_idx_v2_path(*rs, seg_id); - download_idx_file(idx_path); + file_size = idx_file_info.has_index_size() ? idx_file_info.index_size() + : -1; + submit_download_tasks(idx_path, file_size, storage_resource.value()->fs, + expiration_time, wait); } } } } - timespec time; - time.tv_sec = UnixSeconds() + WAIT_TIME_SECONDS; - if (!wait->timed_wait(time)) { - LOG_WARNING("Warm up tablet {} take a long time", tablet_meta->tablet_id()); - } + } + + timespec time; + time.tv_sec = UnixSeconds() + WAIT_TIME_SECONDS; + if (wait->timed_wait(time)) { + LOG_WARNING("Warm up {} tablets take a long time", cur_job->tablet_ids.size()); } { std::unique_lock lock(_mtx); diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 219dedc58065a6..356d7284f6f3ee 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -17,6 +17,8 @@ #pragma once +#include + #include #include #include @@ -69,7 +71,9 @@ class CloudWarmUpManager { private: void handle_jobs(); - + void submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, + int64_t expiration_time, + std::shared_ptr wait); std::mutex _mtx; std::condition_variable _cond; int64_t _cur_job_id {0}; diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 3df7058c12d1ce..2fcedc2fd235d3 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1128,6 +1128,9 @@ DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000"); DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "3000"); DEFINE_mInt64(file_cache_background_ttl_gc_batch, "1000"); +DEFINE_Int32(file_cache_downloader_thread_num_min, "32"); +DEFINE_Int32(file_cache_downloader_thread_num_max, "32"); + DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800"); DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600"); // inverted index searcher cache size diff --git a/be/src/common/config.h b/be/src/common/config.h index 0fa93d0f8f8ef1..5f5a8f14c2dd81 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1152,6 +1152,7 @@ DECLARE_mBool(enbale_dump_error_file); DECLARE_mInt64(file_cache_error_log_limit_bytes); DECLARE_mInt64(cache_lock_wait_long_tail_threshold_us); DECLARE_mInt64(cache_lock_held_long_tail_threshold_us); + // Base compaction may retrieve and produce some less frequently accessed data, // potentially affecting the file cache hit rate. // This configuration determines whether to retain the output within the file cache. @@ -1165,6 +1166,8 @@ DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); DECLARE_mInt64(file_cache_background_monitor_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms); DECLARE_mInt64(file_cache_background_ttl_gc_batch); +DECLARE_Int32(file_cache_downloader_thread_num_min); +DECLARE_Int32(file_cache_downloader_thread_num_max); // inverted index searcher cache // cache entry stay time after lookup diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index b9944e39989d2b..05c18e0b945ce3 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -43,8 +43,8 @@ namespace doris::io { FileCacheBlockDownloader::FileCacheBlockDownloader(CloudStorageEngine& engine) : _engine(engine) { _poller = std::thread(&FileCacheBlockDownloader::polling_download_task, this); auto st = ThreadPoolBuilder("FileCacheBlockDownloader") - .set_min_threads(4) - .set_max_threads(16) + .set_min_threads(config::file_cache_downloader_thread_num_min) + .set_max_threads(config::file_cache_downloader_thread_num_max) .build(&_workers); CHECK(st.ok()) << "failed to create FileCacheBlockDownloader"; } diff --git a/be/src/io/cache/block_file_cache_downloader.h b/be/src/io/cache/block_file_cache_downloader.h index 30827b69580553..c9a4689167363f 100644 --- a/be/src/io/cache/block_file_cache_downloader.h +++ b/be/src/io/cache/block_file_cache_downloader.h @@ -92,7 +92,7 @@ class FileCacheBlockDownloader { // tablet id -> inflight block num of tablet std::unordered_map _inflight_tablets; - static inline constexpr size_t _max_size {10240}; + static inline constexpr size_t _max_size {102400}; }; } // namespace doris::io diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index de2604ad011a13..b3d92cad6f75ac 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3210,6 +3210,9 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, masterOnly = true) public static int cloud_warm_up_job_scheduler_interval_millisecond = 1000; // 1 seconds + @ConfField(mutable = true, masterOnly = true) + public static long cloud_warm_up_job_max_bytes_per_batch = 21474836480L; // 20GB + @ConfField(mutable = true, masterOnly = true) public static boolean enable_fetch_cluster_cache_hotspot = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index b73e467836d91c..4e073b21473566 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -365,7 +365,7 @@ Long getFileCacheCapacity(String clusterName) throws RuntimeException { } private Map>> splitBatch(Map> beToWarmUpTablets) { - final Long maxSizePerBatch = 10737418240L; // 10G + final Long maxSizePerBatch = Config.cloud_warm_up_job_max_bytes_per_batch; Map>> beToTabletIdBatches = new HashMap<>(); for (Map.Entry> entry : beToWarmUpTablets.entrySet()) { List> batches = new ArrayList<>(); From 374c49c13872205765a9d6ce3129657d78a92a03 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 1 Jul 2025 20:54:28 +0800 Subject: [PATCH 118/572] branch-3.0: [opt](meta-service) Add changed value hex for set_value API #52050 (#52593) Cherry-picked from #52050 Co-authored-by: deardeng --- cloud/src/meta-service/http_encode_key.cpp | 38 +++++++++++++++++++--- cloud/test/meta_service_http_test.cpp | 3 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/cloud/src/meta-service/http_encode_key.cpp b/cloud/src/meta-service/http_encode_key.cpp index b851d0956f6bda..fcb2b81a6a8f00 100644 --- a/cloud/src/meta-service/http_encode_key.cpp +++ b/cloud/src/meta-service/http_encode_key.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include +#include "common/config.h" #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" @@ -326,6 +328,34 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const brpc::URI& uri) { return http_text_reply(MetaServiceCode::OK, "", readable_value); } +std::string handle_kv_output(std::string_view key, std::string_view value, + std::string_view original_value_json, + std::string_view serialized_value_to_save) { + std::stringstream final_output; + final_output << "original_value_hex=" << hex(value) << "\n" + << "key_hex=" << hex(key) << "\n" + << "original_value_json=" << original_value_json << "\n" + << "changed_value_hex=" << hex(serialized_value_to_save) << "\n"; + std::string final_json_str = final_output.str(); + LOG(INFO) << final_json_str; + if (final_json_str.size() > 25000) { + std::string file_path = fmt::format("/tmp/{}.txt", hex(key)); + LOG(INFO) << "write to file=" << file_path << ", key=" << hex(key) + << " size=" << final_json_str.size(); + try { + std::ofstream kv_file(file_path); + if (kv_file.is_open()) { + kv_file << final_json_str; + kv_file.close(); + } + } catch (...) { + LOG(INFO) << "write tmp file failed."; + } + } + + return final_json_str; +} + HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) { const brpc::URI& uri = cntl->http_request().uri(); std::string body = cntl->request_attachment().to_string(); @@ -426,12 +456,10 @@ HttpResponse process_http_set_value(TxnKv* txn_kv, brpc::Controller* cntl) { } LOG(WARNING) << "set_value saved, key=" << hex(key); - std::stringstream final_json; - final_json << "original_value_hex=" << hex(value.value()) << "\n" - << "key_hex=" << hex(key) << "\n" - << "original_value_json=" << original_value_json << "\n"; + std::string final_json_str = + handle_kv_output(key, value.value(), original_value_json, serialized_value_to_save); - return http_text_reply(MetaServiceCode::OK, "", final_json.str()); + return http_text_reply(MetaServiceCode::OK, "", final_json_str); } HttpResponse process_http_encode_key(const brpc::URI& uri) { diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp index 64a9a47515fcbe..325770b658b0b4 100644 --- a/cloud/test/meta_service_http_test.cpp +++ b/cloud/test/meta_service_http_test.cpp @@ -1865,7 +1865,8 @@ TEST(HttpEncodeKeyTest, ProcessHttpSetValue) { std::stringstream final_json; final_json << "original_value_hex=" << hex(initial_rowset_meta.SerializeAsString()) << "\n" << "key_hex=" << hex(initial_key) << "\n" - << "original_value_json=" << proto_to_json(initial_rowset_meta) << "\n"; + << "original_value_json=" << proto_to_json(initial_rowset_meta) << "\n" + << "changed_value_hex=" << hex(new_rowset_meta.SerializeAsString()) << "\n"; // std::cout << "xxx " << final_json.str() << std::endl; EXPECT_EQ(response.body, final_json.str()); From d006e3e3cde850290b346ea00f0775cafa530761 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 09:56:22 +0800 Subject: [PATCH 119/572] branch-3.0: [improve](statistics)Specify column name when insert into stat table. #52441 (#52501) Cherry-picked from #52441 Co-authored-by: James --- .../java/org/apache/doris/statistics/AnalysisJob.java | 2 +- .../apache/doris/statistics/StatisticConstants.java | 10 ++++++++++ .../apache/doris/statistics/StatisticsRepository.java | 7 ++++--- .../org/apache/doris/statistics/AnalysisJobTest.java | 2 +- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java index c7c7077f6bb0c2..bc43f80cbc21a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java @@ -122,7 +122,7 @@ protected void flushBuffer() { // buf could be empty when nothing need to do,r for example user submit an analysis task for table with no data // change if (!buf.isEmpty()) { - String insertStmt = "INSERT INTO " + StatisticConstants.FULL_QUALIFIED_STATS_TBL_NAME + " VALUES "; + String insertStmt = StatisticConstants.INSERT_INTO_COLUMN_STATS_PREFIX; StringJoiner values = new StringJoiner(","); for (ColStatsData data : buf) { values.add(data.toSQL(true)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java index 414a590fb772da..166ebbbec9674b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java @@ -17,8 +17,10 @@ package org.apache.doris.statistics; +import org.apache.doris.analysis.ColumnDef; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.InfoSchemaDb; +import org.apache.doris.catalog.InternalSchema; import org.apache.doris.catalog.MysqlDb; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; @@ -28,6 +30,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; public class StatisticConstants { @@ -73,6 +76,13 @@ public class StatisticConstants { public static final String FULL_QUALIFIED_PARTITION_STATS_TBL_NAME = InternalCatalog.INTERNAL_CATALOG_NAME + "." + FeConstants.INTERNAL_DB_NAME + "." + PARTITION_STATISTIC_TBL_NAME; + public static final String INSERT_INTO_COLUMN_STATS_PREFIX = + "INSERT INTO " + FULL_QUALIFIED_STATS_TBL_NAME + "(" + + InternalSchema.TABLE_STATS_SCHEMA.stream() + .map(ColumnDef::getName).map(s -> String.format("`%s`", s)) + .collect(Collectors.joining(", ")) + + ") VALUES "; + public static final int STATISTIC_INTERNAL_TABLE_REPLICA_NUM = 3; public static final int RETRY_LOAD_QUEUE_SIZE = 1000; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java index ac4704b54c6aae..ca904827af974c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java @@ -82,9 +82,10 @@ public class StatisticsRepository { + FULL_QUALIFIED_COLUMN_HISTOGRAM_NAME + " WHERE `id` = '${id}' AND `catalog_id` = '${catalogId}' AND `db_id` = '${dbId}'"; - private static final String INSERT_INTO_COLUMN_STATISTICS_FOR_ALTER = "INSERT INTO " - + FULL_QUALIFIED_COLUMN_STATISTICS_NAME + " VALUES('${id}', ${catalogId}, ${dbId}, ${tblId}, '${idxId}'," - + "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount}, ${min}, ${max}, ${dataSize}, NOW())"; + private static final String INSERT_INTO_COLUMN_STATISTICS_FOR_ALTER = + StatisticConstants.INSERT_INTO_COLUMN_STATS_PREFIX + + "('${id}', ${catalogId}, ${dbId}, ${tblId}, '${idxId}'," + + "'${colId}', ${partId}, ${count}, ${ndv}, ${nullCount}, ${min}, ${max}, ${dataSize}, NOW())"; private static final String DELETE_TABLE_STATISTICS_BY_COLUMN_TEMPLATE = "DELETE FROM " + FeConstants.INTERNAL_DB_NAME + "." + StatisticConstants.TABLE_STATISTIC_TBL_NAME diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java index 68ce20212a32bf..5b7b430a2cda90 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisJobTest.java @@ -238,7 +238,7 @@ protected void syncLoadStats() { job.flushBuffer(); Assertions.assertEquals(0, job.queryFinished.size()); Assertions.assertEquals(0, job.buf.size()); - Assertions.assertEquals("ffd6aa73b79f9228c737a6da0f4b2834", job.stmtExecutor.getContext().getSqlHash()); + Assertions.assertEquals("d8c3a1696769a1862331eec30d2c1ecb", job.stmtExecutor.getContext().getSqlHash()); } } From 79be96b4afd2d8f919de4fb200d033b8ec3b9fda Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 10:32:08 +0800 Subject: [PATCH 120/572] branch-3.0: [fix](be) Fix `check_storage_vault` deadlock #52541 (#52602) Cherry-picked from #52541 Co-authored-by: Lei Zhang --- be/src/cloud/cloud_storage_engine.cpp | 32 +++++++++++++-------------- be/src/cloud/cloud_storage_engine.h | 4 +++- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 1c07c4924a9c86..74a8f5068a49c2 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -190,7 +190,7 @@ struct RefreshFSVaultVisitor { }; Status CloudStorageEngine::open() { - sync_storage_vault(config::enable_check_storage_vault); + sync_storage_vault(); // TODO(plat1ko): DeleteBitmapTxnManager @@ -335,25 +335,14 @@ Status CloudStorageEngine::start_bg_threads() { return Status::OK(); } -void CloudStorageEngine::sync_storage_vault(bool check_storage_vault) { +void CloudStorageEngine::sync_storage_vault() { cloud::StorageVaultInfos vault_infos; bool enable_storage_vault = false; - auto st = Status::OK(); - while (true) { - st = _meta_mgr->get_storage_vault_info(&vault_infos, &enable_storage_vault); - if (st.ok()) { - break; - } - - if (!check_storage_vault) { - LOG(WARNING) << "failed to get storage vault info. err=" << st; - return; - } - LOG(WARNING) << "failed to get storage vault info from ms, err=" << st - << " sleep 200ms retry or add enable_check_storage_vault=false to be.conf" - << " to skip the check."; - std::this_thread::sleep_for(std::chrono::milliseconds(200)); + auto st = _meta_mgr->get_storage_vault_info(&vault_infos, &enable_storage_vault); + if (!st.ok()) { + LOG(WARNING) << "failed to get storage vault info. err=" << st; + return; } if (vault_infos.empty()) { @@ -361,6 +350,15 @@ void CloudStorageEngine::sync_storage_vault(bool check_storage_vault) { return; } + bool check_storage_vault = false; + bool expected = false; + if (first_sync_storage_vault.compare_exchange_strong(expected, true)) { + check_storage_vault = config::enable_check_storage_vault; + LOG(INFO) << "first sync storage vault info, BE try to check iam role connectivity, " + "check_storage_vault=" + << check_storage_vault; + } + for (auto& [id, vault_info, path_format] : vault_infos) { auto fs = get_filesystem(id); auto status = diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index 9e63be6c36c47f..f21e443a77e076 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -137,7 +137,7 @@ class CloudStorageEngine final : public BaseStorageEngine { std::shared_ptr cumu_compaction_policy( std::string_view compaction_policy); - void sync_storage_vault(bool check = false); + void sync_storage_vault(); io::FileCacheBlockDownloader& file_cache_block_downloader() const { return *_file_cache_block_downloader; @@ -219,6 +219,8 @@ class CloudStorageEngine final : public BaseStorageEngine { using CumuPolices = std::unordered_map>; CumuPolices _cumulative_compaction_policies; + + std::atomic_bool first_sync_storage_vault {true}; }; } // namespace doris From 5551e5862dbb24996df004db77bdecb51b2616de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 10:32:58 +0800 Subject: [PATCH 121/572] branch-3.0: [enhance](compaction) optimize mow base compaction parameters #52321 (#52605) Cherry-picked from #52321 Co-authored-by: Luwei --- be/src/cloud/cloud_base_compaction.cpp | 6 +++++- be/src/cloud/config.cpp | 2 +- be/src/common/config.cpp | 1 + be/src/common/config.h | 1 + be/src/olap/base_compaction.cpp | 6 +++++- 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp index 963ce1fa587308..3b861e1c944139 100644 --- a/be/src/cloud/cloud_base_compaction.cpp +++ b/be/src/cloud/cloud_base_compaction.cpp @@ -198,9 +198,13 @@ Status CloudBaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; + int64_t max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() + ? config::mow_base_compaction_max_compaction_score + : config::base_compaction_max_compaction_score; while (rowset_cnt < _input_rowsets.size()) { score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score(); - if (score > config::base_compaction_max_compaction_score) { + if (score > max_compaction_score) { break; } } diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index d4ae356236b416..234993ff0ab928 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -42,7 +42,7 @@ DEFINE_mInt32(tablet_sync_interval_s, "1800"); DEFINE_mInt32(init_scanner_sync_rowsets_parallelism, "10"); DEFINE_mInt64(min_compaction_failure_interval_ms, "5000"); -DEFINE_mInt64(base_compaction_freeze_interval_s, "7200"); +DEFINE_mInt64(base_compaction_freeze_interval_s, "1800"); DEFINE_mInt64(compaction_load_max_freeze_interval_s, "1200"); DEFINE_mInt64(cumu_compaction_interval_s, "1800"); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 2fcedc2fd235d3..0f9a60aec88e34 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -440,6 +440,7 @@ DEFINE_mInt32(max_single_replica_compaction_threads, "-1"); DEFINE_Bool(enable_base_compaction_idle_sched, "true"); DEFINE_mInt64(base_compaction_min_rowset_num, "5"); DEFINE_mInt64(base_compaction_max_compaction_score, "20"); +DEFINE_mInt64(mow_base_compaction_max_compaction_score, "200"); DEFINE_mDouble(base_compaction_min_data_ratio, "0.3"); DEFINE_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 5f5a8f14c2dd81..af916f1f7bd996 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -488,6 +488,7 @@ DECLARE_mInt32(max_single_replica_compaction_threads); DECLARE_Bool(enable_base_compaction_idle_sched); DECLARE_mInt64(base_compaction_min_rowset_num); DECLARE_mInt64(base_compaction_max_compaction_score); +DECLARE_mInt64(mow_base_compaction_max_compaction_score); DECLARE_mDouble(base_compaction_min_data_ratio); DECLARE_mInt64(base_compaction_dup_key_max_file_size_mbytes); diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp index f09851e201d1ff..f0d961ae45405d 100644 --- a/be/src/olap/base_compaction.cpp +++ b/be/src/olap/base_compaction.cpp @@ -178,9 +178,13 @@ Status BaseCompaction::pick_rowsets_to_compact() { int score = 0; int rowset_cnt = 0; + int64_t max_compaction_score = _tablet->keys_type() == KeysType::UNIQUE_KEYS && + _tablet->enable_unique_key_merge_on_write() + ? config::mow_base_compaction_max_compaction_score + : config::base_compaction_max_compaction_score; while (rowset_cnt < _input_rowsets.size()) { score += _input_rowsets[rowset_cnt++]->rowset_meta()->get_compaction_score(); - if (score > config::base_compaction_max_compaction_score) { + if (score > max_compaction_score) { break; } } From 6693a90a986c46141a95df9870710e2c90115986 Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 2 Jul 2025 10:34:17 +0800 Subject: [PATCH 122/572] branch-3.0: [fix](cloud)Fix auto start affected by daemon jobs #51729 (#52519) cherry pick from #51729 --- .../src/main/java/org/apache/doris/common/Config.java | 9 +++++++++ .../doris/cloud/system/CloudSystemInfoService.java | 9 ++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index b3d92cad6f75ac..35d04db31fe469 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3310,6 +3310,15 @@ public static int metaServiceRpcRetryTimes() { "In cloud mode, the retry number when the FE requests the meta service times out is 1 by default"}) public static int meta_service_rpc_timeout_retry_times = 1; + @ConfField(mutable = true, description = {"存算分离模式下自动启停功能，对于该配置中的数据库名不进行唤醒操作，" + + "用于内部作业的数据库，例如统计信息用到的数据库，" + + "举例: auto_start_ignore_db_names=__internal_schema, information_schema", + "In the cloud mode, the automatic start and stop ignores the DB name of the internal job," + + "used for databases involved in internal jobs, such as those used for statistics, " + + "For example: auto_start_ignore_db_names=__internal_schema, information_schema" + }) + public static String[] auto_start_ignore_resume_db_names = {"__internal_schema", "information_schema"}; + // ATTN: DONOT add any config not related to cloud mode here // ATTN: DONOT add any config not related to cloud mode here // ATTN: DONOT add any config not related to cloud mode here diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index 6603c29b1e4198..f3583fa54877cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -56,6 +56,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.HashSet; @@ -1024,9 +1025,11 @@ public String waitForAutoStart(String clusterName) throws DdlException { LOG.debug("auto start wait cluster {} status {}", clusterName, clusterStatus); if (Cloud.ClusterStatus.valueOf(clusterStatus) != Cloud.ClusterStatus.NORMAL) { // ATTN: prevent `Automatic Analyzer` daemon threads from pulling up clusters - // root ? see StatisticsUtil.buildConnectContext - if (ConnectContext.get() != null && ConnectContext.get().getUserIdentity().isRootUser()) { - LOG.warn("auto start daemon thread run in root, not resume cluster {}-{}", clusterName, clusterStatus); + // FeConstants.INTERNAL_DB_NAME ? see StatisticsUtil.buildConnectContext + List ignoreDbNameList = Arrays.asList(Config.auto_start_ignore_resume_db_names); + if (ConnectContext.get() != null && ignoreDbNameList.contains(ConnectContext.get().getDatabase())) { + LOG.warn("auto start daemon thread db {}, not resume cluster {}-{}", + ConnectContext.get().getDatabase(), clusterName, clusterStatus); return null; } Cloud.AlterClusterRequest.Builder builder = Cloud.AlterClusterRequest.newBuilder(); From 4b6aa72772eba0de341e9c5996cf93bd8f31c5f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 10:36:13 +0800 Subject: [PATCH 123/572] branch-3.0: [chore](regression-test) remove `variant_ratio_of_defaults_as_sparse_column` in test cases #52413 (#52580) Cherry-picked from #52413 Co-authored-by: lihangyu --- .../load.out | 0 .../sql/affinityByIssuesAndPRs1.out | 0 .../sql/affinityByIssuesAndPRs2.out | 0 .../sql/authorsWithTheMostPushes.out | 0 .../sql/countingStar1.out | 0 .../sql/countingStar2.out | 0 .../sql/countingStar3.out | 0 .../distributionOfRepositoriesByStarCount.out | 0 .../sql/githubRoulette.out | 0 ...tOfTopRepositoriesChangedOverTheYears1.out | 0 ...tOfTopRepositoriesChangedOverTheYears2.out | 0 ...tOfTopRepositoriesChangedOverTheYears3.out | 0 ...tOfTopRepositoriesChangedOverTheYears4.out | 0 ...tOfTopRepositoriesChangedOverTheYears5.out | 0 ...tOfTopRepositoriesChangedOverTheYears6.out | 0 ...asTheTotalNumberOfStarsChangedOverTime.out | 0 .../sql/issuesWithTheMostComments1.out | 0 .../sql/issuesWithTheMostComments2.out | 0 .../sql/issuesWithTheMostComments3.out | 0 .../sql/issuesWithTheMostComments4.out | 0 .../sql/issuesWithTheMostComments5.out | 0 .../sql/issuesWithTheMostComments6.out | 0 .../sql/issuesWithTheMostComments7.out | 0 .../sql/mostForkedRepositories.out | 0 .../sql/mostPopularCommentsOnGithub.out | 0 ...organizationsByTheNumberOfRepositories.out | 0 .../sql/organizationsByTheNumberOfStars.out | 0 .../sql/proportionsBetweenStarsAndForks1.out | 0 .../sql/proportionsBetweenStarsAndForks2.out | 0 .../sql/proportionsBetweenStarsAndForks3.out | 0 .../sql/proportionsBetweenStarsAndForks4.out | 0 .../sql/proportionsBetweenStarsAndForks5.out | 0 .../repositoriesByAmountOfModifiedCode.out | 0 .../sql/repositoriesByTheNumberOfPushes.out | 0 ...toriesWithClickhouse_related_comments1.out | 0 ...toriesWithClickhouse_related_comments2.out | 0 ...epositoriesWithDoris_related_comments1.out | 0 ...epositoriesWithDoris_related_comments2.out | 0 .../repositoriesWithTheHighestGrowthYoY.out | 0 ...ositoriesWithTheMaximumAmountOfIssues1.out | 0 ...ositoriesWithTheMaximumAmountOfIssues2.out | 0 ...ositoriesWithTheMaximumAmountOfIssues3.out | 0 ...ositoriesWithTheMaximumAmountOfIssues4.out | 0 ...iesWithTheMaximumAmountOfPullRequests1.out | 0 ...iesWithTheMaximumAmountOfPullRequests2.out | 0 ...hTheMaximumNumberOfAcceptedInvitations.out | 0 ...iesWithTheMostPeopleWhoHavePushAccess1.out | 0 ...epositoriesWithTheMostStarsOverOneDay1.out | 0 ...itoriesWithTheMostSteadyGrowthOverTime.out | 0 ...positoriesWithTheWorstStagnation_order.out | 0 .../sql/repositoryAffinityList1.out | 0 .../sql/repositoryAffinityList2.out | 0 .../sql/starsFromHeavyGithubUsers1.out | 0 .../sql/starsFromHeavyGithubUsers2.out | 0 .../sql/test_distinct_streaming_agg.out | 0 .../sql/theLongestRepositoryNames1.out | 0 .../sql/theLongestRepositoryNames2.out | 0 .../sql/theMostToughCodeReviews.out | 0 .../theTotalNumberOfRepositoriesOnGithub.out | 0 .../sql/theTotalNumberOfUsersOnGithub1.out | 0 .../sql/theTotalNumberOfUsersOnGithub2.out | 0 .../sql/theTotalNumberOfUsersOnGithub3.out | 0 .../sql/theTotalNumberOfUsersOnGithub4.out | 0 .../sql/topRepositoriesByStars.out | 0 .../whatIsTheBestDayOfTheWeekToCatchAStar.out | 0 .../sql/whoAreAllThosePeopleGivingStars1.out | 0 .../sql/whoAreAllThosePeopleGivingStars2.out | 0 .../sql/whoAreAllThosePeopleGivingStars3.out | 0 regression-test/data/variant_p0/desc.out | 15 ++++++ .../test_compaction_extract_root.out | 6 +-- ...index_file_http_action_with_variant.groovy | 1 - .../load.groovy | 12 +---- .../load.groovy | 1 - .../suites/variant_log_data_p2/load.groovy | 4 -- regression-test/suites/variant_p0/desc.groovy | 6 +-- .../suites/variant_p0/with_index/load.groovy | 16 +------ .../with_index/test_array_index_write.groovy | 4 +- .../compaction_sparse_column.groovy | 46 +------------------ .../test_compaction_extract_root.groovy | 44 +----------------- regression-test/suites/variant_p2/load.groovy | 13 ------ 80 files changed, 25 insertions(+), 143 deletions(-) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/load.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/affinityByIssuesAndPRs1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/affinityByIssuesAndPRs2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/authorsWithTheMostPushes.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/countingStar1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/countingStar2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/countingStar3.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/distributionOfRepositoriesByStarCount.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/githubRoulette.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/howHasTheTotalNumberOfStarsChangedOverTime.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments3.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments4.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments5.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments6.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/issuesWithTheMostComments7.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/mostForkedRepositories.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/mostPopularCommentsOnGithub.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/organizationsByTheNumberOfRepositories.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/organizationsByTheNumberOfStars.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/proportionsBetweenStarsAndForks1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/proportionsBetweenStarsAndForks2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/proportionsBetweenStarsAndForks3.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/proportionsBetweenStarsAndForks4.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/proportionsBetweenStarsAndForks5.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesByAmountOfModifiedCode.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesByTheNumberOfPushes.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithClickhouse_related_comments1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithClickhouse_related_comments2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithDoris_related_comments1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithDoris_related_comments2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheHighestGrowthYoY.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumAmountOfIssues1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumAmountOfIssues2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumAmountOfIssues3.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumAmountOfIssues4.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMostStarsOverOneDay1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheMostSteadyGrowthOverTime.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoriesWithTheWorstStagnation_order.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoryAffinityList1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/repositoryAffinityList2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/starsFromHeavyGithubUsers1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/starsFromHeavyGithubUsers2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/test_distinct_streaming_agg.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theLongestRepositoryNames1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theLongestRepositoryNames2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theMostToughCodeReviews.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theTotalNumberOfRepositoriesOnGithub.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theTotalNumberOfUsersOnGithub1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theTotalNumberOfUsersOnGithub2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theTotalNumberOfUsersOnGithub3.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/theTotalNumberOfUsersOnGithub4.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/topRepositoriesByStars.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/whoAreAllThosePeopleGivingStars1.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/whoAreAllThosePeopleGivingStars2.out (100%) rename regression-test/data/{variant_github_events_new_p2 => variant_github_events_new_p0}/sql/whoAreAllThosePeopleGivingStars3.out (100%) rename regression-test/suites/{variant_github_events_new_p2 => variant_github_events_new_p0}/load.groovy (93%) diff --git a/regression-test/data/variant_github_events_new_p2/load.out b/regression-test/data/variant_github_events_new_p0/load.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/load.out rename to regression-test/data/variant_github_events_new_p0/load.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/affinityByIssuesAndPRs1.out b/regression-test/data/variant_github_events_new_p0/sql/affinityByIssuesAndPRs1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/affinityByIssuesAndPRs1.out rename to regression-test/data/variant_github_events_new_p0/sql/affinityByIssuesAndPRs1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/affinityByIssuesAndPRs2.out b/regression-test/data/variant_github_events_new_p0/sql/affinityByIssuesAndPRs2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/affinityByIssuesAndPRs2.out rename to regression-test/data/variant_github_events_new_p0/sql/affinityByIssuesAndPRs2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/authorsWithTheMostPushes.out b/regression-test/data/variant_github_events_new_p0/sql/authorsWithTheMostPushes.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/authorsWithTheMostPushes.out rename to regression-test/data/variant_github_events_new_p0/sql/authorsWithTheMostPushes.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/countingStar1.out b/regression-test/data/variant_github_events_new_p0/sql/countingStar1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/countingStar1.out rename to regression-test/data/variant_github_events_new_p0/sql/countingStar1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/countingStar2.out b/regression-test/data/variant_github_events_new_p0/sql/countingStar2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/countingStar2.out rename to regression-test/data/variant_github_events_new_p0/sql/countingStar2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/countingStar3.out b/regression-test/data/variant_github_events_new_p0/sql/countingStar3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/countingStar3.out rename to regression-test/data/variant_github_events_new_p0/sql/countingStar3.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/distributionOfRepositoriesByStarCount.out b/regression-test/data/variant_github_events_new_p0/sql/distributionOfRepositoriesByStarCount.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/distributionOfRepositoriesByStarCount.out rename to regression-test/data/variant_github_events_new_p0/sql/distributionOfRepositoriesByStarCount.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/githubRoulette.out b/regression-test/data/variant_github_events_new_p0/sql/githubRoulette.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/githubRoulette.out rename to regression-test/data/variant_github_events_new_p0/sql/githubRoulette.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out b/regression-test/data/variant_github_events_new_p0/sql/howHasTheTotalNumberOfStarsChangedOverTime.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out rename to regression-test/data/variant_github_events_new_p0/sql/howHasTheTotalNumberOfStarsChangedOverTime.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments1.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments1.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments2.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments2.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments3.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments3.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments3.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments4.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments4.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments4.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments4.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments5.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments5.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments5.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments5.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments6.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments6.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments6.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments6.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments7.out b/regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments7.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/issuesWithTheMostComments7.out rename to regression-test/data/variant_github_events_new_p0/sql/issuesWithTheMostComments7.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/mostForkedRepositories.out b/regression-test/data/variant_github_events_new_p0/sql/mostForkedRepositories.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/mostForkedRepositories.out rename to regression-test/data/variant_github_events_new_p0/sql/mostForkedRepositories.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/mostPopularCommentsOnGithub.out b/regression-test/data/variant_github_events_new_p0/sql/mostPopularCommentsOnGithub.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/mostPopularCommentsOnGithub.out rename to regression-test/data/variant_github_events_new_p0/sql/mostPopularCommentsOnGithub.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/organizationsByTheNumberOfRepositories.out b/regression-test/data/variant_github_events_new_p0/sql/organizationsByTheNumberOfRepositories.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/organizationsByTheNumberOfRepositories.out rename to regression-test/data/variant_github_events_new_p0/sql/organizationsByTheNumberOfRepositories.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/organizationsByTheNumberOfStars.out b/regression-test/data/variant_github_events_new_p0/sql/organizationsByTheNumberOfStars.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/organizationsByTheNumberOfStars.out rename to regression-test/data/variant_github_events_new_p0/sql/organizationsByTheNumberOfStars.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks1.out b/regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks1.out rename to regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks2.out b/regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks2.out rename to regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks3.out b/regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks3.out rename to regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks3.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks4.out b/regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks4.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks4.out rename to regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks4.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks5.out b/regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks5.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/proportionsBetweenStarsAndForks5.out rename to regression-test/data/variant_github_events_new_p0/sql/proportionsBetweenStarsAndForks5.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesByAmountOfModifiedCode.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesByAmountOfModifiedCode.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesByAmountOfModifiedCode.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesByAmountOfModifiedCode.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesByTheNumberOfPushes.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesByTheNumberOfPushes.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesByTheNumberOfPushes.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesByTheNumberOfPushes.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithClickhouse_related_comments1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithClickhouse_related_comments1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments2.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithClickhouse_related_comments2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithClickhouse_related_comments2.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithClickhouse_related_comments2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithDoris_related_comments1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithDoris_related_comments1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments2.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithDoris_related_comments2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithDoris_related_comments2.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithDoris_related_comments2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheHighestGrowthYoY.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheHighestGrowthYoY.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheHighestGrowthYoY.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheHighestGrowthYoY.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues3.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues4.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfIssues4.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMostStarsOverOneDay1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMostStarsOverOneDay1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMostSteadyGrowthOverTime.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheMostSteadyGrowthOverTime.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheWorstStagnation_order.out b/regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheWorstStagnation_order.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoriesWithTheWorstStagnation_order.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoriesWithTheWorstStagnation_order.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoryAffinityList1.out b/regression-test/data/variant_github_events_new_p0/sql/repositoryAffinityList1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoryAffinityList1.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoryAffinityList1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/repositoryAffinityList2.out b/regression-test/data/variant_github_events_new_p0/sql/repositoryAffinityList2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/repositoryAffinityList2.out rename to regression-test/data/variant_github_events_new_p0/sql/repositoryAffinityList2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers1.out b/regression-test/data/variant_github_events_new_p0/sql/starsFromHeavyGithubUsers1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers1.out rename to regression-test/data/variant_github_events_new_p0/sql/starsFromHeavyGithubUsers1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers2.out b/regression-test/data/variant_github_events_new_p0/sql/starsFromHeavyGithubUsers2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/starsFromHeavyGithubUsers2.out rename to regression-test/data/variant_github_events_new_p0/sql/starsFromHeavyGithubUsers2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/test_distinct_streaming_agg.out b/regression-test/data/variant_github_events_new_p0/sql/test_distinct_streaming_agg.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/test_distinct_streaming_agg.out rename to regression-test/data/variant_github_events_new_p0/sql/test_distinct_streaming_agg.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theLongestRepositoryNames1.out b/regression-test/data/variant_github_events_new_p0/sql/theLongestRepositoryNames1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theLongestRepositoryNames1.out rename to regression-test/data/variant_github_events_new_p0/sql/theLongestRepositoryNames1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theLongestRepositoryNames2.out b/regression-test/data/variant_github_events_new_p0/sql/theLongestRepositoryNames2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theLongestRepositoryNames2.out rename to regression-test/data/variant_github_events_new_p0/sql/theLongestRepositoryNames2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theMostToughCodeReviews.out b/regression-test/data/variant_github_events_new_p0/sql/theMostToughCodeReviews.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theMostToughCodeReviews.out rename to regression-test/data/variant_github_events_new_p0/sql/theMostToughCodeReviews.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfRepositoriesOnGithub.out b/regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfRepositoriesOnGithub.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfRepositoriesOnGithub.out rename to regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfRepositoriesOnGithub.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub1.out b/regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub1.out rename to regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub2.out b/regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub2.out rename to regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub3.out b/regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub3.out rename to regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub3.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub4.out b/regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub4.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/theTotalNumberOfUsersOnGithub4.out rename to regression-test/data/variant_github_events_new_p0/sql/theTotalNumberOfUsersOnGithub4.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/topRepositoriesByStars.out b/regression-test/data/variant_github_events_new_p0/sql/topRepositoriesByStars.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/topRepositoriesByStars.out rename to regression-test/data/variant_github_events_new_p0/sql/topRepositoriesByStars.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out b/regression-test/data/variant_github_events_new_p0/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out rename to regression-test/data/variant_github_events_new_p0/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars1.out b/regression-test/data/variant_github_events_new_p0/sql/whoAreAllThosePeopleGivingStars1.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars1.out rename to regression-test/data/variant_github_events_new_p0/sql/whoAreAllThosePeopleGivingStars1.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars2.out b/regression-test/data/variant_github_events_new_p0/sql/whoAreAllThosePeopleGivingStars2.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars2.out rename to regression-test/data/variant_github_events_new_p0/sql/whoAreAllThosePeopleGivingStars2.out diff --git a/regression-test/data/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars3.out b/regression-test/data/variant_github_events_new_p0/sql/whoAreAllThosePeopleGivingStars3.out similarity index 100% rename from regression-test/data/variant_github_events_new_p2/sql/whoAreAllThosePeopleGivingStars3.out rename to regression-test/data/variant_github_events_new_p0/sql/whoAreAllThosePeopleGivingStars3.out diff --git a/regression-test/data/variant_p0/desc.out b/regression-test/data/variant_p0/desc.out index ffd87b9d05bc37..1eff52e44842f3 100644 --- a/regression-test/data/variant_p0/desc.out +++ b/regression-test/data/variant_p0/desc.out @@ -3,14 +3,21 @@ k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE +v.b json Yes false \N NONE +v.c.c smallint Yes false \N NONE +v.c.e double Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_2 -- k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE +v.b json Yes false \N NONE +v.c.c smallint Yes false \N NONE +v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE v.ddd.mxmxm json Yes false \N NONE +v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_3 -- @@ -26,14 +33,21 @@ v.xxxx text Yes false \N NONE k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE +v.b json Yes false \N NONE +v.c.c smallint Yes false \N NONE +v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE v.ddd.mxmxm json Yes false \N NONE +v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_6_2 -- k bigint Yes true \N v variant Yes false \N NONE v.a smallint Yes false \N NONE +v.b json Yes false \N NONE +v.c.c smallint Yes false \N NONE +v.c.e double Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_6_3 -- @@ -53,6 +67,7 @@ v.c.c smallint Yes false \N NONE v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE v.ddd.mxmxm json Yes false \N NONE +v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE -- !sql_7 -- diff --git a/regression-test/data/variant_p1/compaction/test_compaction_extract_root.out b/regression-test/data/variant_p1/compaction/test_compaction_extract_root.out index c8a0cf24017832..5d16b15d26ab1d 100644 --- a/regression-test/data/variant_p1/compaction/test_compaction_extract_root.out +++ b/regression-test/data/variant_p1/compaction/test_compaction_extract_root.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select_b_1 -- -12291 +24582 -- !select_b_2 -- 12288 @@ -9,11 +9,11 @@ {"code":2,"state":"open"} -- !select_b_3 -- -12288 +24588 -- !select_b_4 -- 12288 -- !select_1 -- -\N +{"code":2,"state":"open"} diff --git a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy index e242f858872c3e..f86223b70692ba 100644 --- a/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy +++ b/regression-test/suites/inverted_index_p0/test_show_nested_index_file_http_action_with_variant.groovy @@ -59,7 +59,6 @@ suite("test_show_nested_index_file_http_action_with_variant", "nonConcurrent,p0" } set_be_config.call("memory_limitation_per_thread_for_schema_change_bytes", "6294967296") - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") def run_test = { format -> def tableName = "test_show_nested_index_file_http_action_with_variant_" + format diff --git a/regression-test/suites/variant_github_events_new_p2/load.groovy b/regression-test/suites/variant_github_events_new_p0/load.groovy similarity index 93% rename from regression-test/suites/variant_github_events_new_p2/load.groovy rename to regression-test/suites/variant_github_events_new_p0/load.groovy index 5721a0eb10a345..582f3bec5652ed 100644 --- a/regression-test/suites/variant_github_events_new_p2/load.groovy +++ b/regression-test/suites/variant_github_events_new_p0/load.groovy @@ -15,16 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - def set_be_config = { key, value -> - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } +suite("regression_test_variant_github_events_p0", "p0"){ def load_json_data = {table_name, file_name -> // load the json data streamLoad { @@ -53,7 +44,6 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ } } } - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy index e7252d35ad56a7..316ebabbb55a60 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy @@ -160,7 +160,6 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ DISTRIBUTED BY HASH(k) BUCKETS 4 properties("replication_num" = "1", "disable_auto_compaction" = "true", "bloom_filter_columns" = "v", "variant_enable_flatten_nested" = "true", "inverted_index_storage_format"= "v2"); """ - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") // 2015 load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") diff --git a/regression-test/suites/variant_log_data_p2/load.groovy b/regression-test/suites/variant_log_data_p2/load.groovy index 8e6d15e172d494..f1fa6caf17b075 100644 --- a/regression-test/suites/variant_log_data_p2/load.groovy +++ b/regression-test/suites/variant_log_data_p2/load.groovy @@ -72,21 +72,18 @@ suite("regression_test_variant_logdata", "nonConcurrent,p2"){ create_table.call(table_name, "DUPLICATE", "4") // sql "set enable_two_phase_read_opt = false;" // no sparse columns - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") load_json_data.call(table_name, """${getS3Url() + '/regression/load/logdata.json'}""") qt_sql_32 """ select json_extract(v, "\$.json.parseFailed") from logdata where json_extract(v, "\$.json.parseFailed") != 'null' order by k limit 1;""" qt_sql_32_1 """select cast(v['json']['parseFailed'] as string) from logdata where cast(v['json']['parseFailed'] as string) is not null and k = 162 limit 1;""" sql "truncate table ${table_name}" // 0.95 default ratio - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") load_json_data.call(table_name, """${getS3Url() + '/regression/load/logdata.json'}""") qt_sql_33 """ select json_extract(v,"\$.json.parseFailed") from logdata where json_extract(v,"\$.json.parseFailed") != 'null' order by k limit 1;""" qt_sql_33_1 """select cast(v['json']['parseFailed'] as string) from logdata where cast(v['json']['parseFailed'] as string) is not null and k = 162 limit 1;""" sql "truncate table ${table_name}" // always sparse column - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") load_json_data.call(table_name, """${getS3Url() + '/regression/load/logdata.json'}""") qt_sql_34 """ select json_extract(v, "\$.json.parseFailed") from logdata where json_extract(v,"\$.json.parseFailed") != 'null' order by k limit 1;""" sql "truncate table ${table_name}" @@ -94,5 +91,4 @@ suite("regression_test_variant_logdata", "nonConcurrent,p2"){ qt_sql_35_1 """select cast(v['json']['parseFailed'] as string) from logdata where cast(v['json']['parseFailed'] as string) is not null and k = 162 limit 1;""" // TODO add test case that some certain columns are materialized in some file while others are not materilized(sparse) // unique table - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") } \ No newline at end of file diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy index 868a3e1307fc08..59d0cd261da21c 100644 --- a/regression-test/suites/variant_p0/desc.groovy +++ b/regression-test/suites/variant_p0/desc.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("regression_test_variant_desc", "nonConcurrent"){ +suite("regression_test_variant_desc", "p0"){ // if (isCloudMode()) { // return // } @@ -97,7 +97,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){ // sparse columns def table_name = "sparse_columns" create_table table_name - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") sql """set describe_extend_variant_column = true""" sql """insert into sparse_columns select 0, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str union all select 0, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "kaana"}' as json_str from numbers("number" = "4096") limit 4096 ;""" @@ -115,7 +114,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){ table_name = "no_sparse_columns" create_table.call(table_name, "4") sql "set enable_two_phase_read_opt = false;" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0") sql """insert into ${table_name} select 0, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str union all select 0, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "kaana"}' as json_str from numbers("number" = "4096") limit 4096 ;""" sql "select * from no_sparse_columns limit 1" @@ -126,7 +124,6 @@ suite("regression_test_variant_desc", "nonConcurrent"){ table_name = "partition_data" create_table_partition.call(table_name, "4") sql "set enable_two_phase_read_opt = false;" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") sql """insert into ${table_name} select 2500, '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}' as json_str union all select 2500, '{"a" : 1234, "xxxx" : "kaana", "ddd" : {"aaa" : 123, "mxmxm" : [456, "789"]}}' as json_str from numbers("number" = "4096") limit 4096 ;""" sql """insert into ${table_name} select 45000, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str @@ -274,6 +271,5 @@ suite("regression_test_variant_desc", "nonConcurrent"){ sql "desc large_tablets" } finally { // reset flags - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") } } diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy index 4eeff1bfacb72d..dcfdc7ebccc138 100644 --- a/regression-test/suites/variant_p0/with_index/load.groovy +++ b/regression-test/suites/variant_p0/with_index/load.groovy @@ -15,18 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("regression_test_variant_with_index", "nonConcurrent"){ - def set_be_config = { key, value -> - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - +suite("regression_test_variant_with_index", "p0"){ def timeout = 60000 def delta_time = 1000 def alter_res = "null" @@ -45,8 +34,6 @@ suite("regression_test_variant_with_index", "nonConcurrent"){ } assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") } - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0") - set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "0") def table_name = "var_with_index" sql "DROP TABLE IF EXISTS var_with_index" sql """ @@ -68,7 +55,6 @@ suite("regression_test_variant_with_index", "nonConcurrent"){ qt_sql_inv_3 """select * from var_with_index where inv match 'hello' and cast(v["a"] as int) > 0 order by k""" sql "truncate table var_with_index" // set back configs - set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "2048") // sql "truncate table ${table_name}" sql """insert into var_with_index values(1, '{"a1" : 0, "b1": 3}', 'hello world'), (2, '{"a2" : 123}', 'world'),(3, '{"a3" : 123}', 'hello world')""" sql """insert into var_with_index values(4, '{"b1" : 0, "b2": 3}', 'hello world'), (5, '{"b2" : 123}', 'world'),(6, '{"b3" : 123}', 'hello world')""" diff --git a/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy b/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy index 306a98606169a3..a5f5d9e97eac08 100644 --- a/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy +++ b/regression-test/suites/variant_p0/with_index/test_array_index_write.groovy @@ -28,8 +28,8 @@ suite("test_array_index_write", "nonConcurrent"){ stmt += strTmp stmt = stmt.substring(0, stmt.length()-2) def storageFormat = new Random().nextBoolean() ? "V1" : "V2" - if (storageFormat == "V1" && isCloudMode()) { - return; + if (isCloudMode()) { + storageFormat = "V2"; } stmt += ") \nENGINE=OLAP\n" + "DUPLICATE KEY(`k1`)\n" + diff --git a/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy b/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy index 13ef63e9fe6c37..ebe2aac8642567 100644 --- a/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy +++ b/regression-test/suites/variant_p1/compaction/compaction_sparse_column.groovy @@ -18,38 +18,10 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import org.awaitility.Awaitility -suite("test_compaction_sparse_column", "p1,nonConcurrent") { +suite("test_compaction_sparse_column", "p1") { def tableName = "test_compaction" try { - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) - } - } - - def set_be_config = { key, value -> - (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") - set_be_config.call("write_buffer_size", "10240") - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE ${tableName} ( @@ -110,19 +82,6 @@ suite("test_compaction_sparse_column", "p1,nonConcurrent") { // trigger compactions for all tablets in ${tableName} trigger_and_wait_compaction(tableName, "cumulative") - int rowCount = 0 - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - (code, out, err) = curl("GET", tablet.CompactionStatus) - logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def tabletJson = parseJson(out.trim()) - assert tabletJson.rowsets instanceof List - for (String rowset in (List) tabletJson.rowsets) { - rowCount += Integer.parseInt(rowset.split(" ")[1]) - } - } - assert (rowCount <= 8) qt_select_b """ SELECT count(cast(v['b'] as int)) FROM ${tableName};""" qt_select_xxxx """ SELECT count(cast(v['xxxx'] as string)) FROM ${tableName};""" qt_select_point """ SELECT count(cast(v['point'] as bigint)) FROM ${tableName};""" @@ -140,8 +99,5 @@ suite("test_compaction_sparse_column", "p1,nonConcurrent") { qt_select_6_1 """ SELECT count(cast(v['b'] as int)) FROM ${tableName} where cast(v['b'] as int) = 42005;""" qt_select_all """SELECT k, v['a'], v['b'], v['xxxx'], v['point'], v['ddddd'] from ${tableName} where (cast(v['point'] as int) = 1);""" } finally { - // try_sql("DROP TABLE IF EXISTS ${tableName}") - set_be_config.call("write_buffer_size", "209715200") - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") } } diff --git a/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy b/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy index 015ab9baa1d03f..d1b4e8e93da344 100644 --- a/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy +++ b/regression-test/suites/variant_p1/compaction/test_compaction_extract_root.groovy @@ -17,34 +17,8 @@ import org.codehaus.groovy.runtime.IOGroovyMethods -suite("test_compaction_extract_root", "p1,nonConcurrent") { +suite("test_compaction_extract_root", "p1") { def tableName = "test_t" - - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - def set_be_config = { key, value -> - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } - String backend_id; - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def configList = parseJson(out.trim()) - assert configList instanceof List - - boolean disableAutoCompaction = true - for (Object ele in (List) configList) { - assert ele instanceof List - if (((List) ele)[0] == "disable_auto_compaction") { - disableAutoCompaction = Boolean.parseBoolean(((List) ele)[2]) - } - } - sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE ${tableName} ( @@ -59,8 +33,6 @@ suite("test_compaction_extract_root", "p1,nonConcurrent") { ); """ - set_be_config.call("enable_vertical_segment_writer", "true") - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") sql """insert into ${tableName} select 0, '{"a": 11245, "b" : {"state" : "open", "code" : 2}}' as json_str union all select 8, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "aaaaa"}' as json_str from numbers("number" = "4096") limit 4096 ;""" @@ -98,19 +70,6 @@ suite("test_compaction_extract_root", "p1,nonConcurrent") { // trigger compactions for all tablets in ${tableName} trigger_and_wait_compaction(tableName, "cumulative") - int rowCount = 0 - for (def tablet in tablets) { - String tablet_id = tablet.TabletId - (code, out, err) = curl("GET", tablet.CompactionStatus) - logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def tabletJson = parseJson(out.trim()) - assert tabletJson.rowsets instanceof List - for (String rowset in (List) tabletJson.rowsets) { - rowCount += Integer.parseInt(rowset.split(" ")[1]) - } - } - assert (rowCount <= 8) // fix cast to string tobe {} qt_select_b_3 """ SELECT count(cast(v['b'] as string)) FROM test_t""" qt_select_b_4 """ SELECT count(cast(v['b'] as int)) FROM test_t""" @@ -118,5 +77,4 @@ suite("test_compaction_extract_root", "p1,nonConcurrent") { // qt_select_b_5 """ select v['b'] from test_t where cast(v['b'] as string) != '42005' and cast(v['b'] as string) != '42004' and cast(v['b'] as string) != '42003' order by cast(v['b'] as string); """ qt_select_1 """select v['b'] from test_t where k = 0 and cast(v['a'] as int) = 11245;""" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1") } diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index 7bc05663e6b9d1..056bb666e341c3 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -69,17 +69,6 @@ suite("load_p2", "variant_type,p2"){ """ } - def set_be_config = { key, value -> - String backend_id; - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - - backend_id = backendId_to_backendIP.keySet()[0] - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - // Configuration for the number of threads def numberOfThreads = 10 // Set this to your desired number of threads @@ -88,7 +77,6 @@ suite("load_p2", "variant_type,p2"){ try { def table_name = "github_events" - set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0") def s3load_paral_wait = {tbl, fmt, path, paral -> String ak = getS3AK() String sk = getS3SK() @@ -167,6 +155,5 @@ suite("load_p2", "variant_type,p2"){ qt_sql("select count() from github_events") } finally { // reset flags - // set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") } } From 0adfc3314687605b3249a520199d285d3cd639c3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Jul 2025 10:38:20 +0800 Subject: [PATCH 124/572] branch-3.0: [fix](doris compose) be disable java support #52412 (#52524) Cherry-picked from #52412 Co-authored-by: yujun --- docker/runtime/doris-compose/cluster.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py index 87313546c69004..00fe9ddc3866c1 100644 --- a/docker/runtime/doris-compose/cluster.py +++ b/docker/runtime/doris-compose/cluster.py @@ -588,6 +588,9 @@ def init(self): def get_add_init_config(self): cfg = super().get_add_init_config() + cfg += [ + 'enable_java_support = false', + ] if self.cluster.be_config: cfg += self.cluster.be_config if self.cluster.is_cloud: From 9e19b86accd7e15a3ae4eb7e0a8957a8fbe8e8ba Mon Sep 17 00:00:00 2001 From: morrySnow Date: Wed, 2 Jul 2025 10:40:13 +0800 Subject: [PATCH 125/572] branch-3.0: [fix](audit) update audit table schema do not work as expected #51363 (#52436) pick park from #51363 --- .../catalog/InternalSchemaInitializer.java | 79 +++++++++++-------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java index ca16d498f36714..cb1c60823208d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchemaInitializer.java @@ -17,11 +17,11 @@ package org.apache.doris.catalog; +import org.apache.doris.analysis.AddColumnsClause; import org.apache.doris.analysis.AlterClause; import org.apache.doris.analysis.AlterTableStmt; import org.apache.doris.analysis.ColumnDef; import org.apache.doris.analysis.ColumnNullableType; -import org.apache.doris.analysis.ColumnPosition; import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DbName; @@ -33,6 +33,7 @@ import org.apache.doris.analysis.ModifyPartitionClause; import org.apache.doris.analysis.PartitionDesc; import org.apache.doris.analysis.RangePartitionDesc; +import org.apache.doris.analysis.ReorderColumnsClause; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TypeDef; import org.apache.doris.common.AnalysisException; @@ -54,10 +55,12 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; public class InternalSchemaInitializer extends Thread { @@ -364,41 +367,55 @@ private boolean created() { // 4. check and update audit table schema OlapTable auditTable = (OlapTable) optionalStatsTbl.get(); - List expectedSchema = InternalSchema.AUDIT_SCHEMA; // 5. check if we need to add new columns + return alterAuditSchemaIfNeeded(auditTable); + } + + private boolean alterAuditSchemaIfNeeded(OlapTable auditTable) { + List expectedSchema = InternalSchema.AUDIT_SCHEMA; + List expectedColumnNames = expectedSchema.stream() + .map(ColumnDef::getName) + .map(String::toLowerCase) + .collect(Collectors.toList()); + List currentColumns = auditTable.getBaseSchema(); + List currentColumnNames = currentColumns.stream() + .map(Column::getName) + .map(String::toLowerCase) + .collect(Collectors.toList()); + // check if all expected columns are exists and in the right order + if (currentColumnNames.size() >= expectedColumnNames.size() + && expectedColumnNames.equals(currentColumnNames.subList(0, expectedColumnNames.size()))) { + return true; + } + List alterClauses = Lists.newArrayList(); - for (int i = 0; i < expectedSchema.size(); i++) { - ColumnDef def = expectedSchema.get(i); - if (auditTable.getColumn(def.getName()) == null) { - // add column if it doesn't exist - try { - ColumnDef columnDef = new ColumnDef(def.getName(), def.getTypeDef(), def.isAllowNull()); - // find the previous column name to determine the position - String afterColumn = null; - if (i > 0) { - for (int j = i - 1; j >= 0; j--) { - String prevColName = expectedSchema.get(j).getName(); - if (auditTable.getColumn(prevColName) != null) { - afterColumn = prevColName; - break; - } - } - } - ColumnPosition position = afterColumn == null ? ColumnPosition.FIRST : - new ColumnPosition(afterColumn); - ModifyColumnClause clause = new ModifyColumnClause(columnDef, position, null, - Maps.newHashMap()); - clause.setColumn(columnDef.toColumn()); - alterClauses.add(clause); - } catch (Exception e) { - LOG.warn("Failed to create alter clause for column: " + def.getName(), e); - return false; - } + // add new columns + List addColumnsDef = Lists.newArrayList(); + for (ColumnDef expected : expectedSchema) { + if (!currentColumnNames.contains(expected.getName().toLowerCase())) { + addColumnsDef.add(expected); } } - - // apply schema changes if needed + if (!addColumnsDef.isEmpty()) { + AddColumnsClause addColumnsClause = new AddColumnsClause(addColumnsDef, null, Collections.emptyMap()); + try { + addColumnsClause.analyze(null); + } catch (Exception e) { + LOG.warn("Failed to alter audit table schema", e); + return false; + } + alterClauses.add(addColumnsClause); + } + // reorder columns + List removedColumnNames = Lists.newArrayList(currentColumnNames); + removedColumnNames.removeAll(expectedColumnNames); + List newColumnOrders = Lists.newArrayList(expectedColumnNames); + newColumnOrders.addAll(removedColumnNames); + if (!newColumnOrders.isEmpty()) { + ReorderColumnsClause reorderColumnsOp = new ReorderColumnsClause(newColumnOrders, null, Maps.newHashMap()); + alterClauses.add(reorderColumnsOp); + } if (!alterClauses.isEmpty()) { try { TableName tableName = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, From 32ac14f04135f3298899efcd877d130173cc703c Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Wed, 2 Jul 2025 10:41:58 +0800 Subject: [PATCH 126/572] branch-3.0: [Fix](field) Fix potential memory leak and wrong binary reading about JsonbField (#50174) (#52542) pick https://github.com/apache/doris/pull/50174 --- be/src/runtime/jsonb_value.h | 10 +- be/src/vec/columns/column_string.h | 2 + be/src/vec/common/string_buffer.hpp | 3 + be/src/vec/core/field.cpp | 7 +- be/src/vec/core/field.h | 62 +++++------ be/src/vec/data_types/data_type_jsonb.h | 25 +++-- be/src/vec/io/io_helper.h | 12 ++- be/src/vec/io/var_int.h | 35 ++++-- be/test/vec/core/field_test.cpp | 138 +++++++++++++++++++++++- 9 files changed, 227 insertions(+), 67 deletions(-) diff --git a/be/src/runtime/jsonb_value.h b/be/src/runtime/jsonb_value.h index 1df9469e1720cd..951b91b2f34805 100644 --- a/be/src/runtime/jsonb_value.h +++ b/be/src/runtime/jsonb_value.h @@ -42,8 +42,8 @@ struct JsonBinaryValue { size_t len = 0; JsonbParser parser; - JsonBinaryValue() : ptr(nullptr), len(0) {} - JsonBinaryValue(char* ptr, int len) { + JsonBinaryValue() = default; + JsonBinaryValue(char* ptr, size_t len) { static_cast(from_json_string(const_cast(ptr), len)); } JsonBinaryValue(const std::string& s) { @@ -51,11 +51,11 @@ struct JsonBinaryValue { } JsonBinaryValue(const char* ptr, int len) { static_cast(from_json_string(ptr, len)); } - const char* value() { return ptr; } + const char* value() const { return ptr; } - size_t size() { return len; } + size_t size() const { return len; } - void replace(char* ptr, int len) { + void replace(const char* ptr, int len) { this->ptr = ptr; this->len = len; } diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h index 94c835f600b310..3c9c5829e35ba8 100644 --- a/be/src/vec/columns/column_string.h +++ b/be/src/vec/columns/column_string.h @@ -165,6 +165,8 @@ class ColumnStr final : public COWHelper> { check_chars_length(new_size, old_size + 1); chars.resize(new_size); + DCHECK(s.data != nullptr); + DCHECK(chars.data() != nullptr); memcpy(chars.data() + old_size, s.data, size_to_append); offsets.push_back(new_size); sanity_check_simple(); diff --git a/be/src/vec/common/string_buffer.hpp b/be/src/vec/common/string_buffer.hpp index 8dca6f057a26d3..6319f6fe5f2a64 100644 --- a/be/src/vec/common/string_buffer.hpp +++ b/be/src/vec/common/string_buffer.hpp @@ -25,6 +25,8 @@ namespace doris::vectorized { +// store and commit data. only after commit the data is effective on its' base(ColumnString) +// everytime commit, the _data add one row. class BufferWritable final { public: explicit BufferWritable(ColumnString& vector) @@ -64,6 +66,7 @@ class BufferWritable final { using VectorBufferWriter = BufferWritable; using BufferWriter = BufferWritable; +// There is consumption of the buffer in the read method. class BufferReadable { public: explicit BufferReadable(StringRef& ref) : _data(ref.data) {} diff --git a/be/src/vec/core/field.cpp b/be/src/vec/core/field.cpp index ac92605007d030..d45a1a719ac7fd 100644 --- a/be/src/vec/core/field.cpp +++ b/be/src/vec/core/field.cpp @@ -26,14 +26,9 @@ #include "vec/io/io_helper.h" #include "vec/io/var_int.h" -namespace doris { -namespace vectorized { +namespace doris::vectorized { class BufferReadable; class BufferWritable; -} // namespace vectorized -} // namespace doris - -namespace doris::vectorized { void read_binary(Array& x, BufferReadable& buf) { size_t size; diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h index 922f9abb13e03e..cceb71e53eebd3 100644 --- a/be/src/vec/core/field.h +++ b/be/src/vec/core/field.h @@ -156,59 +156,61 @@ DEFINE_FIELD_VECTOR(Map); using VariantMap = std::map; +//TODO: rethink if we really need this? it only save one pointer from std::string +// not POD type so could only use read/write_json_binary instead of read/write_binary class JsonbField { public: JsonbField() = default; + ~JsonbField() = default; // unique_ptr will handle cleanup automatically - JsonbField(const char* ptr, uint32_t len) : size(len) { - data = new char[size]; + JsonbField(const char* ptr, size_t len) : size(len) { + data = std::make_unique(size); if (!data) { LOG(FATAL) << "new data buffer failed, size: " << size; } - memcpy(data, ptr, size); + if (size > 0) { + memcpy(data.get(), ptr, size); + } } JsonbField(const JsonbField& x) : size(x.size) { - data = new char[size]; + data = std::make_unique(size); if (!data) { LOG(FATAL) << "new data buffer failed, size: " << size; } - memcpy(data, x.data, size); + if (size > 0) { + memcpy(data.get(), x.data.get(), size); + } } - JsonbField(JsonbField&& x) : data(x.data), size(x.size) { - x.data = nullptr; - x.size = 0; - } + JsonbField(JsonbField&& x) noexcept : data(std::move(x.data)), size(x.size) { x.size = 0; } + // dispatch for all type of storage. so need this. but not really used now. JsonbField& operator=(const JsonbField& x) { - data = new char[size]; - if (!data) { - LOG(FATAL) << "new data buffer failed, size: " << size; + if (this != &x) { + data = std::make_unique(x.size); + if (!data) { + LOG(FATAL) << "new data buffer failed, size: " << x.size; + } + if (x.size > 0) { + memcpy(data.get(), x.data.get(), x.size); + } + size = x.size; } - memcpy(data, x.data, size); return *this; } - JsonbField& operator=(JsonbField&& x) { - if (data) { - delete[] data; + JsonbField& operator=(JsonbField&& x) noexcept { + if (this != &x) { + data = std::move(x.data); + size = x.size; + x.size = 0; } - data = x.data; - size = x.size; - x.data = nullptr; - x.size = 0; return *this; } - ~JsonbField() { - if (data) { - delete[] data; - } - } - - const char* get_value() const { return data; } - uint32_t get_size() const { return size; } + const char* get_value() const { return data.get(); } + size_t get_size() const { return size; } bool operator<(const JsonbField& r) const { LOG(FATAL) << "comparing between JsonbField is not supported"; @@ -246,8 +248,8 @@ class JsonbField { } private: - char* data = nullptr; - uint32_t size = 0; + std::unique_ptr data = nullptr; + size_t size = 0; }; template diff --git a/be/src/vec/data_types/data_type_jsonb.h b/be/src/vec/data_types/data_type_jsonb.h index 3d681e3ce79754..12d7aafbca2d1e 100644 --- a/be/src/vec/data_types/data_type_jsonb.h +++ b/be/src/vec/data_types/data_type_jsonb.h @@ -18,9 +18,9 @@ #pragma once #include -#include -#include +#include +#include #include #include @@ -36,15 +36,13 @@ #include "vec/data_types/serde/data_type_serde.h" #include "vec/data_types/serde/data_type_string_serde.h" -namespace doris { -namespace vectorized { +namespace doris::vectorized { +#include "common/compile_check_begin.h" + class BufferWritable; class IColumn; class ReadBuffer; -} // namespace vectorized -} // namespace doris -namespace doris::vectorized { class DataTypeJsonb final : public IDataType { public: using ColumnType = ColumnString; @@ -68,10 +66,13 @@ class DataTypeJsonb final : public IDataType { MutableColumnPtr create_column() const override; - virtual Field get_default() const override { + Field get_default() const override { std::string default_json = "null"; - JsonBinaryValue binary_val(default_json.c_str(), default_json.size()); - return JsonbField(binary_val.value(), binary_val.size()); + // convert default_json to binary + JsonBinaryValue binary_val(default_json.c_str(), static_cast(default_json.size())); + // Throw exception if default_json.size() is large than INT32_MAX + // JsonbField keeps its own memory + return JsonbField(binary_val.value(), static_cast(binary_val.size())); } Field get_field(const TExprNode& node) const override { @@ -100,4 +101,6 @@ class DataTypeJsonb final : public IDataType { private: DataTypeString data_type_string; }; -} // namespace doris::vectorized \ No newline at end of file + +#include "common/compile_check_end.h" +} // namespace doris::vectorized diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h index d5ca522146a1cb..f3dc3d1b131924 100644 --- a/be/src/vec/io/io_helper.h +++ b/be/src/vec/io/io_helper.h @@ -30,6 +30,7 @@ #include "vec/common/string_buffer.hpp" #include "vec/common/string_ref.h" #include "vec/common/uint128.h" +#include "vec/core/field.h" #include "vec/core/types.h" #include "vec/io/reader_buffer.h" #include "vec/io/var_int.h" @@ -126,7 +127,7 @@ inline void write_string_binary(const char* s, BufferWritable& buf) { write_string_binary(StringRef {std::string(s)}, buf); } -inline void write_json_binary(JsonbField s, BufferWritable& buf) { +inline void write_json_binary(const JsonbField& s, BufferWritable& buf) { write_string_binary(StringRef {s.get_value(), s.get_size()}, buf); } @@ -200,13 +201,14 @@ inline StringRef read_string_binary_into(Arena& arena, BufferReadable& buf) { char* data = arena.alloc(size); buf.read(data, size); - return StringRef(data, size); + return {data, size}; } -inline void read_json_binary(JsonbField val, BufferReadable& buf, +inline void read_json_binary(JsonbField& val, BufferReadable& buf, size_t MAX_JSON_SIZE = DEFAULT_MAX_JSON_SIZE) { - StringRef jrf = StringRef {val.get_value(), val.get_size()}; - read_string_binary(jrf, buf); + StringRef result; + read_string_binary(result, buf); + val = JsonbField(result.data, result.size); } template diff --git a/be/src/vec/io/var_int.h b/be/src/vec/io/var_int.h index eda4b6b3873aa8..eb06a1ccb1d147 100644 --- a/be/src/vec/io/var_int.h +++ b/be/src/vec/io/var_int.h @@ -93,35 +93,44 @@ inline void read_var_uint(UInt64& x, std::istream& istr) { for (size_t i = 0; i < 9; ++i) { UInt64 byte = istr.get(); x |= (byte & 0x7F) << (7 * i); - if (!(byte & 0x80)) return; + if (!(byte & 0x80)) { + return; + } } } inline void write_var_uint(UInt64 x, std::ostream& ostr) { for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; - if (x > 0x7F) byte |= 0x80; + if (x > 0x7F) { + byte |= 0x80; + } ostr.put(byte); x >>= 7; - if (!x) return; + if (!x) { + return; + } } } // TODO: do real implement in the future inline void read_var_uint(UInt64& x, BufferReadable& buf) { x = 0; + // get length from first byte firstly uint8_t len = 0; buf.read((char*)&len, 1); auto ref = buf.read(len); - + // read data and set it to x per byte. char* bytes = const_cast(ref.data); for (size_t i = 0; i < 9; ++i) { UInt64 byte = bytes[i]; x |= (byte & 0x7F) << (7 * i); - if (!(byte & 0x80)) return; + if (!(byte & 0x80)) { + return; + } } } @@ -130,12 +139,16 @@ inline void write_var_uint(UInt64 x, BufferWritable& ostr) { uint8_t i = 0; while (i < 9) { uint8_t byte = x & 0x7F; - if (x > 0x7F) byte |= 0x80; + if (x > 0x7F) { + byte |= 0x80; + } bytes[i++] = byte; x >>= 7; - if (!x) break; + if (!x) { + break; + } } ostr.write((char*)&i, 1); ostr.write(bytes, i); @@ -144,13 +157,17 @@ inline void write_var_uint(UInt64 x, BufferWritable& ostr) { inline char* write_var_uint(UInt64 x, char* ostr) { for (size_t i = 0; i < 9; ++i) { uint8_t byte = x & 0x7F; - if (x > 0x7F) byte |= 0x80; + if (x > 0x7F) { + byte |= 0x80; + } *ostr = byte; ++ostr; x >>= 7; - if (!x) return ostr; + if (!x) { + return ostr; + } } return ostr; diff --git a/be/test/vec/core/field_test.cpp b/be/test/vec/core/field_test.cpp index 71d26ea4979bde..4509a4e6fe67b5 100644 --- a/be/test/vec/core/field_test.cpp +++ b/be/test/vec/core/field_test.cpp @@ -22,8 +22,13 @@ #include -#include "gtest/gtest_pred_impl.h" +#include "gtest/gtest_pred_impl.h" // IWYU pragma: keep +#include "runtime/define_primitive_type.h" +#include "vec/columns/column_string.h" +#include "vec/common/string_buffer.hpp" +#include "vec/common/string_ref.h" #include "vec/core/types.h" +#include "vec/io/io_helper.h" namespace doris::vectorized { TEST(VFieldTest, field_string) { @@ -43,4 +48,135 @@ TEST(VFieldTest, field_string) { ASSERT_EQ(f.get()[0].get(), "Hello, world (6)"); } +TEST(VFieldTest, jsonb_field_unique_ptr) { + // Test default constructor + JsonbField empty; + ASSERT_EQ(empty.get_value(), nullptr); + ASSERT_EQ(empty.get_size(), 0); + + // Test constructor with data + const char* test_data = R"({ "key": "value" })"; + size_t test_size = strlen(test_data); + JsonbField jf1(test_data, test_size); + ASSERT_NE(jf1.get_value(), nullptr); + ASSERT_EQ(jf1.get_size(), test_size); + ASSERT_EQ(std::string(jf1.get_value(), jf1.get_size()), std::string(test_data)); + + // Test copy constructor + JsonbField jf2(jf1); + ASSERT_NE(jf2.get_value(), nullptr); + ASSERT_NE(jf2.get_value(), jf1.get_value()); // Different memory locations + ASSERT_EQ(jf2.get_size(), jf1.get_size()); + ASSERT_EQ(std::string(jf2.get_value(), jf2.get_size()), + std::string(jf1.get_value(), jf1.get_size())); + + // Test move constructor + JsonbField jf3(std::move(jf2)); + ASSERT_NE(jf3.get_value(), nullptr); + ASSERT_EQ(jf2.get_value(), nullptr); // jf2 should be empty after move + ASSERT_EQ(jf2.get_size(), 0); // jf2 size should be 0 after move + ASSERT_EQ(jf3.get_size(), test_size); + ASSERT_EQ(std::string(jf3.get_value(), jf3.get_size()), std::string(test_data)); + + // Test copy assignment + JsonbField jf4; + jf4 = jf1; + ASSERT_NE(jf4.get_value(), nullptr); + ASSERT_NE(jf4.get_value(), jf1.get_value()); // Different memory locations + ASSERT_EQ(jf4.get_size(), jf1.get_size()); + ASSERT_EQ(std::string(jf4.get_value(), jf4.get_size()), + std::string(jf1.get_value(), jf1.get_size())); + + // Test move assignment + JsonbField jf5; + jf5 = std::move(jf4); + ASSERT_NE(jf5.get_value(), nullptr); + ASSERT_EQ(jf4.get_value(), nullptr); // jf4 should be empty after move + ASSERT_EQ(jf4.get_size(), 0); // jf4 size should be 0 after move + ASSERT_EQ(jf5.get_size(), test_size); + ASSERT_EQ(std::string(jf5.get_value(), jf5.get_size()), std::string(test_data)); + + // Test JsonbField with Field + Field field_jf = jf1; + ASSERT_EQ(field_jf.get_type(), Field::Types::JSONB); + ASSERT_NE(field_jf.get().get_value(), nullptr); + ASSERT_EQ(field_jf.get().get_size(), test_size); + ASSERT_EQ(std::string(field_jf.get().get_value(), + field_jf.get().get_size()), + std::string(test_data)); +} + +// Test for JsonbField I/O operations +TEST(VFieldTest, jsonb_field_io) { + // Prepare a JsonbField + const char* test_data = R"({ "key": "value" })"; + size_t test_size = strlen(test_data); + JsonbField original(test_data, test_size); + + // TEST 1: write_json_binary - From JsonbField to buffer + // Create a ColumnString to use with BufferWritable + ColumnString column_str; + + // Write the JsonbField to the buffer + { + BufferWritable buf(column_str); + write_json_binary(original, buf); + buf.commit(); // Important: commit the write operation + } + + // Verify data was written + ASSERT_GT(column_str.size(), 0); + + // Read the JsonbField back using BufferReadable + { + // Get the StringRef from ColumnString + StringRef str_ref = column_str.get_data_at(0); + + // Create a BufferReadable from StringRef + BufferReadable read_buf(str_ref); + + // Read the data back into a new JsonbField + JsonbField read_field; + read_json_binary(read_field, read_buf); + + // Verify the data + ASSERT_NE(read_field.get_value(), nullptr); + ASSERT_EQ(read_field.get_size(), original.get_size()); + ASSERT_EQ(std::string(read_field.get_value(), read_field.get_size()), + std::string(original.get_value(), original.get_size())); + } + + // Test with JsonbField as a Field and serde it + { + ColumnString field_column; + + // ser + { + BufferWritable field_buf(field_column); + write_json_binary(original, field_buf); + field_buf.commit(); + } + + // Verify field was written + ASSERT_GT(field_column.size(), 0); + + // de + { + StringRef field_str_ref = field_column.get_data_at(0); + BufferReadable read_field_buf(field_str_ref); + + // we can't use read_binary because of the JsonbField is not POD type + JsonbField jsonb_from_field; + read_json_binary(jsonb_from_field, read_field_buf); + Field f2 = jsonb_from_field; + + ASSERT_EQ(f2.get_type(), Field::Types::JSONB); + ASSERT_NE(f2.get().get_value(), nullptr); + ASSERT_EQ( + std::string(f2.get().get_value(), f2.get().get_size()), + std::string(test_data)); + } + } +} + } // namespace doris::vectorized From 6ba198b9c7bae0c284de727210be0f8ca0692fea Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Wed, 2 Jul 2025 10:46:09 +0800 Subject: [PATCH 127/572] [branch-3.0](timezone) Fix incorrect DST handling (#51454) (#52421) pick https://github.com/apache/doris/pull/51454 --- be/src/vec/runtime/vdatetime_value.cpp | 57 ++++++++----------- be/src/vec/runtime/vdatetime_value.h | 20 +++---- .../datatype_p0/datetimev2/test_timezone.out | 16 ++---- .../datetimev2/test_tz_streamload.out | 2 +- .../datetimev2/test_timezone.groovy | 5 ++ 5 files changed, 45 insertions(+), 55 deletions(-) diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index d161c64fd1fba3..ef6aa7e95eb388 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -2041,7 +2041,7 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale int field_idx = 0; int field_len = year_len; - long sec_offset = 0; + int sec_offset = 0; bool need_use_timezone = false; while (ptr < end && isdigit(*ptr) && field_idx < MAX_DATE_PARTS) { @@ -2206,40 +2206,29 @@ bool DateV2Value::from_date_str_base(const char* date_str, int len, int scale if (!TimezoneUtils::find_cctz_time_zone(std::string {ptr, end}, given_tz)) { return false; // invalid format } - auto given = cctz::convert(cctz::civil_second {}, given_tz); - auto local = cctz::convert(cctz::civil_second {}, *local_time_zone); - // these two values is absolute time. so they are negative. need to use (-local) - (-given) - sec_offset = std::chrono::duration_cast(given - local).count(); - } - - // In check_range_and_set_time, for Date type the time part will be truncated. So if the timezone offset should make - // rounding to date part, it would be lost. To avoid this, we use a Datetime type to do these calc. It will save the - // time part and apply the offset. Then convert to Date type back. - // see https://github.com/apache/doris/pull/33553 for more details. - if constexpr (!is_datetime) { - if (sec_offset) { - DateV2Value tmp; - if (!tmp.check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], - date_val[4], date_val[5], date_val[6])) { - return false; - } - if (!tmp.date_add_interval( - TimeInterval {TimeUnit::SECOND, sec_offset, false})) { - return false; - } - this->assign_from(tmp); - return true; + if (is_invalid(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4], date_val[5], + date_val[6])) { + return false; } - } - - if (!check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4], - date_val[5], date_val[6])) { - return false; - } - - return sec_offset ? date_add_interval( - TimeInterval {TimeUnit::SECOND, sec_offset, false}) - : true; + // will carring on the bits in cctz::civil_second. if day is 70, will carry to month. + cctz::civil_second cs {date_val[0], date_val[1], date_val[2], + date_val[3], date_val[4], date_val[5]}; + + auto given = cctz::convert(cs, given_tz); + auto local = cctz::convert(given, *local_time_zone); + date_val[0] = local.year(); + date_val[1] = local.month(); + date_val[2] = local.day(); + date_val[3] = local.hour(); + date_val[4] = local.minute(); + date_val[5] = local.second(); + } + + return check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4], + date_val[5], date_val[6]) && + (sec_offset ? date_add_interval( + TimeInterval {TimeUnit::SECOND, sec_offset, false}) + : true); } template diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index b337fea46a2051..12e961f5fa91bc 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -1024,8 +1024,8 @@ class DateV2Value { } bool operator==(const VecDateTimeValue& other) const { - int64_t ts1; - int64_t ts2; + int64_t ts1 = 0; + int64_t ts2 = 0; this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); return ts1 == ts2; @@ -1040,8 +1040,8 @@ class DateV2Value { bool operator<=(const DateV2Value& other) const { return !(*this > other); } bool operator<=(const VecDateTimeValue& other) const { - int64_t ts1; - int64_t ts2; + int64_t ts1 = 0; + int64_t ts2 = 0; this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); return ts1 <= ts2; @@ -1050,8 +1050,8 @@ class DateV2Value { bool operator>=(const DateV2Value& other) const { return !(*this < other); } bool operator>=(const VecDateTimeValue& other) const { - int64_t ts1; - int64_t ts2; + int64_t ts1 = 0; + int64_t ts2 = 0; this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); return ts1 >= ts2; @@ -1062,8 +1062,8 @@ class DateV2Value { } bool operator<(const VecDateTimeValue& other) const { - int64_t ts1; - int64_t ts2; + int64_t ts1 = 0; + int64_t ts2 = 0; this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); return ts1 < ts2; @@ -1074,8 +1074,8 @@ class DateV2Value { } bool operator>(const VecDateTimeValue& other) const { - int64_t ts1; - int64_t ts2; + int64_t ts1 = 0; + int64_t ts2 = 0; this->unix_timestamp(&ts1, TimezoneUtils::default_time_zone); other.unix_timestamp(&ts2, TimezoneUtils::default_time_zone); return ts1 > ts2; diff --git a/regression-test/data/datatype_p0/datetimev2/test_timezone.out b/regression-test/data/datatype_p0/datetimev2/test_timezone.out index 1fae14def399b1..6b1b3ebb41de81 100644 --- a/regression-test/data/datatype_p0/datetimev2/test_timezone.out +++ b/regression-test/data/datatype_p0/datetimev2/test_timezone.out @@ -1,14 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !legacy -- -2022-01-01T01:02:55 2022-01-01 -2022-02-01T03:02:55 2022-02-01 -2022-02-28T19:02:55 2022-03-01 -2022-04-01T09:02:55 2022-03-31 -2022-05-01T00:32:55 2022-05-01 -2022-05-31T22:32:55 2022-06-01 -2022-06-30T20:02:55 2022-07-01 -2022-07-31T21:00 2022-08-01 - -- !nereids -- 2022-01-01T01:02:55 2022-01-01 2022-02-01T03:02:55 2022-02-01 @@ -25,3 +15,9 @@ -- !fold3 -- 2020-12-12T13:12:12 +-- !nodst -- +2010-01-05T10:15:30 + +-- !dst -- +2010-08-05T09:15:30 + diff --git a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out index ab103c3a306f46..a05ac54d3056dd 100644 --- a/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out +++ b/regression-test/data/datatype_p0/datetimev2/test_tz_streamload.out @@ -15,7 +15,7 @@ 3 2023-08-17T17:41:18 4 2023-08-17T14:41:18 5 2023-08-17T09:41:18 -6 2023-08-18T01:41:18 +6 2023-08-18T00:41:18 7 2023-08-17T17:41:18 8 2023-08-17T19:41:18 diff --git a/regression-test/suites/datatype_p0/datetimev2/test_timezone.groovy b/regression-test/suites/datatype_p0/datetimev2/test_timezone.groovy index 746a78de4417b3..5d3d7c272ca8ec 100644 --- a/regression-test/suites/datatype_p0/datetimev2/test_timezone.groovy +++ b/regression-test/suites/datatype_p0/datetimev2/test_timezone.groovy @@ -43,4 +43,9 @@ suite("test_timezone") { qt_fold1 """ select cast('2020-12-12T12:12:12asia/shanghai' as datetime); """ qt_fold2 """ select cast('2020-12-12T12:12:12america/los_angeLES' as datetime); """ qt_fold3 """ select cast('2020-12-12T12:12:12Europe/pARIS' as datetime); """ + + qt_nodst "select cast('2010-01-05 08:15:30Europe/London' as datetime);" + qt_dst "select cast('2010-08-05 08:15:30Europe/London' as datetime);" + testFoldConst ("select cast('2010-01-05 08:15:30Europe/London' as datetime);") + testFoldConst ("select cast('2010-08-05 08:15:30Europe/London' as datetime);") } From 4b12a7555b34b9d081b0bda4dcf3482f81c94230 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Wed, 2 Jul 2025 10:55:57 +0800 Subject: [PATCH 128/572] [test](json) fix cases in TestJsonFunctions.sql #52302 (#52326) ### What problem does this PR solve? pick #52302 Co-authored-by: zzzxl --- .../rowset/segment_v2/inverted_index_file_writer_test.cpp | 1 + be/test/util/jsonb_parser_simd_test.cpp | 2 +- .../sql/presto/scalar/TestJsonFunctions.sql | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp index 2320108af2cf18..44c282b88c9622 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp @@ -391,6 +391,7 @@ TEST_F(InvertedIndexFileWriterTest, PrepareSortedFilesTest) { } } } + /*TEST_F(InvertedIndexFileWriterTest, CopyFileTest_OpenInputFailure) { auto mock_dir = std::make_shared(); std::string local_fs_index_path = InvertedIndexDescriptor::get_temporary_index_path( diff --git a/be/test/util/jsonb_parser_simd_test.cpp b/be/test/util/jsonb_parser_simd_test.cpp index 4fff867d5361ab..dca45a460dec8f 100644 --- a/be/test/util/jsonb_parser_simd_test.cpp +++ b/be/test/util/jsonb_parser_simd_test.cpp @@ -232,7 +232,7 @@ TEST_F(JsonbParserTest, ParseJsonWithLongInt) { TEST_F(JsonbParserTest, ParseInvalidJsonFormat) { std::string_view invalid_json = R"({"key": "value")"; - EXPECT_EQ(parse_json_and_check(invalid_json, invalid_json), JsonbErrType::E_INVALID_KEY_STRING); + EXPECT_EQ(parse_json_and_check(invalid_json, invalid_json), JsonbErrType::E_EXCEPTION); } TEST_F(JsonbParserTest, ParseJsonWithInvalidKeyType) { diff --git a/regression-test/suites/dialect_compatible/sql/presto/scalar/TestJsonFunctions.sql b/regression-test/suites/dialect_compatible/sql/presto/scalar/TestJsonFunctions.sql index 4cfd45753326e1..fde48caa3db062 100644 --- a/regression-test/suites/dialect_compatible/sql/presto/scalar/TestJsonFunctions.sql +++ b/regression-test/suites/dialect_compatible/sql/presto/scalar/TestJsonFunctions.sql @@ -175,8 +175,8 @@ SELECT json_array_get('[{\, :null}]', 0); SELECT json_array_get('[{\, :null}]', -1); -- SELECT json_parse('INVALID'); # error: errCode = 2, detailMessage = (127.0.0.1)[INVALID_ARGUMENT]json parse error: Exception throwed for value: INVALID -- SELECT json_parse('\, : 1'); # error: errCode = 2, detailMessage = (127.0.0.1)[INVALID_ARGUMENT]json parse error: Exception throwed for value: \, : 1 -SELECT json_parse('{}{'); -SELECT json_parse('{}{abc'); +SELECT json_parse('{}'); +SELECT json_parse('{}'); -- SELECT json_parse(''); # error: errCode = 2, detailMessage = (127.0.0.1)[INVALID_ARGUMENT]json parse error: Empty document for value: -- SELECT json_format(JSON '[\, , \, ]'); # error: errCode = 2, detailMessage = Can not found function 'JSON_FORMAT' SELECT json_size('{\, : {\, : 1, \, : 2} }', '$'); @@ -369,8 +369,8 @@ SELECT json_array_get('[{\, :null}]', 0); SELECT json_array_get('[{\, :null}]', -1); -- SELECT json_parse('INVALID'); # error: errCode = 2, detailMessage = (127.0.0.1)[INVALID_ARGUMENT]json parse error: Exception throwed for value: INVALID -- SELECT json_parse('\, : 1'); # error: errCode = 2, detailMessage = (127.0.0.1)[INVALID_ARGUMENT]json parse error: Exception throwed for value: \, : 1 -SELECT json_parse('{}{'); -SELECT json_parse('{}{abc'); +SELECT json_parse('{}'); +SELECT json_parse('{}'); -- SELECT json_parse(''); # error: errCode = 2, detailMessage = (127.0.0.1)[INVALID_ARGUMENT]json parse error: Empty document for value: -- SELECT json_format(JSON '[\, , \, ]'); # error: errCode = 2, detailMessage = Can not found function 'JSON_FORMAT' SELECT json_size('{\, : {\, : 1, \, : 2} }', '$'); From d8e6d4ae8abcef8678261d3cbd8b79cf292e4960 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Fri, 4 Jul 2025 15:12:37 +0800 Subject: [PATCH 129/572] [chore](ci) adjust pipelines (#52507) ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- regression-test/pipeline/cloud_p0/clean.sh | 2 +- regression-test/pipeline/cloud_p0/prepare.sh | 23 +++--- regression-test/pipeline/cloud_p0/run.sh | 9 ++- .../pipeline/common/doris-utils.sh | 76 ++++++++++++++++--- regression-test/pipeline/vault_p0/clean.sh | 2 +- regression-test/pipeline/vault_p0/prepare.sh | 2 +- regression-test/pipeline/vault_p0/run.sh | 2 +- 7 files changed, 89 insertions(+), 27 deletions(-) diff --git a/regression-test/pipeline/cloud_p0/clean.sh b/regression-test/pipeline/cloud_p0/clean.sh index 9707378e27bced..2b67ae9349566f 100644 --- a/regression-test/pipeline/cloud_p0/clean.sh +++ b/regression-test/pipeline/cloud_p0/clean.sh @@ -26,5 +26,5 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "#### Run tpcds test on Doris ####" DORIS_HOME="${teamcity_build_checkoutDir}/output" export DORIS_HOME -stop_doris +export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace clean_fdb "cloud_instance_0" diff --git a/regression-test/pipeline/cloud_p0/prepare.sh b/regression-test/pipeline/cloud_p0/prepare.sh index 44154c2ee26f23..bd08a0b2f18acb 100644 --- a/regression-test/pipeline/cloud_p0/prepare.sh +++ b/regression-test/pipeline/cloud_p0/prepare.sh @@ -35,7 +35,7 @@ if ${DEBUG:-false}; then fi # shellcheck source=/dev/null -# stop_doris, clean_fdb, install_fdb, install_java +# stop_doris, clean_fdb, install_fdb, install_java, clear_coredump source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh # shellcheck source=/dev/null # check_oss_file_exist, download_oss_file @@ -100,17 +100,18 @@ echo "#### 3. try to kill old doris process" DORIS_HOME="${teamcity_build_checkoutDir}/output" export DORIS_HOME stop_doris +clear_coredump echo "#### 4. prepare fundationdb" install_fdb clean_fdb "cloud_instance_0" echo "#### 5. check if binary package ready" -merge_pr_to_target_branch_compiled_commit() { +merge_pr_to_master_commit() { local pr_num_from_trigger="$1" local target_branch="$2" - local target_branch_compiled_commit="$3" - echo "INFO: merge pull request into ${target_branch} ${target_branch_compiled_commit}" + local master_commit="$3" + echo "INFO: merge pull request into ${target_branch} ${master_commit}" if [[ -z "${teamcity_build_checkoutDir}" ]]; then echo "ERROR: env teamcity_build_checkoutDir not set" && return 1 fi @@ -119,10 +120,10 @@ merge_pr_to_target_branch_compiled_commit() { git fetch origin "${target_branch}" git checkout "${target_branch}" git reset --hard origin/"${target_branch}" - git checkout "${target_branch_compiled_commit}" + git checkout "${master_commit}" returnValue=$? if [[ ${returnValue} -ne 0 ]]; then - echo "ERROR: checkout ${target_branch} ${target_branch_compiled_commit} failed. please rebase to the newest version." + echo "ERROR: checkout ${target_branch} ${master_commit} failed. please rebase to the newest version." return 1 fi git rev-parse HEAD @@ -131,7 +132,7 @@ merge_pr_to_target_branch_compiled_commit() { echo "git fetch origin refs/pull/${pr_num_from_trigger}/head" git fetch origin "refs/pull/${pr_num_from_trigger}/head" git merge --no-edit --allow-unrelated-histories FETCH_HEAD - echo "INFO: merge refs/pull/${pr_num_from_trigger}/head into ${target_branch} ${target_branch_compiled_commit}" + echo "INFO: merge refs/pull/${pr_num_from_trigger}/head into ${target_branch} ${master_commit}" # CONFLICTS=$(git ls-files -u | wc -l) if [[ $(git ls-files -u | wc -l) -gt 0 ]]; then echo "ERROR: merge refs/pull/${pr_num_from_trigger}/head into failed. Aborting" @@ -144,11 +145,11 @@ if ! check_oss_file_exist "${pr_num_from_trigger}_${commit_id_from_trigger}.tar. if download_oss_file "${pr_num_from_trigger}_${commit_id_from_trigger}.tar.gz"; then rm -rf "${teamcity_build_checkoutDir}"/output tar -I pigz -xf "${pr_num_from_trigger}_${commit_id_from_trigger}.tar.gz" - target_branch_compiled_commit_file="master.commit" - if [[ -e output/${target_branch_compiled_commit_file} ]]; then + master_commit_file="master.commit" + if [[ -e output/${master_commit_file} ]]; then # checkout to master commit and merge this pr, to ensure binary and case are same version - target_branch_compiled_commit=$(cat output/"${target_branch_compiled_commit_file}") - if merge_pr_to_target_branch_compiled_commit "${pr_num_from_trigger}" "${target_branch}" "${target_branch_compiled_commit}"; then + master_commit=$(cat output/"${master_commit_file}") + if merge_pr_to_master_commit "${pr_num_from_trigger}" "${target_branch}" "${master_commit}"; then echo "INFO: merged done" if [[ "${teamcity_buildType_id:-}" =~ ^Doris_DorisCloudRegression_CloudP1 ]]; then echo "INFO: 用cloud_p1/conf覆盖cloud_p0/conf" diff --git a/regression-test/pipeline/cloud_p0/run.sh b/regression-test/pipeline/cloud_p0/run.sh index 43847f86980c2a..55f0c41901372c 100644 --- a/regression-test/pipeline/cloud_p0/run.sh +++ b/regression-test/pipeline/cloud_p0/run.sh @@ -13,7 +13,7 @@ fi EOF ############################# run.sh content ######################################## # shellcheck source=/dev/null -# check_tpcds_table_rows, restart_doris, set_session_variable, check_tpcds_result +# _monitor_regression_log, print_running_pipeline_tasks source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh # shellcheck source=/dev/null # create_an_issue_comment @@ -47,6 +47,9 @@ export DORIS_HOME exit_flag=0 need_collect_log=false +# monitoring the log files in "${DORIS_HOME}"/regression-test/log/ for keyword 'Reach limit of connections' +_monitor_regression_log & + # shellcheck disable=SC2317 run() { set -e @@ -72,7 +75,6 @@ run() { export JAVA_HOME if "${teamcity_build_checkoutDir}"/run-regression-test.sh \ --teamcity \ - --clean \ --run \ --times "${repeat_times_from_trigger:-1}" \ -parallel 18 \ @@ -105,6 +107,7 @@ export -f run timeout_minutes=$((${repeat_times_from_trigger:-1} * ${BUILD_TIMEOUT_MINUTES:-180}))m timeout "${timeout_minutes}" bash -cx run exit_flag="$?" +if print_running_pipeline_tasks; then :; fi # shellcheck source=/dev/null source "$(cd "${teamcity_build_checkoutDir}" && bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')" @@ -116,7 +119,7 @@ if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then print_doris_fe_log print_doris_be_log fi - stop_doris + export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace if log_file_name=$(archive_doris_logs "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_logs.tar.gz"); then if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then reporting_messages_error "${log_info##*logs.tar.gz to }" diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh index 1ae9d7c1c89b5e..d1b6e71835a812 100644 --- a/regression-test/pipeline/common/doris-utils.sh +++ b/regression-test/pipeline/common/doris-utils.sh @@ -94,7 +94,7 @@ function install_java() { [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null fi - # doris master and branch-3.0 use java-17 + # doris master branch use java-17 if ! java -version >/dev/null || [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-17-*')" ]]; then sudo apt update && sudo apt install openjdk-17-jdk -y >/dev/null @@ -137,10 +137,10 @@ function start_doris_fe() { if [[ -n "${fe_version}" ]] && [[ "${fe_version}" != "NULL" ]]; then echo "INFO: doris fe started, fe version: ${fe_version}" && return 0 else - echo "${i}/60, Wait for Frontend ready, sleep 2 seconds ..." && sleep 2 + echo "${i}/60, Wait for Frontend ready, sleep 5 seconds ..." && sleep 5 fi done - if [[ ${i} -ge 60 ]]; then echo "ERROR: Start Doris Frontend Failed after 2 mins wait..." && return 1; fi + if [[ ${i} -ge 60 ]]; then echo "ERROR: Start Doris Frontend Failed after 5 mins wait..." && return 1; fi } function start_doris_be() { @@ -194,10 +194,10 @@ function check_doris_ready() { [[ ${be_ready_count} -eq 1 ]]; then echo -e "INFO: Doris cluster ready, be version: \n$(${cl} -e 'show backends\G' | grep 'Version')" && break else - echo 'Wait for backends ready, sleep 2 seconds ...' && sleep 2 + echo 'Wait for backends ready, sleep 5 seconds ...' && sleep 5 fi done - if [[ ${i} -ge 60 ]]; then echo "ERROR: Doris cluster not ready after 2 mins wait..." && return 1; fi + if [[ ${i} -ge 60 ]]; then echo "ERROR: Doris cluster not ready after 5 mins wait..." && return 1; fi # wait 10s for doris totally started, otherwize may encounter the error below, # ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium. @@ -206,14 +206,26 @@ function check_doris_ready() { function stop_doris() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi - if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then bash "${DORIS_HOME}"/ms/bin/stop.sh; fi - if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then bash "${DORIS_HOME}"/recycler/bin/stop.sh; fi if "${DORIS_HOME}"/be/bin/stop_be.sh && "${DORIS_HOME}"/fe/bin/stop_fe.sh; then echo "INFO: normally stoped doris" else pgrep -fi doris | xargs kill -9 &>/dev/null echo "WARNING: force stoped doris" fi + if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then bash "${DORIS_HOME}"/ms/bin/stop.sh; fi + if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then bash "${DORIS_HOME}"/recycler/bin/stop.sh; fi +} + +function stop_doris_grace() { + if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi + if "${DORIS_HOME}"/be/bin/stop_be.sh --grace && "${DORIS_HOME}"/fe/bin/stop_fe.sh --grace; then + echo "INFO: normally stoped doris --grace" + else + pgrep -fi doris | xargs kill -9 &>/dev/null + echo "WARNING: force stoped doris" + fi + if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then bash "${DORIS_HOME}"/ms/bin/stop.sh --grace; fi + if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then bash "${DORIS_HOME}"/recycler/bin/stop.sh --grace; fi } function clean_fdb() { @@ -293,10 +305,10 @@ function restart_doris() { [[ ${be_ready_count} -eq 1 ]]; then echo -e "INFO: ${be_ready_count} Backends ready, version: \n$(${cl} -e 'show backends\G' | grep 'Version')" && break else - echo 'Wait for Backends ready, sleep 2 seconds ...' && sleep 2 + echo 'Wait for Backends ready, sleep 5 seconds ...' && sleep 5 fi done - if [[ ${i} -ge 60 ]]; then echo "ERROR: Backend not ready after 2 mins wait..." && return 1; fi + if [[ ${i} -ge 60 ]]; then echo "ERROR: Backend not ready after 5 mins wait..." && return 1; fi # wait 10s for doris totally started, otherwize may encounter the error below, # ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium. @@ -521,6 +533,47 @@ function set_doris_session_variables_from_file() { fi } +_monitor_regression_log() { + if ! command -v inotifywait >/dev/null; then + apt install inotify-tools -y + fi + + # Path to the log directory + local LOG_DIR="${DORIS_HOME}"/regression-test/log + + # keyword to search for in the log files + local KEYWORD="Reach limit of connections" + + local query_port + query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port) + + echo "INFO: start monitoring the log files in ${LOG_DIR} for the keyword '${KEYWORD}'" + + local start_row=1 + local filepath="" + set +x + # Monitor the log directory for new files and changes, only one file + # shellcheck disable=SC2034 + inotifywait -m -e modify "${LOG_DIR}" | while read -r directory events filename; do + filepath="${directory}${filename}" + if [[ ! -f "${filepath}" ]]; then continue; fi + total_rows=$(wc -l "${filepath}" | awk '{print $1}') + if [[ -n ${total_rows} ]] && [[ ${start_row} -ge ${total_rows} ]]; then + start_row=${total_rows} + fi + # shellcheck disable=SC2250 + if sed -n "${start_row},\$p" "${filepath}" | grep -a -q "${KEYWORD}"; then + matched=$(grep -a -n "${KEYWORD}" "${filepath}") + start_row=$(echo "${matched}" | tail -n1 | cut -d: -f1) + echo "WARNING: find '${matched}' in ${filepath}, run 'show processlist;' to check the connections" | tee -a "${DORIS_HOME}"/fe/log/monitor_regression_log.out + mysql -h127.0.0.1 -P"${query_port}" -uroot -e'show processlist;' | tee -a "${DORIS_HOME}"/fe/log/monitor_regression_log.out + fi + start_row=$((start_row + 1)) + # echo "start_row ${start_row}" | tee -a "${DORIS_HOME}"/fe/log/monitor_regression_log.out + done + +} + archive_doris_logs() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi local archive_name="$1" @@ -592,6 +645,11 @@ wait_coredump_file_ready() { done } +clear_coredump() { + echo -e "INFO: clear coredump files \n$(ls /var/lib/apport/coredump/)" + rm -rf /var/lib/apport/coredump/* +} + archive_doris_coredump() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi archive_name="$1" diff --git a/regression-test/pipeline/vault_p0/clean.sh b/regression-test/pipeline/vault_p0/clean.sh index 309c9b8e5332ca..1500bd48bf4821 100644 --- a/regression-test/pipeline/vault_p0/clean.sh +++ b/regression-test/pipeline/vault_p0/clean.sh @@ -26,7 +26,7 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "#### stop doris and clean fdb ####" DORIS_HOME="${teamcity_build_checkoutDir}/output" export DORIS_HOME -stop_doris +export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace clean_fdb "cloud_instance_0" echo "#### docker-compose down ####" diff --git a/regression-test/pipeline/vault_p0/prepare.sh b/regression-test/pipeline/vault_p0/prepare.sh index 00f6878ee593d4..3f348052337834 100644 --- a/regression-test/pipeline/vault_p0/prepare.sh +++ b/regression-test/pipeline/vault_p0/prepare.sh @@ -71,7 +71,7 @@ if [[ "${target_branch}" == "master" || "${target_branch}" == "branch-3.0" ]]; t echo "INFO: PR target branch ${target_branch}" install_java else - echo "WARNING: PR target branch ${target_branch} is NOT in (master, branch-3.0), skip pipeline." + echo "WARNING: PR target branch ${target_branch} is NOT in (master,branch-3.0), skip pipeline." bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'set' "export skip_pipeline=true" exit 0 fi diff --git a/regression-test/pipeline/vault_p0/run.sh b/regression-test/pipeline/vault_p0/run.sh index 8322e458a7adc0..7cf48010f95bb0 100644 --- a/regression-test/pipeline/vault_p0/run.sh +++ b/regression-test/pipeline/vault_p0/run.sh @@ -156,7 +156,7 @@ if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then print_doris_fe_log print_doris_be_log fi - stop_doris + export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace if log_file_name=$(archive_doris_logs "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_logs.tar.gz"); then if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then reporting_messages_error "${log_info##*logs.tar.gz to }" From e486dd387b929e94174bb7903e8f58b1d4aaae42 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Fri, 4 Jul 2025 15:13:43 +0800 Subject: [PATCH 130/572] [fix](json) Add . after in JSON path to support correct token parsing (#52543) (#52744) Boost tokenizer requires explicit "." after "$" to correctly extract JSON path tokens. Without this, expressions like "$[0].key" cannot be properly split, causing issues in downstream logic. This commit ensures a "." is automatically added after "$" to maintain consistent token parsing behavior. ### What problem does this PR solve? pick #52543 Issue Number: close #xxx Related PR: #52543 Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/functions/function_json.cpp | 106 +++---- be/test/vec/function/function_json_test.cpp | 16 +- .../json_p0/test_json_load_and_function.out | 300 +++++++++--------- .../jsonb_p0/test_jsonb_load_and_function.out | 150 ++++----- .../nereids_function_p0/scalar_function/J.out | 132 ++++---- .../json_functions/test_json_extract.out | 2 + .../json_functions/test_json_extract.groovy | 8 + 7 files changed, 352 insertions(+), 362 deletions(-) diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp index 6901dfb2b69297..ab1ff616f2b1e7 100644 --- a/be/src/vec/functions/function_json.cpp +++ b/be/src/vec/functions/function_json.cpp @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include @@ -144,45 +145,7 @@ rapidjson::Value* match_value(const std::vector& parsed_paths, rapidjs const std::string& col = parsed_paths[i].key; int index = parsed_paths[i].idx; if (LIKELY(!col.empty())) { - if (root->IsArray()) { - array_obj = static_cast( - mem_allocator.Malloc(sizeof(rapidjson::Value))); - array_obj->SetArray(); - bool is_null = true; - - // if array ,loop the array,find out all Objects,then find the results from the objects - for (int j = 0; j < root->Size(); j++) { - rapidjson::Value* json_elem = &((*root)[j]); - - if (json_elem->IsArray() || json_elem->IsNull()) { - continue; - } else { - if (!json_elem->IsObject()) { - continue; - } - if (!json_elem->HasMember(col.c_str())) { - if (is_insert_null) { // not found item, then insert a null object. - is_null = false; - rapidjson::Value nullObject(rapidjson::kNullType); - array_obj->PushBack(nullObject, mem_allocator); - } - continue; - } - rapidjson::Value* obj = &((*json_elem)[col.c_str()]); - if (obj->IsArray()) { - is_null = false; - for (int k = 0; k < obj->Size(); k++) { - array_obj->PushBack((*obj)[k], mem_allocator); - } - } else if (!obj->IsNull()) { - is_null = false; - array_obj->PushBack(*obj, mem_allocator); - } - } - } - - root = is_null ? &(array_obj->SetNull()) : array_obj; - } else if (root->IsObject()) { + if (root->IsObject()) { if (!root->HasMember(col.c_str())) { return nullptr; } else { @@ -233,8 +196,17 @@ rapidjson::Value* get_json_object(std::string_view json_string, std::string_view //Cannot use '\' as the last character, return NULL if (path_string.back() == '\\') { - document->SetNull(); - return document; + return nullptr; + } + + std::string fixed_string; + if (path_string.size() >= 2 && path_string[0] == '$' && path_string[1] != '.') { + // Boost tokenizer requires explicit "." after "$" to correctly extract JSON path tokens. + // Without this, expressions like "$[0].key" cannot be properly split. + // This commit ensures a "." is automatically added after "$" to maintain consistent token parsing behavior. + fixed_string = "$."; + fixed_string += path_string.substr(1); + path_string = fixed_string; } try { @@ -251,13 +223,13 @@ rapidjson::Value* get_json_object(std::string_view json_string, std::string_view } } catch (boost::escaped_list_error&) { // meet unknown escape sequence, example '$.name\k' - return document; + return nullptr; } parsed_paths = &tmp_parsed_paths; if (!(*parsed_paths)[0].is_valid) { - return document; + return nullptr; } if (UNLIKELY((*parsed_paths).size() == 1)) { @@ -272,8 +244,7 @@ rapidjson::Value* get_json_object(std::string_view json_string, std::string_view if (UNLIKELY(document->HasParseError())) { // VLOG_CRITICAL << "Error at offset " << document->GetErrorOffset() << ": " // << GetParseError_En(document->GetParseError()); - document->SetNull(); - return document; + return nullptr; } return match_value(*parsed_paths, document, document->GetAllocator()); @@ -858,9 +829,10 @@ template struct FunctionJsonExtractImpl { static constexpr auto name = Name::name; - static rapidjson::Value parse_json(const ColumnString* json_col, const ColumnString* path_col, - rapidjson::Document::AllocatorType& allocator, const int row, - const int col, std::vector& column_is_consts) { + static std::pair parse_json( + const ColumnString* json_col, const ColumnString* path_col, + rapidjson::Document::AllocatorType& allocator, const int row, const int col, + std::vector& column_is_consts) { rapidjson::Value value; rapidjson::Document document; @@ -869,10 +841,13 @@ struct FunctionJsonExtractImpl { const auto path = path_col->get_data_at(index_check_const(row, column_is_consts[col])); std::string_view path_string(path.data, path.size); auto* root = get_json_object(json_string, path_string, &document); + bool found = false; if (root != nullptr) { + found = true; value.CopyFrom(*root, allocator); } - return value; + + return {found, std::move(value)}; } static rapidjson::Value* get_document(const ColumnString* path_col, @@ -913,8 +888,9 @@ struct FunctionJsonExtractImpl { rapidjson::StringBuffer buf; rapidjson::Writer writer(buf); const auto* json_col = data_columns[0]; - auto insert_result_lambda = [&](rapidjson::Value& value, int row) { - if (value.IsNull()) { + + auto insert_result_lambda = [&](rapidjson::Value& value, bool is_null, int row) { + if (is_null) { null_map[row] = 1; result_column.insert_default(); } else { @@ -935,12 +911,13 @@ struct FunctionJsonExtractImpl { } }; if (data_columns.size() == 2) { - rapidjson::Value value; if (column_is_consts[1]) { std::vector parsed_paths; auto* root = get_document(data_columns[1], &document, parsed_paths, 0, column_is_consts[1]); for (size_t row = 0; row < input_rows_count; row++) { + bool is_null = false; + rapidjson::Value value; if (root != nullptr) { const auto& obj = json_col->get_data_at(row); std::string_view json_string(obj.data, obj.size); @@ -957,17 +934,18 @@ struct FunctionJsonExtractImpl { if (root_val != nullptr) { value.CopyFrom(*root_val, allocator); } else { - rapidjson::Value tmp; - value.Swap(tmp); + is_null = true; } + } else { + is_null = true; } - insert_result_lambda(value, row); + insert_result_lambda(value, is_null, row); } } else { for (size_t row = 0; row < input_rows_count; row++) { - value = parse_json(json_col, data_columns[1], allocator, row, 1, - column_is_consts); - insert_result_lambda(value, row); + auto result = parse_json(json_col, data_columns[1], allocator, row, 1, + column_is_consts); + insert_result_lambda(result.second, !result.first, row); } } @@ -977,12 +955,16 @@ struct FunctionJsonExtractImpl { value.Reserve(data_columns.size() - 1, allocator); for (size_t row = 0; row < input_rows_count; row++) { value.Clear(); + bool found_any = false; for (size_t col = 1; col < data_columns.size(); ++col) { - value.PushBack(parse_json(json_col, data_columns[col], allocator, row, col, - column_is_consts), - allocator); + auto result = parse_json(json_col, data_columns[col], allocator, row, col, + column_is_consts); + if (result.first) { + found_any = true; + value.PushBack(std::move(result.second), allocator); + } } - insert_result_lambda(value, row); + insert_result_lambda(value, !found_any, row); } } } diff --git a/be/test/vec/function/function_json_test.cpp b/be/test/vec/function/function_json_test.cpp index 988d4ca731a582..8f28a474958537 100644 --- a/be/test/vec/function/function_json_test.cpp +++ b/be/test/vec/function/function_json_test.cpp @@ -72,15 +72,13 @@ TEST(FunctionJsonTEST, GetJsonStringTest) { std::string func_name = "get_json_string"; InputTypeSet input_types = {TypeIndex::String, TypeIndex::String}; DataSet data_set = { - {{VARCHAR("{\"k1\":\"v1\", \"k2\":\"v2\"}"), VARCHAR("$.k1")}, VARCHAR("v1")}, - {{VARCHAR("{\"k1\":\"v1\", \"my.key\":[\"e1\", \"e2\", \"e3\"]}"), - VARCHAR("$.\"my.key\"[1]")}, + {{VARCHAR(R"({"k1":"v1", "k2":"v2"})"), VARCHAR("$.k1")}, VARCHAR("v1")}, + {{VARCHAR(R"({"k1":"v1", "my.key":["e1", "e2", "e3"]})"), VARCHAR("$.\"my.key\"[1]")}, VARCHAR("e2")}, - {{VARCHAR("{\"k1.key\":{\"k2\":[\"v1\", \"v2\"]}}"), VARCHAR("$.\"k1.key\".k2[0]")}, + {{VARCHAR(R"({"k1.key":{"k2":["v1", "v2"]}})"), VARCHAR("$.\"k1.key\".k2[0]")}, VARCHAR("v1")}, - {{VARCHAR("[{\"k1\":\"v1\"}, {\"k2\":\"v2\"}, {\"k1\":\"v3\"}, {\"k1\":\"v4\"}]"), - VARCHAR("$.k1")}, - VARCHAR("[\"v1\",\"v3\",\"v4\"]")}}; + {{VARCHAR(R"([{"k1":"v1"}, {"k2":"v2"}, {"k1":"v3"}, {"k1":"v4"}])"), VARCHAR("$.k1")}, + Null()}}; static_cast(check_function(func_name, input_types, data_set)); } @@ -93,7 +91,7 @@ TEST(FunctionJsonTEST, JsonExtractTest) { // json_extract root DataSet data_set = { {{Null(), STRING("$")}, Null()}, - {{STRING("null"), STRING("$")}, Null()}, + {{STRING("null"), STRING("$")}, STRING("null")}, {{STRING("true"), STRING("$")}, STRING("true")}, {{STRING("false"), STRING("$")}, STRING("false")}, {{STRING("100"), STRING("$")}, STRING("100")}, //int8 @@ -127,7 +125,7 @@ TEST(FunctionJsonTEST, JsonExtractTest) { data_set = { {{Null(), STRING("$")}, Null()}, - {{STRING("null"), STRING("$")}, Null()}, + {{STRING("null"), STRING("$")}, STRING("null")}, {{STRING("true"), STRING("$")}, STRING("true")}, {{STRING("false"), STRING("$")}, STRING("false")}, {{STRING("100"), STRING("$")}, STRING("100")}, //int8 diff --git a/regression-test/data/json_p0/test_json_load_and_function.out b/regression-test/data/json_p0/test_json_load_and_function.out index de25adfef2e758..b6098cfbf640ad 100644 --- a/regression-test/data/json_p0/test_json_load_and_function.out +++ b/regression-test/data/json_p0/test_json_load_and_function.out @@ -6189,7 +6189,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -6203,61 +6203,61 @@ false -- !select -- 1 \N \N -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,456] -15 ["abc","def"] [null,"def"] -16 [null,true,false,100,6.18,"abc"] [null,true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] [456] +15 ["abc","def"] ["def"] +16 [null,true,false,100,6.18,"abc"] [true] +17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] -32 {"":"v1"} [null,null] -33 {"":1," ":"v1"} [null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- 1 \N \N -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,null] -15 ["abc","def"] [null,null] -16 [null,true,false,100,6.18,"abc"] [null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],null] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] -32 {"":"v1"} [null,null] -33 {"":1," ":"v1"} [null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- 1 \N \N @@ -6290,32 +6290,32 @@ false -- !select -- 1 \N \N -2 null [null,null,null] -3 true [null,null,null] -4 false [null,null,null] -5 100 [null,null,null] -6 10000 [null,null,null] -7 1000000000 [null,null,null] -8 1152921504606846976 [null,null,null] -9 6.18 [null,null,null] -10 "abcd" [null,null,null] -11 {} [null,null,null] -12 {"k1":"v31","k2":300} [null,null,null] -13 [] [null,null,null] -14 [123,456] [null,null,null] -15 ["abc","def"] [null,null,null] -16 [null,true,false,100,6.18,"abc"] [null,null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [null,null,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} ["v41",400,"a"] 26 \N \N -27 {"k1":"v1","k2":200} [null,null,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null,null] -29 12524337771678448270 [null,null,null] -30 -9223372036854775808 [null,null,null] -31 18446744073709551615 [null,null,null] -32 {"":"v1"} [null,null,null] -33 {"":1," ":"v1"} [null,null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null,null] +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- 2 null @@ -11664,7 +11664,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 27 {"k1":"v1","k2":200} "v1" 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N @@ -11676,58 +11676,58 @@ false 34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,456] -15 ["abc","def"] [null,"def"] -16 [null,true,false,100,6.18,"abc"] [null,true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] -32 {"":"v1"} [null,null] -33 {"":1," ":"v1"} [null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] [456] +15 ["abc","def"] ["def"] +16 [null,true,false,100,6.18,"abc"] [true] +17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,null] -15 ["abc","def"] [null,null] -16 [null,true,false,100,6.18,"abc"] [null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],null] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] -32 {"":"v1"} [null,null] -33 {"":1," ":"v1"} [null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- 2 null \N @@ -11757,31 +11757,31 @@ false 34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- -2 null [null,null,null] -3 true [null,null,null] -4 false [null,null,null] -5 100 [null,null,null] -6 10000 [null,null,null] -7 1000000000 [null,null,null] -8 1152921504606846976 [null,null,null] -9 6.18 [null,null,null] -10 "abcd" [null,null,null] -11 {} [null,null,null] -12 {"k1":"v31","k2":300} [null,null,null] -13 [] [null,null,null] -14 [123,456] [null,null,null] -15 ["abc","def"] [null,null,null] -16 [null,true,false,100,6.18,"abc"] [null,null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [null,null,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} ["v41",400,"a"] -27 {"k1":"v1","k2":200} [null,null,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null,null] -29 12524337771678448270 [null,null,null] -30 -9223372036854775808 [null,null,null] -31 18446744073709551615 [null,null,null] -32 {"":"v1"} [null,null,null] -33 {"":1," ":"v1"} [null,null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null,null] +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !sql_json_parse -- {"":"v1"} diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index dbd756dab9d0bb..d7d7611931fdb3 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -8187,7 +8187,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -8201,61 +8201,61 @@ false -- !select -- 1 \N \N -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,456] -15 ["abc","def"] [null,"def"] -16 [null,true,false,100,6.18,"abc"] [null,true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] [456] +15 ["abc","def"] ["def"] +16 [null,true,false,100,6.18,"abc"] [true] +17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] -32 {"":"v1"} [null,null] -33 {"":1," ":"v1"} [null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- 1 \N \N -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,null] -15 ["abc","def"] [null,null] -16 [null,true,false,100,6.18,"abc"] [null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],null] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] -32 {"":"v1"} [null,null] -33 {"":1," ":"v1"} [null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !select -- 1 \N \N @@ -8288,32 +8288,32 @@ false -- !select -- 1 \N \N -2 null [null,null,null] -3 true [null,null,null] -4 false [null,null,null] -5 100 [null,null,null] -6 10000 [null,null,null] -7 1000000000 [null,null,null] -8 1152921504606846976 [null,null,null] -9 6.18 [null,null,null] -10 "abcd" [null,null,null] -11 {} [null,null,null] -12 {"k1":"v31","k2":300} [null,null,null] -13 [] [null,null,null] -14 [123,456] [null,null,null] -15 ["abc","def"] [null,null,null] -16 [null,true,false,100,6.18,"abc"] [null,null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [null,null,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} ["v41",400,"a"] 26 \N \N -27 {"k1":"v1","k2":200} [null,null,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null,null] -29 12524337771678448270 [null,null,null] -30 -9223372036854775808 [null,null,null] -31 18446744073709551615 [null,null,null] -32 {"":"v1"} [null,null,null] -33 {"":1," ":"v1"} [null,null,null] -34 {"":1,"ab":"v1"," ":"v1"," ":2} [null,null,null] +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N +32 {"":"v1"} \N +33 {"":1," ":"v1"} \N +34 {"":1,"ab":"v1"," ":"v1"," ":2} \N -- !sql_json_length -- 1 diff --git a/regression-test/data/nereids_function_p0/scalar_function/J.out b/regression-test/data/nereids_function_p0/scalar_function/J.out index 64a76e24826c10..dc1c34eafe65f6 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/J.out +++ b/regression-test/data/nereids_function_p0/scalar_function/J.out @@ -7348,7 +7348,7 @@ false 14 [123,456] \N 15 ["abc","def"] \N 16 [null,true,false,100,6.18,"abc"] \N -17 [{"k1":"v41","k2":400},1,"a",3.14] ["v41"] +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31" 26 \N \N 27 {"k1":"v1","k2":200} "v1" @@ -7359,55 +7359,55 @@ false -- !select -- 1 \N \N -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,456] -15 ["abc","def"] [null,"def"] -16 [null,true,false,100,6.18,"abc"] [null,true] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],1] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] [456] +15 ["abc","def"] ["def"] +16 [null,true,false,100,6.18,"abc"] [true] +17 [{"k1":"v41","k2":400},1,"a",3.14] [1] +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N -- !select -- 1 \N \N -2 null [null,null] -3 true [null,null] -4 false [null,null] -5 100 [null,null] -6 10000 [null,null] -7 1000000000 [null,null] -8 1152921504606846976 [null,null] -9 6.18 [null,null] -10 "abcd" [null,null] -11 {} [null,null] -12 {"k1":"v31","k2":300} [300,null] -13 [] [null,null] -14 [123,456] [null,null] -15 ["abc","def"] [null,null] -16 [null,true,false,100,6.18,"abc"] [null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [[400],null] -18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} [300] +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N +18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} [300] 26 \N \N -27 {"k1":"v1","k2":200} [200,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null] -29 12524337771678448270 [null,null] -30 -9223372036854775808 [null,null] -31 18446744073709551615 [null,null] +27 {"k1":"v1","k2":200} [200] +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N -- !select -- 1 \N \N @@ -7437,29 +7437,29 @@ false -- !select -- 1 \N \N -2 null [null,null,null] -3 true [null,null,null] -4 false [null,null,null] -5 100 [null,null,null] -6 10000 [null,null,null] -7 1000000000 [null,null,null] -8 1152921504606846976 [null,null,null] -9 6.18 [null,null,null] -10 "abcd" [null,null,null] -11 {} [null,null,null] -12 {"k1":"v31","k2":300} [null,null,null] -13 [] [null,null,null] -14 [123,456] [null,null,null] -15 ["abc","def"] [null,null,null] -16 [null,true,false,100,6.18,"abc"] [null,null,null] -17 [{"k1":"v41","k2":400},1,"a",3.14] [null,null,null] +2 null \N +3 true \N +4 false \N +5 100 \N +6 10000 \N +7 1000000000 \N +8 1152921504606846976 \N +9 6.18 \N +10 "abcd" \N +11 {} \N +12 {"k1":"v31","k2":300} \N +13 [] \N +14 [123,456] \N +15 ["abc","def"] \N +16 [null,true,false,100,6.18,"abc"] \N +17 [{"k1":"v41","k2":400},1,"a",3.14] \N 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} ["v41",400,"a"] 26 \N \N -27 {"k1":"v1","k2":200} [null,null,null] -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} [null,null,null] -29 12524337771678448270 [null,null,null] -30 -9223372036854775808 [null,null,null] -31 18446744073709551615 [null,null,null] +27 {"k1":"v1","k2":200} \N +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N +29 12524337771678448270 \N +30 -9223372036854775808 \N +31 18446744073709551615 \N -- !json_extract_string -- "v31" diff --git a/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract.out b/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract.out index 2b64e92fd3ef58..822470fb2732f3 100644 --- a/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract.out +++ b/regression-test/data/query_p0/sql_functions/json_functions/test_json_extract.out @@ -20,3 +20,5 @@ v31 -- !sql_string7 -- \N +-- !fix_array_path -- +[123] [123] [123] [123] diff --git a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract.groovy b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract.groovy index 41e68111d79d77..b728f7ee517a9b 100644 --- a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract.groovy +++ b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_extract.groovy @@ -28,4 +28,12 @@ suite("test_json_extract") { sql """ SELECT JSON_EXTRACT_STRING('{"id": 123, "name": "doris"}', '\$.'); """ exception "Invalid Json Path for value: \$." } + + qt_fix_array_path """ + select + JSON_EXTRACT('[{"key": [123]}]', '\$[0].key') v1 + , JSON_EXTRACT('[{"key": [123]}]', '\$.[0].key') v2 + , JSONB_EXTRACT('[{"key": [123]}]', '\$[0].key') v3 + , JSONB_EXTRACT('[{"key": [123]}]', '\$.[0].key') v4; + """ } From 37f2b5d33967a8810267854140247709ef353f37 Mon Sep 17 00:00:00 2001 From: zzzxl Date: Fri, 4 Jul 2025 19:48:46 +0800 Subject: [PATCH 131/572] [fix](inverted index) index-tool compile error (#52710) https://github.com/apache/doris/pull/52508 Co-authored-by: Yongqiang YANG --- be/src/index-tools/index_tool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp index ca0575dc545459..e45902c0f24df1 100644 --- a/be/src/index-tools/index_tool.cpp +++ b/be/src/index-tools/index_tool.cpp @@ -562,7 +562,7 @@ int main(int argc, char** argv) { auto dir = std::forward(st).value(); auto analyzer = _CLNEW lucene::analysis::standard95::StandardAnalyzer(); // auto analyzer = _CLNEW lucene::analysis::SimpleAnalyzer(); - auto indexwriter = _CLNEW lucene::index::IndexWriter(dir, analyzer, true, true); + auto indexwriter = _CLNEW lucene::index::IndexWriter(dir.get(), analyzer, true, true); indexwriter->setRAMBufferSizeMB(512); indexwriter->setMaxFieldLength(0x7FFFFFFFL); indexwriter->setMergeFactor(100000000); From 9daec4af9b2b93c45cfe20c1b061a197f71e6e00 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:59:16 +0800 Subject: [PATCH 132/572] branch-3.0: [fix](ui) fix ui builds failed error #52711 (#52775) Cherry-picked from #52711 Co-authored-by: Jeffrey --- ui/package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ui/package.json b/ui/package.json index abf65d58e73809..0888a4483b774d 100644 --- a/ui/package.json +++ b/ui/package.json @@ -34,11 +34,13 @@ "classnames": "^2.2.6", "i18next": "^19.7.0", "i18next-browser-languagedetector": "^6.0.1", + "json-bigint": "^1.0.0", "lodash-decorators": "^6.0.1", "path-to-regexp": "^2.2.0", "react": "^16.13.1", "react-codemirror2": "^7.1.0", "react-dom": "^16.13.1", + "react-draggable": "4.4.6", "react-i18next": "^11.7.2", "react-resizable": "^1.10.1", "react-router": "^5.2.0", @@ -77,5 +79,8 @@ "webpack-cli": "^3.3.3", "webpack-dev-server": "^3.7.1", "webpack-merge": "^4.2.1" + }, + "resolutions": { + "react-draggable": "4.4.6" } } From d6998b1fb4b415926b6c59f6987ec596e8ce93ad Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Mon, 7 Jul 2025 10:07:42 +0800 Subject: [PATCH 133/572] [fix](test) Create mv but not check finished (#52625) pick: https://github.com/apache/doris/pull/52167 --- .../test_backup_restore_mv_write.groovy | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy b/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy index c9e9bddb6a1148..4c4e7ee7f826ad 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_mv_write.groovy @@ -59,6 +59,22 @@ suite("test_backup_restore_mv_write", "backup_restore") { vin """ + def alter_finished = false + for (int i = 0; i < 60 && !alter_finished; i++) { + def result = sql_return_maparray "SHOW ALTER TABLE MATERIALIZED VIEW FROM ${dbName}" + logger.info("result: ${result}") + for (int j = 0; j < result.size(); j++) { + if (result[j]['TableName'] == "${tableNamePrefix}" && + result[j]['RollupIndexName'] == "${viewName}" && + result[j]['State'] == 'FINISHED') { + alter_finished = true + break + } + } + Thread.sleep(3000) + } + assertTrue(alter_finished); + sql """ BACKUP SNAPSHOT ${snapshotName} TO `${repoName}` @@ -105,14 +121,14 @@ suite("test_backup_restore_mv_write", "backup_restore") { // DefineExpr: date_format(hours_add(`k2`, 1), '%Y-%m-%d %H:00:00') // WhereClause: (`k1` > '2025-06-12 00:00:00') - assertTrue(res0.toString() == res.toString()) + assertEquals(res0.toString(), res.toString()) sql "insert into ${tableNamePrefix} values (1, '2025-06-12 01:00:00', '2025-06-12 02:00:00', 'test_vin_0')" sql "insert into ${tableNamePrefix} values (2, '2025-06-13 02:00:00', '2025-06-13 03:00:00', 'test_vin_1')" sql "insert into ${tableNamePrefix} values (3, '2025-06-14 03:00:00', '2025-06-14 04:00:00', 'test_vin_2')" def select_res = sql "select * from ${tableNamePrefix}" - assertTrue(select_res.size() == 3) + assertEquals(select_res.size(), 3) sql "DROP REPOSITORY `${repoName}`" } From 12584baa650d9753d965801a0b95bbbbef82b61a Mon Sep 17 00:00:00 2001 From: meiyi Date: Mon, 7 Jul 2025 11:02:15 +0800 Subject: [PATCH 134/572] branch-3.0: [fix](txn load) fix delete in txn load (#52133) (#52635) pick https://github.com/apache/doris/pull/52133 --- .../rules/rewrite/PruneEmptyPartition.java | 31 +++++++++++++++++++ .../doris/transaction/TransactionEntry.java | 26 ++++++++++++++++ .../insert_p0/transaction/txn_insert.groovy | 22 +++++++++++++ 3 files changed, 79 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java index c7b8f452afbe66..3347518b1652a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneEmptyPartition.java @@ -18,11 +18,16 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.transaction.TransactionEntry; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.List; @@ -30,6 +35,7 @@ * Used to prune empty partition. */ public class PruneEmptyPartition extends OneRewriteRuleFactory { + public static final Logger LOG = LogManager.getLogger(PruneEmptyPartition.class); @Override public Rule build() { @@ -38,6 +44,11 @@ public Rule build() { OlapTable table = scan.getTable(); List partitionIdsToPrune = scan.getSelectedPartitionIds(); List ids = table.selectNonEmptyPartitionIds(partitionIdsToPrune); + if (ctx.connectContext != null && ctx.connectContext.isTxnModel()) { + // In transaction load, need to add empty partitions which have invisible data of sub transactions + selectNonEmptyPartitionIdsForTxnLoad(ctx.connectContext.getTxnEntry(), table, scan.getSelectedIndexId(), + partitionIdsToPrune, ids); + } if (ids.isEmpty()) { return new LogicalEmptyRelation(ConnectContext.get().getStatementContext().getNextRelationId(), scan.getOutput()); @@ -49,4 +60,24 @@ public Rule build() { return scan.withSelectedPartitionIds(ids); }).toRule(RuleType.PRUNE_EMPTY_PARTITION); } + + private void selectNonEmptyPartitionIdsForTxnLoad(TransactionEntry txnEntry, OlapTable table, long indexId, + List selectedPartitions, List nonEmptyPartitionIds) { + for (Long selectedPartitionId : selectedPartitions) { + if (nonEmptyPartitionIds.contains(selectedPartitionId)) { + continue; + } + Partition partition = table.getPartition(selectedPartitionId); + if (partition == null) { + continue; + } + if (!txnEntry.getPartitionSubTxnIds(table.getId(), partition, indexId).isEmpty()) { + nonEmptyPartitionIds.add(selectedPartitionId); + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("add partition for txn load, table: {}, selected partitions: {}, non empty partitions: {}", + table.getId(), selectedPartitions, nonEmptyPartitionIds); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java index 25c4ff4b3b2493..70c281201b6f66 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionEntry.java @@ -22,7 +22,9 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndex; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; import org.apache.doris.cloud.transaction.CloudGlobalTransactionMgr; @@ -501,6 +503,30 @@ public void setTxnLoadInfoInObserver(TTxnLoadInfo txnLoadInfo) throws DdlExcepti + "subTxnStates={}", label, transactionId, dbId, timeoutTimestamp, allSubTxnNum, subTransactionStates); } + public List getPartitionSubTxnIds(long tableId, Partition partition, long indexId) { + List subTxnIds = new ArrayList<>(); + MaterializedIndex index = partition.getIndex(indexId); + if (index == null) { + LOG.error("index={} not found in table={}, partition={}", indexId, tableId, partition.getId()); + return subTxnIds; + } + for (SubTransactionState subTransactionState : subTransactionStates) { + if (subTransactionState.getTable().getId() != tableId) { + continue; + } + for (TTabletCommitInfo tabletCommitInfo : subTransactionState.getTabletCommitInfos()) { + if (index.getTablet(tabletCommitInfo.getTabletId()) != null) { + subTxnIds.add(subTransactionState.getSubTransactionId()); + break; + } + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("table_id={}, partition_id={}, sub_txn_ids={}", tableId, partition.getId(), subTxnIds); + } + return subTxnIds; + } + private void resetByTxnInfo(TTxnLoadInfo txnLoadInfo) throws DdlException { if (txnLoadInfo.isSetDbId()) { this.dbId = txnLoadInfo.getDbId(); diff --git a/regression-test/suites/insert_p0/transaction/txn_insert.groovy b/regression-test/suites/insert_p0/transaction/txn_insert.groovy index 41b8a6d35dd909..319a20fdaba27d 100644 --- a/regression-test/suites/insert_p0/transaction/txn_insert.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert.groovy @@ -793,6 +793,28 @@ suite("txn_insert") { order_qt_select_cu2 """select * from ${unique_table}_2""" order_qt_select_cu3 """select * from ${unique_table}_3""" } + + // 19. delete from empty table + sql """ drop table if exists txn_insert_dt6; """ + sql """ + CREATE TABLE `txn_insert_dt6` ( + `ID` int NOT NULL, + `NAME` varchar(100) NULL, + `SCORE` int NULL + ) ENGINE=OLAP + DUPLICATE KEY(`ID`) + DISTRIBUTED BY HASH(`ID`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ begin; """ + sql """ INSERT INTO txn_insert_dt6 select 1, 'Alice', 100; """ + test { + sql """ delete from txn_insert_dt6 where id = 1; """ + exception """Can not delete because there is a insert operation for the same table""" + } + sql """ rollback; """ } def db_name = "regression_test_insert_p0_transaction" From 0e477b6c8dc5ddf699bc86c32d9ab646871a0f40 Mon Sep 17 00:00:00 2001 From: Siyang Tang Date: Mon, 7 Jul 2025 11:04:12 +0800 Subject: [PATCH 135/572] [opt](rowset) Remote fetch rowsets to avoid -230 error when capturing rowsets (#52440) ### What problem does this PR solve? Problem Summary: Make stale rowsets accessible across BEs to avoid E-230 (versions already merged) in Read-Write Splitting senario. --- be/src/cloud/cloud_base_compaction.cpp | 2 +- be/src/cloud/cloud_cumulative_compaction.cpp | 10 +- be/src/cloud/cloud_full_compaction.cpp | 10 +- be/src/cloud/cloud_meta_mgr.cpp | 2 +- be/src/cloud/cloud_schema_change_job.cpp | 26 +- be/src/cloud/cloud_tablet.cpp | 45 +- be/src/cloud/cloud_tablet.h | 3 - be/src/cloud/cloud_tablet_mgr.cpp | 20 +- be/src/cloud/cloud_tablet_mgr.h | 2 +- be/src/common/config.cpp | 1 + be/src/common/config.h | 1 + be/src/http/action/delete_bitmap_action.cpp | 6 +- be/src/olap/base_tablet.cpp | 47 +- be/src/olap/base_tablet.h | 43 +- be/src/olap/compaction.cpp | 6 +- be/src/olap/data_dir.cpp | 6 +- be/src/olap/full_compaction.cpp | 8 +- be/src/olap/merger.cpp | 8 +- be/src/olap/parallel_scanner_builder.cpp | 21 +- be/src/olap/parallel_scanner_builder.h | 8 +- be/src/olap/rowset/rowset_reader_context.h | 2 +- be/src/olap/rowset_version_mgr.cpp | 452 ++++++++++++++++++ be/src/olap/schema_change.cpp | 22 +- be/src/olap/snapshot_manager.cpp | 21 +- be/src/olap/tablet.cpp | 94 +--- be/src/olap/tablet.h | 9 - be/src/olap/tablet_manager.cpp | 2 +- be/src/olap/tablet_meta.cpp | 54 ++- be/src/olap/tablet_meta.h | 14 +- be/src/olap/tablet_reader.cpp | 15 +- be/src/olap/tablet_reader.h | 14 +- be/src/olap/task/engine_checksum_task.cpp | 13 +- be/src/olap/task/engine_clone_task.cpp | 2 +- .../task/engine_storage_migration_task.cpp | 10 +- be/src/olap/task/index_builder.cpp | 4 +- be/src/pipeline/exec/olap_scan_operator.cpp | 9 +- be/src/pipeline/exec/olap_scan_operator.h | 2 +- be/src/service/internal_service.cpp | 67 +++ be/src/service/internal_service.h | 5 + be/src/vec/exec/scan/new_olap_scanner.cpp | 24 +- be/src/vec/exec/scan/new_olap_scanner.h | 2 +- be/test/olap/delta_writer_test.cpp | 4 +- be/test/olap/segcompaction_mow_test.cpp | 2 +- be/test/olap/tablet_meta_test.cpp | 18 +- be/test/olap/test_data/rowset_meta.json | 4 + .../doris/cloud/catalog/CloudReplica.java | 16 + .../doris/service/FrontendServiceImpl.java | 28 +- gensrc/proto/internal_service.proto | 16 + .../test_cloud_version_already_merged.out | 15 + regression-test/pipeline/p0/conf/be.conf | 2 + .../test_cloud_version_already_merged.groovy | 126 +++++ 51 files changed, 1025 insertions(+), 318 deletions(-) create mode 100644 be/src/olap/rowset_version_mgr.cpp create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_version_already_merged.out create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy diff --git a/be/src/cloud/cloud_base_compaction.cpp b/be/src/cloud/cloud_base_compaction.cpp index 3b861e1c944139..a1f57d8ce3d300 100644 --- a/be/src/cloud/cloud_base_compaction.cpp +++ b/be/src/cloud/cloud_base_compaction.cpp @@ -419,7 +419,7 @@ Status CloudBaseCompaction::modify_rowsets() { // the tablet to be unable to synchronize the rowset meta changes generated by cumu compaction. cloud_tablet()->set_base_compaction_cnt(stats.base_compaction_cnt()); if (output_rowset_delete_bitmap) { - _tablet->tablet_meta()->delete_bitmap().merge(*output_rowset_delete_bitmap); + _tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap); } if (stats.cumulative_compaction_cnt() >= cloud_tablet()->cumulative_compaction_cnt()) { cloud_tablet()->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(), diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp b/be/src/cloud/cloud_cumulative_compaction.cpp index 39954097324a99..71234470298d48 100644 --- a/be/src/cloud/cloud_cumulative_compaction.cpp +++ b/be/src/cloud/cloud_cumulative_compaction.cpp @@ -376,7 +376,7 @@ Status CloudCumulativeCompaction::modify_rowsets() { cloud_tablet()->set_cumulative_compaction_cnt(stats.cumulative_compaction_cnt()); cloud_tablet()->set_cumulative_layer_point(stats.cumulative_point()); if (output_rowset_delete_bitmap) { - _tablet->tablet_meta()->delete_bitmap().merge(*output_rowset_delete_bitmap); + _tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap); } if (stats.base_compaction_cnt() >= cloud_tablet()->base_compaction_cnt()) { cloud_tablet()->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(), @@ -416,7 +416,7 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { rowset->rowset_id().to_string(); DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0}; DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version}; - auto d = _tablet->tablet_meta()->delete_bitmap().get_agg( + auto d = _tablet->tablet_meta()->delete_bitmap()->get_agg( {rowset->rowset_id(), seg_id, pre_max_version}); to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end)); if (d->isEmpty()) { @@ -440,10 +440,10 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() { _input_rowsets.back()->end_version()); for (auto it = new_delete_bitmap->delete_bitmap.begin(); it != new_delete_bitmap->delete_bitmap.end(); it++) { - _tablet->tablet_meta()->delete_bitmap().set(it->first, it->second); + _tablet->tablet_meta()->delete_bitmap()->set(it->first, it->second); } - _tablet->tablet_meta()->delete_bitmap().add_to_remove_queue(version.to_string(), - to_remove_vec); + _tablet->tablet_meta()->delete_bitmap()->add_to_remove_queue(version.to_string(), + to_remove_vec); DBUG_EXECUTE_IF( "CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets", { static_cast(_tablet.get())->delete_expired_stale_rowsets(); }); diff --git a/be/src/cloud/cloud_full_compaction.cpp b/be/src/cloud/cloud_full_compaction.cpp index 6bfab2ec69808d..7358f6d19156a1 100644 --- a/be/src/cloud/cloud_full_compaction.cpp +++ b/be/src/cloud/cloud_full_compaction.cpp @@ -26,6 +26,7 @@ #include "common/status.h" #include "cpp/sync_point.h" #include "gen_cpp/cloud.pb.h" +#include "olap/base_tablet.h" #include "olap/compaction.h" #include "olap/rowset/beta_rowset.h" #include "olap/tablet_meta.h" @@ -273,7 +274,7 @@ Status CloudFullCompaction::modify_rowsets() { cloud_tablet()->set_base_compaction_cnt(stats.base_compaction_cnt()); cloud_tablet()->set_cumulative_layer_point(stats.cumulative_point()); if (output_rowset_delete_bitmap) { - _tablet->tablet_meta()->delete_bitmap().merge(*output_rowset_delete_bitmap); + _tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap); } if (stats.cumulative_compaction_cnt() >= cloud_tablet()->cumulative_compaction_cnt()) { cloud_tablet()->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(), @@ -340,8 +341,9 @@ Status CloudFullCompaction::_cloud_full_compaction_update_delete_bitmap(int64_t int64_t max_version = cloud_tablet()->max_version().second; DCHECK(max_version >= _output_rowset->version().second); if (max_version > _output_rowset->version().second) { - RETURN_IF_ERROR(cloud_tablet()->capture_consistent_rowsets_unlocked( - {_output_rowset->version().second + 1, max_version}, &tmp_rowsets)); + auto ret = DORIS_TRY(cloud_tablet()->capture_consistent_rowsets_unlocked( + {_output_rowset->version().second + 1, max_version}, CaptureRowsetOps {})); + tmp_rowsets = std::move(ret.rowsets); } for (const auto& it : tmp_rowsets) { int64_t cur_version = it->rowset_meta()->start_version(); @@ -372,7 +374,7 @@ Status CloudFullCompaction::_cloud_full_compaction_update_delete_bitmap(int64_t .tag("input_segments", _input_segments) .tag("input_rowsets_total_size", _input_rowsets_total_size) .tag("update_bitmap_size", delete_bitmap->delete_bitmap.size()); - _tablet->tablet_meta()->delete_bitmap().merge(*delete_bitmap); + _tablet->tablet_meta()->delete_bitmap()->merge(*delete_bitmap); return Status::OK(); } diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index dff85b3f639258..684d89cb01bb44 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -577,7 +577,7 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, .error(st); return st; } - tablet->tablet_meta()->delete_bitmap().merge(delete_bitmap); + tablet->tablet_meta()->delete_bitmap()->merge(delete_bitmap); if (config::enable_mow_verbose_log && !resp.rowset_meta().empty() && delete_bitmap.cardinality() > 0) { std::vector new_rowset_msgs; diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index a5707e51bb64a6..405dcbe1a0d3cc 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -27,6 +27,7 @@ #include "cloud/cloud_meta_mgr.h" #include "cloud/cloud_tablet_mgr.h" #include "common/status.h" +#include "olap/base_tablet.h" #include "olap/delete_handler.h" #include "olap/olap_define.h" #include "olap/rowset/beta_rowset.h" @@ -186,7 +187,7 @@ Status CloudSchemaChangeJob::process_alter_tablet(const TAlterTabletReqV2& reque reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx(); reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS; reader_context.batch_size = ALTER_TABLE_BATCH_SIZE; - reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap(); + reader_context.delete_bitmap = _base_tablet->tablet_meta()->delete_bitmap(); reader_context.version = Version(0, start_resp.alter_version()); for (auto& split : rs_splits) { @@ -457,7 +458,7 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version, .tag("alter_version", alter_version); RETURN_IF_ERROR(_cloud_storage_engine.register_compaction_stop_token(_new_tablet, initiator)); TabletMetaSharedPtr tmp_meta = std::make_shared(*(_new_tablet->tablet_meta())); - tmp_meta->delete_bitmap().delete_bitmap.clear(); + tmp_meta->delete_bitmap()->delete_bitmap.clear(); std::shared_ptr tmp_tablet = std::make_shared(_cloud_storage_engine, tmp_meta); { @@ -466,22 +467,21 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version, } // step 1, process incremental rowset without delete bitmap update lock - std::vector incremental_rowsets; RETURN_IF_ERROR(_cloud_storage_engine.meta_mgr().sync_tablet_rowsets(tmp_tablet.get())); int64_t max_version = tmp_tablet->max_version().second; LOG(INFO) << "alter table for mow table, calculate delete bitmap of " << "incremental rowsets without lock, version: " << start_calc_delete_bitmap_version << "-" << max_version << " new_table_id: " << _new_tablet->tablet_id(); if (max_version >= start_calc_delete_bitmap_version) { - RETURN_IF_ERROR(tmp_tablet->capture_consistent_rowsets_unlocked( - {start_calc_delete_bitmap_version, max_version}, &incremental_rowsets)); + auto ret = DORIS_TRY(tmp_tablet->capture_consistent_rowsets_unlocked( + {start_calc_delete_bitmap_version, max_version}, CaptureRowsetOps {})); DBUG_EXECUTE_IF("CloudSchemaChangeJob::_process_delete_bitmap.after.capture_without_lock", DBUG_BLOCK); { std::unique_lock wlock(tmp_tablet->get_header_lock()); tmp_tablet->add_rowsets(_output_rowsets, true, wlock); } - for (auto rowset : incremental_rowsets) { + for (auto rowset : ret.rowsets) { RETURN_IF_ERROR(CloudTablet::update_delete_bitmap_without_lock(tmp_tablet, rowset)); } } @@ -497,15 +497,14 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version, LOG(INFO) << "alter table for mow table, calculate delete bitmap of " << "incremental rowsets with lock, version: " << max_version + 1 << "-" << new_max_version << " new_tablet_id: " << _new_tablet->tablet_id(); - std::vector new_incremental_rowsets; if (new_max_version > max_version) { - RETURN_IF_ERROR(tmp_tablet->capture_consistent_rowsets_unlocked( - {max_version + 1, new_max_version}, &new_incremental_rowsets)); + auto ret = DORIS_TRY(tmp_tablet->capture_consistent_rowsets_unlocked( + {max_version + 1, new_max_version}, CaptureRowsetOps {})); { std::unique_lock wlock(tmp_tablet->get_header_lock()); tmp_tablet->add_rowsets(_output_rowsets, true, wlock); } - for (auto rowset : new_incremental_rowsets) { + for (auto rowset : ret.rowsets) { RETURN_IF_ERROR(CloudTablet::update_delete_bitmap_without_lock(tmp_tablet, rowset)); } } @@ -522,13 +521,14 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version, } }); - auto& delete_bitmap = tmp_tablet->tablet_meta()->delete_bitmap(); + auto delete_bitmap = tmp_tablet->tablet_meta()->delete_bitmap(); // step4, store delete bitmap RETURN_IF_ERROR(_cloud_storage_engine.meta_mgr().update_delete_bitmap( - *_new_tablet, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, initiator, &delete_bitmap)); + *_new_tablet, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID, initiator, delete_bitmap.get())); - _new_tablet->tablet_meta()->delete_bitmap() = delete_bitmap; + auto original_dbm = _new_tablet->tablet_meta()->delete_bitmap(); + *original_dbm = std::move(*delete_bitmap); return Status::OK(); } diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 78663de6ed86d6..85b60e1588afaf 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -37,6 +37,7 @@ #include "common/logging.h" #include "io/cache/block_file_cache_downloader.h" #include "io/cache/block_file_cache_factory.h" +#include "olap/base_tablet.h" #include "olap/compaction.h" #include "olap/cumulative_compaction_time_series_policy.h" #include "olap/olap_define.h" @@ -69,23 +70,6 @@ bool CloudTablet::exceed_version_limit(int32_t limit) { return _approximate_num_rowsets.load(std::memory_order_relaxed) > limit; } -Status CloudTablet::capture_consistent_rowsets_unlocked( - const Version& spec_version, std::vector* rowsets) const { - Versions version_path; - auto st = _timestamped_version_tracker.capture_consistent_versions(spec_version, &version_path); - if (!st.ok()) { - // Check no missed versions or req version is merged - auto missed_versions = get_missed_versions(spec_version.second); - if (missed_versions.empty()) { - st.set_code(VERSION_ALREADY_MERGED); // Reset error code - } - st.append(" tablet_id=" + std::to_string(tablet_id())); - return st; - } - VLOG_DEBUG << "capture consitent versions: " << version_path; - return _capture_consistent_rowsets_unlocked(version_path, rowsets); -} - std::string CloudTablet::tablet_path() const { return ""; } @@ -97,25 +81,10 @@ Status CloudTablet::capture_rs_readers(const Version& spec_version, LOG_WARNING("CloudTablet.capture_rs_readers.return e-230").tag("tablet_id", tablet_id()); return Status::Error(-230, "injected error"); }); - Versions version_path; std::shared_lock rlock(_meta_lock); - auto st = _timestamped_version_tracker.capture_consistent_versions(spec_version, &version_path); - if (!st.ok()) { - rlock.unlock(); // avoid logging in lock range - // Check no missed versions or req version is merged - auto missed_versions = get_missed_versions(spec_version.second); - if (missed_versions.empty()) { - st.set_code(VERSION_ALREADY_MERGED); // Reset error code - st.append(" versions are already compacted, "); - } - st.append(" tablet_id=" + std::to_string(tablet_id())); - // clang-format off - LOG(WARNING) << st << '\n' << [this]() { std::string json; get_compaction_status(&json); return json; }(); - // clang-format on - return st; - } - VLOG_DEBUG << "capture consitent versions: " << version_path; - return capture_rs_readers_unlocked(version_path, rs_splits); + *rs_splits = DORIS_TRY(capture_rs_readers_unlocked( + spec_version, CaptureRowsetOps {.skip_missing_versions = skip_missing_version})); + return Status::OK(); } Status CloudTablet::merge_rowsets_schema() { @@ -469,7 +438,7 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() { } _reconstruct_version_tracker_if_necessary(); } - _tablet_meta->delete_bitmap().remove_stale_delete_bitmap_from_queue(version_to_delete); + _tablet_meta->delete_bitmap()->remove_stale_delete_bitmap_from_queue(version_to_delete); recycle_cached_data(expired_rowsets); if (config::enable_mow_verbose_log) { LOG_INFO("finish delete_expired_stale_rowset for tablet={}", tablet_id()); @@ -977,7 +946,7 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( std::size_t missed_rows_size = 0; calc_compaction_output_rowset_delete_bitmap( input_rowsets, rowid_conversion, 0, version.second + 1, missed_rows.get(), - location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); + location_map.get(), *tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); if (missed_rows) { missed_rows_size = missed_rows->size(); if (!allow_delete_in_cumu_compaction) { @@ -1013,7 +982,7 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( calc_compaction_output_rowset_delete_bitmap( input_rowsets, rowid_conversion, version.second, UINT64_MAX, missed_rows.get(), - location_map.get(), tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); + location_map.get(), *tablet_meta()->delete_bitmap(), output_rowset_delete_bitmap.get()); int64_t t4 = MonotonicMicros(); if (location_map) { RETURN_IF_ERROR(check_rowid_conversion(output_rowset, *location_map)); diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index 2dd1d3c4425a3a..dc357eb7249c95 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -55,9 +55,6 @@ class CloudTablet final : public BaseTablet { Status capture_rs_readers(const Version& spec_version, std::vector* rs_splits, bool skip_missing_version) override; - Status capture_consistent_rowsets_unlocked( - const Version& spec_version, std::vector* rowsets) const override; - size_t tablet_footprint() override { return _approximate_data_size.load(std::memory_order_relaxed); } diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 25f50f7ef4bd3b..1f155635c65485 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -25,6 +25,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/config.h" +#include "common/logging.h" #include "common/status.h" #include "olap/lru_cache.h" #include "runtime/memory/cache_policy.h" @@ -158,7 +159,7 @@ void set_tablet_access_time_ms(CloudTablet* tablet) { Result> CloudTabletMgr::get_tablet(int64_t tablet_id, bool warmup_data, bool sync_delete_bitmap, SyncRowsetStats* sync_stats, - bool force_use_cache) { + bool local_only) { // LRU value type. `Value`'s lifetime MUST NOT be longer than `CloudTabletMgr` class Value : public LRUCacheValueBase { public: @@ -176,12 +177,17 @@ Result> CloudTabletMgr::get_tablet(int64_t tablet_i CacheKey key(tablet_id_str); auto* handle = _cache->lookup(key); - if (handle == nullptr && force_use_cache) { - return ResultError( - Status::InternalError("failed to get cloud tablet from cache {}", tablet_id)); - } - if (handle == nullptr) { + if (local_only) { + LOG(INFO) << "tablet=" << tablet_id + << "does not exists in local tablet cache, because param local_only=true, " + "treat it as an error"; + return ResultError(Status::InternalError( + "tablet={} does not exists in local tablet cache, because param " + "local_only=true, " + "treat it as an error", + tablet_id)); + } if (sync_stats) { ++sync_stats->tablet_meta_cache_miss; } @@ -485,7 +491,7 @@ void CloudTabletMgr::get_topn_tablet_delete_bitmap_score( auto t = tablet_wk.lock(); if (!t) return; uint64_t delete_bitmap_count = - t.get()->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); + t.get()->tablet_meta()->delete_bitmap()->get_delete_bitmap_count(); total_delete_map_count += delete_bitmap_count; if (delete_bitmap_count > *max_delete_bitmap_score) { max_delete_bitmap_score_tablet_id = t->tablet_id(); diff --git a/be/src/cloud/cloud_tablet_mgr.h b/be/src/cloud/cloud_tablet_mgr.h index a1ce6d2b8cfb1b..2e0ea79f4244f0 100644 --- a/be/src/cloud/cloud_tablet_mgr.h +++ b/be/src/cloud/cloud_tablet_mgr.h @@ -47,7 +47,7 @@ class CloudTabletMgr { Result> get_tablet(int64_t tablet_id, bool warmup_data = false, bool sync_delete_bitmap = true, SyncRowsetStats* sync_stats = nullptr, - bool force_use_cache = false); + bool local_only = false); void erase_tablet(int64_t tablet_id); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 0f9a60aec88e34..dc81ca48373dd2 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1515,6 +1515,7 @@ DEFINE_mBool(enable_compaction_pause_on_high_memory, "true"); DEFINE_mBool(enable_calc_delete_bitmap_between_segments_concurrently, "false"); +DEFINE_mBool(enable_fetch_rowsets_from_peer_replicas, "false"); // the max length of segments key bounds, in bytes // ATTENTION: as long as this conf has ever been enabled, cluster downgrade and backup recovery will no longer be supported. DEFINE_mInt32(segments_key_bounds_truncation_threshold, "-1"); diff --git a/be/src/common/config.h b/be/src/common/config.h index af916f1f7bd996..a182049472561e 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1589,6 +1589,7 @@ DECLARE_mBool(enable_compaction_pause_on_high_memory); DECLARE_mBool(enable_calc_delete_bitmap_between_segments_concurrently); +DECLARE_mBool(enable_fetch_rowsets_from_peer_replicas); // the max length of segments key bounds, in bytes // ATTENTION: as long as this conf has ever been enabled, cluster downgrade and backup recovery will no longer be supported. DECLARE_mInt32(segments_key_bounds_truncation_threshold); diff --git a/be/src/http/action/delete_bitmap_action.cpp b/be/src/http/action/delete_bitmap_action.cpp index 59783d1c055535..5fc4d0f4388e78 100644 --- a/be/src/http/action/delete_bitmap_action.cpp +++ b/be/src/http/action/delete_bitmap_action.cpp @@ -159,7 +159,7 @@ Status DeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* r if (tablet == nullptr) { return Status::NotFound("Tablet not found. tablet_id={}", tablet_id); } - auto dm = tablet->tablet_meta()->delete_bitmap().snapshot(); + auto dm = tablet->tablet_meta()->delete_bitmap()->snapshot(); _show_delete_bitmap(dm, verbose, json_result); return Status::OK(); } @@ -183,7 +183,7 @@ Status DeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req, LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st; return st; } - auto dm = tablet->tablet_meta()->delete_bitmap().snapshot(); + auto dm = tablet->tablet_meta()->delete_bitmap()->snapshot(); _show_delete_bitmap(dm, verbose, json_result); return Status::OK(); } @@ -210,4 +210,4 @@ void DeleteBitmapAction::handle(HttpRequest* req) { } #include "common/compile_check_end.h" -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 1aa74582d84d9c..56ffc2d100e628 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -516,7 +516,7 @@ Status BaseTablet::lookup_row_key(const Slice& encoded_key, TabletSchema* latest if (!s.ok() && !s.is()) { return s; } - if (s.ok() && _tablet_meta->delete_bitmap().contains_agg_without_cache( + if (s.ok() && _tablet_meta->delete_bitmap()->contains_agg_without_cache( {loc.rowset_id, loc.segment_id, version}, loc.row_id)) { // if has sequence col, we continue to compare the sequence_id of // all rowsets, util we find an existing key. @@ -1114,37 +1114,6 @@ void BaseTablet::_rowset_ids_difference(const RowsetIdUnorderedSet& cur, } } -Status BaseTablet::_capture_consistent_rowsets_unlocked( - const std::vector& version_path, std::vector* rowsets) const { - DCHECK(rowsets != nullptr); - rowsets->reserve(version_path.size()); - for (const auto& version : version_path) { - bool is_find = false; - do { - auto it = _rs_version_map.find(version); - if (it != _rs_version_map.end()) { - is_find = true; - rowsets->push_back(it->second); - break; - } - - auto it_expired = _stale_rs_version_map.find(version); - if (it_expired != _stale_rs_version_map.end()) { - is_find = true; - rowsets->push_back(it_expired->second); - break; - } - } while (false); - - if (!is_find) { - return Status::Error( - "fail to find Rowset for version. tablet={}, version={}", tablet_id(), - version.to_string()); - } - } - return Status::OK(); -} - Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr delete_bitmap, int64_t max_version, int64_t txn_id, const RowsetIdUnorderedSet& rowset_ids, @@ -1564,7 +1533,7 @@ Status BaseTablet::update_delete_bitmap_without_lock( delete_bitmap->remove_sentinel_marks(); } for (auto& iter : delete_bitmap->delete_bitmap) { - self->_tablet_meta->delete_bitmap().merge( + self->_tablet_meta->delete_bitmap()->merge( {std::get<0>(iter.first), std::get<1>(iter.first), cur_version}, iter.second); } @@ -1723,7 +1692,7 @@ void BaseTablet::get_base_rowset_delete_bitmap_count( } base_found = true; uint64_t base_rowset_delete_bitmap_count = - this->tablet_meta()->delete_bitmap().get_count_with_range( + this->tablet_meta()->delete_bitmap()->get_count_with_range( {rowset->rowset_id(), 0, 0}, {rowset->rowset_id(), UINT32_MAX, UINT64_MAX}); if (base_rowset_delete_bitmap_count > *max_base_rowset_delete_bitmap_score) { @@ -1737,6 +1706,16 @@ void BaseTablet::get_base_rowset_delete_bitmap_count( } } +void TabletReadSource::fill_delete_predicates() { + DCHECK_EQ(delete_predicates.size(), 0); + auto delete_pred_view = + rs_splits | std::views::transform([](auto&& split) { + return split.rs_reader->rowset()->rowset_meta(); + }) | + std::views::filter([](const auto& rs_meta) { return rs_meta->has_delete_predicate(); }); + delete_predicates = {delete_pred_view.begin(), delete_pred_view.end()}; +} + int32_t BaseTablet::max_version_config() { int32_t max_version = tablet_meta()->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY ? config::time_series_max_tablet_version_num diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 9dd69d0bd9a20c..6c797e0478bcf7 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -42,6 +42,9 @@ class SegmentCacheHandle; class RowIdConversion; struct PartialUpdateInfo; class PartialUpdateReadPlan; +struct CaptureRowsetOps; +struct CaptureRowsetResult; +struct TabletReadSource; struct TabletWithVersion { BaseTabletSPtr tablet; @@ -107,9 +110,6 @@ class BaseTablet { virtual Result> create_rowset_writer(RowsetWriterContext& context, bool vertical) = 0; - virtual Status capture_consistent_rowsets_unlocked( - const Version& spec_version, std::vector* rowsets) const = 0; - virtual Status capture_rs_readers(const Version& spec_version, std::vector* rs_splits, bool skip_missing_version) = 0; @@ -309,6 +309,20 @@ class BaseTablet { return Status::OK(); } + [[nodiscard]] Result capture_consistent_rowsets_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const; + + [[nodiscard]] Result> capture_consistent_versions_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const; + + [[nodiscard]] Result> capture_rs_readers_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const; + + [[nodiscard]] Result capture_read_source(const Version& version_range, + const CaptureRowsetOps& options); + + Result _remote_capture_rowsets(const Version& version_range) const; + protected: // Find the missed versions until the spec_version. // @@ -326,9 +340,6 @@ class BaseTablet { const RowsetIdUnorderedSet& pre, RowsetIdUnorderedSet* to_add, RowsetIdUnorderedSet* to_del); - Status _capture_consistent_rowsets_unlocked(const std::vector& version_path, - std::vector* rowsets) const; - Status sort_block(vectorized::Block& in_block, vectorized::Block& output_block); mutable std::shared_mutex _meta_lock; @@ -369,4 +380,24 @@ class BaseTablet { Status last_compaction_status = Status::OK(); }; +struct CaptureRowsetOps { + bool skip_missing_versions = false; + bool quiet = false; + bool include_stale_rowsets = true; + bool enable_fetch_rowsets_from_peers = false; +}; + +struct CaptureRowsetResult { + std::vector rowsets; + std::shared_ptr delete_bitmap; +}; + +struct TabletReadSource { + std::vector rs_splits; + std::vector delete_predicates; + std::shared_ptr delete_bitmap; + // Fill delete predicates with `rs_splits` + void fill_delete_predicates(); +}; + } /* namespace doris */ diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 7e312c5847f1dd..77af2b30fe1b34 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1069,7 +1069,7 @@ Status CompactionMixin::modify_rowsets() { std::size_t missed_rows_size = 0; tablet()->calc_compaction_output_rowset_delete_bitmap( _input_rowsets, *_rowid_conversion, 0, version.second + 1, missed_rows.get(), - location_map.get(), _tablet->tablet_meta()->delete_bitmap(), + location_map.get(), *_tablet->tablet_meta()->delete_bitmap(), &output_rowset_delete_bitmap); if (missed_rows) { missed_rows_size = missed_rows->size(); @@ -1112,7 +1112,7 @@ Status CompactionMixin::modify_rowsets() { ss << ", debug info: "; DeleteBitmap subset_map(_tablet->tablet_id()); for (auto rs : _input_rowsets) { - _tablet->tablet_meta()->delete_bitmap().subset( + _tablet->tablet_meta()->delete_bitmap()->subset( {rs->rowset_id(), 0, 0}, {rs->rowset_id(), rs->num_segments(), version.second + 1}, &subset_map); @@ -1187,7 +1187,7 @@ Status CompactionMixin::modify_rowsets() { // incremental data. tablet()->calc_compaction_output_rowset_delete_bitmap( _input_rowsets, *_rowid_conversion, version.second, UINT64_MAX, - missed_rows.get(), location_map.get(), _tablet->tablet_meta()->delete_bitmap(), + missed_rows.get(), location_map.get(), *_tablet->tablet_meta()->delete_bitmap(), &output_rowset_delete_bitmap); if (missed_rows) { diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index 5cb2a6105d9d55..6389f1ad1334d5 100644 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -635,14 +635,14 @@ Status DataDir::load() { } ++dbm_cnt; auto seg_id = delete_bitmap_pb.segment_ids(i); - auto iter = tablet->tablet_meta()->delete_bitmap().delete_bitmap.find( + auto iter = tablet->tablet_meta()->delete_bitmap()->delete_bitmap.find( {rst_id, seg_id, version}); // This version of delete bitmap already exists - if (iter != tablet->tablet_meta()->delete_bitmap().delete_bitmap.end()) { + if (iter != tablet->tablet_meta()->delete_bitmap()->delete_bitmap.end()) { continue; } auto bitmap = delete_bitmap_pb.segment_delete_bitmaps(i).data(); - tablet->tablet_meta()->delete_bitmap().delete_bitmap[{rst_id, seg_id, version}] = + tablet->tablet_meta()->delete_bitmap()->delete_bitmap[{rst_id, seg_id, version}] = roaring::Roaring::read(bitmap); } return true; diff --git a/be/src/olap/full_compaction.cpp b/be/src/olap/full_compaction.cpp index 06641e7a22c2a9..7d3c497f2a214b 100644 --- a/be/src/olap/full_compaction.cpp +++ b/be/src/olap/full_compaction.cpp @@ -27,6 +27,7 @@ #include "common/config.h" #include "common/status.h" +#include "olap/base_tablet.h" #include "olap/compaction.h" #include "olap/cumulative_compaction_policy.h" #include "olap/olap_common.h" @@ -143,8 +144,9 @@ Status FullCompaction::modify_rowsets() { int64_t max_version = tablet()->max_version().second; DCHECK(max_version >= _output_rowset->version().second); if (max_version > _output_rowset->version().second) { - RETURN_IF_ERROR(_tablet->capture_consistent_rowsets_unlocked( - {_output_rowset->version().second + 1, max_version}, &tmp_rowsets)); + auto ret = DORIS_TRY(_tablet->capture_consistent_rowsets_unlocked( + {_output_rowset->version().second + 1, max_version}, CaptureRowsetOps {})); + tmp_rowsets = std::move(ret.rowsets); } for (const auto& it : tmp_rowsets) { @@ -219,7 +221,7 @@ Status FullCompaction::_full_compaction_calc_delete_bitmap(const RowsetSharedPtr [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); }); for (const auto& [k, v] : delete_bitmap->delete_bitmap) { if (std::get<1>(k) != DeleteBitmap::INVALID_SEGMENT_ID) { - _tablet->tablet_meta()->delete_bitmap().merge( + _tablet->tablet_meta()->delete_bitmap()->merge( {std::get<0>(k), std::get<1>(k), cur_version}, v); } } diff --git a/be/src/olap/merger.cpp b/be/src/olap/merger.cpp index 01f36d77dc2cc5..c3f59872f62acb 100644 --- a/be/src/olap/merger.cpp +++ b/be/src/olap/merger.cpp @@ -68,7 +68,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, reader_params.tablet = tablet; reader_params.reader_type = reader_type; - TabletReader::ReadSource read_source; + TabletReadSource read_source; read_source.rs_splits.reserve(src_rowset_readers.size()); for (const RowsetReaderSharedPtr& rs_reader : src_rowset_readers) { read_source.rs_splits.emplace_back(rs_reader); @@ -87,7 +87,7 @@ Status Merger::vmerge_rowsets(BaseTabletSPtr tablet, ReaderType reader_type, } reader_params.tablet_schema = merge_tablet_schema; if (!tablet->tablet_schema()->cluster_key_idxes().empty()) { - reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); + reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap(); } if (stats_output && stats_output->rowid_conversion) { @@ -249,7 +249,7 @@ Status Merger::vertical_compact_one_group( reader_params.tablet = tablet; reader_params.reader_type = reader_type; - TabletReader::ReadSource read_source; + TabletReadSource read_source; read_source.rs_splits.reserve(src_rowset_readers.size()); for (const RowsetReaderSharedPtr& rs_reader : src_rowset_readers) { read_source.rs_splits.emplace_back(rs_reader); @@ -268,7 +268,7 @@ Status Merger::vertical_compact_one_group( reader_params.tablet_schema = merge_tablet_schema; if (!tablet->tablet_schema()->cluster_key_idxes().empty()) { - reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); + reader_params.delete_bitmap = tablet->tablet_meta()->delete_bitmap(); } if (is_key && stats_output && stats_output->rowid_conversion) { diff --git a/be/src/olap/parallel_scanner_builder.cpp b/be/src/olap/parallel_scanner_builder.cpp index 103e6341d7c8b3..f4cd210bbd205a 100644 --- a/be/src/olap/parallel_scanner_builder.cpp +++ b/be/src/olap/parallel_scanner_builder.cpp @@ -57,7 +57,7 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& // `rs_splits` in `entire read source` will be devided into several partitial read sources // to build several parallel scanners, based on segment rows number. All the partitial read sources // share the same delete predicates from their corresponding entire read source. - TabletReader::ReadSource partitial_read_source; + TabletReadSource partitial_read_source; int64_t rows_collected = 0; for (auto& rs_split : entire_read_source.rs_splits) { auto reader = rs_split.rs_reader; @@ -106,10 +106,11 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& partitial_read_source.rs_splits.emplace_back(std::move(split)); - scanners.emplace_back( - _build_scanner(tablet, version, _key_ranges, - {std::move(partitial_read_source.rs_splits), - entire_read_source.delete_predicates})); + scanners.emplace_back(_build_scanner( + tablet, version, _key_ranges, + {.rs_splits = std::move(partitial_read_source.rs_splits), + .delete_predicates = entire_read_source.delete_predicates, + .delete_bitmap = entire_read_source.delete_bitmap})); partitial_read_source = {}; split = RowSetSplits(reader->clone()); @@ -150,9 +151,11 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& split.segment_offsets.second - split.segment_offsets.first); } #endif - scanners.emplace_back(_build_scanner(tablet, version, _key_ranges, - {std::move(partitial_read_source.rs_splits), - entire_read_source.delete_predicates})); + scanners.emplace_back( + _build_scanner(tablet, version, _key_ranges, + {.rs_splits = std::move(partitial_read_source.rs_splits), + .delete_predicates = entire_read_source.delete_predicates, + .delete_bitmap = entire_read_source.delete_bitmap})); } } @@ -199,7 +202,7 @@ Status ParallelScannerBuilder::_load() { std::shared_ptr ParallelScannerBuilder::_build_scanner( BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, - TabletReader::ReadSource&& read_source) { + TabletReadSource&& read_source) { NewOlapScanner::Params params {_state, _scanner_profile.get(), key_ranges, std::move(tablet), version, std::move(read_source), _limit, _is_preaggregation}; return NewOlapScanner::create_shared(_parent, std::move(params)); diff --git a/be/src/olap/parallel_scanner_builder.h b/be/src/olap/parallel_scanner_builder.h index 1f371e3129a04f..a746ff5ba5d77d 100644 --- a/be/src/olap/parallel_scanner_builder.h +++ b/be/src/olap/parallel_scanner_builder.h @@ -44,7 +44,7 @@ class ParallelScannerBuilder { public: ParallelScannerBuilder(pipeline::OlapScanLocalState* parent, const std::vector& tablets, - std::vector& read_sources, + std::vector& read_sources, const std::shared_ptr& profile, const std::vector& key_ranges, RuntimeState* state, int64_t limit, bool is_dup_mow_key, bool is_preaggregation) @@ -71,7 +71,7 @@ class ParallelScannerBuilder { std::shared_ptr _build_scanner( BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, - TabletReader::ReadSource&& read_source); + TabletReadSource&& read_source); pipeline::OlapScanLocalState* _parent; @@ -94,8 +94,8 @@ class ParallelScannerBuilder { bool _is_preaggregation; std::vector _tablets; std::vector _key_ranges; - std::unordered_map _all_read_sources; - std::vector& _read_sources; + std::unordered_map _all_read_sources; + std::vector& _read_sources; }; } // namespace doris diff --git a/be/src/olap/rowset/rowset_reader_context.h b/be/src/olap/rowset/rowset_reader_context.h index fd3b4fed56f967..2faed632349c63 100644 --- a/be/src/olap/rowset/rowset_reader_context.h +++ b/be/src/olap/rowset/rowset_reader_context.h @@ -74,7 +74,7 @@ struct RowsetReaderContext { uint64_t* merged_rows = nullptr; // for unique key merge on write bool enable_unique_key_merge_on_write = false; - const DeleteBitmap* delete_bitmap = nullptr; + std::shared_ptr delete_bitmap = nullptr; bool record_rowids = false; RowIdConversion* rowid_conversion; bool is_vertical_compaction = false; diff --git a/be/src/olap/rowset_version_mgr.cpp b/be/src/olap/rowset_version_mgr.cpp new file mode 100644 index 00000000000000..27df2ede4b2dec --- /dev/null +++ b/be/src/olap/rowset_version_mgr.cpp @@ -0,0 +1,452 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "cloud/config.h" +#include "common/status.h" +#include "cpp/sync_point.h" +#include "olap/base_tablet.h" +#include "olap/olap_common.h" +#include "olap/rowset/rowset.h" +#include "olap/rowset/rowset_factory.h" +#include "olap/rowset/rowset_reader.h" +#include "runtime/client_cache.h" +#include "service/backend_options.h" +#include "service/internal_service.h" +#include "util/brpc_client_cache.h" +#include "util/debug_points.h" +#include "util/thrift_rpc_helper.h" +#include "util/time.h" + +namespace doris { + +using namespace ErrorCode; +using namespace std::ranges; + +static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_single_request_latency( + "remote_fetch_rowsets_single_rpc"); +static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch_rowsets"); + +[[nodiscard]] Result> BaseTablet::capture_consistent_versions_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const { + std::vector version_path; + auto st = + _timestamped_version_tracker.capture_consistent_versions(version_range, &version_path); + if (!st && !options.quiet) { + auto missed_versions = get_missed_versions_unlocked(version_range.second); + if (missed_versions.empty()) { + LOG(WARNING) << fmt::format( + "version already has been merged. version_range={}, max_version={}, " + "tablet_id={}", + version_range.to_string(), _tablet_meta->max_version().second, tablet_id()); + return ResultError(Status::Error( + "missed versions is empty, version_range={}, max_version={}, tablet_id={}", + version_range.to_string(), _tablet_meta->max_version().second, tablet_id())); + } + LOG(WARNING) << fmt::format("missed version for version_range={}, tablet_id={}, st={}", + version_range.to_string(), tablet_id(), st); + _print_missed_versions(missed_versions); + if (!options.skip_missing_versions) { + return ResultError(std::move(st)); + } + LOG(WARNING) << "force skipping missing version for tablet:" << tablet_id(); + } + DBUG_EXECUTE_IF("Tablet::capture_consistent_versions.inject_failure", { + auto tablet_id = dp->param("tablet_id", -1); + auto skip_by_option = dp->param("skip_by_option", false); + if (skip_by_option && !options.enable_fetch_rowsets_from_peers) { + return version_path; + } + if (tablet_id != -1 && (tablet_id == _tablet_meta->tablet_id()) || tablet_id == -2) { + return ResultError(Status::Error("version already merged")); + } + }); + return version_path; +} + +[[nodiscard]] Result BaseTablet::capture_consistent_rowsets_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const { + CaptureRowsetResult result; + auto& rowsets = result.rowsets; + auto maybe_versions = capture_consistent_versions_unlocked(version_range, options); + if (maybe_versions) { + const auto& version_paths = maybe_versions.value(); + rowsets.reserve(version_paths.size()); + + auto rowset_for_version = [&](const Version& version, + bool include_stale) -> Result { + if (auto it = _rs_version_map.find(version); it != _rs_version_map.end()) { + return it->second; + } else { + VLOG_NOTICE << "fail to find Rowset in rs_version for version. tablet=" + << tablet_id() << ", version='" << version.first << "-" + << version.second; + } + if (include_stale) { + if (auto it = _stale_rs_version_map.find(version); + it != _stale_rs_version_map.end()) { + return it->second; + } else { + LOG(WARNING) << fmt::format( + "fail to find Rowset in stale_rs_version for version. tablet={}, " + "version={}-{}", + tablet_id(), version.first, version.second); + } + } + return ResultError(Status::Error( + "failed to find rowset for version={}", version.to_string())); + }; + + for (const auto& version : version_paths) { + auto ret = rowset_for_version(version, options.include_stale_rowsets); + if (!ret) { + return ResultError(std::move(ret.error())); + } + + rowsets.push_back(std::move(ret.value())); + } + if (keys_type() == KeysType::UNIQUE_KEYS && enable_unique_key_merge_on_write()) { + result.delete_bitmap = _tablet_meta->delete_bitmap(); + } + return result; + } + + if (!config::is_cloud_mode() || !options.enable_fetch_rowsets_from_peers) { + return ResultError(std::move(maybe_versions.error())); + } + auto ret = _remote_capture_rowsets(version_range); + if (!ret) { + auto st = Status::Error( + "version already merged, meet error during remote capturing rowsets, " + "error={}, version_range={}", + ret.error().to_string(), version_range.to_string()); + return ResultError(std::move(st)); + } + return ret; +} + +[[nodiscard]] Result> BaseTablet::capture_rs_readers_unlocked( + const Version& version_range, const CaptureRowsetOps& options) const { + auto maybe_rs_list = capture_consistent_rowsets_unlocked(version_range, options); + if (!maybe_rs_list) { + return ResultError(std::move(maybe_rs_list.error())); + } + const auto& rs_list = maybe_rs_list.value().rowsets; + std::vector rs_splits; + rs_splits.reserve(rs_list.size()); + for (const auto& rs : rs_list) { + RowsetReaderSharedPtr rs_reader; + auto st = rs->create_reader(&rs_reader); + if (!st) { + return ResultError(Status::Error( + "failed to create reader for rowset={}, reason={}", rs->rowset_id().to_string(), + st.to_string())); + } + rs_splits.emplace_back(std::move(rs_reader)); + } + return rs_splits; +} + +[[nodiscard]] Result BaseTablet::capture_read_source( + const Version& version_range, const CaptureRowsetOps& options) { + std::shared_lock rdlock(get_header_lock()); + auto maybe_result = capture_consistent_rowsets_unlocked(version_range, options); + if (!maybe_result) { + return ResultError(std::move(maybe_result.error())); + } + auto rowsets_result = std::move(maybe_result.value()); + TabletReadSource read_source; + read_source.delete_bitmap = std::move(rowsets_result.delete_bitmap); + const auto& rowsets = rowsets_result.rowsets; + read_source.rs_splits.reserve(rowsets.size()); + for (const auto& rs : rowsets) { + RowsetReaderSharedPtr rs_reader; + auto st = rs->create_reader(&rs_reader); + if (!st) { + return ResultError(Status::Error( + "failed to create reader for rowset={}, reason={}", rs->rowset_id().to_string(), + st.to_string())); + } + read_source.rs_splits.emplace_back(std::move(rs_reader)); + } + return read_source; +} + +template +bool call_bthread(bthread_t& th, const bthread_attr_t* attr, Fn&& fn, Args&&... args) { + auto p_wrap_fn = new auto([=] { fn(args...); }); + auto call_back = [](void* ar) -> void* { + auto f = reinterpret_cast(ar); + (*f)(); + delete f; + return nullptr; + }; + return bthread_start_background(&th, attr, call_back, p_wrap_fn) == 0; +} + +struct GetRowsetsCntl : std::enable_shared_from_this { + struct RemoteGetRowsetResult { + std::vector rowsets; + std::unique_ptr delete_bitmap; + }; + + Status start_req_bg() { + task_cnt = req_addrs.size(); + for (const auto& [ip, port] : req_addrs) { + bthread_t tid; + bthread_attr_t attr = BTHREAD_ATTR_NORMAL; + + bool succ = call_bthread(tid, &attr, [self = shared_from_this(), &ip, port]() { + LOG(INFO) << "start to get tablet rowsets from peer BE, ip=" << ip; + Defer defer_log {[&ip, port]() { + LOG(INFO) << "finish to get rowsets from peer BE, ip=" << ip + << ", port=" << port; + }}; + + PGetTabletRowsetsRequest req; + req.set_tablet_id(self->tablet_id); + req.set_version_start(self->version_range.first); + req.set_version_end(self->version_range.second); + if (self->delete_bitmap_keys.has_value()) { + req.mutable_delete_bitmap_keys()->CopyFrom(self->delete_bitmap_keys.value()); + } + brpc::Controller cntl; + cntl.set_timeout_ms(60000); + cntl.set_max_retry(3); + PGetTabletRowsetsResponse response; + auto start_tm_us = MonotonicMicros(); +#ifndef BE_TEST + std::shared_ptr stub = + ExecEnv::GetInstance()->brpc_internal_client_cache()->get_client(ip, port); + if (stub == nullptr) { + self->result = ResultError(Status::InternalError( + "failed to fetch get_tablet_rowsets stub, ip={}, port={}", ip, port)); + return; + } + stub->get_tablet_rowsets(&cntl, &req, &response, nullptr); +#else + TEST_SYNC_POINT_CALLBACK("get_tablet_rowsets", &response); +#endif + g_remote_fetch_tablet_rowsets_single_request_latency + << MonotonicMicros() - start_tm_us; + + std::unique_lock l(self->butex); + if (self->done) { + return; + } + --self->task_cnt; + auto resp_st = Status::create(response.status()); + DBUG_EXECUTE_IF("GetRowsetCntl::start_req_bg.inject_failure", + { resp_st = Status::InternalError("inject error"); }); + if (cntl.Failed() || !resp_st) { + if (self->task_cnt != 0) { + return; + } + std::stringstream reason; + reason << "failed to get rowsets from all replicas, tablet_id=" + << self->tablet_id; + if (cntl.Failed()) { + reason << ", reason=[" << cntl.ErrorCode() << "] " << cntl.ErrorText(); + } else { + reason << ", reason=" << resp_st.to_string(); + } + self->result = ResultError(Status::InternalError(reason.str())); + self->done = true; + self->event.signal(); + return; + } + + Defer done_cb {[&]() { + self->done = true; + self->event.signal(); + }}; + std::vector rs_metas; + for (auto&& rs_pb : response.rowsets()) { + auto rs_meta = std::make_shared(); + if (!rs_meta->init_from_pb(rs_pb)) { + self->result = + ResultError(Status::InternalError("failed to init rowset from pb")); + return; + } + rs_metas.push_back(std::move(rs_meta)); + } + CaptureRowsetResult result; + self->result->rowsets = std::move(rs_metas); + + if (response.has_delete_bitmap()) { + self->result->delete_bitmap = std::make_unique( + DeleteBitmap::from_pb(response.delete_bitmap(), self->tablet_id)); + } + }); + + if (!succ) { + return Status::InternalError( + "failed to create bthread when request rowsets for tablet={}", tablet_id); + } + if (bthread_join(tid, nullptr) != 0) { + return Status::InternalError("failed to join bthread tid={}", tid); + } + } + return Status::OK(); + } + + Result wait_for_ret() { + event.wait(); + return std::move(result); + } + + int64_t tablet_id; + std::vector> req_addrs; + Version version_range; + std::optional delete_bitmap_keys = std::nullopt; + +private: + size_t task_cnt; + + bthread::Mutex butex; + bthread::CountdownEvent event {1}; + bool done = false; + + Result result; +}; + +Result>> get_peer_replicas_addresses( + const int64_t tablet_id) { + auto* cluster_info = ExecEnv::GetInstance()->cluster_info(); + DCHECK_NE(cluster_info, nullptr); + auto master_addr = cluster_info->master_fe_addr; + TGetTabletReplicaInfosRequest req; + req.tablet_ids.push_back(tablet_id); + TGetTabletReplicaInfosResult resp; + auto st = ThriftRpcHelper::rpc( + master_addr.hostname, master_addr.port, + [&](FrontendServiceConnection& client) { client->getTabletReplicaInfos(resp, req); }); + if (!st) { + return ResultError(Status::InternalError( + "failed to get tablet replica infos, rpc error={}, tablet_id={}", st.to_string(), + tablet_id)); + } + + auto it = resp.tablet_replica_infos.find(tablet_id); + if (it == resp.tablet_replica_infos.end()) { + return ResultError(Status::InternalError("replicas not found, tablet_id={}", tablet_id)); + } + auto replicas = it->second; + auto local_host = BackendOptions::get_localhost(); + bool include_local_host = false; + DBUG_EXECUTE_IF("get_peer_replicas_address.enable_local_host", { include_local_host = true; }); + auto ret_view = + replicas | std::views::filter([&local_host, include_local_host](const auto& replica) { + return local_host.find(replica.host) == std::string::npos || include_local_host; + }) | + std::views::transform([](auto& replica) { + return std::make_pair(std::move(replica.host), replica.brpc_port); + }); + return std::vector(ret_view.begin(), ret_view.end()); +} + +Result BaseTablet::_remote_capture_rowsets( + const Version& version_range) const { + auto start_tm_us = MonotonicMicros(); + Defer defer { + [&]() { g_remote_fetch_tablet_rowsets_latency << MonotonicMicros() - start_tm_us; }}; +#ifndef BE_TEST + auto maybe_be_addresses = get_peer_replicas_addresses(tablet_id()); +#else + Result>> maybe_be_addresses; + TEST_SYNC_POINT_CALLBACK("get_peer_replicas_addresses", &maybe_be_addresses); +#endif + DBUG_EXECUTE_IF("Tablet::_remote_get_rowsets_meta.inject_replica_address_fail", + { maybe_be_addresses = ResultError(Status::InternalError("inject failure")); }); + if (!maybe_be_addresses) { + return ResultError(std::move(maybe_be_addresses.error())); + } + auto be_addresses = std::move(maybe_be_addresses.value()); + if (be_addresses.empty()) { + LOG(WARNING) << "no peers replica for tablet=" << tablet_id(); + return ResultError(Status::InternalError("no replicas for tablet={}", tablet_id())); + } + + auto cntl = std::make_shared(); + cntl->tablet_id = tablet_id(); + cntl->req_addrs = std::move(be_addresses); + cntl->version_range = version_range; + bool is_mow = keys_type() == KeysType::UNIQUE_KEYS && enable_unique_key_merge_on_write(); + CaptureRowsetResult result; + if (is_mow) { + result.delete_bitmap = + std::make_unique(_tablet_meta->delete_bitmap()->snapshot()); + DeleteBitmapPB delete_bitmap_keys; + auto keyset = result.delete_bitmap->delete_bitmap | + std::views::transform([](const auto& kv) { return kv.first; }); + for (const auto& key : keyset) { + const auto& [rs_id, seg_id, version] = key; + delete_bitmap_keys.mutable_rowset_ids()->Add(rs_id.to_string()); + delete_bitmap_keys.mutable_segment_ids()->Add(seg_id); + delete_bitmap_keys.mutable_versions()->Add(version); + } + cntl->delete_bitmap_keys = std::move(delete_bitmap_keys); + } + + RETURN_IF_ERROR_RESULT(cntl->start_req_bg()); + auto maybe_meta = cntl->wait_for_ret(); + if (!maybe_meta) { + auto err = Status::InternalError( + "tried to get rowsets from peer replicas and failed, " + "reason={}", + maybe_meta.error()); + return ResultError(std::move(err)); + } + + auto& remote_meta = maybe_meta.value(); + const auto& rs_metas = remote_meta.rowsets; + for (const auto& rs_meta : rs_metas) { + RowsetSharedPtr rs; + auto st = RowsetFactory::create_rowset(_tablet_meta->tablet_schema(), {}, rs_meta, &rs); + if (!st) { + return ResultError(std::move(st)); + } + result.rowsets.push_back(std::move(rs)); + } + if (is_mow) { + DCHECK_NE(result.delete_bitmap, nullptr); + result.delete_bitmap->merge(*remote_meta.delete_bitmap); + } + return result; +} + +} // namespace doris diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index ae285a94aff1d7..7fd6e96730ee14 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -496,12 +496,12 @@ Status LinkedSchemaChange::process(RowsetReaderSharedPtr rowset_reader, RowsetWr // copy delete bitmap to new tablet. if (new_tablet->keys_type() == UNIQUE_KEYS && new_tablet->enable_unique_key_merge_on_write()) { DeleteBitmap origin_delete_bitmap(base_tablet->tablet_id()); - base_tablet->tablet_meta()->delete_bitmap().subset( + base_tablet->tablet_meta()->delete_bitmap()->subset( {rowset_reader->rowset()->rowset_id(), 0, 0}, {rowset_reader->rowset()->rowset_id(), UINT32_MAX, INT64_MAX}, &origin_delete_bitmap); for (auto& iter : origin_delete_bitmap.delete_bitmap) { - int ret = new_tablet->tablet_meta()->delete_bitmap().set( + int ret = new_tablet->tablet_meta()->delete_bitmap()->set( {rowset_writer->rowset_id(), std::get<1>(iter.first), std::get<2>(iter.first)}, iter.second); DCHECK(ret == 1); @@ -958,7 +958,7 @@ Status SchemaChangeJob::_do_process_alter_tablet(const TAlterTabletReqV2& reques reader_context.sequence_id_idx = reader_context.tablet_schema->sequence_col_idx(); reader_context.is_unique = _base_tablet->keys_type() == UNIQUE_KEYS; reader_context.batch_size = ALTER_TABLE_BATCH_SIZE; - reader_context.delete_bitmap = &_base_tablet->tablet_meta()->delete_bitmap(); + reader_context.delete_bitmap = _base_tablet->tablet_meta()->delete_bitmap(); reader_context.version = Version(0, end_version); for (auto& rs_split : rs_splits) { res = rs_split.rs_reader->init(&reader_context); @@ -1073,10 +1073,8 @@ Status SchemaChangeJob::_get_versions_to_be_changed(std::vector* versio _base_tablet->tablet_id()); } *max_rowset = rowset; - - RETURN_IF_ERROR(_base_tablet->capture_consistent_versions_unlocked( - Version(0, rowset->version().second), versions_to_be_changed, false, false)); - + *versions_to_be_changed = DORIS_TRY(_base_tablet->capture_consistent_versions_unlocked( + Version(0, rowset->version().second), {})); return Status::OK(); } @@ -1478,8 +1476,9 @@ Status SchemaChangeJob::_calc_delete_bitmap_for_mow_table(int64_t alter_version) << "double write rowsets for version: " << alter_version + 1 << "-" << max_version << " new_tablet=" << _new_tablet->tablet_id(); std::shared_lock rlock(_new_tablet->get_header_lock()); - RETURN_IF_ERROR(_new_tablet->capture_consistent_rowsets_unlocked( - {alter_version + 1, max_version}, &rowsets)); + auto ret = DORIS_TRY(_new_tablet->capture_consistent_rowsets_unlocked( + {alter_version + 1, max_version}, CaptureRowsetOps {})); + rowsets = std::move(ret.rowsets); } for (auto rowset_ptr : rowsets) { std::lock_guard rwlock(_new_tablet->get_rowset_update_lock()); @@ -1497,8 +1496,9 @@ Status SchemaChangeJob::_calc_delete_bitmap_for_mow_table(int64_t alter_version) LOG(INFO) << "alter table for unique with merge-on-write, calculate delete bitmap of " << "incremental rowsets for version: " << max_version + 1 << "-" << new_max_version << " new_tablet=" << _new_tablet->tablet_id(); - RETURN_IF_ERROR(_new_tablet->capture_consistent_rowsets_unlocked( - {max_version + 1, new_max_version}, &rowsets)); + auto ret = DORIS_TRY(_new_tablet->capture_consistent_rowsets_unlocked( + {max_version + 1, new_max_version}, CaptureRowsetOps {})); + rowsets = std::move(ret.rowsets); } for (auto&& rowset_ptr : rowsets) { RETURN_IF_ERROR(Tablet::update_delete_bitmap_without_lock(_new_tablet, rowset_ptr)); diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp index a59ed36bb827cc..6d2c956ac04445 100644 --- a/be/src/olap/snapshot_manager.cpp +++ b/be/src/olap/snapshot_manager.cpp @@ -40,6 +40,7 @@ #include "common/logging.h" #include "common/status.h" #include "io/fs/local_file_system.h" +#include "olap/base_tablet.h" #include "olap/data_dir.h" #include "olap/olap_common.h" #include "olap/olap_define.h" @@ -567,13 +568,23 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet res = check_version_continuity(consistent_rowsets); if (res.ok() && max_cooldowned_version < version) { // Pick consistent rowsets of remaining required version - res = ref_tablet->capture_consistent_rowsets_unlocked( - {max_cooldowned_version + 1, version}, &consistent_rowsets); + auto ret = ref_tablet->capture_consistent_rowsets_unlocked( + {max_cooldowned_version + 1, version}, CaptureRowsetOps {}); + if (ret) { + consistent_rowsets = std::move(ret->rowsets); + } else { + res = std::move(ret.error()); + } } } else { // get shortest version path - res = ref_tablet->capture_consistent_rowsets_unlocked(Version(0, version), - &consistent_rowsets); + auto ret = ref_tablet->capture_consistent_rowsets_unlocked(Version(0, version), + CaptureRowsetOps {}); + if (ret) { + consistent_rowsets = std::move(ret->rowsets); + } else { + res = std::move(ret.error()); + } } if (!res.ok()) { LOG(WARNING) << "fail to select versions to span. res=" << res; @@ -594,7 +605,7 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet if (ref_tablet->keys_type() == UNIQUE_KEYS && ref_tablet->enable_unique_key_merge_on_write()) { delete_bitmap_snapshot = - ref_tablet->tablet_meta()->delete_bitmap().snapshot(version); + ref_tablet->tablet_meta()->delete_bitmap()->snapshot(version); } } diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 5c4770e3a3344f..403438f1dadad3 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -391,12 +391,14 @@ Status Tablet::revise_tablet_meta(const std::vector& to_add, } if (calc_delete_bitmap_ver.first <= calc_delete_bitmap_ver.second) { - calc_bm_status = capture_consistent_rowsets_unlocked(calc_delete_bitmap_ver, - &calc_delete_bitmap_rowsets); - if (!calc_bm_status.ok()) { - LOG(WARNING) << "fail to capture_consistent_rowsets, res: " << calc_bm_status; + auto ret = capture_consistent_rowsets_unlocked(calc_delete_bitmap_ver, + CaptureRowsetOps {}); + if (!ret) { + LOG(WARNING) << "fail to capture_consistent_rowsets, res: " << ret.error(); + calc_bm_status = std::move(ret.error()); break; } + calc_delete_bitmap_rowsets = std::move(ret->rowsets); // FIXME(plat1ko): Use `const TabletSharedPtr&` as parameter auto self = _engine.tablet_manager()->get_tablet(tablet_id()); CHECK(self); @@ -451,17 +453,16 @@ Status Tablet::revise_tablet_meta(const std::vector& to_add, // that we can capture by version if (keys_type() == UNIQUE_KEYS && enable_unique_key_merge_on_write()) { Version full_version = Version(0, max_version_unlocked()); - std::vector expected_rowsets; - auto st = capture_consistent_rowsets_unlocked(full_version, &expected_rowsets); - DCHECK(st.ok()) << st; - DCHECK_EQ(base_rowsets_for_full_clone.size(), expected_rowsets.size()); - if (st.ok() && base_rowsets_for_full_clone.size() != expected_rowsets.size()) - [[unlikely]] { + auto ret = capture_consistent_rowsets_unlocked(full_version, CaptureRowsetOps {}); + DCHECK(ret) << ret.error(); + DCHECK_EQ(base_rowsets_for_full_clone.size(), ret->rowsets.size()); + + if (ret && base_rowsets_for_full_clone.size() != ret->rowsets.size()) [[unlikely]] { LOG(WARNING) << "full clone succeeded, but the count(" << base_rowsets_for_full_clone.size() << ") of base rowsets used for delete bitmap calculation is not match " "expect count(" - << expected_rowsets.size() << ") we capture from tablet meta"; + << ret->rowsets.size() << ") we capture from tablet meta"; } } } @@ -747,10 +748,9 @@ void Tablet::delete_expired_stale_rowset() { Version test_version = Version(0, lastest_delta->end_version()); stale_version_path_map[*path_id_iter] = version_path; - Status status = - capture_consistent_versions_unlocked(test_version, nullptr, false, false); + auto ret = capture_consistent_versions_unlocked(test_version, {}); // 1. When there is no consistent versions, we must reconstruct the tracker. - if (!status.ok()) { + if (!ret) { // 2. fetch missing version after delete Versions after_missed_versions = get_missed_versions_unlocked(lastest_delta->end_version()); @@ -865,51 +865,11 @@ void Tablet::delete_expired_stale_rowset() { { _engine.start_delete_unused_rowset(); }); } -Status Tablet::capture_consistent_versions_unlocked(const Version& spec_version, - Versions* version_path, - bool skip_missing_version, bool quiet) const { - Status status = - _timestamped_version_tracker.capture_consistent_versions(spec_version, version_path); - if (!status.ok() && !quiet) { - Versions missed_versions = get_missed_versions_unlocked(spec_version.second); - if (missed_versions.empty()) { - // if version_path is null, it may be a compaction check logic. - // so to avoid print too many logs. - if (version_path != nullptr) { - LOG(WARNING) << "tablet:" << tablet_id() - << ", version already has been merged. spec_version: " << spec_version - << ", max_version: " << max_version_unlocked(); - } - status = Status::Error( - "versions are already compacted, spec_version " - "{}, max_version {}, tablet_id {}", - spec_version.second, max_version_unlocked(), tablet_id()); - } else { - if (version_path != nullptr) { - LOG(WARNING) << "status:" << status << ", tablet:" << tablet_id() - << ", missed version for version:" << spec_version; - _print_missed_versions(missed_versions); - if (skip_missing_version) { - LOG(WARNING) << "force skipping missing version for tablet:" << tablet_id(); - return Status::OK(); - } - } - } - } - - DBUG_EXECUTE_IF("TTablet::capture_consistent_versions.inject_failure", { - auto tablet_id = dp->param("tablet_id", -1); - if (tablet_id != -1 && tablet_id == _tablet_meta->tablet_id()) { - status = Status::Error("version already merged"); - } - }); - - return status; -} - Status Tablet::check_version_integrity(const Version& version, bool quiet) { std::shared_lock rdlock(_meta_lock); - return capture_consistent_versions_unlocked(version, nullptr, false, quiet); + [[maybe_unused]] auto _versions = DORIS_TRY( + capture_consistent_versions_unlocked(version, CaptureRowsetOps {.quiet = quiet})); + return Status::OK(); } bool Tablet::exceed_version_limit(int32_t limit) { @@ -939,22 +899,12 @@ void Tablet::acquire_version_and_rowsets( } } -Status Tablet::capture_consistent_rowsets_unlocked(const Version& spec_version, - std::vector* rowsets) const { - std::vector version_path; - RETURN_IF_ERROR( - capture_consistent_versions_unlocked(spec_version, &version_path, false, false)); - RETURN_IF_ERROR(_capture_consistent_rowsets_unlocked(version_path, rowsets)); - return Status::OK(); -} - Status Tablet::capture_rs_readers(const Version& spec_version, std::vector* rs_splits, bool skip_missing_version) { std::shared_lock rlock(_meta_lock); std::vector version_path; - RETURN_IF_ERROR(capture_consistent_versions_unlocked(spec_version, &version_path, - skip_missing_version, false)); - RETURN_IF_ERROR(capture_rs_readers_unlocked(version_path, rs_splits)); + *rs_splits = DORIS_TRY(capture_rs_readers_unlocked( + spec_version, CaptureRowsetOps {.skip_missing_versions = skip_missing_version})); return Status::OK(); } @@ -2514,8 +2464,8 @@ Status Tablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, for (auto& [key, bitmap] : delete_bitmap->delete_bitmap) { // skip sentinel mark, which is used for delete bitmap correctness check if (std::get<1>(key) != DeleteBitmap::INVALID_SEGMENT_ID) { - _tablet_meta->delete_bitmap().merge({std::get<0>(key), std::get<1>(key), cur_version}, - bitmap); + _tablet_meta->delete_bitmap()->merge({std::get<0>(key), std::get<1>(key), cur_version}, + bitmap); } } @@ -2523,7 +2473,7 @@ Status Tablet::save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t txn_id, } void Tablet::merge_delete_bitmap(const DeleteBitmap& delete_bitmap) { - _tablet_meta->delete_bitmap().merge(delete_bitmap); + _tablet_meta->delete_bitmap()->merge(delete_bitmap); } bool Tablet::check_all_rowset_segment() { diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 467fc51f98517c..a8c9df89ff0889 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -180,21 +180,12 @@ class Tablet final : public BaseTablet { /// need to delete flag. void delete_expired_stale_rowset(); - // Given spec_version, find a continuous version path and store it in version_path. - // If quiet is true, then only "does this path exist" is returned. - // If skip_missing_version is true, return ok even there are missing versions. - Status capture_consistent_versions_unlocked(const Version& spec_version, Versions* version_path, - bool skip_missing_version, bool quiet) const; - // if quiet is true, no error log will be printed if there are missing versions Status check_version_integrity(const Version& version, bool quiet = false); bool check_version_exist(const Version& version) const; void acquire_version_and_rowsets( std::vector>* version_rowsets) const; - Status capture_consistent_rowsets_unlocked( - const Version& spec_version, std::vector* rowsets) const override; - // If skip_missing_version is true, skip versions if they are missing. Status capture_rs_readers(const Version& spec_version, std::vector* rs_splits, bool skip_missing_version) override; diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index f74cf2bf1f670e..3e7d48e44af97e 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -1764,7 +1764,7 @@ void TabletManager::get_topn_tablet_delete_bitmap_score( buf.reserve(n + 1); auto handler = [&](const TabletSharedPtr& tablet) { uint64_t delete_bitmap_count = - tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count(); + tablet->tablet_meta()->delete_bitmap()->get_delete_bitmap_count(); total_delete_map_count += delete_bitmap_count; if (delete_bitmap_count > *max_delete_bitmap_score) { max_delete_bitmap_score_tablet_id = tablet->tablet_id(); diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 3a0ff3419ee09c..c1dbc8a93948df 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -455,12 +456,12 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco void TabletMeta::remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version) { if (_enable_unique_key_merge_on_write) { - delete_bitmap().remove({rowset_id, 0, 0}, {rowset_id, UINT32_MAX, 0}); + delete_bitmap()->remove({rowset_id, 0, 0}, {rowset_id, UINT32_MAX, 0}); if (config::enable_mow_verbose_log) { LOG_INFO("delete rowset delete bitmap. tablet={}, rowset={}, version={}", tablet_id(), rowset_id.to_string(), version.to_string()); } - size_t rowset_cache_version_size = delete_bitmap().remove_rowset_cache_version(rowset_id); + size_t rowset_cache_version_size = delete_bitmap()->remove_rowset_cache_version(rowset_id); _check_mow_rowset_cache_version_size(rowset_cache_version_size); } } @@ -705,7 +706,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) { auto seg_id = tablet_meta_pb.delete_bitmap().segment_ids(i); auto ver = tablet_meta_pb.delete_bitmap().versions(i); auto bitmap = tablet_meta_pb.delete_bitmap().segment_delete_bitmaps(i).data(); - delete_bitmap().delete_bitmap[{rst_id, seg_id, ver}] = roaring::Roaring::read(bitmap); + delete_bitmap()->delete_bitmap[{rst_id, seg_id, ver}] = roaring::Roaring::read(bitmap); } } @@ -789,7 +790,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) { stale_rs_ids.insert(rowset->rowset_id()); } DeleteBitmapPB* delete_bitmap_pb = tablet_meta_pb->mutable_delete_bitmap(); - for (auto& [id, bitmap] : delete_bitmap().snapshot().delete_bitmap) { + for (auto& [id, bitmap] : delete_bitmap()->snapshot().delete_bitmap) { auto& [rowset_id, segment_id, ver] = id; if (stale_rs_ids.count(rowset_id) != 0) { continue; @@ -1121,6 +1122,35 @@ DeleteBitmap& DeleteBitmap::operator=(DeleteBitmap&& o) { return *this; } +DeleteBitmap DeleteBitmap::from_pb(const DeleteBitmapPB& pb, int64_t tablet_id) { + size_t len = pb.rowset_ids().size(); + DCHECK_EQ(len, pb.segment_ids().size()); + DCHECK_EQ(len, pb.versions().size()); + DeleteBitmap delete_bitmap(tablet_id); + for (int32_t i = 0; i < len; ++i) { + RowsetId rs_id; + rs_id.init(pb.rowset_ids(i)); + BitmapKey key = {rs_id, pb.segment_ids(i), pb.versions(i)}; + delete_bitmap.delete_bitmap[key] = + roaring::Roaring::read(pb.segment_delete_bitmaps(i).data()); + } + return delete_bitmap; +} + +DeleteBitmapPB DeleteBitmap::to_pb() { + std::shared_lock l(lock); + DeleteBitmapPB ret; + for (const auto& [k, v] : delete_bitmap) { + ret.mutable_rowset_ids()->Add(std::get<0>(k).to_string()); + ret.mutable_segment_ids()->Add(std::get<1>(k)); + ret.mutable_versions()->Add(std::get<2>(k)); + std::string bitmap_data(v.getSizeInBytes(), '\0'); + v.write(bitmap_data.data()); + ret.mutable_segment_delete_bitmaps()->Add(std::move(bitmap_data)); + } + return ret; +} + DeleteBitmap DeleteBitmap::snapshot() const { std::shared_lock l(lock); return DeleteBitmap(*this); @@ -1508,6 +1538,22 @@ std::shared_ptr DeleteBitmap::get_agg_without_cache(const Bitm return bitmap; } +DeleteBitmap DeleteBitmap::diffset(const std::set& key_set) const { + std::shared_lock l(lock); + auto diff_key_set_view = + delete_bitmap | std::ranges::views::transform([](const auto& kv) { return kv.first; }) | + std::ranges::views::filter( + [&key_set](const auto& key) { return !key_set.contains(key); }); + + DeleteBitmap dbm(_tablet_id); + for (const auto& key : diff_key_set_view) { + const auto* bitmap = get(key); + DCHECK_NE(bitmap, nullptr); + dbm.delete_bitmap[key] = *bitmap; + } + return dbm; +} + std::atomic DeleteBitmap::AggCache::s_repr {nullptr}; std::string tablet_state_name(TabletState state) { diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 388ddc439dc31e..63ad0124f862a6 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -241,7 +241,7 @@ class TabletMeta : public MetadataAdder { static void init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn, ColumnPB* column); - DeleteBitmap& delete_bitmap() { return *_delete_bitmap; } + std::shared_ptr delete_bitmap() { return _delete_bitmap; } void remove_rowset_delete_bitmap(const RowsetId& rowset_id, const Version& version); bool enable_unique_key_merge_on_write() const { return _enable_unique_key_merge_on_write; } @@ -417,6 +417,10 @@ class DeleteBitmap { DeleteBitmap(DeleteBitmap&& r); DeleteBitmap& operator=(DeleteBitmap&& r); + static DeleteBitmap from_pb(const DeleteBitmapPB& pb, int64_t tablet_id); + + DeleteBitmapPB to_pb(); + /** * Makes a snapshot of delete bitmap, read lock will be acquired in this * process @@ -566,6 +570,14 @@ class DeleteBitmap { std::set get_rowset_cache_version(); + /** + * Calculate diffset with given `key_set`. All entries with keys contained in this delete bitmap but not + * in given key_set will be added to the output delete bitmap. + * + * @return Deletebitmap containning all entries in diffset + */ + DeleteBitmap diffset(const std::set& key_set) const; + class AggCachePolicy : public LRUCachePolicy { public: AggCachePolicy(size_t capacity) diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp index a453634ca8381f..7654a37de16b1f 100644 --- a/be/src/olap/tablet_reader.cpp +++ b/be/src/olap/tablet_reader.cpp @@ -100,16 +100,6 @@ std::string TabletReader::KeysParam::to_string() const { return ss.str(); } -void TabletReader::ReadSource::fill_delete_predicates() { - DCHECK_EQ(delete_predicates.size(), 0); - for (auto&& split : rs_splits) { - auto& rs_meta = split.rs_reader->rowset()->rowset_meta(); - if (rs_meta->has_delete_predicate()) { - delete_predicates.push_back(rs_meta); - } - } -} - TabletReader::~TabletReader() { for (auto* pred : _col_predicates) { delete pred; @@ -658,7 +648,7 @@ Status TabletReader::init_reader_params_and_create_block( reader_params->version = Version(input_rowsets.front()->start_version(), input_rowsets.back()->end_version()); - ReadSource read_source; + TabletReadSource read_source; for (const auto& rowset : input_rowsets) { RowsetReaderSharedPtr rs_reader; RETURN_IF_ERROR(rowset->create_reader(&rs_reader)); @@ -680,9 +670,6 @@ Status TabletReader::init_reader_params_and_create_block( merge_tablet_schema->merge_dropped_columns(*del_pred->tablet_schema()); } reader_params->tablet_schema = merge_tablet_schema; - if (tablet->enable_unique_key_merge_on_write()) { - reader_params->delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); - } reader_params->return_columns.resize(read_tablet_schema->num_columns()); std::iota(reader_params->return_columns.begin(), reader_params->return_columns.end(), 0); diff --git a/be/src/olap/tablet_reader.h b/be/src/olap/tablet_reader.h index 87af3bb08eb36e..d5aac0b89b5211 100644 --- a/be/src/olap/tablet_reader.h +++ b/be/src/olap/tablet_reader.h @@ -34,6 +34,7 @@ #include "exprs/function_filter.h" #include "gutil/strings/substitute.h" #include "io/io_common.h" +#include "olap/base_tablet.h" #include "olap/delete_handler.h" #include "olap/iterators.h" #include "olap/olap_common.h" @@ -91,12 +92,6 @@ class TabletReader { }; public: - struct ReadSource { - std::vector rs_splits; - std::vector delete_predicates; - // Fill delete predicates with `rs_splits` - void fill_delete_predicates(); - }; // Params for Reader, // mainly include tablet, data version and fetch range. struct ReaderParams { @@ -117,9 +112,12 @@ class TabletReader { return BeExecVersionManager::get_newest_version(); } - void set_read_source(ReadSource read_source) { + void set_read_source(TabletReadSource read_source, bool skip_delete_bitmap = false) { rs_splits = std::move(read_source.rs_splits); delete_predicates = std::move(read_source.delete_predicates); + if (tablet->enable_unique_key_merge_on_write() && !skip_delete_bitmap) { + delete_bitmap = std::move(read_source.delete_bitmap); + } } BaseTabletSPtr tablet; @@ -148,7 +146,7 @@ class TabletReader { std::vector rs_splits; // For unique key table with merge-on-write - DeleteBitmap* delete_bitmap = nullptr; + std::shared_ptr delete_bitmap = nullptr; // return_columns is init from query schema std::vector return_columns; diff --git a/be/src/olap/task/engine_checksum_task.cpp b/be/src/olap/task/engine_checksum_task.cpp index 05ecfc0401b6d0..c6cf69d54d9eef 100644 --- a/be/src/olap/task/engine_checksum_task.cpp +++ b/be/src/olap/task/engine_checksum_task.cpp @@ -24,6 +24,7 @@ #include #include "io/io_common.h" +#include "olap/base_tablet.h" #include "olap/olap_common.h" #include "olap/olap_define.h" #include "olap/rowset/rowset.h" @@ -81,13 +82,15 @@ Status EngineChecksumTask::_compute_checksum() { vectorized::Block block; { std::shared_lock rdlock(tablet->get_header_lock()); - Status acquire_reader_st = - tablet->capture_consistent_rowsets_unlocked(version, &input_rowsets); - if (!acquire_reader_st.ok()) { + auto ret = tablet->capture_consistent_rowsets_unlocked(version, CaptureRowsetOps {}); + if (ret) { + input_rowsets = std::move(ret->rowsets); + } else { LOG(WARNING) << "fail to captute consistent rowsets. tablet=" << tablet->tablet_id() - << "res=" << acquire_reader_st; - return acquire_reader_st; + << "res=" << ret.error(); + return std::move(ret.error()); } + RETURN_IF_ERROR(TabletReader::init_reader_params_and_create_block( tablet, ReaderType::READER_CHECKSUM, input_rowsets, &reader_params, &block)); } diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index ecf1bdfc6d5c7d..6a9e66f1d383cd 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -984,7 +984,7 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet, } } if (tablet->enable_unique_key_merge_on_write()) { - tablet->tablet_meta()->delete_bitmap().merge(cloned_tablet_meta->delete_bitmap()); + tablet->tablet_meta()->delete_bitmap()->merge(*cloned_tablet_meta->delete_bitmap()); } return tablet->revise_tablet_meta(to_add, to_delete, false); // TODO(plat1ko): write cooldown meta to remote if this replica is cooldown replica diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp index 210aa6a8c56f08..d9352a0ea9d0bb 100644 --- a/be/src/olap/task/engine_storage_migration_task.cpp +++ b/be/src/olap/task/engine_storage_migration_task.cpp @@ -32,8 +32,10 @@ #include "common/config.h" #include "common/logging.h" +#include "common/status.h" #include "gutil/strings/numbers.h" #include "io/fs/local_file_system.h" +#include "olap/base_tablet.h" #include "olap/data_dir.h" #include "olap/olap_common.h" #include "olap/olap_define.h" @@ -87,8 +89,10 @@ Status EngineStorageMigrationTask::_get_versions(int32_t start_version, int32_t* << ", start_version=" << start_version << ", end_version=" << *end_version; return Status::OK(); } - return _tablet->capture_consistent_rowsets_unlocked(Version(start_version, *end_version), - consistent_rowsets); + auto ret = DORIS_TRY(_tablet->capture_consistent_rowsets_unlocked( + Version(start_version, *end_version), CaptureRowsetOps {})); + *consistent_rowsets = std::move(ret.rowsets); + return Status::OK(); } bool EngineStorageMigrationTask::_is_timeout() { @@ -354,7 +358,7 @@ void EngineStorageMigrationTask::_generate_new_header( } new_tablet_meta->revise_rs_metas(std::move(rs_metas)); if (_tablet->keys_type() == UNIQUE_KEYS && _tablet->enable_unique_key_merge_on_write()) { - DeleteBitmap bm = _tablet->tablet_meta()->delete_bitmap().snapshot(end_version); + DeleteBitmap bm = _tablet->tablet_meta()->delete_bitmap()->snapshot(end_version); new_tablet_meta->revise_delete_bitmap_unlocked(bm); } new_tablet_meta->set_shard_id(new_shard); diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 9f72056af8d1c9..ffb226bee58da9 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -822,7 +822,7 @@ Status IndexBuilder::modify_rowsets(const Merger::Statistics* stats) { for (auto i = 0; i < _input_rowsets.size(); ++i) { RowsetId input_rowset_id = _input_rowsets[i]->rowset_id(); RowsetId output_rowset_id = _output_rowsets[i]->rowset_id(); - for (const auto& [k, v] : _tablet->tablet_meta()->delete_bitmap().delete_bitmap) { + for (const auto& [k, v] : _tablet->tablet_meta()->delete_bitmap()->delete_bitmap) { RowsetId rs_id = std::get<0>(k); if (rs_id == input_rowset_id) { DeleteBitmap::BitmapKey output_rs_key = {output_rowset_id, std::get<1>(k), @@ -832,7 +832,7 @@ Status IndexBuilder::modify_rowsets(const Merger::Statistics* stats) { } } } - _tablet->tablet_meta()->delete_bitmap().merge(*delete_bitmap); + _tablet->tablet_meta()->delete_bitmap()->merge(*delete_bitmap); // modify_rowsets will remove the delete_bitmap for input rowsets, // should call it after merge delete_bitmap diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp index 53a7d8cd0f3490..13d2044cc2ab6c 100644 --- a/be/src/pipeline/exec/olap_scan_operator.cpp +++ b/be/src/pipeline/exec/olap_scan_operator.cpp @@ -27,6 +27,7 @@ #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_hotspot.h" #include "cloud/config.h" +#include "common/config.h" #include "olap/parallel_scanner_builder.h" #include "olap/storage_engine.h" #include "olap/tablet_manager.h" @@ -528,9 +529,11 @@ Status OlapScanLocalState::hold_tablets() { } for (size_t i = 0; i < _scan_ranges.size(); i++) { - RETURN_IF_ERROR(_tablets[i].tablet->capture_rs_readers( - {0, _tablets[i].version}, &_read_sources[i].rs_splits, - RuntimeFilterConsumer::_state->skip_missing_version())); + _read_sources[i] = DORIS_TRY(_tablets[i].tablet->capture_read_source( + {0, _tablets[i].version}, + {.skip_missing_versions = RuntimeFilterConsumer::_state->skip_missing_version(), + .enable_fetch_rowsets_from_peers = + config::enable_fetch_rowsets_from_peer_replicas})); if (!PipelineXLocalState<>::_state->skip_delete_predicate()) { _read_sources[i].fill_delete_predicates(); } diff --git a/be/src/pipeline/exec/olap_scan_operator.h b/be/src/pipeline/exec/olap_scan_operator.h index 24a1b1b876a354..e1eea0c7822f88 100644 --- a/be/src/pipeline/exec/olap_scan_operator.h +++ b/be/src/pipeline/exec/olap_scan_operator.h @@ -229,7 +229,7 @@ class OlapScanLocalState final : public ScanLocalState { std::mutex _profile_mtx; std::vector _tablets; - std::vector _read_sources; + std::vector _read_sources; }; class OlapScanOperatorX final : public ScanOperatorX { diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 423c62c6c4b409..6d5d9055fc3a3e 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -53,6 +53,9 @@ #include #include +#include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "common/config.h" #include "common/consts.h" #include "common/exception.h" @@ -162,6 +165,8 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(arrow_flight_work_active_threads, MetricUnit: DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(arrow_flight_work_pool_max_queue_size, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(arrow_flight_work_max_threads, MetricUnit::NOUNIT); +static bvar::LatencyRecorder g_process_remote_fetch_rowsets_latency("process_remote_fetch_rowsets"); + bthread_key_t btls_key; static void thread_context_deleter(void* d) { @@ -2200,4 +2205,66 @@ void PInternalService::get_be_resource(google::protobuf::RpcController* controll } } +void PInternalService::get_tablet_rowsets(google::protobuf::RpcController* controller, + const PGetTabletRowsetsRequest* request, + PGetTabletRowsetsResponse* response, + google::protobuf::Closure* done) { + DCHECK(config::is_cloud_mode()); + auto start_time = GetMonoTimeMicros(); + Defer defer { + [&]() { g_process_remote_fetch_rowsets_latency << GetMonoTimeMicros() - start_time; }}; + brpc::ClosureGuard closure_guard(done); + LOG(INFO) << "process get tablet rowsets, request=" << request->ShortDebugString(); + if (!request->has_tablet_id() || !request->has_version_start() || !request->has_version_end()) { + Status::InvalidArgument("missing params tablet/version_start/version_end") + .to_protobuf(response->mutable_status()); + return; + } + CloudStorageEngine& storage = ExecEnv::GetInstance()->storage_engine().to_cloud(); + + auto maybe_tablet = + storage.tablet_mgr().get_tablet(request->tablet_id(), /*warmup data*/ false, + /*syn_delete_bitmap*/ false, /*delete_bitmap*/ nullptr, + /*local_only*/ true); + if (!maybe_tablet) { + maybe_tablet.error().to_protobuf(response->mutable_status()); + return; + } + auto tablet = maybe_tablet.value(); + Result ret; + { + std::shared_lock l(tablet->get_header_lock()); + ret = tablet->capture_consistent_rowsets_unlocked( + {request->version_start(), request->version_end()}, + CaptureRowsetOps {.enable_fetch_rowsets_from_peers = false}); + } + if (!ret) { + ret.error().to_protobuf(response->mutable_status()); + return; + } + auto rowsets = std::move(ret.value().rowsets); + for (const auto& rs : rowsets) { + RowsetMetaPB meta; + rs->rowset_meta()->to_rowset_pb(&meta); + response->mutable_rowsets()->Add(std::move(meta)); + } + if (request->has_delete_bitmap_keys()) { + DCHECK(tablet->enable_unique_key_merge_on_write()); + auto delete_bitmap = std::move(ret.value().delete_bitmap); + auto keys_pb = request->delete_bitmap_keys(); + size_t len = keys_pb.rowset_ids().size(); + DCHECK_EQ(len, keys_pb.segment_ids().size()); + DCHECK_EQ(len, keys_pb.versions().size()); + std::set keys; + for (size_t i = 0; i < len; ++i) { + RowsetId rs_id; + rs_id.init(keys_pb.rowset_ids(i)); + keys.emplace(rs_id, keys_pb.segment_ids(i), keys_pb.versions(i)); + } + auto diffset = delete_bitmap->diffset(keys).to_pb(); + *response->mutable_delete_bitmap() = std::move(diffset); + } + Status::OK().to_protobuf(response->mutable_status()); +} + } // namespace doris diff --git a/be/src/service/internal_service.h b/be/src/service/internal_service.h index 66a0f867393793..7262f4eed2da80 100644 --- a/be/src/service/internal_service.h +++ b/be/src/service/internal_service.h @@ -240,6 +240,11 @@ class PInternalService : public PBackendService { const PGetBeResourceRequest* request, PGetBeResourceResponse* response, google::protobuf::Closure* done) override; + void get_tablet_rowsets(google::protobuf::RpcController* controller, + const PGetTabletRowsetsRequest* request, + PGetTabletRowsetsResponse* response, + google::protobuf::Closure* done) override; + private: void _exec_plan_fragment_in_pthread(google::protobuf::RpcController* controller, const PExecPlanFragmentRequest* request, diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index e9c199074ecc4f..c8008c9852583b 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -68,7 +68,7 @@ namespace doris::vectorized { -using ReadSource = TabletReader::ReadSource; +using ReadSource = TabletReadSource; NewOlapScanner::NewOlapScanner(pipeline::ScanLocalStateBase* parent, NewOlapScanner::Params&& params) @@ -97,7 +97,8 @@ NewOlapScanner::NewOlapScanner(pipeline::ScanLocalStateBase* parent, .filter_block_conjuncts {}, .key_group_cluster_key_idxes {}, }) { - _tablet_reader_params.set_read_source(std::move(params.read_source)); + _tablet_reader_params.set_read_source(std::move(params.read_source), + _state->skip_delete_bitmap()); _is_init = false; } @@ -193,12 +194,14 @@ Status NewOlapScanner::init() { ExecEnv::GetInstance()->storage_engine().to_cloud().tablet_hotspot().count(*tablet); } - auto st = tablet->capture_rs_readers(_tablet_reader_params.version, - &read_source.rs_splits, - _state->skip_missing_version()); - if (!st.ok()) { - LOG(WARNING) << "fail to init reader.res=" << st; - return st; + auto maybe_read_source = tablet->capture_read_source( + _tablet_reader_params.version, + {.skip_missing_versions = _state->skip_missing_version(), + .enable_fetch_rowsets_from_peers = + config::enable_fetch_rowsets_from_peer_replicas}); + if (!maybe_read_source) { + LOG(WARNING) << "fail to init reader. res=" << maybe_read_source.error(); + return maybe_read_source.error(); } if (config::enable_mow_verbose_log && tablet->enable_unique_key_merge_on_write()) { LOG_INFO("finish capture_rs_readers for tablet={}, query_id={}", @@ -309,7 +312,6 @@ Status NewOlapScanner::_init_tablet_reader_params( std::inserter(_tablet_reader_params.function_filters, _tablet_reader_params.function_filters.begin())); - auto& tablet = _tablet_reader_params.tablet; auto& tablet_schema = _tablet_reader_params.tablet_schema; // Merge the columns in delete predicate that not in latest schema in to current tablet schema for (auto& del_pred : _tablet_reader_params.delete_predicates) { @@ -369,10 +371,6 @@ Status NewOlapScanner::_init_tablet_reader_params( _tablet_reader_params.use_page_cache = _state->enable_page_cache(); - if (tablet->enable_unique_key_merge_on_write() && !_state->skip_delete_bitmap()) { - _tablet_reader_params.delete_bitmap = &tablet->tablet_meta()->delete_bitmap(); - } - DBUG_EXECUTE_IF("NewOlapScanner::_init_tablet_reader_params.block", DBUG_BLOCK); if (!_state->skip_storage_engine_merge()) { diff --git a/be/src/vec/exec/scan/new_olap_scanner.h b/be/src/vec/exec/scan/new_olap_scanner.h index fd1246b120ba77..a997ae2adf5af8 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.h +++ b/be/src/vec/exec/scan/new_olap_scanner.h @@ -62,7 +62,7 @@ class NewOlapScanner : public VScanner { std::vector key_ranges; BaseTabletSPtr tablet; int64_t version; - TabletReader::ReadSource read_source; + TabletReadSource read_source; int64_t limit; bool aggregation; }; diff --git a/be/test/olap/delta_writer_test.cpp b/be/test/olap/delta_writer_test.cpp index 7f6aadd6070cce..d3e9115baf1239 100644 --- a/be/test/olap/delta_writer_test.cpp +++ b/be/test/olap/delta_writer_test.cpp @@ -996,7 +996,7 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { StorageReadOptions opts; opts.stats = &stats; opts.tablet_schema = rowset1->tablet_schema(); - opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( + opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap()->get_agg( {rowset1->rowset_id(), 0, cur_version})); std::unique_ptr iter; std::shared_ptr schema = std::make_shared(rowset1->tablet_schema()); @@ -1024,7 +1024,7 @@ TEST_F(TestDeltaWriter, vec_sequence_col_concurrent_write) { StorageReadOptions opts; opts.stats = &stats; opts.tablet_schema = rowset2->tablet_schema(); - opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap().get_agg( + opts.delete_bitmap.emplace(0, tablet->tablet_meta()->delete_bitmap()->get_agg( {rowset2->rowset_id(), 0, cur_version})); std::unique_ptr iter; std::shared_ptr schema = std::make_shared(rowset2->tablet_schema()); diff --git a/be/test/olap/segcompaction_mow_test.cpp b/be/test/olap/segcompaction_mow_test.cpp index 62a3232889dede..efe40dcb859bca 100644 --- a/be/test/olap/segcompaction_mow_test.cpp +++ b/be/test/olap/segcompaction_mow_test.cpp @@ -237,7 +237,7 @@ class SegCompactionMoWTest : public ::testing::TestWithParam { std::vector return_columns = {0, 1, 2}; reader_context.return_columns = &return_columns; reader_context.stats = &_stats; - reader_context.delete_bitmap = delete_bitmap.get(); + reader_context.delete_bitmap = delete_bitmap; std::vector segment_num_rows; Status s; diff --git a/be/test/olap/tablet_meta_test.cpp b/be/test/olap/tablet_meta_test.cpp index b85c63ef714e8e..b350e5e00616a5 100644 --- a/be/test/olap/tablet_meta_test.cpp +++ b/be/test/olap/tablet_meta_test.cpp @@ -73,23 +73,23 @@ TEST(TabletMetaTest, TestReviseMeta) { } ASSERT_EQ(4, tablet_meta.all_rs_metas().size()); - tablet_meta.delete_bitmap().add({rsids[0], 1, 1}, 1); - tablet_meta.delete_bitmap().add({rsids[1], 0, 2}, 2); - tablet_meta.delete_bitmap().add({rsids[2], 1, 1}, 1); - tablet_meta.delete_bitmap().add({rsids[3], 0, 2}, 3); - tablet_meta.delete_bitmap().add({rsids[3], 0, 4}, 4); - ASSERT_EQ(5, tablet_meta.delete_bitmap().delete_bitmap.size()); + tablet_meta.delete_bitmap()->add({rsids[0], 1, 1}, 1); + tablet_meta.delete_bitmap()->add({rsids[1], 0, 2}, 2); + tablet_meta.delete_bitmap()->add({rsids[2], 1, 1}, 1); + tablet_meta.delete_bitmap()->add({rsids[3], 0, 2}, 3); + tablet_meta.delete_bitmap()->add({rsids[3], 0, 4}, 4); + ASSERT_EQ(5, tablet_meta.delete_bitmap()->delete_bitmap.size()); std::vector new_rowsets; new_rowsets.push_back(src_rowsets[2]->rowset_meta()); new_rowsets.push_back(src_rowsets[3]->rowset_meta()); tablet_meta.revise_rs_metas(std::move(new_rowsets)); // Take a snapshot with max_version=3. - DeleteBitmap snap = tablet_meta.delete_bitmap().snapshot(3); + DeleteBitmap snap = tablet_meta.delete_bitmap()->snapshot(3); tablet_meta.revise_delete_bitmap_unlocked(snap); ASSERT_EQ(2, tablet_meta.all_rs_metas().size()); - ASSERT_EQ(2, tablet_meta.delete_bitmap().delete_bitmap.size()); - for (auto entry : tablet_meta.delete_bitmap().delete_bitmap) { + ASSERT_EQ(2, tablet_meta.delete_bitmap()->delete_bitmap.size()); + for (auto entry : tablet_meta.delete_bitmap()->delete_bitmap) { RowsetId rsid = std::get<0>(entry.first); ASSERT_TRUE(rsid == rsids[2] || rsid == rsids[3]); int64_t version = std::get<2>(entry.first); diff --git a/be/test/olap/test_data/rowset_meta.json b/be/test/olap/test_data/rowset_meta.json index d446e2a34e971d..4fe585978aa620 100644 --- a/be/test/olap/test_data/rowset_meta.json +++ b/be/test/olap/test_data/rowset_meta.json @@ -12,6 +12,10 @@ "data_disk_size": 0, "index_disk_size": 0, "empty": true, + "load_id": { + "hi": 0, + "lo": 0 + }, "creation_time": 1552911435, "tablet_uid": { "hi": 10, diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java index 358fc1023b297a..fce12193e3571f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java @@ -602,6 +602,22 @@ public void clearClusterToBe(String cluster) { secondaryClusterToBackends.remove(cluster); } + public List getAllPrimaryBes() { + List result = new ArrayList(); + primaryClusterToBackends.keySet().forEach(clusterId -> { + List backendIds = primaryClusterToBackends.get(clusterId); + if (backendIds == null || backendIds.isEmpty()) { + return; + } + Long beId = backendIds.get(0); + if (beId != -1) { + Backend backend = Env.getCurrentSystemInfo().getBackend(beId); + result.add(backend); + } + }); + return result; + } + // ATTN: This func is only used by redundant tablet report clean in bes. // Only the master node will do the diff logic, // so just only need to clean up secondaryClusterToBackends on the master node. diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 92cfe98e535f5f..ddb49a0ee0ab7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -48,6 +48,7 @@ import org.apache.doris.catalog.TabletMeta; import org.apache.doris.catalog.View; import org.apache.doris.cloud.catalog.CloudPartition; +import org.apache.doris.cloud.catalog.CloudReplica; import org.apache.doris.cloud.catalog.CloudTablet; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; import org.apache.doris.cluster.ClusterNamespace; @@ -2756,15 +2757,24 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos LOG.warn("replica {} not normal", replica.getId()); continue; } - Backend backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); - if (backend != null) { - TReplicaInfo replicaInfo = new TReplicaInfo(); - replicaInfo.setHost(backend.getHost()); - replicaInfo.setBePort(backend.getBePort()); - replicaInfo.setHttpPort(backend.getHttpPort()); - replicaInfo.setBrpcPort(backend.getBrpcPort()); - replicaInfo.setReplicaId(replica.getId()); - replicaInfos.add(replicaInfo); + List backends; + if (Config.isCloudMode()) { + CloudReplica cloudReplica = (CloudReplica) replica; + backends = cloudReplica.getAllPrimaryBes(); + } else { + Backend backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); + backends = Lists.newArrayList(backend); + } + for (Backend backend : backends) { + if (backend != null) { + TReplicaInfo replicaInfo = new TReplicaInfo(); + replicaInfo.setHost(backend.getHost()); + replicaInfo.setBePort(backend.getBePort()); + replicaInfo.setHttpPort(backend.getHttpPort()); + replicaInfo.setBrpcPort(backend.getBrpcPort()); + replicaInfo.setReplicaId(replica.getId()); + replicaInfos.add(replicaInfo); + } } } tabletReplicaInfos.put(tabletId, replicaInfos); diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 150c07fcf9a9ed..20de386b3eba7b 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -989,6 +989,21 @@ message PGetBeResourceResponse { optional PGlobalResourceUsage global_be_resource_usage = 2; } +message PGetTabletRowsetsRequest { + optional int64 tablet_id = 1; + optional int64 version_start = 2; + optional int64 version_end = 3; + + optional DeleteBitmapPB delete_bitmap_keys = 4; +} + +message PGetTabletRowsetsResponse { + required PStatus status = 1; + repeated RowsetMetaPB rowsets = 2; + + optional DeleteBitmapPB delete_bitmap = 3; +} + service PBackendService { rpc transmit_data(PTransmitDataParams) returns (PTransmitDataResult); rpc transmit_data_by_http(PEmptyRequest) returns (PTransmitDataResult); @@ -1041,5 +1056,6 @@ service PBackendService { rpc test_jdbc_connection(PJdbcTestConnectionRequest) returns (PJdbcTestConnectionResult); rpc alter_vault_sync(PAlterVaultSyncRequest) returns (PAlterVaultSyncResponse); rpc get_be_resource(PGetBeResourceRequest) returns (PGetBeResourceResponse); + rpc get_tablet_rowsets(PGetTabletRowsetsRequest) returns (PGetTabletRowsetsResponse); }; diff --git a/regression-test/data/fault_injection_p0/cloud/test_cloud_version_already_merged.out b/regression-test/data/fault_injection_p0/cloud/test_cloud_version_already_merged.out new file mode 100644 index 00000000000000..78964812ebfb24 --- /dev/null +++ b/regression-test/data/fault_injection_p0/cloud/test_cloud_version_already_merged.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 + +-- !sql -- +1 1 1 1 +2 2 2 2 +3 3 3 3 +4 4 4 4 +5 5 5 5 + diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf index 5627f048905a27..9b0bc0f8e7184b 100644 --- a/regression-test/pipeline/p0/conf/be.conf +++ b/regression-test/pipeline/p0/conf/be.conf @@ -80,3 +80,5 @@ large_cumu_compaction_task_min_thread_num=3 # This feature has bug, so by default is false, only open it in pipeline to observe enable_parquet_page_index=true + +enable_fetch_rowsets_from_peer_replicas = true diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy new file mode 100644 index 00000000000000..e1c6669b11b8ea --- /dev/null +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.NodeType + +suite("test_cloud_version_already_merged", "nonConcurrent") { + if (!isCloudMode()) { + return + } + def tblName = "test_cloud_version_already_merged" + sql """ DROP TABLE IF EXISTS ${tblName} FORCE; """ + sql """ + CREATE TABLE IF NOT EXISTS ${tblName} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1"); + """ + + sql "insert into ${tblName} values(1,-1,-1,-1);" + sql "insert into ${tblName} values(2,-2,-2,-2);" + sql "insert into ${tblName} values(3,-3,-3,-3);" + sql "insert into ${tblName} values(4,-4,-4,-4)" + sql "insert into ${tblName} values(5,-5,-5,-5)" + sql "insert into ${tblName} values(1,1,1,1);" + sql "insert into ${tblName} values(2,2,2,2);" + sql "insert into ${tblName} values(3,3,3,3);" + sql "insert into ${tblName} values(4,4,4,4)" + sql "insert into ${tblName} values(5,5,5,5)" + + + sql "sync;" + qt_sql "select * from ${tblName} order by k1;" + + def backends = sql_return_maparray('show backends') + def tabletStats = sql_return_maparray("show tablets from ${tblName};") + assert tabletStats.size() == 1 + def tabletId = tabletStats[0].TabletId + def tabletBackendId = tabletStats[0].BackendId + def tabletBackend + for (def be : backends) { + if (be.BackendId == tabletBackendId) { + tabletBackend = be + break; + } + } + logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); + + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + + try { + GetDebugPoint().enableDebugPoint(tabletBackend.Host, tabletBackend.HttpPort as int, NodeType.BE, "Tablet::capture_consistent_versions.inject_failure", [tablet_id: tabletId, skip_by_option: true]) + GetDebugPoint().enableDebugPointForAllBEs("get_peer_replicas_address.enable_local_host") + + qt_sql """ SELECT * from ${tblName} ORDER BY k1 """ + + } finally { + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("Tablet::capture_consistent_versions.inject_failure", [tablet_id: tabletId]) + GetDebugPoint().enableDebugPointForAllBEs("get_peer_replicas_address.enable_local_host") + + test { + sql """ SELECT * from ${tblName} ORDER BY k1 """ + exception "version already merged, meet error during remote capturing rowsets" + } + + } finally { + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + } + + try { + GetDebugPoint().enableDebugPoint(tabletBackend.Host, tabletBackend.HttpPort as int, NodeType.BE, "Tablet::capture_consistent_versions.inject_failure", [tablet_id: tabletId, skip_by_option: true]) + GetDebugPoint().enableDebugPointForAllBEs("get_peer_replicas_address.enable_local_host") + GetDebugPoint().enableDebugPointForAllBEs("GetRowsetCntl::start_req_bg.inject_failure"); + + test { + sql """ SELECT * from ${tblName} ORDER BY k1 """ + exception "version already merged, meet error during remote capturing rowsets" + } + + } finally { + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + } + + try { + GetDebugPoint().enableDebugPoint(tabletBackend.Host, tabletBackend.HttpPort as int, NodeType.BE, "Tablet::capture_consistent_versions.inject_failure", [tablet_id: tabletId, skip_by_option: true]) + GetDebugPoint().enableDebugPointForAllBEs("get_peer_replicas_address.enable_local_host") + GetDebugPoint().enableDebugPointForAllBEs("Tablet::_remote_get_rowsets_meta.inject_replica_address_fail"); + + test { + sql """ SELECT * from ${tblName} ORDER BY k1 """ + exception "version already merged, meet error during remote capturing rowsets" + } + + } finally { + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + } +} From 9b0bdc8c02078e8ef92f88b47735caed421e5b2d Mon Sep 17 00:00:00 2001 From: meiyi Date: Mon, 7 Jul 2025 14:25:42 +0800 Subject: [PATCH 136/572] branch-3.0: [fix](fe) multi statements different behavior in master and follower (#52144) (#52630) pick https://github.com/apache/doris/pull/52144 --- .../org/apache/doris/qe/ConnectProcessor.java | 4 ++- regression-test/data/insert_p0/insert.out | 6 +++++ .../suites/insert_p0/insert.groovy | 27 +++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 06f5397d3dccd8..6b3640c1d887f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -381,7 +381,9 @@ public void executeQuery(String originStmt) throws Exception { auditAfterExec(auditStmt, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); // execute failed, skip remaining stmts - if (ctx.getState().getStateType() == MysqlStateType.ERR) { + if (ctx.getState().getStateType() == MysqlStateType.ERR || (!Env.getCurrentEnv().isMaster() + && ctx.executor != null && ctx.executor.isForwardToMaster() + && ctx.executor.getProxyStatusCode() != 0)) { break; } } catch (Throwable throwable) { diff --git a/regression-test/data/insert_p0/insert.out b/regression-test/data/insert_p0/insert.out index b582f45123dc66..bd2216290c7080 100644 --- a/regression-test/data/insert_p0/insert.out +++ b/regression-test/data/insert_p0/insert.out @@ -202,3 +202,9 @@ 44980 113.8777 again 1987-04-09 2010-01-02T04:03:06 false -0.01 0E-10 44980 113.8777 again 1987-04-09 2010-01-02T04:03:06 false -0.01 0E-10 +-- !select1 -- +1994-12-08 1 1 +1994-12-14 1 1 +1994-12-14 2 1 +2000-12-08 1 1 + diff --git a/regression-test/suites/insert_p0/insert.groovy b/regression-test/suites/insert_p0/insert.groovy index bcab9956c1d4ec..6a61ea9388ae0c 100644 --- a/regression-test/suites/insert_p0/insert.groovy +++ b/regression-test/suites/insert_p0/insert.groovy @@ -112,4 +112,31 @@ suite("insert") { b as (select * from a) select id from a; """ + sql """ + DROP TABLE IF EXISTS source; + CREATE TABLE source ( + l_shipdate DATE NOT NULL, + l_orderkey bigint NOT NULL, + l_linenumber int not null + )ENGINE=OLAP + DUPLICATE KEY(`l_shipdate`, `l_orderkey`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + PROPERTIES ( + "replication_num" = "1" + ); + + insert into source values('1994-12-08', 1,1) , ('1994-12-14',1,1), ('1994-12-14',2,1); + + + """ + + try { + sql """ insert into source values('2000-12-08', 1, 1); + insert into source values('2000-12-09', 1, 1, 100); + insert into source values('2000-12-10', 1, 1); """ + } catch (Exception e) { + logger.info("exception: " + e.getMessage()) + } + order_qt_select1 """ select * from source; """ } From 6f4da0192e9c20d72f3954a758ce7fbf076b09d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:31:49 +0800 Subject: [PATCH 137/572] branch-3.0: [fix](session variables) Limit query_timeout to within LoadTimeout #51222 (#52841) Cherry-picked from #51222 Co-authored-by: Uniqueyou --- .../CloudGlobalTransactionMgr.java | 5 +- .../transaction/GlobalTransactionMgr.java | 6 +- .../GlobalTransactionMgrIface.java | 22 +++++-- .../insert_p0/test_insert_timeout.groovy | 59 +++++++++++++++++++ 4 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 regression-test/suites/insert_p0/test_insert_timeout.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index fb9c7d293a709e..944099409dc397 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -217,10 +217,11 @@ public long beginTransaction(long dbId, List tableIdList, String label, TU switch (sourceType) { case BACKEND_STREAMING: checkValidTimeoutSecond(timeoutSecond, Config.max_stream_load_timeout_second, - Config.min_load_timeout_second); + Config.min_load_timeout_second, sourceType); break; default: - checkValidTimeoutSecond(timeoutSecond, Config.max_load_timeout_second, Config.min_load_timeout_second); + checkValidTimeoutSecond(timeoutSecond, Config.max_load_timeout_second, + Config.min_load_timeout_second, sourceType); } BeginTxnResponse beginTxnResponse = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index 6552c65d412464..98e6f5c26f292d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -164,16 +164,16 @@ public long beginTransaction(long dbId, List tableIdList, String label, TU switch (sourceType) { case BACKEND_STREAMING: checkValidTimeoutSecond(timeoutSecond, Config.max_stream_load_timeout_second, - Config.min_load_timeout_second); + Config.min_load_timeout_second, sourceType); break; default: checkValidTimeoutSecond(timeoutSecond, Config.max_load_timeout_second, - Config.min_load_timeout_second); + Config.min_load_timeout_second, sourceType); } DatabaseTransactionMgr dbTransactionMgr = getDatabaseTransactionMgr(dbId); return dbTransactionMgr.beginTransaction(tableIdList, label, requestId, - coordinator, sourceType, listenerId, timeoutSecond); + coordinator, sourceType, listenerId, timeoutSecond); } catch (DuplicatedRequestException e) { throw e; } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index 38c8cdd5a6d07e..943af245698b11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -46,11 +46,23 @@ public interface GlobalTransactionMgrIface extends Writable { default void checkValidTimeoutSecond(long timeoutSecond, int maxLoadTimeoutSecond, - int minLoadTimeOutSecond) throws AnalysisException { - if (timeoutSecond > maxLoadTimeoutSecond || timeoutSecond < minLoadTimeOutSecond) { - throw new AnalysisException("Invalid timeout: " + timeoutSecond + ". Timeout should between " - + minLoadTimeOutSecond + " and " + maxLoadTimeoutSecond - + " seconds"); + int minLoadTimeOutSecond, LoadJobSourceType sourceType) + throws AnalysisException { + if (timeoutSecond < minLoadTimeOutSecond) { + throw new AnalysisException("Invalid timeout: " + timeoutSecond + + ". Timeout should be higher than Config.min_load_timeout_second: " + + minLoadTimeOutSecond); + } else if (timeoutSecond > maxLoadTimeoutSecond) { + switch (sourceType) { + case BACKEND_STREAMING: + throw new AnalysisException("Invalid timeout: " + timeoutSecond + + ". Timeout should be lower than Config.max_stream_load_timeout_second: " + + maxLoadTimeoutSecond); + default: + throw new AnalysisException("Invalid timeout: " + timeoutSecond + + ". Timeout should be lower than Config.max_load_timeout_second: " + + maxLoadTimeoutSecond); + } } } diff --git a/regression-test/suites/insert_p0/test_insert_timeout.groovy b/regression-test/suites/insert_p0/test_insert_timeout.groovy new file mode 100644 index 00000000000000..a5f906c82eb2c1 --- /dev/null +++ b/regression-test/suites/insert_p0/test_insert_timeout.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// The cases is copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +suite("test_insert_timeout") { + def tableName = "test_insert_timeout_tbl" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `id` int(11) NOT NULL, + `name` varchar(50) NULL, + `score` int(11) NULL default "-1" + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `name`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + try { + def timeout = sql "SHOW FRONTEND CONFIG like '%max_load_timeout_second%';" + + timeout = Integer.parseInt(timeout[0][1]) + + logger.info("${timeout}") + + def invalid_timeout = timeout + 200 + + sql " set query_timeout = ${invalid_timeout} " + + test { + sql " INSERT INTO ${tableName} VALUES (1, 'Alice', 90), (2, 'Bob', 85), (3, 'Charlie', 95) " + exception "begin transaction failed. errCode = 2, detailMessage = Invalid timeout: ${invalid_timeout}. Timeout should be lower than Config.max_load_timeout_second: ${timeout}" + } + + } finally { + sql """ + UNSET VARIABLE query_timeout; + """ + } +} From 4177ad9ab084265e6e0ad1833c720478745e6ebb Mon Sep 17 00:00:00 2001 From: zhangdong Date: Mon, 7 Jul 2025 14:32:52 +0800 Subject: [PATCH 138/572] =?UTF-8?q?branch-3.0:[fix](restapi)=20Unify=20Per?= =?UTF-8?q?mission=20Requirements=20for=20Executing=20SHOW=20FRONT?= =?UTF-8?q?=E2=80=A6=20(#52226)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ENDS/BACKENDS And NODE RestAPI (#50140) pick: https://github.com/apache/doris/pull/50140 --------- Co-authored-by: zhaorongsheng --- .../doris/analysis/ShowBackendsStmt.java | 10 +-- .../doris/analysis/ShowFrontendsStmt.java | 10 +-- .../doris/httpv2/rest/BackendsAction.java | 9 ++- .../doris/httpv2/rest/manager/NodeAction.java | 15 ++-- .../org/apache/doris/qe/ShowExecutor.java | 14 +++- .../BackendsTableValuedFunction.java | 12 ++++ .../FrontendsDisksTableValuedFunction.java | 12 ++++ .../FrontendsTableValuedFunction.java | 12 ++++ .../doris/analysis/ShowBackendsStmtTest.java | 2 +- .../doris/analysis/ShowFrontendsStmtTest.java | 68 +++++++++++++++++++ .../auth_call/test_show_backend_auth.groovy | 14 +++- .../auth_call/test_show_frontend_auth.groovy | 13 +++- 12 files changed, 166 insertions(+), 25 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ShowFrontendsStmtTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java index 8e1e589477d2dc..06652d2dada50e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackendsStmt.java @@ -19,11 +19,13 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; import org.apache.doris.common.proc.BackendsProcDir; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; @@ -37,10 +39,10 @@ public ShowBackendsStmt() { public void analyze(Analyzer analyzer) throws UserException { super.analyze(analyzer); - if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) - && !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), - PrivPredicate.OPERATOR)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), + InternalCatalog.INTERNAL_CATALOG_NAME, InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR, + PrivPredicate.SELECT.getPrivs().toString(), InfoSchemaDb.DATABASE_NAME); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java index d8b5c454ff9d08..4e968843477e14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowFrontendsStmt.java @@ -19,11 +19,13 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.ScalarType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.proc.FrontendsProcNode; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSetMetaData; @@ -46,10 +48,10 @@ public String getDetailType() { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN) - && !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), - PrivPredicate.OPERATOR)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN/OPERATOR"); + if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), + InternalCatalog.INTERNAL_CATALOG_NAME, InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR, + PrivPredicate.SELECT.getPrivs().toString(), InfoSchemaDb.DATABASE_NAME); } if (detail != null && !detail.equalsIgnoreCase("disks")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java index cef898326b9acf..81e44421500b50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/BackendsAction.java @@ -18,7 +18,10 @@ package org.apache.doris.httpv2.rest; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.Backend; import com.fasterxml.jackson.annotation.JsonProperty; @@ -66,12 +69,8 @@ public class BackendsAction extends RestBaseController { @RequestMapping(path = "/api/backends", method = {RequestMethod.GET}) public Object getBackends(HttpServletRequest request, HttpServletResponse response) { - /** - * As required, the interface should require user have GlobalAuth-PrivPredicate.ADMIN permission. - * However, a user who uses spark-doris-connector/flink-doris-connector does not have corresponding permission. - * To ensure that the connector works properly, we do not verify the permission of the interface. - */ executeCheckPassword(request, response); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); boolean needAlive = false; String isAlive = request.getParameter(IS_ALIVE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java index 4658f936db5f1f..36620c36e92159 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/NodeAction.java @@ -18,6 +18,7 @@ package org.apache.doris.httpv2.rest.manager; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.common.Config; import org.apache.doris.common.ConfigBase; import org.apache.doris.common.MarkedCountDownLatch; @@ -108,7 +109,7 @@ public class NodeAction extends RestBaseController { @RequestMapping(path = "/frontends", method = RequestMethod.GET) public Object frontends_info(HttpServletRequest request, HttpServletResponse response) throws Exception { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); return fetchNodeInfo(request, response, "/frontends"); } @@ -117,7 +118,7 @@ public Object frontends_info(HttpServletRequest request, HttpServletResponse res @RequestMapping(path = "/backends", method = RequestMethod.GET) public Object backends_info(HttpServletRequest request, HttpServletResponse response) throws Exception { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); return fetchNodeInfo(request, response, "/backends"); } @@ -126,7 +127,7 @@ public Object backends_info(HttpServletRequest request, HttpServletResponse resp @RequestMapping(path = "/brokers", method = RequestMethod.GET) public Object brokers_info(HttpServletRequest request, HttpServletResponse response) throws Exception { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); return fetchNodeInfo(request, response, "/brokers"); } @@ -181,7 +182,7 @@ public NodeInfo(List columnNames, List> rows) { @RequestMapping(path = "/configuration_name", method = RequestMethod.GET) public Object configurationName(HttpServletRequest request, HttpServletResponse response) { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); Map> result = Maps.newHashMap(); try { @@ -220,7 +221,7 @@ public Object configurationName(HttpServletRequest request, HttpServletResponse @RequestMapping(path = "/node_list", method = RequestMethod.GET) public Object nodeList(HttpServletRequest request, HttpServletResponse response) { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); Map> result = Maps.newHashMap(); result.put("frontend", getFeList()); @@ -247,7 +248,7 @@ private static List getBeList() { @RequestMapping(path = "/config", method = RequestMethod.GET) public Object config(HttpServletRequest request, HttpServletResponse response) { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkDbAuth(ConnectContext.get().getCurrentUserIdentity(), InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT); List> configs = ConfigBase.getConfigInfo(null); // Sort all configs by config key. @@ -308,7 +309,7 @@ public Object configurationInfo(HttpServletRequest request, HttpServletResponse @RequestParam(value = "type") String type, @RequestBody(required = false) ConfigInfoRequestBody requestBody) { executeCheckPassword(request, response); - checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN_OR_NODE); initHttpExecutor(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 179ad443ec28fb..c2330cb84d7af2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -2263,8 +2263,13 @@ private void handleShowExport() throws AnalysisException { resultSet = new ShowResultSet(showExportStmt.getMetaData(), infos); } - private void handleShowBackends() { + private void handleShowBackends() throws AnalysisException { final ShowBackendsStmt showStmt = (ShowBackendsStmt) stmt; + try { + showStmt.analyze(null); + } catch (Exception e) { + throw (AnalysisException) e; + } List> backendInfos = BackendsProcDir.getBackendInfos(); backendInfos.sort(new Comparator>() { @@ -2277,8 +2282,13 @@ public int compare(List o1, List o2) { resultSet = new ShowResultSet(showStmt.getMetaData(), backendInfos); } - private void handleShowFrontends() { + private void handleShowFrontends() throws AnalysisException { final ShowFrontendsStmt showStmt = (ShowFrontendsStmt) stmt; + try { + showStmt.analyze(null); + } catch (Exception e) { + throw (AnalysisException) e; + } List> infos = Lists.newArrayList(); FrontendsProcNode.getFrontendsInfo(Env.getCurrentEnv(), showStmt.getDetailType(), infos); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java index 04ea7d01eae3dd..a40ec67c0dc72d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/BackendsTableValuedFunction.java @@ -18,9 +18,15 @@ package org.apache.doris.tablefunction; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TBackendsMetadataParams; import org.apache.doris.thrift.TMetaScanRange; import org.apache.doris.thrift.TMetadataType; @@ -83,6 +89,12 @@ public BackendsTableValuedFunction(Map params) throws AnalysisEx if (params.size() != 0) { throw new AnalysisException("backends table-valued-function does not support any params"); } + if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), + InternalCatalog.INTERNAL_CATALOG_NAME, InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT)) { + String message = ErrorCode.ERR_DB_ACCESS_DENIED_ERROR.formatErrorMsg( + PrivPredicate.SELECT.getPrivs().toString(), InfoSchemaDb.DATABASE_NAME); + throw new AnalysisException(message); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java index cc7ff82b8fb0e1..66ac253e88d00c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsDisksTableValuedFunction.java @@ -18,8 +18,14 @@ package org.apache.doris.tablefunction; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFrontendsMetadataParams; import org.apache.doris.thrift.TMetaScanRange; import org.apache.doris.thrift.TMetadataType; @@ -67,6 +73,12 @@ public FrontendsDisksTableValuedFunction(Map params) throws Anal if (params.size() != 0) { throw new AnalysisException("frontends_disks table-valued-function does not support any params"); } + if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), + InternalCatalog.INTERNAL_CATALOG_NAME, InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT)) { + String message = ErrorCode.ERR_DB_ACCESS_DENIED_ERROR.formatErrorMsg( + PrivPredicate.SELECT.getPrivs().toString(), InfoSchemaDb.DATABASE_NAME); + throw new AnalysisException(message); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java index aded1076a83d03..06d323bc66c802 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FrontendsTableValuedFunction.java @@ -18,8 +18,14 @@ package org.apache.doris.tablefunction; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TFrontendsMetadataParams; import org.apache.doris.thrift.TMetaScanRange; import org.apache.doris.thrift.TMetadataType; @@ -76,6 +82,12 @@ public FrontendsTableValuedFunction(Map params) throws AnalysisE if (params.size() != 0) { throw new AnalysisException("frontends table-valued-function does not support any params"); } + if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), + InternalCatalog.INTERNAL_CATALOG_NAME, InfoSchemaDb.DATABASE_NAME, PrivPredicate.SELECT)) { + String message = ErrorCode.ERR_DB_ACCESS_DENIED_ERROR.formatErrorMsg( + PrivPredicate.SELECT.getPrivs().toString(), InfoSchemaDb.DATABASE_NAME); + throw new AnalysisException(message); + } } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java index aeb19ef73ef171..c46c6428b96be3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java @@ -55,7 +55,7 @@ public void testAnalyze() throws Exception { AtomicBoolean privilege = new AtomicBoolean(false); new MockUp() { @Mock - public boolean checkGlobalPriv(ConnectContext ctx, PrivPredicate wanted) { + public boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, PrivPredicate wanted) { return privilege.get(); } }; diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowFrontendsStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowFrontendsStmtTest.java new file mode 100644 index 00000000000000..ebbe555d8cb1fe --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowFrontendsStmtTest.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import mockit.Mock; +import mockit.MockUp; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.jupiter.api.Assertions; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class ShowFrontendsStmtTest { + private Analyzer analyzer; + private ConnectContext ctx = new ConnectContext(); + + @Before + public void setUp() { + analyzer = AccessTestUtil.fetchAdminAnalyzer(true); + ctx.setSessionVariable(new SessionVariable()); + ctx.setThreadLocalInfo(); + } + + @After + public void tearDown() { + ConnectContext.remove(); + } + + @Test + public void testAnalyze() throws Exception { + AtomicBoolean privilege = new AtomicBoolean(false); + new MockUp() { + @Mock + public boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, PrivPredicate wanted) { + return privilege.get(); + } + }; + + ShowFrontendsStmt stmt = new ShowFrontendsStmt(); + Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer)); + + privilege.set(true); + stmt.analyze(analyzer); + } + +} diff --git a/regression-test/suites/auth_call/test_show_backend_auth.groovy b/regression-test/suites/auth_call/test_show_backend_auth.groovy index 13e1fa4fbe480c..627e28c57d1bd7 100644 --- a/regression-test/suites/auth_call/test_show_backend_auth.groovy +++ b/regression-test/suites/auth_call/test_show_backend_auth.groovy @@ -32,14 +32,26 @@ suite("test_show_backend_auth","p0,auth_call") { try_sql("DROP USER ${user}") sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" + sql """grant select_priv on internal.information_schema.* to ${user}""" + def show_grants_result = sql """show grants for ${user}""" + logger.info("show grants result: " + show_grants_result) + sql """revoke select_priv on internal.information_schema.* from ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { + + try { + def show_result = sql """SHOW BACKENDS""" + logger.info("show_result: " + show_result) + } catch (Exception e) { + logger.info("show_result: " + e) + e.printStackTrace() + } test { sql """SHOW BACKENDS""" exception "denied" } } - sql """grant node_priv on *.*.* to ${user}""" + sql """grant select_priv on internal.information_schema.* to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { def res = sql """SHOW BACKENDS""" assertTrue(res.size() > 0) diff --git a/regression-test/suites/auth_call/test_show_frontend_auth.groovy b/regression-test/suites/auth_call/test_show_frontend_auth.groovy index 87f8e9c7c09a68..120698039705b2 100644 --- a/regression-test/suites/auth_call/test_show_frontend_auth.groovy +++ b/regression-test/suites/auth_call/test_show_frontend_auth.groovy @@ -32,8 +32,19 @@ suite("test_show_frontend_auth","p0,auth_call") { try_sql("DROP USER ${user}") sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" + sql """grant select_priv on internal.information_schema.* to ${user}""" + def show_grants_result = sql """show grants for ${user}""" + logger.info("show grants result: " + show_grants_result) + sql """revoke select_priv on internal.information_schema.* from ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { + try { + def show_result = sql """SHOW frontends""" + logger.info("show_result: " + show_result) + } catch (Exception e) { + logger.info("show_result: " + e) + e.printStackTrace() + } test { sql """SHOW frontends""" exception "denied" @@ -43,7 +54,7 @@ suite("test_show_frontend_auth","p0,auth_call") { exception "denied" } } - sql """grant node_priv on *.*.* to ${user}""" + sql """grant select_priv on internal.information_schema.* to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { def res = sql """SHOW frontends""" assertTrue(res.size() > 0) From d3a7f6186fb0d22b67d099f61b2b317e1e8190d5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:33:54 +0800 Subject: [PATCH 139/572] branch-3.0: [improve](hive row count)Log stack info when fetch hive row count fail. #52795 (#52839) Cherry-picked from #52795 Co-authored-by: James --- .../apache/doris/datasource/ExternalRowCountCache.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index fc955c4964a30c..b30c14f695ba59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -86,9 +86,15 @@ protected Optional doLoad(RowCountKey rowCountKey) { TableIf table = StatisticsUtil.findTable(rowCountKey.catalogId, rowCountKey.dbId, rowCountKey.tableId); return Optional.of(table.fetchRowCount()); } catch (Exception e) { - LOG.warn("Failed to get table row count with catalogId {}, dbId {}, tableId {}. Reason {}", + String message = String.format("Failed to get table row count with catalogId %s, dbId %s, tableId %s. " + + "Reason %s", rowCountKey.catalogId, rowCountKey.dbId, rowCountKey.tableId, e.getMessage()); - LOG.debug(e); + if (LOG.isDebugEnabled()) { + LOG.debug(message, e); + } else { + LOG.warn(message); + } + // Return Optional.empty() will cache this empty value in memory, // so we can't try to load the row count until the cache expire. // Throw an exception here will cause too much stack log in fe.out. From ad3ae7c96405d60f79949e1119a585082f806acd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:34:57 +0800 Subject: [PATCH 140/572] branch-3.0: [fix](cache) Add conf `segment_cache_enable_prune` #52513 (#52641) Cherry-picked from #52513 Co-authored-by: Xinyi Zou --- be/src/common/config.cpp | 1 + be/src/common/config.h | 2 ++ be/src/olap/segment_loader.h | 3 ++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index dc81ca48373dd2..9f14823e97523c 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1192,6 +1192,7 @@ DEFINE_Int32(segment_cache_capacity, "-1"); DEFINE_Int32(segment_cache_fd_percentage, "20"); DEFINE_mInt32(estimated_mem_per_column_reader, "512"); DEFINE_Int32(segment_cache_memory_percentage, "5"); +DEFINE_Bool(enable_segment_cache_prune, "true"); // enable feature binlog, default false DEFINE_Bool(enable_feature_binlog, "false"); diff --git a/be/src/common/config.h b/be/src/common/config.h index a182049472561e..34fee6cabed955 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1231,6 +1231,8 @@ DECLARE_mInt32(schema_cache_sweep_time_sec); DECLARE_Int32(segment_cache_capacity); DECLARE_Int32(segment_cache_fd_percentage); DECLARE_Int32(segment_cache_memory_percentage); +DECLARE_Bool(enable_segment_cache_prune); + DECLARE_mInt32(estimated_mem_per_column_reader); // enable binlog diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h index 2c5b1ed200dde7..792e66fdd7f8c0 100644 --- a/be/src/olap/segment_loader.h +++ b/be/src/olap/segment_loader.h @@ -83,7 +83,8 @@ class SegmentCache : public LRUCachePolicy { SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit) : LRUCachePolicy(CachePolicy::CacheType::SEGMENT_CACHE, memory_bytes_limit, LRUCacheType::SIZE, config::tablet_rowset_stale_sweep_time_sec, - DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit) {} + DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit, + config::enable_segment_cache_prune) {} // Lookup the given segment in the cache. // If the segment is found, the cache entry will be written into handle. From 67fb00de80cee23c3ae4bb06d810416ffd24abae Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Mon, 7 Jul 2025 14:35:58 +0800 Subject: [PATCH 141/572] branch-3.0: [fix](type) Fix TYPE_DATETIMEV2 upgrade and downgrade compatibility (#52533) Introduced from https://github.com/apache/doris/pull/49643 --- be/src/runtime/types.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp index f0112ebd2bcd89..b64fd139eacd4e 100644 --- a/be/src/runtime/types.cpp +++ b/be/src/runtime/types.cpp @@ -167,6 +167,8 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const { scalar_type.__set_scale(scale); } else if (type == TYPE_DATETIMEV2) { DCHECK_NE(scale, -1); + scalar_type.__set_precision( + precision); // -1, adapting upgrade and downgrade compatibility. scalar_type.__set_scale(scale); } } @@ -187,6 +189,7 @@ void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const { scalar_type->set_scale(scale); } else if (type == TYPE_DATETIMEV2) { DCHECK_NE(scale, -1); + scalar_type->set_precision(precision); // -1, adapting upgrade and downgrade compatibility. scalar_type->set_scale(scale); } else if (type == TYPE_ARRAY) { node->set_type(TTypeNodeType::ARRAY); From 3a6a2b681276a7bce657709f3339c860f329296a Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 7 Jul 2025 14:38:02 +0800 Subject: [PATCH 142/572] branch-3.0 [fix](nereids) fix bug in LogicalIntersect datatrait OutofBounds exception (#52170) pick part of #49947 --- .../doris/nereids/trees/plans/logical/LogicalIntersect.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java index 06f0f7d0ce6c4e..610a40ac96af3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIntersect.java @@ -107,9 +107,7 @@ public LogicalIntersect withNewOutputs(List newOutputs) { Map constructReplaceMap() { Map replaceMap = new HashMap<>(); for (int i = 0; i < children.size(); i++) { - List originOutputs = this.regularChildrenOutputs.size() == children.size() - ? child(i).getOutput() - : regularChildrenOutputs.get(i); + List originOutputs = regularChildrenOutputs.get(i); for (int j = 0; j < originOutputs.size(); j++) { replaceMap.put(originOutputs.get(j), getOutput().get(j)); } From 5e3a14cb8950b3019b15f9ac6c823cf057774cb9 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 7 Jul 2025 14:40:42 +0800 Subject: [PATCH 143/572] branch-3.0: [fix](inverted index) fix error when writing empty index file #51984 (#52861) cherry pick from #51984 --- be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index fd6ae6d54aff76..36d82bad1f2d51 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -205,6 +205,10 @@ void InvertedIndexFileWriter::copyFile(const char* fileName, lucene::store::Dire err.set(CL_ERR_IO, "debug point: copyFile_openInput_error"); }); if (!open) { + if (err.number() == CL_ERR_EmptyIndexSegment) { + LOG(WARNING) << "InvertedIndexFileWriter::copyFile: " << fileName << " is empty"; + return; + } throw err; } From 774808f7ec154d4b39d2bfc72371ccae0c748c2f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:43:39 +0800 Subject: [PATCH 144/572] branch-3.0: [fix](be java) be hang while enable_java_support #52818 (#52823) Cherry-picked from #52818 Co-authored-by: camby <104178625@qq.com> --- be/src/olap/rowset/rowset_meta.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp index c9851cdc5fc64b..0da212ea8812d4 100644 --- a/be/src/olap/rowset/rowset_meta.cpp +++ b/be/src/olap/rowset/rowset_meta.cpp @@ -229,7 +229,7 @@ void RowsetMeta::set_segments_key_bounds(const std::vector& segment int32_t truncation_threshold = config::segments_key_bounds_truncation_threshold; if (config::random_segments_key_bounds_truncation) { - static thread_local std::mt19937 generator(std::random_device {}()); + std::mt19937 generator(std::random_device {}()); std::uniform_int_distribution distribution(-10, 40); truncation_threshold = distribution(generator); } From e900594935d3bf9f1e4973b613aaaf3806987e1d Mon Sep 17 00:00:00 2001 From: wangbo Date: Mon, 7 Jul 2025 14:45:25 +0800 Subject: [PATCH 145/572] [branch-3.0]Fix losing audit log when statistics queue is full (#52700) ### What problem does this PR solve? pick #47358 Issue Number: close #xx --- .../workloadschedpolicy/WorkloadRuntimeStatusMgr.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java index 0c2e4f458bc391..cb8d2bbaaac808 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadRuntimeStatusMgr.java @@ -109,11 +109,12 @@ protected void runAfterCatalogReady() { public void submitFinishQueryToAudit(AuditEvent event) { queryAuditEventLogWriteLock(); try { - if (queryAuditEventList.size() >= Config.audit_event_log_queue_size) { + if (queryAuditEventList.size() > Config.audit_event_log_queue_size) { LOG.warn("audit log event queue size {} is full, this may cause audit log missing statistics." - + "you can check whether qps is too high or " - + "set audit_event_log_queue_size to a larger value in fe.conf. query id: {}", + + "you can check whether qps is too high " + + "or reset audit_event_log_queue_size. query id: {}", queryAuditEventList.size(), event.queryId); + Env.getCurrentAuditEventProcessor().handleAuditEvent(event); return; } event.pushToAuditLogQueueTime = System.currentTimeMillis(); From 269691727f36618a0e1da44edbc6bbbd6c86bb0d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:47:45 +0800 Subject: [PATCH 146/572] branch-3.0: [fix](ci) exclude case with known issue of memory leak #52784 (#52803) Cherry-picked from #52784 Co-authored-by: Dongyang Li --- .../pipeline/nonConcurrent/conf/regression-conf.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy index c1c1ad91865aed..954beab5fa313f 100644 --- a/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy +++ b/regression-test/pipeline/nonConcurrent/conf/regression-conf.groovy @@ -64,6 +64,7 @@ excludeGroups = "p1,p2" // this suites will not be executed excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line "test_write_inverted_index_exception_fault_injection," + // cause core dump + "test_index_compaction_exception_fault_injection," + // cause memory leak "zzz_the_end_sentinel_do_not_touch"// keep this line as the last line // this directories will not be executed From dd3595aa94f5ac12b73a6af9eb659e0db49f781a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:48:22 +0800 Subject: [PATCH 147/572] branch-3.0: [fix](move-memtable) gracefully shutdown flush token in TabletStream::pre_close #52719 (#52779) Cherry-picked from #52719 Co-authored-by: Kaijie Chen --- be/src/runtime/load_stream.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp index 1a386341f891a7..42db372c1708e9 100644 --- a/be/src/runtime/load_stream.cpp +++ b/be/src/runtime/load_stream.cpp @@ -296,6 +296,8 @@ Status TabletStream::_run_in_heavy_work_pool(std::function fn) { void TabletStream::pre_close() { if (!_status.ok()) { + // cancel all pending tasks, wait all running tasks to finish + _flush_token->shutdown(); return; } From ecb241af10a17bc2b6266b9c985c1ee439ef498c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:50:00 +0800 Subject: [PATCH 148/572] branch-3.0: [regression-test](Variant) add escaped chars cases #52657 (#52697) Cherry-picked from #52657 Co-authored-by: lihangyu --- .../data/variant_p0/escaped_chars.out | 31 ++++++++ .../suites/variant_p0/escaped_chars.groovy | 71 +++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 regression-test/data/variant_p0/escaped_chars.out create mode 100644 regression-test/suites/variant_p0/escaped_chars.groovy diff --git a/regression-test/data/variant_p0/escaped_chars.out b/regression-test/data/variant_p0/escaped_chars.out new file mode 100644 index 00000000000000..ddfa3abdfc974e --- /dev/null +++ b/regression-test/data/variant_p0/escaped_chars.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 {"a":123,"b":"test with escape \\" characters"} +2 {"a":456,"b":"another test with escape \\\\ characters"} +3 {"a":789,"b":"test with single quote ' characters"} +4 {"a":101112,"b":"test with newline \\n characters"} +5 {"a":131415,"b":"test with tab \\t characters"} +6 {"a":161718,"b":"test with backslash \\b characters"} + +-- !select -- +test with escape " characters +another test with escape \\ characters +test with single quote ' characters +test with newline \n characters +test with tab \ characters +test with backslash  characters + +-- !select -- +test with escape " characters +another test with escape \\ characters +test with single quote ' characters +test with newline \n characters +test with tab \ characters +test with backslash  characters + +-- !select -- +1 {"c_json":{"a":"a\\nb"}} {"c_json":{"quote":"\\"Helvetica tofu try-hard gluten-free gentrify leggings.\\" - Remington Trantow"}} {"c_json":{"quote":"\\"Helvetica tofu try-hard gluten-free gentrify leggings.\\" - Remington Trantow"}} {"c_json":{"a":"a\\nb"}} + +-- !select -- +{"a":"a\\nb"} {"a":"a\\nb"} + diff --git a/regression-test/suites/variant_p0/escaped_chars.groovy b/regression-test/suites/variant_p0/escaped_chars.groovy new file mode 100644 index 00000000000000..f92797d03bbdef --- /dev/null +++ b/regression-test/suites/variant_p0/escaped_chars.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("regression_test_variant_escaped_chars", "p0"){ + def tableName = "variant_escape_chars" + + sql """ DROP TABLE IF EXISTS variant_escape_chars """ + + sql """ + CREATE TABLE IF NOT EXISTS variant_escape_chars ( + `id` INT, + `description` VARIANT + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'This is a test table with escape characters in description' + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO variant_escape_chars VALUES + (1, '{"a" : 123, "b" : "test with escape \\\\" characters"}'), + (2, '{"a" : 456, "b" : "another test with escape \\\\\\\\ characters"}'), + (3, '{"a" : 789, "b" : "test with single quote \\\' characters"}'), + (4, '{"a" : 101112, "b" : "test with newline \\\\n characters"}'), + (5, '{"a" : 131415, "b" : "test with tab \\\\t characters"}'), + (6, '{"a" : 161718, "b" : "test with backslash \\\\b characters"}'); + """ + + // test json value with escaped characters + qt_select """ SELECT * FROM variant_escape_chars ORDER BY id """ + qt_select """ SELECT description['b'] FROM variant_escape_chars ORDER BY id """ + qt_select """ SELECT CAST(description['b'] AS TEXT) FROM variant_escape_chars ORDER BY id """ + + sql """ + drop table if exists t01; + create table t01(id int, b json, c json, d variant, e variant) properties ("replication_num" = "1"); + insert into t01 values (1, '{"c_json":{"a":"a\\\\nb"}}', '{"c_json": {"quote":"\\\\"Helvetica tofu try-hard gluten-free gentrify leggings.\\\\" - Remington Trantow"}}', '{"c_json": {"quote":"\\\\"Helvetica tofu try-hard gluten-free gentrify leggings.\\\\" - Remington Trantow"}}', '{"c_json":{"a":"a\\\\nb"}}'); + """ + qt_select """ SELECT * FROM t01 """ + qt_select """select json_extract(b, "\$.c_json"), e["c_json"] from t01;""" + + // test json keys with escaped characters, FIXED in 3.1.0 + // sql "truncate table variant_escape_chars" + // sql """ + // INSERT INTO variant_escape_chars VALUES + // (1, '{"test with escape \\\\" characters" : 123}'), + // (2, '{"another test with escape \\\\\\\\ characters" : 123}'), + // (3, '{"test with single quote \\\' characters" : 123}'), + // (4, '{"test with newline \\\\n characters":123}'), + // (5, '{"test with tab \\\\t characters" : 123}'), + // (6, '{"test with backslash \\\\b characters" : 123}'); + // """ + // qt_select """ SELECT * FROM variant_escape_chars ORDER BY id """ +} From ed8a4cfa45d3034b84bb8b9a661ff822bd86a1de Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:50:36 +0800 Subject: [PATCH 149/572] branch-3.0: [fix](nereids) fix tanh cosntant folding #52100 (#52526) Cherry-picked from #52100 Co-authored-by: admiring_xm <86338598+wumeibanfa@users.noreply.github.com> --- .../expressions/functions/executable/NumericArithmetic.java | 2 +- .../fold_constant/fold_constant_numeric_arithmatic.groovy | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java index 57eda59ee7d137..1368d8adcf7049 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java @@ -1050,7 +1050,7 @@ public static Expression cosh(DoubleLiteral first) { /** * tanh */ - @ExecFunction(name = "cosh") + @ExecFunction(name = "tanh") public static Expression tanh(DoubleLiteral first) { return checkOutputBoundary(new DoubleLiteral(Math.tanh(first.getValue()))); } diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy index bc1cb91ca9d243..7aa09d8a18666c 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy @@ -470,6 +470,9 @@ suite("fold_constant_numeric_arithmatic") { testFoldConst("SELECT TANH(0) AS tanh_case_1") //tanh(0) = 0 testFoldConst("SELECT TANH(1) AS tanh_case_2") //tanh(1) testFoldConst("SELECT TANH(-1) AS tanh_case_3") //tanh(-1) + testFoldConst("SELECT TANH(NULL)") + testFoldConst("SELECT TANH(-0.5), TANH(0.5), TANH(10), TANH(-10)") + testFoldConst("SELECT TANH(-20), TANH(20), TANH(1E-7), TANH(-1E-7)") //Truncate function cases testFoldConst("SELECT TRUNCATE(123.456, 2) AS truncate_case_1") //truncate(123.456, 2) = 123.45 From 80c48d69f390bbf5a057bd65b7b54437031b9826 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Mon, 7 Jul 2025 14:52:30 +0800 Subject: [PATCH 150/572] branch-3.0: [opt](explain) use full qualified name in scan node (#52342) (#52706) bp #52342 --- .../org/apache/doris/catalog/TableIf.java | 13 ++++-- .../apache/doris/datasource/FileScanNode.java | 2 +- .../iceberg/test_iceberg_filter.groovy | 1 + .../tvf/test_tvf_view.groovy | 15 ++++++- .../tvf/test_tvf_view_count.groovy | 42 ------------------- 5 files changed, 26 insertions(+), 47 deletions(-) delete mode 100644 regression-test/suites/external_table_p0/tvf/test_tvf_view_count.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index c185284744e468..a45e5fa47a5586 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -488,9 +488,15 @@ default List getFullQualifiers() { } default String getNameWithFullQualifiers() { - return String.format("%s.%s.%s", getDatabase().getCatalog().getName(), - ClusterNamespace.getNameFromFullName(getDatabase().getFullName()), - getName()); + DatabaseIf db = getDatabase(); + // Some kind of table like FunctionGenTable does not belong to any database + if (db == null) { + return "null.null." + getName(); + } else { + return db.getCatalog().getName() + + "." + ClusterNamespace.getNameFromFullName(db.getFullName()) + + "." + getName(); + } } default boolean isManagedTable() { @@ -522,3 +528,4 @@ default boolean isPartitionedTable() { TableIndexes getTableIndexes(); } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index b7d34312313308..6ce56be6f94205 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -92,7 +92,7 @@ public long getPushDownCount() { @Override public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); - output.append(prefix).append("table: ").append(desc.getTable().getName()).append("\n"); + output.append(prefix).append("table: ").append(desc.getTable().getNameWithFullQualifiers()).append("\n"); if (!conjuncts.isEmpty()) { output.append(prefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n"); } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_filter.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_filter.groovy index 7e654175f9cbe3..de75a204eacb64 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_filter.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_filter.groovy @@ -64,6 +64,7 @@ suite("test_iceberg_filter", "p0,external,doris,external_docker,external_docker_ explain { sql("select * from ${tb_ts_filter} where ts < '2024-05-30 20:34:56'") contains "inputSplitNum=0" + contains "table: test_iceberg_filter.multi_catalog.tb_ts_filter" } explain { sql("select * from ${tb_ts_filter} where ts < '2024-05-30 20:34:56.12'") diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_view.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_view.groovy index 25f70961074755..0a59397bab466f 100644 --- a/regression-test/suites/external_table_p0/tvf/test_tvf_view.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_tvf_view.groovy @@ -44,6 +44,7 @@ suite("test_tvf_view", "p0,external,tvf,external_docker,hive") { contains("_table_valued_function_hdfs.p_container") contains("_table_valued_function_hdfs.p_retailprice") contains("_table_valued_function_hdfs.p_comment") + contains("table: null.null.HDFSTableValuedFunction") } explain{ sql("select * from hdfs (\n" + @@ -61,7 +62,19 @@ suite("test_tvf_view", "p0,external,tvf,external_docker,hive") { contains("_table_valued_function_hdfs.p_comment") } - sql """drop database if exists test_tvf_view_p2""" + sql """create view tvf_view_count as select * from hdfs ( + "uri"="hdfs://${nameNodeHost}:${hdfsPort}/user/doris/tpch1.db/tpch1_parquet/part/part-00000-cb9099f7-a053-4f9a-80af-c659cfa947cc-c000.snappy.parquet", + "hadoop.username" = "hadoop", + "format"="parquet");""" + + explain { + verbose true + sql("select count(1) from tvf_view_count") + contains "SlotDescriptor{id=0," + notContains "SlotDescriptor{id=1," + } + + // sql """drop database if exists test_tvf_view_p2""" } } diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_view_count.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_view_count.groovy deleted file mode 100644 index d0d6d80fd5eba2..00000000000000 --- a/regression-test/suites/external_table_p0/tvf/test_tvf_view_count.groovy +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_tvf_view_count", "p0,external,tvf,external_docker,hive") { - String enabled = context.config.otherConfigs.get("enableHiveTest") - if (enabled != null && enabled.equalsIgnoreCase("true")) { - String nameNodeHost = context.config.otherConfigs.get("externalEnvIp") - String hdfsPort = context.config.otherConfigs.get("hive2HdfsPort") - - sql """drop database if exists test_tvf_view_count_p2""" - sql """create database test_tvf_view_count_p2""" - sql """use test_tvf_view_count_p2""" - sql """create view tvf_view_count as select * from hdfs ( - "uri"="hdfs://${nameNodeHost}:${hdfsPort}/user/doris/tpch1.db/tpch1_parquet/part/part-00000-cb9099f7-a053-4f9a-80af-c659cfa947cc-c000.snappy.parquet", - "hadoop.username" = "hadoop", - "format"="parquet");""" - - explain { - verbose true - sql("select count(1) from tvf_view_count") - contains "SlotDescriptor{id=0," - notContains "SlotDescriptor{id=1," - } - - sql """drop database if exists test_tvf_view_count_p2""" - } -} - From e78970fbef898b7af99b166eccd54e0b887b4ea5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:54:19 +0800 Subject: [PATCH 151/572] branch-3.0: [Exec](alias) support stddev alias to std #52819 (#52838) Cherry-picked from #52819 Co-authored-by: HappenLee --- .../vec/aggregate_functions/aggregate_function_stddev.cpp | 1 + .../java/org/apache/doris/analysis/FunctionCallExpr.java | 3 ++- .../apache/doris/catalog/BuiltinAggregateFunctions.java | 2 +- .../main/java/org/apache/doris/catalog/FunctionSet.java | 8 ++++++++ .../doris/nereids/rules/analysis/GenerateFunction.java | 1 + .../aggregate_functions/test_aggregate_all_functions.out | 5 +++++ .../test_aggregate_all_functions.groovy | 1 + 7 files changed, 19 insertions(+), 2 deletions(-) diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp index f9fe2dca748b20..5c9127f39e91dc 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp +++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.cpp @@ -91,6 +91,7 @@ void register_aggregate_function_stddev_variance_pop(AggregateFunctionSimpleFact factory.register_alias("variance", "variance_pop"); factory.register_function_both("stddev", create_aggregate_function_stddev_pop); factory.register_alias("stddev", "stddev_pop"); + factory.register_alias("stddev", "std"); } void register_aggregate_function_stddev_variance_samp_old(AggregateFunctionSimpleFactory& factory) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 62353ba32879c5..cc231312d2ac51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -81,7 +81,8 @@ public class FunctionCallExpr extends Expr { public static final ImmutableSet STDDEV_FUNCTION_SET = new ImmutableSortedSet.Builder( String.CASE_INSENSITIVE_ORDER) .add("stddev").add("stddev_val").add("stddev_samp").add("stddev_pop").add("variance").add("variance_pop") - .add("variance_pop").add("var_samp").add("var_pop").add("variance_samp").add("avg_weighted").build(); + .add("variance_pop").add("var_samp").add("var_pop").add("variance_samp").add("avg_weighted") + .add("std").build(); public static final Map, Type, Type>> PRECISION_INFER_RULE; public static final java.util.function.BiFunction, Type, Type> DEFAULT_PRECISION_INFER_RULE; public static final ImmutableSet ROUND_FUNCTION_SET = new ImmutableSortedSet.Builder( diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java index 4535ebdd02d007..c1f31f79d40250 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinAggregateFunctions.java @@ -150,7 +150,7 @@ public class BuiltinAggregateFunctions implements FunctionHelper { agg(SequenceCount.class, "sequence_count"), agg(SequenceMatch.class, "sequence_match"), agg(Skew.class, "skew", "skew_pop", "skewness"), - agg(Stddev.class, "stddev_pop", "stddev"), + agg(Stddev.class, "stddev_pop", "stddev", "std"), agg(StddevSamp.class, "stddev_samp"), agg(Sum.class, "sum"), agg(Sum0.class, "sum0"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index 926ab77ac650d5..de1d1178ed65c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -1196,6 +1196,14 @@ private void initAggregateBuiltins() { null, null, null, "", false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin("stddev", + Lists.newArrayList(t), STDDEV_RETTYPE_SYMBOL.get(t), t, + "", + "", + "", + null, null, null, + "", + false, true, false, true)); addBuiltin(AggregateFunction.createBuiltin("stddev_samp", Lists.newArrayList(t), STDDEV_RETTYPE_SYMBOL.get(t), t, "", diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/GenerateFunction.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/GenerateFunction.java index 105ab00f39502b..0c189a7ca6a2c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/GenerateFunction.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/GenerateFunction.java @@ -182,6 +182,7 @@ public class GenerateFunction { .put("any", "any_value") .put("char_length", "character_length") .put("stddev_pop", "stddev") + .put("std", "stddev") .put("percentile_cont", "percentile") .put("var_pop", "variance") .put("variance_pop", "variance") diff --git a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out index 90953b0a11c84f..55298776e38037 100644 --- a/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out +++ b/regression-test/data/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.out @@ -211,6 +211,11 @@ beijing chengdu shanghai 2 216.5 3 36.0 +-- !select29 -- +1 0.0 +2 216.5 +3 36.0 + -- !select30 -- 1 0.0 2 216.5 diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy index e9683ffb92bbbc..06b8ba8e1ef2d4 100644 --- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy +++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_aggregate_all_functions.groovy @@ -381,6 +381,7 @@ suite("test_aggregate_all_functions", "arrow_flight_sql") { sql "INSERT INTO ${tableName_15} values(1,10), (2,8), (2,441) ,(1,10) ,(3,29) ,(3,101)" qt_select29 "select id,stddev(level) from ${tableName_15} group by id order by id" + qt_select29 "select id,std(level) from ${tableName_15} group by id order by id" qt_select30 "select id,stddev_pop(level) from ${tableName_15} group by id order by id" sql "DROP TABLE IF EXISTS ${tableName_15}" From 4edfac512057a503925c79a183139f128bb9225d Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 7 Jul 2025 19:32:54 +0800 Subject: [PATCH 152/572] [Feature](function) support like with escape clause (#52146) (#52540) pick from #52146 --- be/src/vec/functions/like.cpp | 35 ++++++++- be/src/vec/functions/like.h | 30 +++++++- .../org/apache/doris/nereids/DorisLexer.g4 | 1 + .../org/apache/doris/nereids/DorisParser.g4 | 4 +- .../nereids/parser/LogicalPlanBuilder.java | 14 ++-- .../expression/rules/LikeToEqualRewrite.java | 4 ++ .../doris/nereids/trees/expressions/Like.java | 55 +++++++++++++- .../string_functions/test_like_escape.out | 40 +++++++++++ .../string_functions/test_like_escape.groovy | 71 +++++++++++++++++++ 9 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_like_escape.out create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_like_escape.groovy diff --git a/be/src/vec/functions/like.cpp b/be/src/vec/functions/like.cpp index dcb4fc6dd0dbe3..4ed14280e4c51e 100644 --- a/be/src/vec/functions/like.cpp +++ b/be/src/vec/functions/like.cpp @@ -669,16 +669,30 @@ VPatternSearchStateSPtr FunctionLikeBase::pattern_type_recognition(const ColumnS Status FunctionLikeBase::vector_non_const(const ColumnString& values, const ColumnString& patterns, ColumnUInt8::Container& result, LikeState* state, size_t input_rows_count) const { + ColumnString::MutablePtr replaced_patterns; VPatternSearchStateSPtr vector_search_state; if (state->is_like_pattern) { - vector_search_state = pattern_type_recognition(patterns); + if (state->has_custom_escape) { + replaced_patterns = ColumnString::create(); + for (int i = 0; i < input_rows_count; ++i) { + std::string val = + replace_pattern_by_escape(patterns.get_data_at(i), state->escape_char); + replaced_patterns->insert_data(val.c_str(), val.size()); + } + vector_search_state = pattern_type_recognition(*replaced_patterns); + } else { + vector_search_state = pattern_type_recognition(patterns); + } } else { vector_search_state = pattern_type_recognition(patterns); } + + const ColumnString& real_pattern = state->has_custom_escape ? *replaced_patterns : patterns; + if (vector_search_state == nullptr) { // pattern type recognition failed, use default case for (int i = 0; i < input_rows_count; ++i) { - const auto pattern_val = patterns.get_data_at(i); + const auto pattern_val = real_pattern.get_data_at(i); const auto value_val = values.get_data_at(i); RETURN_IF_ERROR((state->scalar_function)(&state->search_state, value_val, pattern_val, &result[i])); @@ -815,7 +829,12 @@ void verbose_log_match(const std::string& str, const std::string& pattern_name, Status FunctionLike::construct_like_const_state(FunctionContext* context, const StringRef& pattern, std::shared_ptr& state, bool try_hyperscan) { - std::string pattern_str = pattern.to_string(); + std::string pattern_str; + if (state->has_custom_escape) { + pattern_str = replace_pattern_by_escape(pattern, state->escape_char); + } else { + pattern_str = pattern.to_string(); + } state->search_state.pattern_str = pattern_str; std::string search_string; @@ -920,6 +939,16 @@ Status FunctionLike::open(FunctionContext* context, FunctionContext::FunctionSta state->is_like_pattern = true; state->function = like_fn; state->scalar_function = like_fn_scalar; + if (context->is_col_constant(2)) { + state->has_custom_escape = true; + const auto escape_col = context->get_constant_col(2)->column_ptr; + const auto& escape = escape_col->get_data_at(0); + if (escape.size != 1) { + return Status::InternalError("Escape character must be a single character, got: {}", + escape.to_string()); + } + state->escape_char = escape.data[0]; + } if (context->is_col_constant(1)) { const auto pattern_col = context->get_constant_col(1)->column_ptr; const auto& pattern = pattern_col->get_data_at(0); diff --git a/be/src/vec/functions/like.h b/be/src/vec/functions/like.h index 435e2742788497..d9f95123cead7d 100644 --- a/be/src/vec/functions/like.h +++ b/be/src/vec/functions/like.h @@ -51,6 +51,31 @@ class Block; namespace doris::vectorized { +inline std::string replace_pattern_by_escape(const StringRef& pattern, char escape_char) { + std::string result; + result.reserve(pattern.size); + for (size_t i = 0; i < pattern.size; ++i) { + if (i + 1 < pattern.size && pattern.data[i] == escape_char && + (pattern.data[i + 1] == escape_char || pattern.data[i + 1] == '%' || + pattern.data[i + 1] == '_')) { + // "^^" -> "^" + // "^%" -> "\%" + // "^_" -> "\_" + if ((pattern.data[i + 1] == '%' || pattern.data[i + 1] == '_')) { + result.push_back('\\'); + } + result.push_back(pattern.data[i + 1]); + ++i; // skip next char + } else if (pattern.data[i] == '\\') { + // "\" -> "\\" + result.append("\\\\"); + } else { + result.push_back(pattern.data[i]); + } + } + return result; +} + // TODO: replace with std::string_view when `LikeSearchState.substring_pattern` can // construct from std::string_view. struct LikeSearchState { @@ -123,6 +148,8 @@ using VectorLikeFn = std::function; class FunctionLikeBase : public IFunction { public: - size_t get_number_of_arguments() const override { return 2; } + size_t get_number_of_arguments() const override { return 0; } + bool is_variadic() const override { return true; } DataTypePtr get_return_type_impl(const DataTypes& /*arguments*/) const override { return std::make_shared(); diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 47a45b67aa7b36..0fe651d9675523 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -235,6 +235,7 @@ ENGINE: 'ENGINE'; ENGINES: 'ENGINES'; ENTER: 'ENTER'; ERRORS: 'ERRORS'; +ESCAPE: 'ESCAPE'; EVENTS: 'EVENTS'; EVERY: 'EVERY'; EXCEPT: 'EXCEPT'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 7b3ecbafa7e27e..bc205f6686f817 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1455,7 +1455,8 @@ rowConstructorItem predicate : NOT? kind=BETWEEN lower=valueExpression AND upper=valueExpression - | NOT? kind=(LIKE | REGEXP | RLIKE) pattern=valueExpression + | NOT? kind=(REGEXP | RLIKE) pattern=valueExpression + | NOT? kind=LIKE pattern=valueExpression (ESCAPE escape=valueExpression)? | NOT? kind=(MATCH | MATCH_ANY | MATCH_ALL | MATCH_PHRASE | MATCH_PHRASE_PREFIX | MATCH_REGEXP | MATCH_PHRASE_EDGE) pattern=valueExpression | NOT? kind=IN LEFT_PAREN query RIGHT_PAREN | NOT? kind=IN LEFT_PAREN expression (COMMA expression)* RIGHT_PAREN @@ -1893,6 +1894,7 @@ nonReserved | ENGINE | ENGINES | ERRORS + | ESCAPE | EVENTS | EVERY | EXCLUDE diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 6d21441c95f3a4..39dc52a03fd1f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3505,10 +3505,16 @@ private Expression withPredicate(Expression valueExpression, PredicateContext ct } break; case DorisParser.LIKE: - outExpression = new Like( - valueExpression, - getExpression(ctx.pattern) - ); + if (ctx.ESCAPE() == null) { + outExpression = new Like( + valueExpression, + getExpression(ctx.pattern)); + } else { + outExpression = new Like( + valueExpression, + getExpression(ctx.pattern), + getExpression(ctx.escape)); + } break; case DorisParser.RLIKE: case DorisParser.REGEXP: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/LikeToEqualRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/LikeToEqualRewrite.java index e2836204cdc033..e532deb3901a31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/LikeToEqualRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/LikeToEqualRewrite.java @@ -51,6 +51,10 @@ private static Expression rewriteLikeToEqual(Like like) { StringBuilder sb = new StringBuilder(); int len = str.length(); char escapeChar = '\\'; + if (like.arity() == 3) { + escapeChar = ((VarcharLiteral) like.child(2)).value.charAt(0); + } + for (int i = 0; i < len;) { char c = str.charAt(i); if (c == escapeChar && (i + 1) < len diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java index 84b6ffa984fff4..10f25fb0ebca98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Like.java @@ -17,7 +17,12 @@ package org.apache.doris.nereids.trees.expressions; +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.VarcharType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -28,10 +33,20 @@ * like expression: a like 'xxx%'. */ public class Like extends StringRegexPredicate { + + private static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(BooleanType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT), + FunctionSignature.ret(BooleanType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, + VarcharType.SYSTEM_DEFAULT)); + public Like(Expression left, Expression right) { this(ImmutableList.of(left, right)); } + public Like(Expression left, Expression right, Expression escape) { + this(ImmutableList.of(left, right, escape)); + } + private Like(List children) { this(children, false); } @@ -40,9 +55,32 @@ private Like(List children, boolean inferred) { super("like", children, inferred); } + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public String computeToSql() { + if (arity() == 2) { + return super.computeToSql(); + } + return '(' + left().toSql() + ' ' + getName() + ' ' + right().toSql() + " escape " + child(2).toSql() + + ')'; + } + + @Override + public String toString() { + if (arity() == 2) { + return super.computeToSql(); + } + return "(" + left() + " " + getName() + " " + right() + " escape " + child(2) + + ")"; + } + @Override public Like withChildren(List children) { - Preconditions.checkArgument(children.size() == 2); + Preconditions.checkArgument(children.size() == 2 || children.size() == 3); return new Like(children); } @@ -54,4 +92,19 @@ public R accept(ExpressionVisitor visitor, C context) { public Expression withInferred(boolean inferred) { return new Like(this.children, inferred); } + + @Override + public void checkLegalityBeforeTypeCoercion() { + if (arity() == 3) { + if (child(2) instanceof StringLikeLiteral) { + String escapeChar = ((StringLikeLiteral) child(2)).getStringValue(); + if (escapeChar.getBytes().length != 1) { + throw new AnalysisException( + "like escape character must be a single ascii character: " + escapeChar); + } + } else { + throw new AnalysisException("like escape character must be a string literal: " + this.toSql()); + } + } + } } diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_like_escape.out b/regression-test/data/query_p0/sql_functions/string_functions/test_like_escape.out new file mode 100644 index 00000000000000..59f0007906ef3e --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_like_escape.out @@ -0,0 +1,40 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test -- +true + +-- !test -- +true + +-- !test -- +true + +-- !test -- +true + +-- !test -- +true + +-- !test -- +false + +-- !test -- +false + +-- !test -- +true + +-- !test -- +true + +-- !test -- +false + +-- !test -- +true + +-- !test -- +false + +-- !test -- +true + diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_like_escape.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_like_escape.groovy new file mode 100644 index 00000000000000..106d2709a850b7 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_like_escape.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_like_escapes") { + qt_test """ + select "%a" like "a%_" ESCAPE "a"; + """ + qt_test """ + select "%_" like "a%_" ESCAPE "a"; + """ + qt_test """ + select "a" like "a" ESCAPE "a"; + """ + qt_test """ + select "a" like "aa" ESCAPE "a"; + """ + qt_test """ + select "%a" like "a%a" ESCAPE "a"; + """ + qt_test """ + select "%_" like "a%a" ESCAPE "a"; + """ + qt_test """ + select "%a" like "a%a_" ESCAPE "a"; + """ + qt_test """ + select "%_" like "a%a_" ESCAPE "a"; + """ + + test { + sql """select "啊啊" like "啊啊" ESCAPE "啊";""" + exception "like escape character must be a single ascii character" + } + test { + sql """select "a" like "aa" ESCAPE "aa";""" + exception "like escape character must be a single ascii character" + } + test { + sql """select "a" like "aa" ESCAPE 1;""" + exception "like escape character must be a string literal" + } + qt_test """ + select "啊%a" like "啊a%_" ESCAPE "a"; + """ + qt_test """ + select "%a" like "a%_" ESCAPE "A"; + """ + qt_test """ + select "\\\\" like "\\\\%" ESCAPE "A"; + """ + qt_test """ + select "\\\\" like "\\\\A%" ESCAPE "A"; + """ + qt_test """ + select "\\\\%" like "\\\\A%" ESCAPE "A"; + """ +} \ No newline at end of file From 5fe28cc309498b54322780a02e61f3dfd1efbf69 Mon Sep 17 00:00:00 2001 From: Siyang Tang Date: Mon, 7 Jul 2025 20:56:54 +0800 Subject: [PATCH 153/572] [fix](compile) Fix gcc compile error about parentheses (#52869) ### What problem does this PR solve? Problem Summary: ``` 12:05:31 /root/doris/be/src/olap/rowset_version_mgr.cpp: In member function 'doris::Result > doris::BaseTablet::capture_consistent_versions_unlocked(const doris::Version&, const doris::CaptureRowsetOps&) const': 12:05:31 /root/doris/be/src/olap/rowset_version_mgr.cpp:94:29: error: suggest parentheses around '&&' within '||' [-Werror=parentheses] 12:05:31 94 | if (tablet_id != -1 && (tablet_id == _tablet_meta->tablet_id()) || tablet_id == -2) { 12:05:31 | ~~~~~~~~~~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 12:05:31 /root/doris/be/src/util/debug_points.h:41:15: note: in definition of macro 'DBUG_EXECUTE_IF' 12:05:31 41 | { code; } \ 12:05:31 | ^~~~ 12:05:31 cc1plus: all warnings being treated as errors ``` --- be/src/olap/rowset_version_mgr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset_version_mgr.cpp b/be/src/olap/rowset_version_mgr.cpp index 27df2ede4b2dec..731287dcb0049b 100644 --- a/be/src/olap/rowset_version_mgr.cpp +++ b/be/src/olap/rowset_version_mgr.cpp @@ -91,7 +91,7 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch if (skip_by_option && !options.enable_fetch_rowsets_from_peers) { return version_path; } - if (tablet_id != -1 && (tablet_id == _tablet_meta->tablet_id()) || tablet_id == -2) { + if ((tablet_id != -1 && tablet_id == _tablet_meta->tablet_id()) || tablet_id == -2) { return ResultError(Status::Error("version already merged")); } }); From 2f80be49afd32e175bc84eb6843b21491f526cd8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:24:16 +0800 Subject: [PATCH 154/572] branch-3.0: [fix](cloud)Fix modify the cluster public and private network causing the node to be temporarily offline #52294 (#52655) Cherry-picked from #52294 Co-authored-by: deardeng --- .../cloud/catalog/CloudClusterChecker.java | 42 ++++++-- .../regression/suite/SuiteCluster.groovy | 1 + .../multi_cluster/test_change_node_net.groovy | 102 ++++++++++++++++++ 3 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java index 49198536444447..731ab0675882bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java @@ -155,7 +155,7 @@ private void checkToDelCluster(Map remoteClusterIdToPB, Set currentBes, List expectedBes) { + private void updateStatus(List currentBes, List expectedBes, ClusterPB remoteClusterPb) { Map currentMap = new HashMap<>(); for (Backend be : currentBes) { String endpoint = be.getHost() + ":" + be.getHeartbeatPort(); @@ -200,6 +200,33 @@ private void updateStatus(List currentBes, List expec // edit log Env.getCurrentEnv().getEditLog().logBackendStateChange(be); } + updateIfComputeNodeEndpointChanged(remoteClusterPb, be); + } + } + + private void updateIfComputeNodeEndpointChanged(ClusterPB remoteClusterPb, Backend be) { + // check PublicEndpoint、PrivateEndpoint is changed? + boolean netChanged = false; + String remotePublicEndpoint = remoteClusterPb.getPublicEndpoint(); + String localPublicEndpoint = be.getTagMap().get(Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT); + if (!localPublicEndpoint.equals(remotePublicEndpoint)) { + LOG.info("be {} has changed public_endpoint from {} to {}", + be, localPublicEndpoint, remotePublicEndpoint); + be.getTagMap().put(Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, remotePublicEndpoint); + netChanged = true; + } + + String remotePrivateEndpoint = remoteClusterPb.getPrivateEndpoint(); + String localPrivateEndpoint = be.getTagMap().get(Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT); + if (!localPrivateEndpoint.equals(remotePrivateEndpoint)) { + LOG.info("be {} has changed private_endpoint from {} to {}", + be, localPrivateEndpoint, remotePrivateEndpoint); + be.getTagMap().put(Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, remotePrivateEndpoint); + netChanged = true; + } + if (netChanged) { + // edit log + Env.getCurrentEnv().getEditLog().logBackendStateChange(be); } } @@ -278,13 +305,12 @@ private void checkDiffNode(Map remoteClusterIdToPB, LOG.info("get cloud cluster, clusterId={} local nodes={} remote nodes={}", cid, currentBeEndpoints, remoteBeEndpoints); - updateStatus(currentBes, expectedBes); + updateStatus(currentBes, expectedBes, remoteClusterIdToPB.get(cid)); diffNodes(toAdd, toDel, () -> { Map currentMap = new HashMap<>(); for (Backend be : currentBes) { - String endpoint = be.getHost() + ":" + be.getHeartbeatPort() - + be.getCloudPublicEndpoint() + be.getCloudPrivateEndpoint(); + String endpoint = be.getHost() + ":" + be.getHeartbeatPort(); currentMap.put(endpoint, be); } return currentMap; @@ -296,9 +322,7 @@ private void checkDiffNode(Map remoteClusterIdToPB, LOG.warn("cant get valid add from ms {}", node); continue; } - String endpoint = host + ":" + node.getHeartbeatPort() - + remoteClusterIdToPB.get(cid).getPublicEndpoint() - + remoteClusterIdToPB.get(cid).getPrivateEndpoint(); + String endpoint = host + ":" + node.getHeartbeatPort(); Backend b = new Backend(Env.getCurrentEnv().getNextId(), host, node.getHeartbeatPort()); if (node.hasIsSmoothUpgrade()) { b.setSmoothUpgradeDst(node.getIsSmoothUpgrade()); @@ -463,10 +487,6 @@ private void checkCloudFes() { continue; } Cloud.NodeInfoPB.NodeType type = node.getNodeType(); - // ATTN: just allow to add follower or observer - if (Cloud.NodeInfoPB.NodeType.FE_MASTER.equals(type)) { - LOG.warn("impossible !!!, get fe node {} type equal master from ms", node); - } FrontendNodeType role = type == Cloud.NodeInfoPB.NodeType.FE_OBSERVER ? FrontendNodeType.OBSERVER : FrontendNodeType.FOLLOWER; Frontend fe = new Frontend(role, diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index ec801c47c01d46..9b948a3c3033da 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -59,6 +59,7 @@ class ClusterOptions { 'max_sys_mem_available_low_water_mark_bytes=0', //no check mem available memory 'report_disk_state_interval_seconds=2', 'report_random_wait=false', + 'enable_java_support=false', ] List msConfigs = [] diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy new file mode 100644 index 00000000000000..3cc6991fa207de --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +suite('test_change_node_net', 'multi_cluster,docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=5', + ] + options.cloudMode = true + + def token = "greedisgood9999" + def update_cluster_endpoint_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/update_cluster_endpoint?token=$token" + body request_body + check check_func + } + } + + def showClusterBackends = { clusterName -> + def bes = sql_return_maparray "show backends" + def clusterBes = bes.findAll { be -> be.Tag.contains(clusterName) } + def backendMap = clusterBes.collectEntries { be -> + [(be.BackendId): be.Tag] + } + logger.info("Collected BackendId and Tag map: {}", backendMap) + backendMap + } + + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName = "newcluster1" + // 添加一个新的cluster add_new_cluster + cluster.addBackend(3, clusterName) + + def result = sql """show clusters""" + logger.info("show cluster1 : {}", result) + + def beforeBackendMap = showClusterBackends.call(clusterName) + + def tag = beforeBackendMap.entrySet().iterator().next().Value + assertNotNull(tag) + def jsonSlurper = new JsonSlurper() + def jsonObject = jsonSlurper.parseText(tag) + def cloudUniqueId = jsonObject.cloud_unique_id + def clusterId = jsonObject.compute_group_id + def before_public_endpoint = jsonObject.public_endpoint + def after_private_endpoint = jsonObject.private_endpoint + + + def changeCluster = [cluster_id: "${clusterId}", public_endpoint: "test_public_endpoint", private_endpoint: "test_private_endpoint"] + def updateClusterEndpointBody = [cloud_unique_id: "${cloudUniqueId}", cluster: changeCluster] + def jsonOutput = new JsonOutput() + def updateClusterEndpointJson = jsonOutput.toJson(updateClusterEndpointBody) + + update_cluster_endpoint_api.call(msHttpPort, updateClusterEndpointJson) { + respCode, body -> + def json = parseJson(body) + log.info("update cluster endpoint result: ${body} ${respCode} ${json}".toString()) + } + + def futrue = thread { + // check 15s + for (def i = 0; i < 15; i++) { + def afterBackendMap = showClusterBackends.call(clusterName) + if (i > 5) { + // cloud_cluster_check_interval_second = 5 + afterBackendMap.each { key, value -> + assert value.contains("test_public_endpoint") : "Value for key ${key} does not contain 'test_public_endpoint'" + assert value.contains("test_private_endpoint") : "Value for key ${key} does not contain 'test_private_endpoint'" + } + } + // check beid not changed + assertEquals(afterBackendMap.keySet(), beforeBackendMap.keySet()) + sleep(1 * 1000) + } + } + futrue.get() + } +} From 87c6b475feab2046d5f43c718b5344499d12caa7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:25:22 +0800 Subject: [PATCH 155/572] branch-3.0: [fix](nereids) fix bug when CTEConsumer is used as runtime filter target #51807 (#52749) Cherry-picked from #51807 Co-authored-by: minghong --- .../translator/RuntimeFilterTranslator.java | 145 ++++++++++-------- .../runtime_filter/cte-runtime-filter.out | 35 +++++ .../runtime_filter/cte-runtime-filter.groovy | 71 +++++++++ 3 files changed, 184 insertions(+), 67 deletions(-) create mode 100644 regression-test/data/nereids_p0/runtime_filter/cte-runtime-filter.out create mode 100644 regression-test/suites/nereids_p0/runtime_filter/cte-runtime-filter.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index 07e0af601739b1..d034145a4631f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -41,6 +41,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.ArrayList; import java.util.List; @@ -50,6 +52,7 @@ * translate runtime filter */ public class RuntimeFilterTranslator { + private static final Logger LOG = LogManager.getLogger(RuntimeFilterTranslator.class); private final RuntimeFilterContext context; @@ -96,77 +99,85 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, JoinNodeBase node, P .getIgnoredRuntimeFilterIds().contains(filter.getId().asInt())) { return; } - Expr src = ExpressionTranslator.translate(filter.getSrcExpr(), ctx); - List targetExprList = new ArrayList<>(); - List>> targetTupleIdMapList = new ArrayList<>(); - List scanNodeList = new ArrayList<>(); - boolean hasInvalidTarget = false; - for (int i = 0; i < filter.getTargetExpressions().size(); i++) { - Slot curTargetSlot = filter.getTargetSlots().get(i); - Expression curTargetExpression = filter.getTargetExpressions().get(i); - SlotRef targetSlotRef = context.getExprIdToOlapScanNodeSlotRef().get(curTargetSlot.getExprId()); - if (targetSlotRef == null) { - context.setTargetNullCount(); - hasInvalidTarget = true; - break; - } - ScanNode scanNode = context.getScanNodeOfLegacyRuntimeFilterTarget().get(curTargetSlot); - Expr targetExpr; - if (curTargetSlot.equals(curTargetExpression)) { - targetExpr = targetSlotRef; - } else { - // map nereids target slot to original planner slot - Preconditions.checkArgument(curTargetExpression.getInputSlots().size() == 1, - "target expression is invalid, input slot num > 1; filter :" + filter); - Slot slotInTargetExpression = curTargetExpression.getInputSlots().iterator().next(); - Preconditions.checkArgument(slotInTargetExpression.equals(curTargetSlot) - || curTargetSlot.equals(context.getAliasTransferMap().get(slotInTargetExpression).second)); - RuntimeFilterExpressionTranslator translator = new RuntimeFilterExpressionTranslator(targetSlotRef); - targetExpr = curTargetExpression.accept(translator, ctx); - } + try { + Expr src = ExpressionTranslator.translate(filter.getSrcExpr(), ctx); + List targetExprList = new ArrayList<>(); + List>> targetTupleIdMapList = new ArrayList<>(); + List scanNodeList = new ArrayList<>(); + boolean hasInvalidTarget = false; + for (int i = 0; i < filter.getTargetExpressions().size(); i++) { + Slot curTargetSlot = filter.getTargetSlots().get(i); + Expression curTargetExpression = filter.getTargetExpressions().get(i); + SlotRef targetSlotRef = context.getExprIdToOlapScanNodeSlotRef().get(curTargetSlot.getExprId()); + if (targetSlotRef == null) { + context.setTargetNullCount(); + hasInvalidTarget = true; + break; + } + ScanNode scanNode = context.getScanNodeOfLegacyRuntimeFilterTarget().get(curTargetSlot); + Expr targetExpr; + if (curTargetSlot.equals(curTargetExpression)) { + targetExpr = targetSlotRef; + } else { + // map nereids target slot to original planner slot + Preconditions.checkArgument(curTargetExpression.getInputSlots().size() == 1, + "target expression is invalid, input slot num > 1; filter :" + filter); + Slot slotInTargetExpression = curTargetExpression.getInputSlots().iterator().next(); + Preconditions.checkArgument(slotInTargetExpression.equals(curTargetSlot) + || curTargetSlot.equals(context.getAliasTransferMap().get(slotInTargetExpression).second)); + RuntimeFilterExpressionTranslator translator = new RuntimeFilterExpressionTranslator(targetSlotRef); + targetExpr = curTargetExpression.accept(translator, ctx); + } - // adjust data type - if (!src.getType().equals(targetExpr.getType()) && filter.getType() != TRuntimeFilterType.BITMAP) { - targetExpr = new CastExpr(src.getType(), targetExpr); + // adjust data type + if (!src.getType().equals(targetExpr.getType()) && filter.getType() != TRuntimeFilterType.BITMAP) { + targetExpr = new CastExpr(src.getType(), targetExpr); + } + TupleId targetTupleId = targetSlotRef.getDesc().getParent().getId(); + SlotId targetSlotId = targetSlotRef.getSlotId(); + scanNodeList.add(scanNode); + targetExprList.add(targetExpr); + targetTupleIdMapList.add(ImmutableMap.of(targetTupleId, ImmutableList.of(targetSlotId))); } - SlotRef targetSlot = targetSlotRef.getSrcSlotRef(); - TupleId targetTupleId = targetSlot.getDesc().getParent().getId(); - SlotId targetSlotId = targetSlot.getSlotId(); - scanNodeList.add(scanNode); - targetExprList.add(targetExpr); - targetTupleIdMapList.add(ImmutableMap.of(targetTupleId, ImmutableList.of(targetSlotId))); - } - if (!hasInvalidTarget) { - org.apache.doris.planner.RuntimeFilter origFilter - = org.apache.doris.planner.RuntimeFilter.fromNereidsRuntimeFilter( - filter, node, src, targetExprList, - targetTupleIdMapList, context.getLimits()); - if (node instanceof HashJoinNode) { - origFilter.setIsBroadcast(((HashJoinNode) node).getDistributionMode() == DistributionMode.BROADCAST); - origFilter.setSingleEq(((HashJoinNode) node).getEqJoinConjuncts().size()); - } else { - // nest loop join - origFilter.setIsBroadcast(true); + if (!hasInvalidTarget) { + org.apache.doris.planner.RuntimeFilter origFilter + = org.apache.doris.planner.RuntimeFilter.fromNereidsRuntimeFilter( + filter, node, src, targetExprList, + targetTupleIdMapList, context.getLimits()); + if (node instanceof HashJoinNode) { + origFilter.setIsBroadcast( + ((HashJoinNode) node).getDistributionMode() == DistributionMode.BROADCAST); + origFilter.setSingleEq(((HashJoinNode) node).getEqJoinConjuncts().size()); + } else { + // nest loop join + origFilter.setIsBroadcast(true); + } + boolean isLocalTarget = scanNodeList.stream().allMatch(e -> + !(e instanceof CTEScanNode) && e.getFragmentId().equals(node.getFragmentId())); + for (int i = 0; i < targetExprList.size(); i++) { + ScanNode scanNode = scanNodeList.get(i); + Expr targetExpr = targetExprList.get(i); + origFilter.addTarget(new RuntimeFilterTarget( + scanNode, targetExpr, true, isLocalTarget)); + } + origFilter.setBitmapFilterNotIn(filter.isBitmapFilterNotIn()); + origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); + org.apache.doris.planner.RuntimeFilter finalizedFilter = finalize(origFilter); + scanNodeList.stream().filter(e -> e.getStatisticalType() == StatisticalType.CTE_SCAN_NODE) + .forEach(f -> { + DataStreamSink sink = context.getPlanNodeIdToCTEDataSinkMap().get(f.getId()); + if (sink != null) { + sink.addRuntimeFilter(finalizedFilter); + } + }); + context.getLegacyFilters().add(finalizedFilter); } - boolean isLocalTarget = scanNodeList.stream().allMatch(e -> - !(e instanceof CTEScanNode) && e.getFragmentId().equals(node.getFragmentId())); - for (int i = 0; i < targetExprList.size(); i++) { - ScanNode scanNode = scanNodeList.get(i); - Expr targetExpr = targetExprList.get(i); - origFilter.addTarget(new RuntimeFilterTarget( - scanNode, targetExpr, true, isLocalTarget)); + } catch (Exception e) { + LOG.info("failed to translate runtime filter: " + e.getMessage()); + // throw exception in debug mode + if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().feDebug) { + throw e; } - origFilter.setBitmapFilterNotIn(filter.isBitmapFilterNotIn()); - origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); - org.apache.doris.planner.RuntimeFilter finalizedFilter = finalize(origFilter); - scanNodeList.stream().filter(e -> e.getStatisticalType() == StatisticalType.CTE_SCAN_NODE) - .forEach(f -> { - DataStreamSink sink = context.getPlanNodeIdToCTEDataSinkMap().get(f.getId()); - if (sink != null) { - sink.addRuntimeFilter(finalizedFilter); - } - }); - context.getLegacyFilters().add(finalizedFilter); } } diff --git a/regression-test/data/nereids_p0/runtime_filter/cte-runtime-filter.out b/regression-test/data/nereids_p0/runtime_filter/cte-runtime-filter.out new file mode 100644 index 00000000000000..cff6f05e5b397c --- /dev/null +++ b/regression-test/data/nereids_p0/runtime_filter/cte-runtime-filter.out @@ -0,0 +1,35 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !shape_onerow -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalOneRowRelation +--PhysicalResultSink +----PhysicalDistribute[DistributionSpecGather] +------PhysicalProject +--------hashJoin[INNER_JOIN shuffle] hashCondition=((expr_cast(id as BIGINT) = b.user_id)) otherCondition=() build RFs:RF0 user_id->[cast(id as BIGINT)] +----------PhysicalProject +------------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF0 +----------PhysicalProject +------------filter((cte_runtime_filter_table.__DORIS_DELETE_SIGN__ = 0)) +--------------PhysicalOlapScan[cte_runtime_filter_table] + +-- !exec_onerow -- +1 1 1 1 + +-- !shape_cte_cte -- +PhysicalCteAnchor ( cteId=CTEId#0 ) +--PhysicalCteProducer ( cteId=CTEId#0 ) +----PhysicalProject +------filter((cte_runtime_filter_table.__DORIS_DELETE_SIGN__ = 0)) +--------PhysicalOlapScan[cte_runtime_filter_table] +--PhysicalResultSink +----PhysicalDistribute[DistributionSpecGather] +------hashJoin[INNER_JOIN shuffle] hashCondition=((a.user_id = b.user_id)) otherCondition=() build RFs:RF0 user_id->[user_id] +--------PhysicalCteConsumer ( cteId=CTEId#0 ) apply RFs: RF0 +--------PhysicalProject +----------filter((cte_runtime_filter_table.__DORIS_DELETE_SIGN__ = 0)) +------------PhysicalOlapScan[cte_runtime_filter_table] + +-- !exec_cte_cte -- +1 1 1 1 1 1 + diff --git a/regression-test/suites/nereids_p0/runtime_filter/cte-runtime-filter.groovy b/regression-test/suites/nereids_p0/runtime_filter/cte-runtime-filter.groovy new file mode 100644 index 00000000000000..b5ce12e0851992 --- /dev/null +++ b/regression-test/suites/nereids_p0/runtime_filter/cte-runtime-filter.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('cte-runtime-filter') { + + sql ''' + drop table if exists cte_runtime_filter_table; + CREATE TABLE `cte_runtime_filter_table` ( + `part_dt` bigint NOT NULL COMMENT '日期', + `group_id` bigint NOT NULL COMMENT '客群id', + `user_id` bigint NOT NULL COMMENT '用户id' + ) ENGINE=OLAP + UNIQUE KEY(`part_dt`, `group_id`, `user_id`) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + + insert into cte_runtime_filter_table values (1, 1, 1); + + set inline_cte_referenced_threshold=0; + set disable_join_reorder = true; + set enable_runtime_filter_prune=false; + set runtime_filter_mode=global; + set runtime_filter_type=2; + ''' + + qt_shape_onerow ''' + explain shape plan + with cte as ((select 1 as id)) + select * + from cte a + join cte_runtime_filter_table b on a.id=b.user_id ; + ''' + + qt_exec_onerow''' + with cte as ((select 1 as id)) + select * + from cte a + join cte_runtime_filter_table b on a.id=b.user_id ; + ''' + + qt_shape_cte_cte ''' + explain shape plan + with cte as ((select * from cte_runtime_filter_table)) + select * + from cte a + join cte_runtime_filter_table b on a.user_id=b.user_id ; + ''' + + qt_exec_cte_cte ''' + with cte as ((select * from cte_runtime_filter_table)) + select * + from cte a + join cte_runtime_filter_table b on a.user_id=b.user_id ; + ''' +} \ No newline at end of file From 2381db21279dee895bb9f7051ca487f22d1b069d Mon Sep 17 00:00:00 2001 From: zhangdong Date: Tue, 8 Jul 2025 09:26:24 +0800 Subject: [PATCH 156/572] branch-3.0:[fix](mtmv) fix when compatible fail, will throw NPE (#49875) (#52275) pick: https://github.com/apache/doris/pull/49875 --- .../java/org/apache/doris/alter/Alter.java | 5 ++ .../java/org/apache/doris/catalog/MTMV.java | 16 +++++ .../org/apache/doris/mtmv/BaseTableInfo.java | 16 ++++- .../apache/doris/mtmv/MTMVPartitionInfo.java | 2 +- .../mtmv/MTMVRefreshPartitionSnapshot.java | 61 ++++++++++--------- .../doris/mtmv/MTMVRefreshSnapshot.java | 2 +- .../org/apache/doris/mtmv/MTMVRelation.java | 4 +- .../doris/mtmv/MTMVRelationManager.java | 43 ++++++------- .../apache/doris/mtmv/MTMVRewriteUtil.java | 5 +- .../org/apache/doris/mtmv/MTMVStatus.java | 6 ++ .../doris/nereids/StatementContext.java | 6 ++ .../rules/analysis/CollectRelation.java | 8 ++- ...tConsistentMaterializationContextHook.java | 8 +-- .../mv/InitMaterializationContextHook.java | 7 +-- .../org/apache/doris/mtmv/AlterMTMVTest.java | 2 +- .../doris/mtmv/MTMVRewriteUtilTest.java | 12 ++-- .../doris/nereids/memo/StructInfoMapTest.java | 18 ++++++ .../doris/nereids/mv/IdStatisticsMapTest.java | 6 ++ .../doris/nereids/mv/MvTableIdIsLongTest.java | 6 ++ .../doris/nereids/util/PlanChecker.java | 11 ++++ .../insert/dml_insert_and_overwrite.groovy | 9 +-- .../mv/dml/outfile/dml_into_outfile.groovy | 5 +- 22 files changed, 170 insertions(+), 88 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 37b1f177d45887..25c6cc1d0c1e8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -1029,6 +1029,11 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) { case ADD_TASK: mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(), isReplay); + // If it is not a replay thread, it means that the current service is already a new version + // and does not require compatibility + if (isReplay) { + mtmv.compatible(Env.getCurrentEnv().getCatalogMgr()); + } break; default: throw new RuntimeException("Unknown type value: " + alterMTMV.getOpType()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 2084703989d42c..4ad92888acb437 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -480,6 +480,10 @@ public void setRefreshSnapshot(MTMVRefreshSnapshot refreshSnapshot) { this.refreshSnapshot = refreshSnapshot; } + public boolean canBeCandidate() { + return getStatus().canBeCandidate(); + } + public void readMvLock() { this.mvRwLock.readLock().lock(); } @@ -557,6 +561,18 @@ public String toInfoString() { * The logic here is to be compatible with older versions by converting ID to name */ public void compatible(CatalogMgr catalogMgr) { + try { + compatibleInternal(catalogMgr); + Env.getCurrentEnv().getMtmvService().unregisterMTMV(this); + Env.getCurrentEnv().getMtmvService().registerMTMV(this, this.getDatabase().getId()); + } catch (Throwable e) { + LOG.warn("MTMV compatible failed, dbName: {}, mvName: {}, errMsg: {}", getDBName(), name, e.getMessage()); + status.setState(MTMVState.SCHEMA_CHANGE); + status.setSchemaChangeDetail("compatible failed, please refresh or recreate it, reason: " + e.getMessage()); + } + } + + private void compatibleInternal(CatalogMgr catalogMgr) throws Exception { if (mvPartitionInfo != null) { mvPartitionInfo.compatible(catalogMgr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java index 625fe7d91870ef..b2b20ec0bab202 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseTableInfo.java @@ -159,10 +159,18 @@ public String toString() { + '}'; } - public void compatible(CatalogMgr catalogMgr) { + public void compatible(CatalogMgr catalogMgr) throws Exception { if (!StringUtils.isEmpty(ctlName)) { return; } + // should not get meta from external catalog when replay, because the timeout period may be very long + if (ctlId != InternalCatalog.INTERNAL_CATALOG_ID) { + String msg = String.format( + "Can not compatibility external table, ctlId: %s, dbId: %s, tableId: %s", + ctlId, dbId, tableId); + LOG.warn(msg); + throw new Exception(msg); + } try { CatalogIf catalog = catalogMgr.getCatalogOrAnalysisException(ctlId); DatabaseIf db = catalog.getDbOrAnalysisException(dbId); @@ -171,7 +179,11 @@ public void compatible(CatalogMgr catalogMgr) { this.dbName = db.getFullName(); this.tableName = table.getName(); } catch (AnalysisException e) { - LOG.warn("MTMV compatible failed, ctlId: {}, dbId: {}, tableId: {}", ctlId, dbId, tableId, e); + String msg = String.format( + "Failed to get name based on id during compatibility process, ctlId: %s, dbId: %s, tableId: %s", + ctlId, dbId, tableId); + LOG.warn(msg, e); + throw new Exception(msg); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java index 7eae44db0af4cc..682273a93cd27d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java @@ -152,7 +152,7 @@ public String toNameString() { } } - public void compatible(CatalogMgr catalogMgr) { + public void compatible(CatalogMgr catalogMgr) throws Exception { if (relatedTable == null) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java index a8de5b6597bc49..2601ace5ed628a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java @@ -20,8 +20,9 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -77,29 +78,28 @@ public String toString() { + '}'; } - public void compatible(MTMV mtmv) { - try { - // snapshot add partitionId resolve problem of insert overwrite - compatiblePartitions(mtmv); - } catch (Throwable e) { - LOG.warn("MTMV compatiblePartitions failed, mtmv: {}", mtmv.getName(), e); - } - try { - // change table id to BaseTableInfo - compatibleTables(mtmv); - } catch (Throwable e) { - LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e); - } - - try { - // snapshot add tableId resolve problem of recreate table - compatibleTablesSnapshot(); - } catch (Throwable e) { - LOG.warn("MTMV compatibleTables failed, mtmv: {}", mtmv.getName(), e); - } + public void compatible(MTMV mtmv) throws Exception { + // snapshot add partitionId resolve problem of insert overwrite + compatiblePartitions(mtmv); + // change table id to BaseTableInfo + compatibleTables(mtmv); + // snapshot add tableId resolve problem of recreate table + compatibleTablesSnapshot(); } private void compatiblePartitions(MTMV mtmv) throws AnalysisException { + if (mtmv.getMvPartitionInfo().getPartitionType().equals(MTMVPartitionType.SELF_MANAGE)) { + return; + } + // Only olapTable has historical data issues that require compatibility + if (mtmv.getMvPartitionInfo().getRelatedTableInfo().getCtlId() != InternalCatalog.INTERNAL_CATALOG_ID) { + return; + } + MTMVRelatedTableIf relatedTableIf = mtmv.getMvPartitionInfo().getRelatedTable(); + // Only olapTable has historical data issues that require compatibility + if (!(relatedTableIf instanceof OlapTable)) { + return; + } if (!checkHasDataWithoutPartitionId()) { return; } @@ -108,6 +108,8 @@ private void compatiblePartitions(MTMV mtmv) throws AnalysisException { MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue(); if (versionSnapshot.getId() == 0) { Partition partition = relatedTable.getPartition(entry.getKey()); + // if not find partition, may be partition has been dropped, + // the impact is that MTMV will consider this partition to be async if (partition != null) { (versionSnapshot).setId(partition.getId()); } @@ -131,12 +133,7 @@ private void compatibleTablesSnapshot() { for (Entry entry : tablesInfo.entrySet()) { MTMVVersionSnapshot versionSnapshot = (MTMVVersionSnapshot) entry.getValue(); if (versionSnapshot.getId() == 0) { - try { - TableIf table = MTMVUtil.getTable(entry.getKey()); - versionSnapshot.setId(table.getId()); - } catch (AnalysisException e) { - LOG.warn("MTMV compatibleTablesSnapshot failed, can not get table by: {}", entry.getKey()); - } + versionSnapshot.setId(entry.getKey().getTableId()); } } } @@ -150,7 +147,7 @@ private boolean checkHasDataWithoutTableId() { return false; } - private void compatibleTables(MTMV mtmv) { + private void compatibleTables(MTMV mtmv) throws Exception { if (tables.size() == tablesInfo.size()) { return; } @@ -164,8 +161,12 @@ private void compatibleTables(MTMV mtmv) { if (tableInfo.isPresent()) { tablesInfo.put(tableInfo.get(), entry.getValue()); } else { - LOG.warn("MTMV compatibleTables failed, tableId: {}, relationTables: {}", entry.getKey(), - relation.getBaseTablesOneLevel()); + String msg = String.format( + "Failed to get table info based on id during compatibility process, " + + "tableId: %s, relationTables: %s", + entry.getKey(), relation.getBaseTablesOneLevel()); + LOG.warn(msg); + throw new Exception(msg); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java index 74fc3cc1c5cfb9..0d9665cb4463b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshSnapshot.java @@ -91,7 +91,7 @@ public String toString() { + '}'; } - public void compatible(MTMV mtmv) { + public void compatible(MTMV mtmv) throws Exception { if (MapUtils.isEmpty(partitionSnapshots)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java index 87a0199f128f88..148d2d008843d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java @@ -65,13 +65,13 @@ public String toInfoString() { + '}'; } - public void compatible(CatalogMgr catalogMgr) { + public void compatible(CatalogMgr catalogMgr) throws Exception { compatible(catalogMgr, baseTables); compatible(catalogMgr, baseViews); compatible(catalogMgr, baseTablesOneLevel); } - private void compatible(CatalogMgr catalogMgr, Set infos) { + private void compatible(CatalogMgr catalogMgr, Set infos) throws Exception { if (CollectionUtils.isEmpty(infos)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index c45558ec8cfdf3..ba498636d73619 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -78,48 +78,43 @@ public Set getMtmvsByBaseTableOneLevel(BaseTableInfo table) { /** * if At least one partition is available, return this mtmv * - * @param tableInfos + * @param candidateMTMVs * @param ctx * @return */ - public Set getAvailableMTMVs(List tableInfos, ConnectContext ctx, + public Set getAvailableMTMVs(Set candidateMTMVs, ConnectContext ctx, boolean forceConsistent, BiPredicate predicate) { Set res = Sets.newLinkedHashSet(); - Set mvInfos = getMTMVInfos(tableInfos); Map, Set> queryUsedPartitions = PartitionCompensator.getQueryUsedPartitions( ctx.getStatementContext(), new BitSet()); - - for (BaseTableInfo tableInfo : mvInfos) { - try { - MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo); - if (predicate.test(ctx, mtmv)) { - continue; - } - if (!mtmv.isUseForRewrite()) { - continue; - } - BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); - if (isMVPartitionValid(mtmv, ctx, forceConsistent, - relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) { - res.add(mtmv); - } - } catch (Exception e) { - // not throw exception to client, just ignore it - LOG.warn("getTable failed: {}", tableInfo.toString(), e); + for (MTMV mtmv : candidateMTMVs) { + if (predicate.test(ctx, mtmv)) { + continue; + } + if (!mtmv.isUseForRewrite()) { + continue; + } + BaseTableInfo relatedTableInfo = mtmv.getMvPartitionInfo().getRelatedTableInfo(); + if (isMVPartitionValid(mtmv, ctx, forceConsistent, + relatedTableInfo == null ? null : queryUsedPartitions.get(relatedTableInfo.toList()))) { + res.add(mtmv); } } return res; } /** - * get all mtmv related to tableInfos. + * get candidate mtmv related to tableInfos. */ - public Set getAllMTMVs(List tableInfos) { + public Set getCandidateMTMVs(List tableInfos) { Set mtmvs = Sets.newLinkedHashSet(); Set mvInfos = getMTMVInfos(tableInfos); for (BaseTableInfo tableInfo : mvInfos) { try { - mtmvs.add((MTMV) MTMVUtil.getTable(tableInfo)); + MTMV mtmv = (MTMV) MTMVUtil.getTable(tableInfo); + if (mtmv.canBeCandidate()) { + mtmvs.add(mtmv); + } } catch (Exception e) { // not throw exception to client, just ignore it LOG.warn("getTable failed: {}", tableInfo.toString(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index afaad55a34bbc4..58b2a37d504810 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -20,8 +20,6 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Partition; import org.apache.doris.common.AnalysisException; -import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState; -import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Lists; @@ -56,8 +54,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne return res; } // check mv is normal - MTMVStatus mtmvStatus = mtmv.getStatus(); - if (mtmvStatus.getState() != MTMVState.NORMAL || mtmvStatus.getRefreshState() == MTMVRefreshState.INIT) { + if (!mtmv.canBeCandidate()) { return res; } // if relatedPartitions is empty but not null, which means query no partitions diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java index b1761b9e97381b..aa058e628c6284 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVStatus.java @@ -82,6 +82,12 @@ public MTMVStatus updateStateAndDetail(MTMVStatus status) { return this; } + public boolean canBeCandidate() { + // MTMVRefreshState.FAIL also can be candidate, because may have some sync partitions + return getState() == MTMVState.NORMAL + && getRefreshState() != MTMVRefreshState.INIT; + } + @Override public String toString() { return "MTMVStatus{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 6c2e0d1ab12201..9401ec102dac9f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; @@ -186,6 +187,7 @@ public enum TableFrom { // if query is: select * from t2 join t5 // mtmvRelatedTables is mv1, mv2, mv3, t1, t2, t3, t4, t5 private final Map, TableIf> mtmvRelatedTables = Maps.newHashMap(); + private final Set candidateMTMVs = Sets.newHashSet(); // insert into target tables private final Map, TableIf> insertTargetTables = Maps.newHashMap(); // save view's def and sql mode to avoid them change before lock @@ -307,6 +309,10 @@ public Map, TableIf> getMtmvRelatedTables() { return mtmvRelatedTables; } + public Set getCandidateMTMVs() { + return candidateMTMVs; + } + public Map, TableIf> getTables() { return tables; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index efcefb2951fd2e..3f5edb81e91ec6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -208,9 +208,12 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex } if (shouldCollect) { Set mtmvSet = Env.getCurrentEnv().getMtmvService().getRelationManager() - .getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table))); - LOG.info("table {} related mv set is {}", new BaseTableInfo(table), mtmvSet); + .getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table))); + if (LOG.isDebugEnabled()) { + LOG.debug("table {} related mv set is {}", new BaseTableInfo(table), mtmvSet); + } for (MTMV mtmv : mtmvSet) { + cascadesContext.getStatementContext().getCandidateMTMVs().add(mtmv); cascadesContext.getStatementContext().getMtmvRelatedTables().put(mtmv.getFullQualifiers(), mtmv); mtmv.readMvLock(); try { @@ -222,6 +225,7 @@ private void collectMTMVCandidates(TableIf table, CascadesContext cascadesContex LOG.debug("mtmv {} related base table include {}", new BaseTableInfo(mtmv), baseTableInfo); } try { + // Collect all base tables and lock them before querying cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(), TableFrom.MTMV); } catch (AnalysisException exception) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java index fbcf4726a1023e..e86cca263d30b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitConsistentMaterializationContextHook.java @@ -20,16 +20,13 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; -import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.PlannerHook; import com.google.common.annotations.VisibleForTesting; -import java.util.List; import java.util.Set; -import java.util.stream.Collectors; /** * If enable query rewrite with mv in dml, should init consistent materialization context after analyze @@ -49,10 +46,9 @@ public void initMaterializationContext(CascadesContext cascadesContext) { } protected Set getAvailableMTMVs(Set usedTables, CascadesContext cascadesContext) { - List usedBaseTables = - usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); return Env.getCurrentEnv().getMtmvService().getRelationManager() - .getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(), + .getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(), + cascadesContext.getConnectContext(), true, ((connectContext, mtmv) -> { return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable() .isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 6270dc9572ec8f..de703ba9fd6fc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -26,7 +26,6 @@ import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; -import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.MTMVUtil; @@ -51,7 +50,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * If enable query rewrite with mv, should init materialization context after analyze @@ -117,10 +115,9 @@ protected void doInitMaterializationContext(CascadesContext cascadesContext) { } protected Set getAvailableMTMVs(Set usedTables, CascadesContext cascadesContext) { - List usedBaseTables = - usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); return Env.getCurrentEnv().getMtmvService().getRelationManager() - .getAvailableMTMVs(usedBaseTables, cascadesContext.getConnectContext(), + .getAvailableMTMVs(cascadesContext.getStatementContext().getCandidateMTMVs(), + cascadesContext.getConnectContext(), false, ((connectContext, mtmv) -> { return MTMVUtil.mtmvContainsExternalTable(mtmv) && (!connectContext.getSessionVariable() .isEnableMaterializedViewRewriteWhenBaseTableUnawareness()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java index 17ec145f583221..342f9fd60c8823 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/AlterMTMVTest.java @@ -51,7 +51,7 @@ public void testAlterMTMV() throws Exception { MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager(); Table table = Env.getCurrentInternalCatalog().getDb("test").get().getTableOrMetaException("stu"); - Set allMTMVs = relationManager.getAllMTMVs(Lists.newArrayList(new BaseTableInfo(table))); + Set allMTMVs = relationManager.getCandidateMTMVs(Lists.newArrayList(new BaseTableInfo(table))); boolean hasMvA = false; boolean hasMvB = false; for (MTMV mtmv : allMTMVs) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 82c7eaac631e81..e4788c18409a5c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -122,6 +122,10 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc MTMVUtil.mtmvContainsExternalTable((MTMV) any); minTimes = 0; result = false; + + mtmv.canBeCandidate(); + minTimes = 0; + result = true; } }; } @@ -279,9 +283,9 @@ public void testGetMTMVCanRewritePartitionsDisableContainExternalTable() { public void testGetMTMVCanRewritePartitionsStateAbnormal() { new Expectations() { { - status.getState(); + mtmv.canBeCandidate(); minTimes = 0; - result = MTMVState.SCHEMA_CHANGE; + result = false; } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil @@ -309,9 +313,9 @@ public void testGetMTMVCanRewritePartitionsRefreshStateAbnormal() { public void testGetMTMVCanRewritePartitionsRefreshStateInit() { new Expectations() { { - status.getRefreshState(); + mtmv.canBeCandidate(); minTimes = 0; - result = MTMVRefreshState.INIT; + result = false; } }; Collection mtmvCanRewritePartitions = MTMVRewriteUtil diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java index 19d1efdbbd82d6..0d3181d15e8893 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/StructInfoMapTest.java @@ -70,6 +70,12 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceCo return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; @@ -129,6 +135,12 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceCo return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" @@ -177,6 +189,12 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean forceCo return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java index 1403a9fee5e254..3e93919ed22321 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/IdStatisticsMapTest.java @@ -59,6 +59,12 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPar return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv100 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java index 4fa0a68e77c6ab..5c9fb3c0392058 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/mv/MvTableIdIsLongTest.java @@ -55,6 +55,12 @@ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx, boolean isMVPar return true; } }; + new MockUp() { + @Mock + public boolean canBeCandidate() { + return true; + } + }; connectContext.getSessionVariable().enableMaterializedViewRewrite = true; connectContext.getSessionVariable().enableMaterializedViewNestRewrite = true; createMvByNereids("create materialized view mv1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index e535e3a8ac5e25..71d0f0101b0413 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.util; import org.apache.doris.analysis.ExplainOptions; +import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.PlanProcess; @@ -49,6 +50,7 @@ import org.apache.doris.nereids.rules.RuleFactory; import org.apache.doris.nereids.rules.RuleSet; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.InitConsistentMaterializationContextHook; import org.apache.doris.nereids.rules.exploration.mv.InitMaterializationContextHook; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; @@ -120,6 +122,9 @@ public PlanChecker parse(String sql) { } public PlanChecker analyze() { + this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE); + this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.setCteContext(new CTEContext()); this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.toMemo(); return this; @@ -127,6 +132,9 @@ public PlanChecker analyze() { public PlanChecker analyze(Plan plan) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); + this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE); + this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.setCteContext(new CTEContext()); Set originDisableRules = connectContext.getSessionVariable().getDisableNereidsRuleNames(); Set disableRuleWithAuth = Sets.newHashSet(originDisableRules); disableRuleWithAuth.add(RuleType.RELATION_AUTHENTICATION.name()); @@ -140,6 +148,9 @@ public PlanChecker analyze(Plan plan) { public PlanChecker analyze(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); + this.cascadesContext.getStatementContext().addPlannerHook(InitConsistentMaterializationContextHook.INSTANCE); + this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.setCteContext(new CTEContext()); this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.toMemo(); return this; diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy index 54e9b57f7e5adb..753217f922da49 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/insert/dml_insert_and_overwrite.groovy @@ -106,7 +106,8 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + // todo: Temporarily turn off, otherwise usable materialized views will not be collected and will need to be changed back in the future + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -155,7 +156,7 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -203,7 +204,7 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -249,7 +250,7 @@ suite("dml_insert_and_overwrite") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; diff --git a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy index 350e49057d08ba..dc23130b990064 100644 --- a/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/dml/outfile/dml_into_outfile.groovy @@ -122,7 +122,8 @@ suite("dml_into_outfile", "p0") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + // todo: Temporarily turn off, otherwise usable materialized views will not be collected and will need to be changed back in the future + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; @@ -185,7 +186,7 @@ suite("dml_into_outfile", "p0") { ps_comment;""") // disable query rewrite by mv - sql "set enable_materialized_view_rewrite=false"; + sql "set enable_materialized_view_rewrite=true"; // enable dml rewrite by mv sql "set enable_dml_materialized_view_rewrite=true"; From 436c04a0f8d1249aa6d438d93321c9eee78ebfc7 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 8 Jul 2025 09:28:06 +0800 Subject: [PATCH 157/572] branch-3.0 [fix](stream loader) do not redirect to https for stream load #50009 (#52332) ### What problem does this PR solve? pick #50009 Issue Number: close #xxx --- .../org/apache/doris/common/profile/Profile.java | 5 +++-- .../org/apache/doris/httpv2/rest/LoadAction.java | 14 -------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java index eb17d3801135e7..84e4ff5b061b6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/Profile.java @@ -47,10 +47,11 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -111,7 +112,7 @@ public class Profile { private PhysicalPlan physicalPlan; public Map rowsProducedMap = new HashMap<>(); - private List physicalRelations = new ArrayList<>(); + private Set physicalRelations = new LinkedHashSet<>(); private String changedSessionVarCache = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java index 11c2b2811111b5..a7dd23c916a241 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java @@ -87,10 +87,6 @@ public class LoadAction extends RestBaseController { @RequestMapping(path = "/api/{" + DB_KEY + "}/{" + TABLE_KEY + "}/_load", method = RequestMethod.PUT) public Object load(HttpServletRequest request, HttpServletResponse response, @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { - if (needRedirect(request.getScheme())) { - return redirectToHttps(request); - } - if (Config.disable_mini_load) { ResponseEntity entity = ResponseEntityBuilder.notFound("The mini load operation has been" + " disabled by default, if you need to add disable_mini_load=false in fe.conf."); @@ -128,9 +124,6 @@ public Object streamLoad(HttpServletRequest request, } } } - if (needRedirect(request.getScheme())) { - return redirectToHttps(request); - } String authToken = request.getHeader("token"); // if auth token is not null, check it first @@ -246,9 +239,6 @@ public Object streamLoad2PC(HttpServletRequest request, HttpServletResponse response, @PathVariable(value = DB_KEY) String db) { LOG.info("streamload action 2PC, db: {}, headers: {}", db, getAllHeaders(request)); - if (needRedirect(request.getScheme())) { - return redirectToHttps(request); - } executeCheckPassword(request, response); return executeStreamLoad2PC(request, db); @@ -260,10 +250,6 @@ public Object streamLoad2PC_table(HttpServletRequest request, @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) { LOG.info("streamload action 2PC, db: {}, tbl: {}, headers: {}", db, table, getAllHeaders(request)); - if (needRedirect(request.getScheme())) { - return redirectToHttps(request); - } - executeCheckPassword(request, response); return executeStreamLoad2PC(request, db); } From 611bc9d916916794532e39a9c75a2dbeb7e9b056 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:28:44 +0800 Subject: [PATCH 158/572] branch-3.0: [Fix](regression) update flink connector version for test case #48292 (#52855) Cherry-picked from #48292 Co-authored-by: wudi --- regression-test/framework/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/framework/pom.xml b/regression-test/framework/pom.xml index 2a470fd2ea9b33..14556af1e1f330 100644 --- a/regression-test/framework/pom.xml +++ b/regression-test/framework/pom.xml @@ -385,7 +385,7 @@ under the License. org.apache.doris flink-doris-connector-1.16 - 24.0.0 + 25.0.0 From 042c45503cb2a2cc68873453d30ba4f1e13c3cc9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:31:15 +0800 Subject: [PATCH 159/572] branch-3.0: [opt](tvf) longer the timeout of getting file list from backend #52774 (#52808) Cherry-picked from #52774 Co-authored-by: Mingyu Chen (Rayner) --- .../apache/doris/tablefunction/LocalTableValuedFunction.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 32f76b233fad68..a95960ff0e0edb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -24,6 +24,7 @@ import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PGlobResponse; import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.rpc.BackendServiceProxy; import org.apache.doris.system.Backend; import org.apache.doris.thrift.TBrokerFileStatus; @@ -112,9 +113,10 @@ private void getFileListFromBackend() throws AnalysisException { TNetworkAddress address = be.getBrpcAddress(); InternalService.PGlobRequest.Builder requestBuilder = InternalService.PGlobRequest.newBuilder(); requestBuilder.setPattern(filePath); + long timeoutS = ConnectContext.get() == null ? 60 : Math.min(ConnectContext.get().getQueryTimeoutS(), 60); try { Future response = proxy.glob(address, requestBuilder.build()); - PGlobResponse globResponse = response.get(5, TimeUnit.SECONDS); + PGlobResponse globResponse = response.get(timeoutS, TimeUnit.SECONDS); if (globResponse.getStatus().getStatusCode() != 0) { throw new AnalysisException( "error code: " + globResponse.getStatus().getStatusCode() From 9110c77d552061f5e225fd26fc3f4e90c2ac3e47 Mon Sep 17 00:00:00 2001 From: hui lai Date: Tue, 8 Jul 2025 09:31:34 +0800 Subject: [PATCH 160/572] branch-3.0: [fix](job) fix routine load task scheduler block for one job can not find any BE (#52654) (#52791) pick (#52654) ### What problem does this PR solve? routine load task will block in following case: 1. The user created a job using the admin user of clusterA, and at some point deleted clusterA, and renamed clusterB to clusterA 2. The cluster ID saved in the job is invalid and can't find any BE 3. This task was repeatedly taken out of the queue and was put back to queue for there was no BE to execute, causing the other tasks to get stuck. ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/load/routineload/RoutineLoadManager.java | 12 +++++++++++- .../load/routineload/RoutineLoadTaskScheduler.java | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index 23b36f11a4b89c..169af97670f6e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -498,8 +498,18 @@ public long getMinTaskBeId(String clusterName) throws LoadException { // check if the specified BE is available for running task // return true if it is available. return false if otherwise. // throw exception if unrecoverable errors happen. - public long getAvailableBeForTask(long jobId, long previousBeId) throws LoadException { + public long getAvailableBeForTask(long jobId, long previousBeId) throws UserException { List availableBeIds = getAvailableBackendIds(jobId); + if (availableBeIds.isEmpty()) { + RoutineLoadJob job = getJob(jobId); + if (job != null) { + String msg = "no available BE found for job " + jobId + + "please check the BE status and user's cluster or tags"; + job.updateState(RoutineLoadJob.JobState.PAUSED, + new ErrorReason(InternalErrorCode.MANUAL_PAUSE_ERR, msg), false /* not replay */); + } + return -1L; + } // check if be has idle slot readLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java index d40a6705626c84..040ca103004e34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java @@ -147,7 +147,7 @@ private void scheduleOneTask(RoutineLoadTaskInfo routineLoadTaskInfo) throws Exc // this should be done before txn begin, or the txn may be begun successfully but failed to be allocated. if (!allocateTaskToBe(routineLoadTaskInfo)) { // allocate failed, push it back to the queue to wait next scheduling - needScheduleTasksQueue.addFirst(routineLoadTaskInfo); + needScheduleTasksQueue.addLast(routineLoadTaskInfo); return; } } catch (UserException e) { @@ -311,7 +311,7 @@ private void submitTask(long beId, TRoutineLoadTask tTask) throws LoadException // 2. If not, try to find a better one with most idle slots. // return true if allocate successfully. return false if failed. // throw exception if unrecoverable errors happen. - private boolean allocateTaskToBe(RoutineLoadTaskInfo routineLoadTaskInfo) throws LoadException { + private boolean allocateTaskToBe(RoutineLoadTaskInfo routineLoadTaskInfo) throws UserException { long beId = routineLoadManager.getAvailableBeForTask(routineLoadTaskInfo.getJobId(), routineLoadTaskInfo.getPreviousBeId()); if (beId == -1L) { From 28c666460900b423d0e13fe163a2a9f83dbba1da Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 8 Jul 2025 09:32:24 +0800 Subject: [PATCH 161/572] branch-3.0: [Opt](cloud-mow) Retry to commit txn when encounter stale calc delete bitmap response regardless of status code (#52547) (#52848) pick https://github.com/apache/doris/pull/52547 --- .../java/org/apache/doris/master/MasterImpl.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java index 213984ae982ae6..ad4f1a1bacdb09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java @@ -683,12 +683,8 @@ private void finishCalcDeleteBitmap(AgentTask task, TFinishTaskRequest request) // and if meta is missing, we no longer need to resend this task try { CalcDeleteBitmapTask calcDeleteBitmapTask = (CalcDeleteBitmapTask) task; - if (request.getTaskStatus().getStatusCode() != TStatusCode.OK) { - calcDeleteBitmapTask.countDownToZero(request.getTaskStatus().getStatusCode(), - "backend: " + task.getBackendId() + ", error_tablet_size: " - + request.getErrorTabletIdsSize() + ", err_msg: " - + request.getTaskStatus().getErrorMsgs().toString()); - } else if (request.isSetRespPartitions() + // check if the request is stale first, if so, let it retry regardless of the status code + if (request.isSetRespPartitions() && calcDeleteBitmapTask.isFinishRequestStale(request.getRespPartitions())) { LOG.warn("get staled response from backend: {}, report version: {}. calcDeleteBitmapTask's" + "partitionInfos: {}. response's partitionInfos: {}", task.getBackendId(), @@ -699,6 +695,11 @@ private void finishCalcDeleteBitmap(AgentTask task, TFinishTaskRequest request) calcDeleteBitmapTask.countDownToZero(TStatusCode.DELETE_BITMAP_LOCK_ERROR, "get staled response from backend " + task.getBackendId() + ", report version: " + request.getReportVersion()); + } else if (request.getTaskStatus().getStatusCode() != TStatusCode.OK) { + calcDeleteBitmapTask.countDownToZero(request.getTaskStatus().getStatusCode(), + "backend: " + task.getBackendId() + ", error_tablet_size: " + request.getErrorTabletIdsSize() + + ", error_tablets: " + request.getErrorTabletIds() + + ", err_msg: " + request.getTaskStatus().getErrorMsgs().toString()); } else { calcDeleteBitmapTask.countDownLatch(task.getBackendId(), calcDeleteBitmapTask.getTransactionId()); if (LOG.isDebugEnabled()) { From 63b07dd291e4a85e14a17c62c25f3605f538b729 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 8 Jul 2025 09:32:48 +0800 Subject: [PATCH 162/572] branch-3.0: [Opt](mow) Move `DeleteBitmapAggCache` to `ExecEnv` (#52649) (#52737) pick https://github.com/apache/doris/pull/52649 --- be/src/olap/tablet_meta.cpp | 57 +++++++++++++++--------------- be/src/olap/tablet_meta.h | 45 ++++++++--------------- be/src/runtime/exec_env.h | 5 +++ be/src/runtime/exec_env_init.cpp | 12 +++++++ be/test/testutil/run_all_tests.cpp | 4 +++ 5 files changed, 63 insertions(+), 60 deletions(-) diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index c1dbc8a93948df..d2fc09455143d5 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1085,21 +1085,21 @@ bool operator!=(const TabletMeta& a, const TabletMeta& b) { return !(a == b); } -DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) { - // The default delete bitmap cache is set to 100MB, - // which can be insufficient and cause performance issues when the amount of user data is large. - // To mitigate the problem of an inadequate cache, - // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. - bool is_percent = false; - int64_t delete_bitmap_agg_cache_cache_limit = - ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit, - MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent); - _agg_cache.reset(new AggCache(delete_bitmap_agg_cache_cache_limit > - config::delete_bitmap_agg_cache_capacity - ? delete_bitmap_agg_cache_cache_limit - : config::delete_bitmap_agg_cache_capacity)); +DeleteBitmapAggCache::DeleteBitmapAggCache(size_t capacity) + : LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity, + LRUCacheType::SIZE, config::delete_bitmap_agg_cache_stale_sweep_time_sec, + 256) {} + +DeleteBitmapAggCache* DeleteBitmapAggCache::instance() { + return ExecEnv::GetInstance()->delete_bitmap_agg_cache(); +} + +DeleteBitmapAggCache* DeleteBitmapAggCache::create_instance(size_t capacity) { + return new DeleteBitmapAggCache(capacity); } +DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {} + DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) { delete_bitmap = o.delete_bitmap; // just copy data _tablet_id = o._tablet_id; @@ -1438,20 +1438,20 @@ static std::string agg_cache_key(int64_t tablet_id, const DeleteBitmap::BitmapKe std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) const { std::string key_str = agg_cache_key(_tablet_id, bmk); // Cache key container CacheKey key(key_str); - Cache::Handle* handle = _agg_cache->repr()->lookup(key); + Cache::Handle* handle = DeleteBitmapAggCache::instance()->lookup(key); - AggCache::Value* val = - handle == nullptr - ? nullptr - : reinterpret_cast(_agg_cache->repr()->value(handle)); + DeleteBitmapAggCache::Value* val = + handle == nullptr ? nullptr + : reinterpret_cast( + DeleteBitmapAggCache::instance()->value(handle)); // FIXME: do we need a mutex here to get rid of duplicated initializations // of cache entries in some cases? if (val == nullptr) { // Renew if needed, put a new Value to cache - val = new AggCache::Value(); + val = new DeleteBitmapAggCache::Value(); Version start_version = config::enable_mow_get_agg_by_cache ? _get_rowset_cache_version(bmk) : 0; if (start_version > 0) { - Cache::Handle* handle2 = _agg_cache->repr()->lookup( + Cache::Handle* handle2 = DeleteBitmapAggCache::instance()->lookup( agg_cache_key(_tablet_id, {std::get<0>(bmk), std::get<1>(bmk), start_version})); DBUG_EXECUTE_IF("DeleteBitmap::get_agg.cache_miss", { @@ -1469,10 +1469,10 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co if (handle2 == nullptr || start_version > std::get<2>(bmk)) { start_version = 0; } else { - val->bitmap |= - reinterpret_cast(_agg_cache->repr()->value(handle2)) - ->bitmap; - _agg_cache->repr()->release(handle2); + val->bitmap |= reinterpret_cast( + DeleteBitmapAggCache::instance()->value(handle2)) + ->bitmap; + DeleteBitmapAggCache::instance()->release(handle2); VLOG_DEBUG << "get agg cache version=" << start_version << " for tablet=" << _tablet_id << ", rowset=" << std::get<0>(bmk).to_string() @@ -1492,8 +1492,9 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co val->bitmap |= bm; } } - size_t charge = val->bitmap.getSizeInBytes() + sizeof(AggCache::Value); - handle = _agg_cache->repr()->insert(key, val, charge, charge, CachePriority::NORMAL); + size_t charge = val->bitmap.getSizeInBytes() + sizeof(DeleteBitmapAggCache::Value); + handle = DeleteBitmapAggCache::instance()->insert(key, val, charge, charge, + CachePriority::NORMAL); if (config::enable_mow_get_agg_by_cache && !val->bitmap.isEmpty()) { std::lock_guard l(_rowset_cache_version_lock); // this version is already agg @@ -1520,7 +1521,7 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co // It is natural for the cache to reclaim the underlying memory return std::shared_ptr( - &val->bitmap, [this, handle](...) { _agg_cache->repr()->release(handle); }); + &val->bitmap, [handle](...) { DeleteBitmapAggCache::instance()->release(handle); }); } std::shared_ptr DeleteBitmap::get_agg_without_cache(const BitmapKey& bmk) const { @@ -1554,8 +1555,6 @@ DeleteBitmap DeleteBitmap::diffset(const std::set& key_set) const { return dbm; } -std::atomic DeleteBitmap::AggCache::s_repr {nullptr}; - std::string tablet_state_name(TabletState state) { switch (state) { case TABLET_NOTREADY: diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 63ad0124f862a6..949ce05fd74881 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -364,6 +364,20 @@ class TabletMeta : public MetadataAdder { mutable std::shared_mutex _meta_lock; }; +class DeleteBitmapAggCache : public LRUCachePolicy { +public: + DeleteBitmapAggCache(size_t capacity); + + static DeleteBitmapAggCache* instance(); + + static DeleteBitmapAggCache* create_instance(size_t capacity); + + class Value : public LRUCacheValueBase { + public: + roaring::Roaring bitmap; + }; +}; + /** * Wraps multiple bitmaps for recording rows (row id) that are deleted or * overwritten. For now, it's only used when unique key merge-on-write property @@ -578,40 +592,9 @@ class DeleteBitmap { */ DeleteBitmap diffset(const std::set& key_set) const; - class AggCachePolicy : public LRUCachePolicy { - public: - AggCachePolicy(size_t capacity) - : LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity, - LRUCacheType::SIZE, - config::delete_bitmap_agg_cache_stale_sweep_time_sec, 256) {} - }; - - class AggCache { - public: - class Value : public LRUCacheValueBase { - public: - roaring::Roaring bitmap; - }; - - AggCache(size_t size_in_bytes) { - static std::once_flag once; - std::call_once(once, [size_in_bytes] { - auto* tmp = new AggCachePolicy(size_in_bytes); - AggCache::s_repr.store(tmp, std::memory_order_release); - }); - - while (!s_repr.load(std::memory_order_acquire)) { - } - } - - static LRUCachePolicy* repr() { return s_repr.load(std::memory_order_acquire); } - static std::atomic s_repr; - }; - private: DeleteBitmap::Version _get_rowset_cache_version(const BitmapKey& bmk) const; - mutable std::shared_ptr _agg_cache; int64_t _tablet_id; mutable std::shared_mutex _rowset_cache_version_lock; mutable std::map> _rowset_cache_version; diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h index ec9fed17b7f85e..86a1e4c6c73066 100644 --- a/be/src/runtime/exec_env.h +++ b/be/src/runtime/exec_env.h @@ -115,6 +115,7 @@ class HeapProfiler; class WalManager; class DNSCache; struct SyncRowsetStats; +class DeleteBitmapAggCache; inline bool k_doris_exit = false; @@ -290,6 +291,7 @@ class ExecEnv { void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; } void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; } void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; } + void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; } void set_tablet_column_object_pool(TabletColumnObjectPool* c) { this->_tablet_column_object_pool = c; } @@ -356,6 +358,8 @@ class ExecEnv { bool check_auth_token(const std::string& auth_token); + DeleteBitmapAggCache* delete_bitmap_agg_cache() { return _delete_bitmap_agg_cache; } + private: ExecEnv(); @@ -485,6 +489,7 @@ class ExecEnv { QueryCache* _query_cache = nullptr; std::shared_ptr _dummy_lru_cache = nullptr; std::unique_ptr _file_cache_open_fd_cache; + DeleteBitmapAggCache* _delete_bitmap_agg_cache {nullptr}; pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr; diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index fd6f61a294d8e2..8f30211602a30b 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -54,6 +54,7 @@ #include "olap/segment_loader.h" #include "olap/storage_engine.h" #include "olap/tablet_column_object_pool.h" +#include "olap/tablet_meta.h" #include "olap/tablet_schema_cache.h" #include "olap/wal/wal_manager.h" #include "pipeline/pipeline_tracing.h" @@ -566,6 +567,16 @@ Status ExecEnv::_init_mem_env() { _query_cache = QueryCache::create_global_cache(config::query_cache_size * 1024L * 1024L); LOG(INFO) << "query cache memory limit: " << config::query_cache_size << "MB"; + // The default delete bitmap cache is set to 100MB, + // which can be insufficient and cause performance issues when the amount of user data is large. + // To mitigate the problem of an inadequate cache, + // we will take the larger of 0.5% of the total memory and 100MB as the delete bitmap cache size. + int64_t delete_bitmap_agg_cache_cache_limit = + ParseUtil::parse_mem_spec(config::delete_bitmap_dynamic_agg_cache_limit, + MemInfo::mem_limit(), MemInfo::physical_mem(), &is_percent); + _delete_bitmap_agg_cache = DeleteBitmapAggCache::create_instance(std::max( + delete_bitmap_agg_cache_cache_limit, config::delete_bitmap_agg_cache_capacity)); + return Status::OK(); } @@ -720,6 +731,7 @@ void ExecEnv::destroy() { SAFE_DELETE(_segment_loader); SAFE_DELETE(_row_cache); SAFE_DELETE(_query_cache); + SAFE_DELETE(_delete_bitmap_agg_cache); // Free resource after threads are stopped. // Some threads are still running, like threads created by _new_load_stream_mgr ... diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp index 810f3e0c28f548..253d17297804a6 100644 --- a/be/test/testutil/run_all_tests.cpp +++ b/be/test/testutil/run_all_tests.cpp @@ -32,6 +32,7 @@ #include "olap/segment_loader.h" #include "olap/storage_engine.h" #include "olap/tablet_column_object_pool.h" +#include "olap/tablet_meta.h" #include "olap/tablet_schema_cache.h" #include "runtime/exec_env.h" #include "runtime/memory/cache_manager.h" @@ -77,6 +78,9 @@ int main(int argc, char** argv) { doris::ExecEnv::GetInstance()->set_tablet_schema_cache( doris::TabletSchemaCache::create_global_schema_cache( doris::config::tablet_schema_cache_capacity)); + doris::ExecEnv::GetInstance()->set_delete_bitmap_agg_cache( + doris::DeleteBitmapAggCache::create_instance( + doris::config::delete_bitmap_agg_cache_capacity)); doris::ExecEnv::GetInstance()->set_tablet_column_object_pool( doris::TabletColumnObjectPool::create_global_column_cache( doris::config::tablet_schema_cache_capacity)); From 7a90b69c95c6dab7885fe8a06f20ab5f2f3946ae Mon Sep 17 00:00:00 2001 From: shuke Date: Tue, 8 Jul 2025 09:33:25 +0800 Subject: [PATCH 163/572] Revert "branch-3.0: [Fix](meta tool) Fix meta tool #51522" (#52728) Reverts apache/doris#51534 no need to pick to 3.0 --- be/src/tools/meta_tool.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp index d233f53966d2e3..076b4fae18b025 100644 --- a/be/src/tools/meta_tool.cpp +++ b/be/src/tools/meta_tool.cpp @@ -34,24 +34,37 @@ #include "io/fs/local_file_system.h" #include "json2pb/pb_to_json.h" #include "olap/data_dir.h" +#include "olap/olap_define.h" #include "olap/options.h" +#include "olap/rowset/segment_v2/binary_plain_page.h" #include "olap/rowset/segment_v2/column_reader.h" #include "olap/storage_engine.h" #include "olap/tablet_meta.h" #include "olap/tablet_meta_manager.h" +#include "olap/utils.h" #include "util/coding.h" #include "util/crc32c.h" +using std::filesystem::path; using doris::DataDir; using doris::StorageEngine; +using doris::OlapMeta; using doris::Status; using doris::TabletMeta; using doris::TabletMetaManager; using doris::Slice; using strings::Substitute; using doris::segment_v2::SegmentFooterPB; +using doris::segment_v2::ColumnReader; +using doris::segment_v2::PageHandle; +using doris::segment_v2::PagePointer; +using doris::segment_v2::ColumnReaderOptions; +using doris::segment_v2::ColumnIteratorOptions; +using doris::segment_v2::PageFooterPB; using doris::io::FileReaderSPtr; +const std::string HEADER_PREFIX = "tabletmeta_"; + DEFINE_string(root_path, "", "storage root path"); DEFINE_string(operation, "get_meta", "valid operation: get_meta, flag, load_meta, delete_meta, show_meta"); @@ -308,7 +321,6 @@ void show_segment_footer(const std::string& file_name) { } int main(int argc, char** argv) { - SCOPED_INIT_THREAD_CONTEXT(); std::string usage = get_usage(argv[0]); gflags::SetUsageMessage(usage); google::ParseCommandLineFlags(&argc, &argv, true); From 5368a6d3c2d83f13c202ad80916d514cbd467389 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:34:28 +0800 Subject: [PATCH 164/572] branch-3.0: [fix](main) check error immediately after flight_server init #52616 (#52621) Cherry-picked from #52616 Co-authored-by: Kaijie Chen --- be/src/service/doris_main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 00935e8fc64bed..67c8e870e70559 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -593,12 +593,12 @@ int main(int argc, char** argv) { std::shared_ptr flight_server = std::move(doris::flight::FlightSqlServer::create()).ValueOrDie(); status = flight_server->init(doris::config::arrow_flight_sql_port); + stop_work_if_error( + status, "Arrow Flight Service did not start correctly, exiting, " + status.to_string()); // 6. start daemon thread to do clean or gc jobs doris::Daemon daemon; daemon.start(); - stop_work_if_error( - status, "Arrow Flight Service did not start correctly, exiting, " + status.to_string()); exec_env->storage_engine().notify_listeners(); From 9a45101932c709a2265f5d636c86beccb36d427a Mon Sep 17 00:00:00 2001 From: koarz Date: Tue, 8 Jul 2025 09:34:52 +0800 Subject: [PATCH 165/572] branch-3.0: [enhance](meta-service)add bvar for ms kv get del put count (#52714) pick: https://github.com/apache/doris/pull/51708 --- cloud/src/common/bvars.cpp | 167 + cloud/src/common/bvars.h | 113 +- cloud/src/common/stats.h | 29 + cloud/src/meta-service/mem_txn_kv.cpp | 15 +- cloud/src/meta-service/mem_txn_kv.h | 7 + cloud/src/meta-service/meta_service.cpp | 105 +- cloud/src/meta-service/meta_service.h | 4 +- cloud/src/meta-service/meta_service_helper.h | 73 +- cloud/src/meta-service/meta_service_job.cpp | 6 +- .../meta-service/meta_service_partition.cpp | 31 +- .../meta-service/meta_service_resource.cpp | 80 +- cloud/src/meta-service/meta_service_txn.cpp | 106 +- cloud/src/meta-service/txn_kv.cpp | 3 + cloud/src/meta-service/txn_kv.h | 8 + cloud/src/meta-service/txn_lazy_committer.cpp | 6 +- cloud/test/CMakeLists.txt | 4 + cloud/test/meta_server_test.cpp | 6 +- cloud/test/rpc_kv_bvar_test.cpp | 3141 +++++++++++++++++ 18 files changed, 3741 insertions(+), 163 deletions(-) create mode 100644 cloud/src/common/stats.h create mode 100644 cloud/test/rpc_kv_bvar_test.cpp diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 29a8d1447bc408..9f7753b4063a52 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -214,4 +214,171 @@ BvarStatusWithTag g_bvar_inverted_checker_leaked_delete_bitmaps("checke BvarStatusWithTag g_bvar_inverted_checker_abnormal_delete_bitmaps("checker", "abnormal_delete_bitmaps"); BvarStatusWithTag g_bvar_inverted_checker_delete_bitmaps_scanned("checker", "delete_bitmap_keys_scanned"); +// rpc kv rw count +// get_rowset +mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_counter("rpc_kv_get_rowset_get_counter",{"instance_id"}); +// get_version +mBvarInt64Adder g_bvar_rpc_kv_get_version_get_counter("rpc_kv_get_version_get_counter",{"instance_id"}); +// get_schema_dict +mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_counter("rpc_kv_get_schema_dict_get_counter",{"instance_id"}); +// create_tablets +mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_counter("rpc_kv_create_tablets_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_counter("rpc_kv_create_tablets_put_counter",{"instance_id"}); +// update_tablet +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_get_counter("rpc_kv_update_tablet_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_put_counter("rpc_kv_update_tablet_put_counter",{"instance_id"}); +// update_tablet_schema +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_get_counter("rpc_kv_update_tablet_schema_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_put_counter("rpc_kv_update_tablet_schema_put_counter",{"instance_id"}); +// get_tablet +mBvarInt64Adder g_bvar_rpc_kv_get_tablet_get_counter("rpc_kv_get_tablet_get_counter",{"instance_id"}); +// prepare_rowset +mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_get_counter("rpc_kv_prepare_rowset_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_put_counter("rpc_kv_prepare_rowset_put_counter",{"instance_id"}); +// commit_rowset +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_get_counter("rpc_kv_commit_rowset_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_put_counter("rpc_kv_commit_rowset_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_del_counter("rpc_kv_commit_rowset_del_counter",{"instance_id"}); +// update_tmp_rowset +mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter("rpc_kv_update_tmp_rowset_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter("rpc_kv_update_tmp_rowset_put_counter",{"instance_id"}); +// get_tablet_stats +mBvarInt64Adder g_bvar_rpc_kv_get_tablet_stats_get_counter("rpc_kv_get_tablet_stats_get_counter",{"instance_id"}); +// update_delete_bitmap +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter("rpc_kv_update_delete_bitmap_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter("rpc_kv_update_delete_bitmap_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter("rpc_kv_update_delete_bitmap_del_counter",{"instance_id"}); +// get_delete_bitmap +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter("rpc_kv_get_delete_bitmap_get_counter",{"instance_id"}); +// get_delete_bitmap_update_lock +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter("rpc_kv_get_delete_bitmap_update_lock_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter("rpc_kv_get_delete_bitmap_update_lock_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter("rpc_kv_get_delete_bitmap_update_lock_del_counter",{"instance_id"}); +// remove_delete_bitmap_update_lock +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter("rpc_kv_remove_delete_bitmap_update_lock_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter("rpc_kv_remove_delete_bitmap_update_lock_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter("rpc_kv_remove_delete_bitmap_update_lock_del_counter",{"instance_id"}); +// remove_delete_bitmap +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter("rpc_kv_remove_delete_bitmap_del_counter",{"instance_id"}); +// start_tablet_job +mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_get_counter("rpc_kv_start_tablet_job_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_put_counter("rpc_kv_start_tablet_job_put_counter",{"instance_id"}); +// finish_tablet_job +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_get_counter("rpc_kv_finish_tablet_job_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_put_counter("rpc_kv_finish_tablet_job_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_del_counter("rpc_kv_finish_tablet_job_del_counter",{"instance_id"}); +// prepare_index +mBvarInt64Adder g_bvar_rpc_kv_prepare_index_get_counter("rpc_kv_prepare_index_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_index_put_counter("rpc_kv_prepare_index_put_counter",{"instance_id"}); +// commit_index +mBvarInt64Adder g_bvar_rpc_kv_commit_index_get_counter("rpc_kv_commit_index_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_put_counter("rpc_kv_commit_index_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_del_counter("rpc_kv_commit_index_del_counter",{"instance_id"}); +// drop_index +mBvarInt64Adder g_bvar_rpc_kv_drop_index_get_counter("rpc_kv_drop_index_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_drop_index_put_counter("rpc_kv_drop_index_put_counter",{"instance_id"}); +// prepare_partition +mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_get_counter("rpc_kv_prepare_partition_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_put_counter("rpc_kv_prepare_partition_put_counter",{"instance_id"}); +// commit_partition +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_get_counter("rpc_kv_commit_partition_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_put_counter("rpc_kv_commit_partition_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_del_counter("rpc_kv_commit_partition_del_counter",{"instance_id"}); +// drop_partition +mBvarInt64Adder g_bvar_rpc_kv_drop_partition_get_counter("rpc_kv_drop_partition_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_drop_partition_put_counter("rpc_kv_drop_partition_put_counter",{"instance_id"}); +// check_kv +mBvarInt64Adder g_bvar_rpc_kv_check_kv_get_counter("rpc_kv_check_kv_get_counter",{"instance_id"}); +// get_obj_store_info +mBvarInt64Adder g_bvar_rpc_kv_get_obj_store_info_get_counter("rpc_kv_get_obj_store_info_get_counter",{"instance_id"}); +// alter_storage_vault +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_get_counter("rpc_kv_alter_storage_vault_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_put_counter("rpc_kv_alter_storage_vault_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_del_counter("rpc_kv_alter_storage_vault_del_counter",{"instance_id"}); +// alter_obj_store_info +mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter("rpc_kv_alter_obj_store_info_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter("rpc_kv_alter_obj_store_info_put_counter",{"instance_id"}); +// update_ak_sk +mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_get_counter("rpc_kv_update_ak_sk_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_put_counter("rpc_kv_update_ak_sk_put_counter",{"instance_id"}); +// create_instance +mBvarInt64Adder g_bvar_rpc_kv_create_instance_get_counter("rpc_kv_create_instance_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_instance_put_counter("rpc_kv_create_instance_put_counter",{"instance_id"}); +// get_instance +mBvarInt64Adder g_bvar_rpc_kv_get_instance_get_counter("rpc_kv_get_instance_get_counter",{"instance_id"}); +// alter_cluster +mBvarInt64Adder g_bvar_rpc_kv_alter_cluster_get_counter("rpc_kv_alter_cluster_get_counter",{"instance_id"}); +// get_cluster +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_get_counter("rpc_kv_get_cluster_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_put_counter("rpc_kv_get_cluster_put_counter",{"instance_id"}); +// create_stage +mBvarInt64Adder g_bvar_rpc_kv_create_stage_get_counter("rpc_kv_create_stage_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_stage_put_counter("rpc_kv_create_stage_put_counter",{"instance_id"}); +// get_stage +mBvarInt64Adder g_bvar_rpc_kv_get_stage_get_counter("rpc_kv_get_stage_get_counter",{"instance_id"}); +// get_iam +mBvarInt64Adder g_bvar_rpc_kv_get_iam_get_counter("rpc_kv_get_iam_get_counter",{"instance_id"}); +// alter_iam +mBvarInt64Adder g_bvar_rpc_kv_alter_iam_get_counter("rpc_kv_alter_iam_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_iam_put_counter("rpc_kv_alter_iam_put_counter",{"instance_id"}); +// alter_ram_user +mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_get_counter("rpc_kv_alter_ram_user_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_put_counter("rpc_kv_alter_ram_user_put_counter",{"instance_id"}); +// begin_copy +mBvarInt64Adder g_bvar_rpc_kv_begin_copy_get_counter("rpc_kv_begin_copy_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_copy_put_counter("rpc_kv_begin_copy_put_counter",{"instance_id"}); +// finish_copy +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_get_counter("rpc_kv_finish_copy_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_put_counter("rpc_kv_finish_copy_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_del_counter("rpc_kv_finish_copy_del_counter",{"instance_id"}); +// get_copy_job +mBvarInt64Adder g_bvar_rpc_kv_get_copy_job_get_counter("rpc_kv_get_copy_job_get_counter",{"instance_id"}); +// get_copy_files +mBvarInt64Adder g_bvar_rpc_kv_get_copy_files_get_counter("rpc_kv_get_copy_files_get_counter",{"instance_id"}); +// filter_copy_files +mBvarInt64Adder g_bvar_rpc_kv_filter_copy_files_get_counter("rpc_kv_filter_copy_files_get_counter",{"instance_id"}); +// get_cluster_status +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_status_get_counter("rpc_kv_get_cluster_status_get_counter",{"instance_id"}); +// begin_txn +mBvarInt64Adder g_bvar_rpc_kv_begin_txn_get_counter("rpc_kv_begin_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_txn_put_counter("rpc_kv_begin_txn_put_counter",{"instance_id"}); +// precommit_txn +mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_get_counter("rpc_kv_precommit_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_put_counter("rpc_kv_precommit_txn_put_counter",{"instance_id"}); +// get_rl_task_commit_attach +mBvarInt64Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter("rpc_kv_get_rl_task_commit_attach_get_counter",{"instance_id"}); +// reset_rl_progress +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_get_counter("rpc_kv_reset_rl_progress_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_put_counter("rpc_kv_reset_rl_progress_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_del_counter("rpc_kv_reset_rl_progress_del_counter",{"instance_id"}); +// commit_txn +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_get_counter("rpc_kv_commit_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_put_counter("rpc_kv_commit_txn_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_del_counter("rpc_kv_commit_txn_del_counter",{"instance_id"}); +// abort_txn +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_get_counter("rpc_kv_abort_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_put_counter("rpc_kv_abort_txn_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_del_counter("rpc_kv_abort_txn_del_counter",{"instance_id"}); +// get_txn +mBvarInt64Adder g_bvar_rpc_kv_get_txn_get_counter("rpc_kv_get_txn_get_counter",{"instance_id"}); +// get_current_max_txn_id +mBvarInt64Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter("rpc_kv_get_current_max_txn_id_get_counter",{"instance_id"}); +// begin_sub_txn +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_get_counter("rpc_kv_begin_sub_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_put_counter("rpc_kv_begin_sub_txn_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_del_counter("rpc_kv_begin_sub_txn_del_counter",{"instance_id"}); +// abort_sub_txn +mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_get_counter("rpc_kv_abort_sub_txn_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter("rpc_kv_abort_sub_txn_put_counter",{"instance_id"}); +// abort_txn_with_coordinator +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter("rpc_kv_abort_txn_with_coordinator_get_counter",{"instance_id"}); +// check_txn_conflict +mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter("rpc_kv_check_txn_conflict_get_counter",{"instance_id"}); +// clean_txn_label +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter("rpc_kv_clean_txn_label_get_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_label_put_counter",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv_clean_txn_label_del_counter",{"instance_id"}); +// get_txn_id +mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv_get_txn_id_get_counter",{"instance_id"}); + // clang-format on diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index e8b5a9a0f5bd9f..2288209531dce6 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -150,10 +150,11 @@ class mBvarWrapper { auto get(const std::initializer_list& dim_values) { BvarType* stats = counter_.get_stats(std::list(dim_values)); + using ReturnType = decltype(stats->get_value()); if (stats) { return stats->get_value(); } - return std::declval(0); + return ReturnType {}; } private: @@ -174,6 +175,7 @@ class mBvarWrapper { }; using mBvarIntAdder = mBvarWrapper>; +using mBvarInt64Adder = mBvarWrapper>; using mBvarDoubleAdder = mBvarWrapper>; using mBvarIntRecorder = mBvarWrapper; using mBvarLatencyRecorder = mBvarWrapper; @@ -351,3 +353,112 @@ extern BvarStatusWithTag g_bvar_inverted_checker_num_check_failed; extern BvarStatusWithTag g_bvar_inverted_checker_leaked_delete_bitmaps; extern BvarStatusWithTag g_bvar_inverted_checker_abnormal_delete_bitmaps; extern BvarStatusWithTag g_bvar_inverted_checker_delete_bitmaps_scanned; + +// rpc kv +extern mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_version_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tablet_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tablet_stats_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_index_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_index_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_index_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_index_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_partition_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_partition_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_check_kv_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_obj_store_info_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_instance_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_instance_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_instance_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_cluster_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_stage_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_create_stage_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_stage_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_iam_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_iam_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_iam_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_copy_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_copy_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_copy_job_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_copy_files_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_filter_copy_files_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_status_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_current_max_txn_id_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter; diff --git a/cloud/src/common/stats.h b/cloud/src/common/stats.h new file mode 100644 index 00000000000000..2d99bf0d3c1671 --- /dev/null +++ b/cloud/src/common/stats.h @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris::cloud { +struct KVStats { + int64_t get_counter {}; + int64_t put_counter {}; + int64_t del_counter {}; +}; + +} // namespace doris::cloud \ No newline at end of file diff --git a/cloud/src/meta-service/mem_txn_kv.cpp b/cloud/src/meta-service/mem_txn_kv.cpp index 84593de3c200f8..72ecbda3254c2d 100644 --- a/cloud/src/meta-service/mem_txn_kv.cpp +++ b/cloud/src/meta-service/mem_txn_kv.cpp @@ -248,6 +248,7 @@ void Transaction::put(std::string_view key, std::string_view val) { writes_.insert_or_assign(k, v); op_list_.emplace_back(ModifyOpType::PUT, k, v); ++num_put_keys_; + kv_->put_count_++; put_bytes_ += key.size() + val.size(); approximate_bytes_ += key.size() + val.size(); } @@ -270,6 +271,8 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, int limit) { TEST_SYNC_POINT_CALLBACK("memkv::Transaction::get", &limit); std::lock_guard l(lock_); + num_get_keys_++; + kv_->get_count_++; std::string begin_k(begin.data(), begin.size()); std::string end_k(end.data(), end.size()); // TODO: figure out what happen if range_get has part of unreadable_keys @@ -344,12 +347,15 @@ TxnErrorCode Transaction::inner_get(const std::string& begin, const std::string& } std::vector> kv_list(kv_map.begin(), kv_map.end()); + num_get_keys_ += kv_list.size(); + kv_->get_count_ += kv_list.size(); *iter = std::make_unique(std::move(kv_list), more); return TxnErrorCode::TXN_OK; } void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_view val) { std::lock_guard l(lock_); + kv_->put_count_++; std::string k(key_prefix.data(), key_prefix.size()); std::string v(val.data(), val.size()); unreadable_keys_.insert(k); @@ -362,6 +368,7 @@ void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_vi void Transaction::atomic_set_ver_value(std::string_view key, std::string_view value) { std::lock_guard l(lock_); + kv_->put_count_++; std::string k(key.data(), key.size()); std::string v(value.data(), value.size()); unreadable_keys_.insert(k); @@ -380,6 +387,7 @@ void Transaction::atomic_add(std::string_view key, int64_t to_add) { op_list_.emplace_back(ModifyOpType::ATOMIC_ADD, std::move(k), std::move(v)); ++num_put_keys_; + kv_->put_count_++; put_bytes_ += key.size() + 8; approximate_bytes_ += key.size() + 8; } @@ -402,6 +410,7 @@ void Transaction::remove(std::string_view key) { remove_ranges_.emplace_back(k, end_key); op_list_.emplace_back(ModifyOpType::REMOVE, k, ""); + kv_->del_count_++; ++num_del_keys_; delete_bytes_ += key.size(); approximate_bytes_ += key.size(); @@ -421,7 +430,9 @@ void Transaction::remove(std::string_view begin, std::string_view end) { remove_ranges_.emplace_back(begin_k, end_k); op_list_.emplace_back(ModifyOpType::REMOVE_RANGE, begin_k, end_k); } - ++num_del_keys_; + kv_->del_count_ += 2; + // same as normal txn + num_del_keys_ += 2; delete_bytes_ += begin.size() + end.size(); approximate_bytes_ += begin.size() + end.size(); } @@ -480,6 +491,8 @@ TxnErrorCode Transaction::batch_get(std::vector>* res auto ret = inner_get(k, &val, opts.snapshot); ret == TxnErrorCode::TXN_OK ? res->push_back(val) : res->push_back(std::nullopt); } + kv_->get_count_ += keys.size(); + num_get_keys_ += keys.size(); return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-service/mem_txn_kv.h b/cloud/src/meta-service/mem_txn_kv.h index 63fb008f586ed8..9be6a2b5573c49 100644 --- a/cloud/src/meta-service/mem_txn_kv.h +++ b/cloud/src/meta-service/mem_txn_kv.h @@ -57,6 +57,10 @@ class MemTxnKv : public TxnKv, public std::enable_shared_from_this { TxnErrorCode get_kv(const std::string& begin, const std::string& end, int64_t version, int limit, bool* more, std::map* kv_list); + int64_t get_count_ {}; + int64_t put_count_ {}; + int64_t del_count_ {}; + private: using OpTuple = std::tuple; TxnErrorCode update(const std::set& read_set, const std::vector& op_list, @@ -189,6 +193,8 @@ class Transaction : public cloud::Transaction { size_t approximate_bytes() const override { return approximate_bytes_; } + size_t num_get_keys() const override { return num_get_keys_; } + size_t num_del_keys() const override { return num_del_keys_; } size_t num_put_keys() const override { return num_put_keys_; } @@ -218,6 +224,7 @@ class Transaction : public cloud::Transaction { int64_t read_version_ = -1; size_t approximate_bytes_ {0}; + size_t num_get_keys_ {0}; size_t num_del_keys_ {0}; size_t num_put_keys_ {0}; size_t delete_bytes_ {0}; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index e1b3195d50b5ed..9b131c58334c5b 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -50,6 +50,7 @@ #include "common/config.h" #include "common/encryption_util.h" #include "common/logging.h" +#include "common/stats.h" #include "common/stopwatch.h" #include "common/string_util.h" #include "common/util.h" @@ -88,7 +89,7 @@ std::string get_instance_id(const std::shared_ptr& rc_mgr, std::vector nodes; std::string err = rc_mgr->get_node(cloud_unique_id, &nodes); - { TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); } + TEST_SYNC_POINT_CALLBACK("get_instance_id_err", &err); std::string instance_id; if (!err.empty()) { // cache can't find cloud_unique_id, so degraded by parse cloud_unique_id @@ -208,7 +209,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, return; } - RPC_PREPROCESS(get_version); + RPC_PREPROCESS(get_version, get); std::string cloud_unique_id; if (request->has_cloud_unique_id()) { cloud_unique_id = request->cloud_unique_id(); @@ -248,7 +249,6 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, } code = MetaServiceCode::OK; - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -285,7 +285,7 @@ void MetaServiceImpl::get_version(::google::protobuf::RpcController* controller, response->set_version(version_pb.version()); response->add_version_update_time_ms(version_pb.update_time_ms()); } - { TEST_SYNC_POINT_CALLBACK("get_version_code", &code); } + TEST_SYNC_POINT_CALLBACK("get_version_code", &code); return; } else if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { msg = "not found"; @@ -300,7 +300,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr const GetVersionRequest* request, GetVersionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_version); + RPC_PREPROCESS(get_version, get); std::string cloud_unique_id; if (request->has_cloud_unique_id()) { @@ -356,7 +356,10 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr code = cast_as(err); break; } - + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + }; for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { size_t limit = (i + BATCH_SIZE < num_acquired) ? i + BATCH_SIZE : num_acquired; version_keys.clear(); @@ -431,7 +434,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode& code, std::string& msg, const doris::TabletMetaCloudPB& meta, std::shared_ptr txn_kv, const std::string& instance_id, - std::set>& saved_schema) { + std::set>& saved_schema, KVStats& stats) { doris::TabletMetaCloudPB tablet_meta(meta); bool has_first_rowset = tablet_meta.rs_metas_size() > 0; @@ -454,6 +457,11 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode msg = "failed to init txn"; return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + }; std::string rs_key, rs_val; if (has_first_rowset) { @@ -567,7 +575,7 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll const CreateTabletsRequest* request, CreateTabletsResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(create_tablets); + RPC_PREPROCESS(create_tablets, get, put); if (request->tablet_metas_size() == 0) { msg = "no tablet meta"; @@ -599,7 +607,7 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll err = txn0->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); - + stats.get_counter++; if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); ss << "failed to get instance, instance_id=" << instance_id << " err=" << err; @@ -650,7 +658,8 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll std::set> saved_schema; TEST_SYNC_POINT_RETURN_WITH_VOID("create_tablets"); for (auto& tablet_meta : request->tablet_metas()) { - internal_create_tablet(request, code, msg, tablet_meta, txn_kv_, instance_id, saved_schema); + internal_create_tablet(request, code, msg, tablet_meta, txn_kv_, instance_id, saved_schema, + stats); if (code != MetaServiceCode::OK) { return; } @@ -722,7 +731,7 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle const UpdateTabletRequest* request, UpdateTabletResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_tablet); + RPC_PREPROCESS(update_tablet, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -731,7 +740,6 @@ void MetaServiceImpl::update_tablet(::google::protobuf::RpcController* controlle return; } RPC_RATE_LIMIT(update_tablet) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -831,7 +839,7 @@ void MetaServiceImpl::update_tablet_schema(::google::protobuf::RpcController* co UpdateTabletSchemaResponse* response, ::google::protobuf::Closure* done) { DCHECK(false) << "should not call update_tablet_schema"; - RPC_PREPROCESS(update_tablet_schema); + RPC_PREPROCESS(update_tablet_schema, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -842,7 +850,6 @@ void MetaServiceImpl::update_tablet_schema(::google::protobuf::RpcController* co RPC_RATE_LIMIT(update_tablet_schema) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -904,7 +911,7 @@ void MetaServiceImpl::update_tablet_schema(::google::protobuf::RpcController* co void MetaServiceImpl::get_tablet(::google::protobuf::RpcController* controller, const GetTabletRequest* request, GetTabletResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_tablet); + RPC_PREPROCESS(get_tablet, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -913,7 +920,6 @@ void MetaServiceImpl::get_tablet(::google::protobuf::RpcController* controller, return; } RPC_RATE_LIMIT(get_tablet) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1122,7 +1128,7 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(prepare_rowset); + RPC_PREPROCESS(prepare_rowset, get, put); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; @@ -1147,7 +1153,6 @@ void MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll int64_t tablet_id = rowset_meta.tablet_id(); const auto& rowset_id = rowset_meta.rowset_id_v2(); auto tmp_rs_key = meta_rowset_tmp_key({instance_id, rowset_meta.txn_id(), tablet_id}); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1267,7 +1272,7 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_rowset); + RPC_PREPROCESS(commit_rowset, get, put, del); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; @@ -1293,7 +1298,6 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle auto tmp_rs_key = meta_rowset_tmp_key({instance_id, rowset_meta.txn_id(), tablet_id}); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1428,7 +1432,7 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr const CreateRowsetRequest* request, CreateRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_tmp_rowset); + RPC_PREPROCESS(update_tmp_rowset, get, put); if (!request->has_rowset_meta()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "no rowset meta"; @@ -1457,7 +1461,6 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr MetaRowsetTmpKeyInfo key_info {instance_id, txn_id, tablet_id}; meta_rowset_tmp_key(key_info, &update_key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1685,7 +1688,7 @@ static bool try_fetch_and_parse_schema(Transaction* txn, RowsetMetaCloudPB& rows void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, const GetRowsetRequest* request, GetRowsetResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_rowset); + RPC_PREPROCESS(get_rowset, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -1721,7 +1724,10 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, LOG(WARNING) << msg; return; } - + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + }; TabletIndexPB idx; // Get tablet id index from kv get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx); @@ -1760,6 +1766,7 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, if (version_pb.pending_txn_ids_size() > 0) { DCHECK(version_pb.pending_txn_ids_size() == 1); + stats.get_counter += txn->num_get_keys(); txn.reset(); TEST_SYNC_POINT_CALLBACK("get_rowset::advance_last_pending_txn_id", &version_pb); @@ -1873,7 +1880,7 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro const GetTabletStatsRequest* request, GetTabletStatsResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_tablet_stats); + RPC_PREPROCESS(get_tablet_stats, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -1883,7 +1890,6 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro } RPC_RATE_LIMIT(get_tablet_stats) - std::unique_ptr txn; for (auto& i : request->tablet_idx()) { TabletIndexPB idx(i); // FIXME(plat1ko): Get all tablet stats in one txn @@ -1893,6 +1899,11 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro msg = fmt::format("failed to create txn, tablet_id={}", idx.tablet_id()); return; } + DORIS_CLOUD_DEFER { + stats.get_counter += txn->num_get_keys(); + // the txn is not a local variable, if not reset will count last res twice + txn.reset(nullptr); + }; if (!(/* idx.has_db_id() && */ idx.has_table_id() && idx.has_index_id() && idx.has_partition_id() && i.has_tablet_id())) { get_tablet_idx(code, msg, txn.get(), instance_id, idx.tablet_id(), idx); @@ -2102,7 +2113,7 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont const UpdateDeleteBitmapRequest* request, UpdateDeleteBitmapResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_delete_bitmap); + RPC_PREPROCESS(update_delete_bitmap, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2123,7 +2134,6 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont auto table_id = request->table_id(); auto tablet_id = request->tablet_id(); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2236,6 +2246,9 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont g_bvar_update_delete_bitmap_fail_counter << 1; return; } + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); current_key_count = 0; current_value_count = 0; TxnErrorCode err = txn_kv_->create_txn(&txn); @@ -2298,7 +2311,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control const GetDeleteBitmapRequest* request, GetDeleteBitmapResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_delete_bitmap); + RPC_PREPROCESS(get_delete_bitmap, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2344,6 +2357,10 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + }; MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], begin_versions[i], 0}; MetaDeleteBitmapInfo end_key_info {instance_id, tablet_id, rowset_ids[i], end_versions[i], @@ -2370,6 +2387,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_err", &round, &err); int64_t retry = 0; while (err == TxnErrorCode::TXN_TOO_OLD && retry < 3) { + stats.get_counter += txn->num_get_keys(); txn = nullptr; err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { @@ -2461,6 +2479,10 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control msg = "failed to init txn"; return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + }; TabletIndexPB idx(request->idx()); TabletStatsPB tablet_stat; internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, @@ -2484,7 +2506,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_delete_bitmap_update_lock); + RPC_PREPROCESS(get_delete_bitmap_update_lock, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2501,7 +2523,6 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl } RPC_RATE_LIMIT(get_delete_bitmap_update_lock) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2581,7 +2602,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl return; } - if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key)) { + if (!get_mow_tablet_stats_and_meta(code, msg, request, response, instance_id, lock_key, + stats)) { return; }; } @@ -2589,8 +2611,8 @@ void MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std::string& msg, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, - std::string& instance_id, - std::string& lock_key) { + std::string& instance_id, std::string& lock_key, + KVStats& stats) { bool require_tablet_stats = request->has_require_compaction_stats() ? request->require_compaction_stats() : false; if (!require_tablet_stats) return true; @@ -2611,6 +2633,12 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: msg = "failed to init txn"; return false; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + }; auto table_id = request->table_id(); std::stringstream ss; if (!config::enable_batch_get_mow_tablet_stats_and_meta) { @@ -2784,7 +2812,7 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock( google::protobuf::RpcController* controller, const RemoveDeleteBitmapUpdateLockRequest* request, RemoveDeleteBitmapUpdateLockResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(remove_delete_bitmap_update_lock); + RPC_PREPROCESS(remove_delete_bitmap_update_lock, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2801,7 +2829,6 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock( } RPC_RATE_LIMIT(remove_delete_bitmap_update_lock) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2863,7 +2890,7 @@ void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController* cont const RemoveDeleteBitmapRequest* request, RemoveDeleteBitmapResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(remove_delete_bitmap); + RPC_PREPROCESS(remove_delete_bitmap, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2892,7 +2919,6 @@ void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController* cont msg = ss.str(); return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to init txn"; @@ -3014,7 +3040,7 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control const GetSchemaDictRequest* request, GetSchemaDictResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_schema_dict); + RPC_PREPROCESS(get_schema_dict, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3031,7 +3057,6 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control RPC_RATE_LIMIT(get_schema_dict) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index a4136ed39be26a..ce8e04fed48f19 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -28,6 +28,7 @@ #include #include "common/config.h" +#include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/txn_kv.h" #include "meta-service/txn_lazy_committer.h" @@ -332,7 +333,8 @@ class MetaServiceImpl : public cloud::MetaService { bool get_mow_tablet_stats_and_meta(MetaServiceCode& code, std::string& msg, const GetDeleteBitmapUpdateLockRequest* request, GetDeleteBitmapUpdateLockResponse* response, - std::string& instance_id, std::string& lock_key); + std::string& instance_id, std::string& lock_key, + KVStats& stats); std::shared_ptr txn_kv_; std::shared_ptr resource_mgr_; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 5e724eae14f619..256816b68bb3cf 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -30,6 +30,7 @@ #include "common/config.h" #include "common/defer.h" #include "common/logging.h" +#include "common/stats.h" #include "common/stopwatch.h" #include "common/util.h" #include "cpp/sync_point.h" @@ -226,24 +227,57 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { } } -#define RPC_PREPROCESS(func_name) \ - StopWatch sw; \ - auto ctrl = static_cast(controller); \ - begin_rpc(#func_name, ctrl, request); \ - brpc::ClosureGuard closure_guard(done); \ - [[maybe_unused]] std::stringstream ss; \ - [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ - [[maybe_unused]] std::string msg; \ - [[maybe_unused]] std::string instance_id; \ - [[maybe_unused]] bool drop_request = false; \ - DORIS_CLOUD_DEFER { \ - response->mutable_status()->set_code(code); \ - response->mutable_status()->set_msg(msg); \ - finish_rpc(#func_name, ctrl, response); \ - closure_guard.reset(nullptr); \ - if (config::use_detailed_metrics && !instance_id.empty() && !drop_request) { \ - g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ - } \ +// don't use these macro it just for defer count, reduce useless variable(some rpc just need one of rw op) +// If we have to write separate code for each RPC, it would be quite troublesome +// After all, adding put, get, and del after the RPC_PREPROCESS macro is simpler than writing a long string of code +#define RPCKVCOUNTHELPER(func_name, op) \ + g_bvar_rpc_kv_##func_name##_##op##_counter.put({instance_id}, stats.op##_counter); +#define RPCKVCOUNT_0(func_name) +#define RPCKVCOUNT_1(func_name, op1) RPCKVCOUNTHELPER(func_name, op1) +#define RPCKVCOUNT_2(func_name, op1, op2) \ + RPCKVCOUNT_1(func_name, op1) RPCKVCOUNTHELPER(func_name, op2) +#define RPCKVCOUNT_3(func_name, op1, op2, op3) \ + RPCKVCOUNT_2(func_name, op1, op2) RPCKVCOUNTHELPER(func_name, op3) +#define GET_RPCKVCOUNT_MACRO(_0, _1, _2, _3, NAME, ...) NAME + +// input func_name, count type(get, put, del), make sure the counter is exist +// about defer_count: +// which means that these bvars will only be counted after stats has finished counting. +// why not cancle KVStats, count directly? +// 1. some RPC operations call functions and function reset txn it also need to be counted +// 2. some function such as `scan_tmp_rowset` it used by RPC(commit_txn) and non rpc +// maybe we can add a bool variable to judge weather we need count, but if have more complex situation +// `func1` used by RPC1, RPC2 and RPC3 judge it or just give func1 a pointer +#define RPC_PREPROCESS(func_name, ...) \ + StopWatch sw; \ + auto ctrl = static_cast(controller); \ + begin_rpc(#func_name, ctrl, request); \ + brpc::ClosureGuard closure_guard(done); \ + [[maybe_unused]] std::stringstream ss; \ + [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ + [[maybe_unused]] std::unique_ptr txn; \ + [[maybe_unused]] std::string msg; \ + [[maybe_unused]] std::string instance_id; \ + [[maybe_unused]] bool drop_request = false; \ + [[maybe_unused]] KVStats stats; \ + DORIS_CLOUD_DEFER { \ + response->mutable_status()->set_code(code); \ + response->mutable_status()->set_msg(msg); \ + finish_rpc(#func_name, ctrl, response); \ + closure_guard.reset(nullptr); \ + if (txn != nullptr) { \ + stats.get_counter += txn->num_get_keys(); \ + stats.put_counter += txn->num_put_keys(); \ + stats.del_counter += txn->num_del_keys(); \ + } \ + if (config::use_detailed_metrics && !instance_id.empty()) { \ + if (!drop_request) { \ + g_bvar_ms_##func_name.put(instance_id, sw.elapsed_us()); \ + } \ + GET_RPCKVCOUNT_MACRO(_0, ##__VA_ARGS__, RPCKVCOUNT_3, RPCKVCOUNT_2, RPCKVCOUNT_1, \ + RPCKVCOUNT_0) \ + (func_name, ##__VA_ARGS__) \ + } \ }; #define RPC_RATE_LIMIT(func_name) \ @@ -272,7 +306,8 @@ int decrypt_instance_info(InstanceInfoPB& instance, const std::string& instance_ /** * Notifies other metaservice to refresh instance */ -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats); void get_tablet_idx(MetaServiceCode& code, std::string& msg, Transaction* txn, const std::string& instance_id, int64_t tablet_id, TabletIndexPB& tablet_idx); diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index e356c198670089..218742713680f6 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -406,7 +406,7 @@ void MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro const StartTabletJobRequest* request, StartTabletJobResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(start_tablet_job); + RPC_PREPROCESS(start_tablet_job, get, put); std::string cloud_unique_id = request->cloud_unique_id(); instance_id = get_instance_id(resource_mgr_, cloud_unique_id); if (instance_id.empty()) { @@ -424,7 +424,6 @@ void MetaServiceImpl::start_tablet_job(::google::protobuf::RpcController* contro return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1390,7 +1389,7 @@ void MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr const FinishTabletJobRequest* request, FinishTabletJobResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(finish_tablet_job); + RPC_PREPROCESS(finish_tablet_job, get, put, del); std::string cloud_unique_id = request->cloud_unique_id(); instance_id = get_instance_id(resource_mgr_, cloud_unique_id); if (instance_id.empty()) { @@ -1410,7 +1409,6 @@ void MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr } bool need_commit = false; - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index d165b8b5e06bb2..379a7d72eaa340 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -77,7 +77,7 @@ static TxnErrorCode check_recycle_key_exist(Transaction* txn, const std::string& void MetaServiceImpl::prepare_index(::google::protobuf::RpcController* controller, const IndexRequest* request, IndexResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(prepare_index); + RPC_PREPROCESS(prepare_index, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -94,7 +94,6 @@ void MetaServiceImpl::prepare_index(::google::protobuf::RpcController* controlle return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -164,7 +163,7 @@ void MetaServiceImpl::prepare_index(::google::protobuf::RpcController* controlle void MetaServiceImpl::commit_index(::google::protobuf::RpcController* controller, const IndexRequest* request, IndexResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_index); + RPC_PREPROCESS(commit_index, get, put, del); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -179,7 +178,6 @@ void MetaServiceImpl::commit_index(::google::protobuf::RpcController* controller return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -248,7 +246,7 @@ void MetaServiceImpl::commit_index(::google::protobuf::RpcController* controller void MetaServiceImpl::drop_index(::google::protobuf::RpcController* controller, const IndexRequest* request, IndexResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(drop_index); + RPC_PREPROCESS(drop_index, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -263,7 +261,6 @@ void MetaServiceImpl::drop_index(::google::protobuf::RpcController* controller, return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -350,7 +347,7 @@ void MetaServiceImpl::prepare_partition(::google::protobuf::RpcController* contr const PartitionRequest* request, PartitionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(prepare_partition); + RPC_PREPROCESS(prepare_partition, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -368,7 +365,6 @@ void MetaServiceImpl::prepare_partition(::google::protobuf::RpcController* contr return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -440,7 +436,7 @@ void MetaServiceImpl::prepare_partition(::google::protobuf::RpcController* contr void MetaServiceImpl::commit_partition(::google::protobuf::RpcController* controller, const PartitionRequest* request, PartitionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_partition); + RPC_PREPROCESS(commit_partition, get, put, del); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -455,7 +451,6 @@ void MetaServiceImpl::commit_partition(::google::protobuf::RpcController* contro return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -528,7 +523,7 @@ void MetaServiceImpl::commit_partition(::google::protobuf::RpcController* contro void MetaServiceImpl::drop_partition(::google::protobuf::RpcController* controller, const PartitionRequest* request, PartitionResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(drop_partition); + RPC_PREPROCESS(drop_partition, get, put); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -544,7 +539,6 @@ void MetaServiceImpl::drop_partition(::google::protobuf::RpcController* controll return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -623,7 +617,7 @@ void MetaServiceImpl::drop_partition(::google::protobuf::RpcController* controll void check_create_table(std::string instance_id, std::shared_ptr txn_kv, const CheckKVRequest* request, CheckKVResponse* response, - MetaServiceCode* code, std::string* msg, + MetaServiceCode* code, std::string* msg, KVStats& stats, check_create_table_type get_check_info) { StopWatch watch; std::unique_ptr txn; @@ -633,6 +627,10 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "failed to create txn"; return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + }; auto& [keys, hint, key_func] = get_check_info(request); if (keys.empty()) { *code = MetaServiceCode::INVALID_ARGUMENT; @@ -652,6 +650,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "prepare and commit rpc not match, recycle key remained"; return; } else if (err == TxnErrorCode::TXN_TOO_OLD) { + stats.get_counter += txn->num_get_keys(); // separate it to several txn for rubustness txn.reset(); TxnErrorCode err = txn_kv->create_txn(&txn); @@ -677,7 +676,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, void MetaServiceImpl::check_kv(::google::protobuf::RpcController* controller, const CheckKVRequest* request, CheckKVResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(check_kv); + RPC_PREPROCESS(check_kv, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -697,7 +696,7 @@ void MetaServiceImpl::check_kv(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(check_kv); switch (request->op()) { case CheckKVRequest::CREATE_INDEX_AFTER_FE_COMMIT: { - check_create_table(instance_id, txn_kv_, request, response, &code, &msg, + check_create_table(instance_id, txn_kv_, request, response, &code, &msg, stats, [](const CheckKVRequest* request) { return std::make_tuple( request->check_keys().index_ids(), "index", @@ -709,7 +708,7 @@ void MetaServiceImpl::check_kv(::google::protobuf::RpcController* controller, } case CheckKVRequest::CREATE_PARTITION_AFTER_FE_COMMIT: { check_create_table( - instance_id, txn_kv_, request, response, &code, &msg, + instance_id, txn_kv_, request, response, &code, &msg, stats, [](const CheckKVRequest* request) { return std::make_tuple( request->check_keys().partition_ids(), "partition", diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 44389a41e2b91f..8b3c922b87633c 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -31,6 +31,7 @@ #include "common/encryption_util.h" #include "common/logging.h" #include "common/network_util.h" +#include "common/stats.h" #include "common/string_util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -195,7 +196,7 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro const GetObjStoreInfoRequest* request, GetObjStoreInfoResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_obj_store_info); + RPC_PREPROCESS(get_obj_store_info, get); TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response); TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return"); // Prepare data @@ -219,7 +220,6 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -929,7 +929,7 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr bool use_path_style; EncryptionInfoPB encryption_info; AkSkPair cipher_ak_sk_pair; - RPC_PREPROCESS(alter_storage_vault); + RPC_PREPROCESS(alter_storage_vault, get, put, del); switch (request->op()) { case AlterObjStoreInfoRequest::ADD_S3_VAULT: case AlterObjStoreInfoRequest::DROP_S3_VAULT: { @@ -1008,7 +1008,6 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1237,7 +1236,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont bool use_path_style; EncryptionInfoPB encryption_info; AkSkPair cipher_ak_sk_pair; - RPC_PREPROCESS(alter_obj_store_info); + RPC_PREPROCESS(alter_obj_store_info, get, put); switch (request->op()) { case AlterObjStoreInfoRequest::ADD_OBJ_INFO: case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: @@ -1287,7 +1286,6 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1464,7 +1462,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont void MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller, const UpdateAkSkRequest* request, UpdateAkSkResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(update_ak_sk); + RPC_PREPROCESS(update_ak_sk, get, put); instance_id = request->has_instance_id() ? request->instance_id() : ""; if (instance_id.empty()) { msg = "instance id not set"; @@ -1483,7 +1481,6 @@ void MetaServiceImpl::update_ak_sk(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1662,7 +1659,7 @@ void MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle const CreateInstanceRequest* request, CreateInstanceResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(create_instance); + RPC_PREPROCESS(create_instance, get, put); if (request->has_ram_user()) { auto& ram_user = request->ram_user(); std::string ram_user_id = ram_user.has_user_id() ? ram_user.user_id() : ""; @@ -1714,7 +1711,6 @@ void MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -1967,7 +1963,7 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller if (request->op() == AlterInstanceRequest::REFRESH) return; auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id); + notify_refresh_instance(txn_kv, instance_id, nullptr); }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -1979,7 +1975,7 @@ void MetaServiceImpl::alter_instance(google::protobuf::RpcController* controller void MetaServiceImpl::get_instance(google::protobuf::RpcController* controller, const GetInstanceRequest* request, GetInstanceResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_instance); + RPC_PREPROCESS(get_instance, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -1999,7 +1995,6 @@ void MetaServiceImpl::get_instance(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2093,7 +2088,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, const AlterClusterRequest* request, AlterClusterResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(alter_cluster); + RPC_PREPROCESS(alter_cluster, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; instance_id = request->has_instance_id() ? request->instance_id() : ""; if (!cloud_unique_id.empty() && instance_id.empty()) { @@ -2444,7 +2439,12 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, if (code != MetaServiceCode::OK) return; auto f = new std::function([instance_id = request->instance_id(), txn_kv = txn_kv_] { - notify_refresh_instance(txn_kv, instance_id); + // the func run with a thread, so if use macro proved stats, maybe cause stack-use-after-return error + KVStats stats; + notify_refresh_instance(txn_kv, instance_id, &stats); + if (config::use_detailed_metrics && !instance_id.empty()) { + g_bvar_rpc_kv_alter_cluster_get_counter.put({instance_id}, stats.get_counter); + } }); bthread_t bid; if (bthread_start_background(&bid, nullptr, run_bthread_work, f) != 0) { @@ -2456,7 +2456,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, const GetClusterRequest* request, GetClusterResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_cluster); + RPC_PREPROCESS(get_cluster, get, put); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; std::string cluster_id = request->has_cluster_id() ? request->cluster_id() : ""; std::string cluster_name = request->has_cluster_name() ? request->cluster_name() : ""; @@ -2510,7 +2510,6 @@ void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2601,7 +2600,7 @@ void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller, const CreateStageRequest* request, CreateStageResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(create_stage); + RPC_PREPROCESS(create_stage, get, put); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2658,7 +2657,6 @@ void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2788,7 +2786,7 @@ void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller, const GetStageRequest* request, GetStageResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_stage); + RPC_PREPROCESS(get_stage, get); TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response); TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return"); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; @@ -2818,7 +2816,6 @@ void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3176,7 +3173,7 @@ void MetaServiceImpl::drop_stage(google::protobuf::RpcController* controller, void MetaServiceImpl::get_iam(google::protobuf::RpcController* controller, const GetIamRequest* request, GetIamResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_iam); + RPC_PREPROCESS(get_iam, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3198,7 +3195,6 @@ void MetaServiceImpl::get_iam(google::protobuf::RpcController* controller, std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3273,7 +3269,7 @@ void MetaServiceImpl::get_iam(google::protobuf::RpcController* controller, void MetaServiceImpl::alter_iam(google::protobuf::RpcController* controller, const AlterIamRequest* request, AlterIamResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(alter_iam); + RPC_PREPROCESS(alter_iam, get, put); std::string arn_id = request->has_account_id() ? request->account_id() : ""; std::string arn_ak = request->has_ak() ? request->ak() : ""; std::string arn_sk = request->has_sk() ? request->sk() : ""; @@ -3285,9 +3281,11 @@ void MetaServiceImpl::alter_iam(google::protobuf::RpcController* controller, RPC_RATE_LIMIT(alter_iam) + // for metric, give it a common instance id + instance_id = "alter_iam_instance"; + std::string key = system_meta_service_arn_info_key(); std::string val; - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3363,7 +3361,7 @@ void MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller const AlterRamUserRequest* request, AlterRamUserResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(alter_ram_user); + RPC_PREPROCESS(alter_ram_user, get, put); instance_id = request->has_instance_id() ? request->instance_id() : ""; if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3383,7 +3381,6 @@ void MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller std::string val; instance_key(key_info, &key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3446,7 +3443,7 @@ void MetaServiceImpl::alter_ram_user(google::protobuf::RpcController* controller void MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller, const BeginCopyRequest* request, BeginCopyResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(begin_copy); + RPC_PREPROCESS(begin_copy, get, put); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3462,7 +3459,6 @@ void MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller, return; } RPC_RATE_LIMIT(begin_copy) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3563,7 +3559,7 @@ void MetaServiceImpl::begin_copy(google::protobuf::RpcController* controller, void MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller, const FinishCopyRequest* request, FinishCopyResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(finish_copy); + RPC_PREPROCESS(finish_copy, get, put, del); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3580,7 +3576,6 @@ void MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller, } RPC_RATE_LIMIT(finish_copy) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3668,7 +3663,7 @@ void MetaServiceImpl::finish_copy(google::protobuf::RpcController* controller, void MetaServiceImpl::get_copy_job(google::protobuf::RpcController* controller, const GetCopyJobRequest* request, GetCopyJobResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_copy_job); + RPC_PREPROCESS(get_copy_job, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3684,7 +3679,6 @@ void MetaServiceImpl::get_copy_job(google::protobuf::RpcController* controller, return; } - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3720,7 +3714,7 @@ void MetaServiceImpl::get_copy_files(google::protobuf::RpcController* controller const GetCopyFilesRequest* request, GetCopyFilesResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_copy_files); + RPC_PREPROCESS(get_copy_files, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3737,7 +3731,6 @@ void MetaServiceImpl::get_copy_files(google::protobuf::RpcController* controller } RPC_RATE_LIMIT(get_copy_files) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3784,7 +3777,7 @@ void MetaServiceImpl::filter_copy_files(google::protobuf::RpcController* control const FilterCopyFilesRequest* request, FilterCopyFilesResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(filter_copy_files); + RPC_PREPROCESS(filter_copy_files, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -3801,7 +3794,6 @@ void MetaServiceImpl::filter_copy_files(google::protobuf::RpcController* control } RPC_RATE_LIMIT(filter_copy_files) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3836,7 +3828,7 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro const GetClusterStatusRequest* request, GetClusterStatusResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_cluster_status); + RPC_PREPROCESS(get_cluster_status, get); if (request->instance_ids().empty() && request->cloud_unique_ids().empty()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "cloud_unique_ids or instance_ids must be given, instance_ids.size: " + @@ -3886,6 +3878,12 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro LOG(WARNING) << "failed to create txn err=" << err; return; } + DORIS_CLOUD_DEFER { + if (config::use_detailed_metrics && txn != nullptr) { + g_bvar_rpc_kv_get_cluster_status_get_counter.put({instance_id}, + txn->num_get_keys()); + } + }; err = txn->get(key, &val); LOG(INFO) << "get instance_key=" << hex(key); @@ -3930,7 +3928,8 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro msg = proto_to_json(*response); } -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id) { +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats) { LOG(INFO) << "begin notify_refresh_instance"; std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -3941,6 +3940,9 @@ void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& i std::string key = system_meta_service_registry_key(); std::string val; err = txn->get(key, &val); + if (config::use_detailed_metrics && stats) { + stats->get_counter++; + } if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to get server registry" << " err=" << err; diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 4243dc4a1b52fe..55e201d43d2bc9 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -23,6 +23,7 @@ #include "common/config.h" #include "common/logging.h" +#include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/doris_txn.h" #include "meta-service/keys.h" @@ -80,7 +81,7 @@ static void calc_table_stats(std::unordered_map& tablet_ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, const BeginTxnRequest* request, BeginTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(begin_txn); + RPC_PREPROCESS(begin_txn, get, put); if (!request->has_txn_info()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid argument, missing txn info"; @@ -110,7 +111,6 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(begin_txn) //1. Generate version stamp for txn id - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -150,6 +150,9 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, msg = ss.str(); return; } + // get count before txn reset, if not we will lose these count + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); //2. Get txn id from version stamp txn.reset(); @@ -358,7 +361,7 @@ void MetaServiceImpl::precommit_txn(::google::protobuf::RpcController* controlle const PrecommitTxnRequest* request, PrecommitTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(precommit_txn); + RPC_PREPROCESS(precommit_txn, get, put); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t db_id = request->has_db_id() ? request->db_id() : -1; if ((txn_id < 0 && db_id < 0)) { @@ -379,7 +382,6 @@ void MetaServiceImpl::precommit_txn(::google::protobuf::RpcController* controlle return; } RPC_RATE_LIMIT(precommit_txn); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -599,7 +601,7 @@ void MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcControlle const GetRLTaskCommitAttachRequest* request, GetRLTaskCommitAttachResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_rl_task_commit_attach); + RPC_PREPROCESS(get_rl_task_commit_attach, get); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -609,7 +611,6 @@ void MetaServiceImpl::get_rl_task_commit_attach(::google::protobuf::RpcControlle } RPC_RATE_LIMIT(get_rl_task_commit_attach) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -668,7 +669,7 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr const ResetRLProgressRequest* request, ResetRLProgressResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(reset_rl_progress); + RPC_PREPROCESS(reset_rl_progress, get, put, del); instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -678,7 +679,6 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr } RPC_RATE_LIMIT(reset_rl_progress) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -769,7 +769,8 @@ void MetaServiceImpl::reset_rl_progress(::google::protobuf::RpcController* contr void scan_tmp_rowset( const std::string& instance_id, int64_t txn_id, std::shared_ptr txn_kv, MetaServiceCode& code, std::string& msg, int64_t* db_id, - std::vector>* tmp_rowsets_meta) { + std::vector>* tmp_rowsets_meta, + KVStats* stats) { // Create a readonly txn for scan tmp rowset std::stringstream ss; std::unique_ptr txn; @@ -781,6 +782,9 @@ void scan_tmp_rowset( LOG(WARNING) << msg; return; } + DORIS_CLOUD_DEFER { + if (stats && txn) stats->get_counter += txn->num_get_keys(); + }; // Get db id with txn id std::string index_val; @@ -937,7 +941,7 @@ void commit_txn_immediately( std::shared_ptr& txn_kv, std::shared_ptr& txn_lazy_committer, MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id, std::vector>& tmp_rowsets_meta, - TxnErrorCode& err) { + TxnErrorCode& err, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); do { @@ -952,7 +956,12 @@ void commit_txn_immediately( LOG(WARNING) << msg; return; } - + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + }; // Get txn info with db_id and txn_id std::string info_val; // Will be reused when saving updated txn const std::string info_key = txn_info_key({instance_id, db_id, txn_id}); @@ -1120,6 +1129,7 @@ void commit_txn_immediately( version_values.clear(); if (last_pending_txn_id > 0) { + stats.get_counter += txn->num_get_keys(); txn.reset(); TEST_SYNC_POINT_CALLBACK("commit_txn_immediately::advance_last_pending_txn_id", &last_pending_txn_id); @@ -1554,7 +1564,8 @@ void commit_txn_eventually( const CommitTxnRequest* request, CommitTxnResponse* response, std::shared_ptr& txn_kv, std::shared_ptr& txn_lazy_committer, MetaServiceCode& code, std::string& msg, const std::string& instance_id, int64_t db_id, - const std::vector>& tmp_rowsets_meta) { + const std::vector>& tmp_rowsets_meta, + KVStats& stats) { StopWatch sw; DORIS_CLOUD_DEFER { if (config::use_detailed_metrics && !instance_id.empty()) { @@ -1578,6 +1589,12 @@ void commit_txn_eventually( LOG(WARNING) << msg; return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + }; // tablet_id -> {table/index/partition}_id std::unordered_map tablet_ids; @@ -1594,6 +1611,7 @@ void commit_txn_eventually( TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::need_repair_tablet_idx", &need_repair_tablet_idx); if (need_repair_tablet_idx) { + stats.get_counter += txn->num_get_keys(); txn.reset(); repair_tablet_index(txn_kv, code, msg, instance_id, db_id, txn_id, tmp_rowsets_meta); if (code != MetaServiceCode::OK) { @@ -1658,6 +1676,7 @@ void commit_txn_eventually( if (last_pending_txn_id > 0) { TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::advance_last_pending_txn_id", &last_pending_txn_id); + stats.get_counter += txn->num_get_keys(); txn.reset(); std::shared_ptr task = txn_lazy_committer->submit(instance_id, last_pending_txn_id); @@ -1973,7 +1992,7 @@ void commit_txn_eventually( */ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* response, std::shared_ptr& txn_kv, MetaServiceCode& code, - std::string& msg, const std::string& instance_id) { + std::string& msg, const std::string& instance_id, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); auto sub_txn_infos = request->sub_txn_infos(); @@ -1987,6 +2006,12 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* LOG(WARNING) << msg; return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + }; // Get db id with txn id std::string index_val; @@ -2079,6 +2104,7 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* sub_txn_to_tmp_rowsets_meta.emplace(sub_txn_id, std::move(tmp_rowsets_meta)); } + stats.get_counter += txn->num_get_keys(); // Create a read/write txn for guarantee consistency txn.reset(); err = txn_kv->create_txn(&txn); @@ -2557,7 +2583,7 @@ static bool force_txn_lazy_commit() { void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, const CommitTxnRequest* request, CommitTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(commit_txn); + RPC_PREPROCESS(commit_txn, get, put, del); if (!request->has_txn_id()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid argument, missing txn id"; @@ -2576,13 +2602,13 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(commit_txn) if (request->has_is_txn_load() && request->is_txn_load()) { - commit_txn_with_sub_txn(request, response, txn_kv_, code, msg, instance_id); + commit_txn_with_sub_txn(request, response, txn_kv_, code, msg, instance_id, stats); return; } int64_t db_id; std::vector> tmp_rowsets_meta; - scan_tmp_rowset(instance_id, txn_id, txn_kv_, code, msg, &db_id, &tmp_rowsets_meta); + scan_tmp_rowset(instance_id, txn_id, txn_kv_, code, msg, &db_id, &tmp_rowsets_meta, &stats); if (code != MetaServiceCode::OK) { LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id << " code=" << code; return; @@ -2601,7 +2627,7 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, } commit_txn_immediately(request, response, txn_kv_, txn_lazy_committer_, code, msg, - instance_id, db_id, tmp_rowsets_meta, err); + instance_id, db_id, tmp_rowsets_meta, err, stats); if (MetaServiceCode::OK == code) { return; @@ -2631,7 +2657,7 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, code = MetaServiceCode::OK; msg.clear(); commit_txn_eventually(request, response, txn_kv_, txn_lazy_committer_, code, msg, instance_id, - db_id, tmp_rowsets_meta); + db_id, tmp_rowsets_meta, stats); } static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* request, @@ -2841,7 +2867,7 @@ static void _abort_txn(const std::string& instance_id, const AbortTxnRequest* re void MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller, const AbortTxnRequest* request, AbortTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(abort_txn); + RPC_PREPROCESS(abort_txn, get, put, del); // Get txn id int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; std::string label = request->has_label() ? request->label() : ""; @@ -2866,7 +2892,6 @@ void MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller, } RPC_RATE_LIMIT(abort_txn); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -2891,7 +2916,7 @@ void MetaServiceImpl::abort_txn(::google::protobuf::RpcController* controller, void MetaServiceImpl::get_txn(::google::protobuf::RpcController* controller, const GetTxnRequest* request, GetTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_txn); + RPC_PREPROCESS(get_txn, get); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t db_id = request->has_db_id() ? request->db_id() : -1; std::string label = request->has_label() ? request->label() : ""; @@ -2913,7 +2938,6 @@ void MetaServiceImpl::get_txn(::google::protobuf::RpcController* controller, } RPC_RATE_LIMIT(get_txn) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3020,7 +3044,7 @@ void MetaServiceImpl::get_current_max_txn_id(::google::protobuf::RpcController* const GetCurrentMaxTxnRequest* request, GetCurrentMaxTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_current_max_txn_id); + RPC_PREPROCESS(get_current_max_txn_id, get); // TODO: For auth instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id()); if (instance_id.empty()) { @@ -3030,7 +3054,6 @@ void MetaServiceImpl::get_current_max_txn_id(::google::protobuf::RpcController* return; } RPC_RATE_LIMIT(get_current_max_txn_id) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -3077,7 +3100,7 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle const BeginSubTxnRequest* request, BeginSubTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(begin_sub_txn); + RPC_PREPROCESS(begin_sub_txn, get, put, del); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t sub_txn_num = request->has_sub_txn_num() ? request->sub_txn_num() : -1; int64_t db_id = request->has_db_id() ? request->db_id() : -1; @@ -3106,7 +3129,6 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle } RPC_RATE_LIMIT(begin_sub_txn) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3144,6 +3166,8 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle msg = ss.str(); return; } + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); // 2. Get sub txn id from version stamp txn.reset(); @@ -3275,7 +3299,7 @@ void MetaServiceImpl::abort_sub_txn(::google::protobuf::RpcController* controlle const AbortSubTxnRequest* request, AbortSubTxnResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(abort_sub_txn); + RPC_PREPROCESS(abort_sub_txn, get, put); int64_t txn_id = request->has_txn_id() ? request->txn_id() : -1; int64_t sub_txn_id = request->has_sub_txn_id() ? request->sub_txn_id() : -1; int64_t sub_txn_num = request->has_sub_txn_num() ? request->sub_txn_num() : -1; @@ -3304,7 +3328,6 @@ void MetaServiceImpl::abort_sub_txn(::google::protobuf::RpcController* controlle } RPC_RATE_LIMIT(abort_sub_txn) - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); @@ -3390,7 +3413,7 @@ void MetaServiceImpl::abort_txn_with_coordinator(::google::protobuf::RpcControll const AbortTxnWithCoordinatorRequest* request, AbortTxnWithCoordinatorResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(abort_txn_with_coordinator); + RPC_PREPROCESS(abort_txn_with_coordinator, get); if (!request->has_id() || !request->has_ip() || !request->has_start_time()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid coordinate id, coordinate ip or coordinate start time."; @@ -3411,7 +3434,6 @@ void MetaServiceImpl::abort_txn_with_coordinator(::google::protobuf::RpcControll std::string end_info_key = txn_info_key({instance_id, INT64_MAX, INT64_MAX}); LOG(INFO) << "begin_info_key:" << hex(begin_info_key) << " end_info_key:" << hex(end_info_key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -3503,7 +3525,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont const CheckTxnConflictRequest* request, CheckTxnConflictResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(check_txn_conflict); + RPC_PREPROCESS(check_txn_conflict, get); if (!request->has_db_id() || !request->has_end_txn_id() || (request->table_ids_size() <= 0)) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "invalid db id, end txn id or table_ids."; @@ -3527,7 +3549,6 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont LOG(INFO) << "begin_running_key:" << hex(begin_running_key) << " end_running_key:" << hex(end_running_key); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { msg = "failed to create txn"; @@ -3637,7 +3658,7 @@ void MetaServiceImpl::check_txn_conflict(::google::protobuf::RpcController* cont * @return TxnErrorCode */ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::string_view instance_id, - int64_t db_id, const std::string_view label_key) { + int64_t db_id, const std::string_view label_key, KVStats& stats) { std::string label_val; TxnLabelPB label_pb; @@ -3653,6 +3674,12 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri << " label_key=" << hex(label_key); return err; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + }; err = txn->get(label_key, &label_val); if (err != TxnErrorCode::TXN_OK && err != TxnErrorCode::TXN_KEY_NOT_FOUND) { @@ -3768,7 +3795,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control const CleanTxnLabelRequest* request, CleanTxnLabelResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(clean_txn_label); + RPC_PREPROCESS(clean_txn_label, get, put, del); if (!request->has_db_id()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "missing db id"; @@ -3808,6 +3835,10 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control << " end=" << hex(end_label_key); return; } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_counter += txn->num_get_keys(); + }; err = txn->get(begin_label_key, end_label_key, &it, snapshot, limit); if (err != TxnErrorCode::TXN_OK) { @@ -3829,7 +3860,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control begin_label_key = k; LOG(INFO) << "iterator has no more kvs. key=" << hex(k); } - err = internal_clean_label(txn_kv_, instance_id, db_id, k); + err = internal_clean_label(txn_kv_, instance_id, db_id, k, stats); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to clean txn label. err={}", err); @@ -3842,7 +3873,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control } else { const std::string& label = request->labels(0); const std::string label_key = txn_label_key({instance_id, db_id, label}); - TxnErrorCode err = internal_clean_label(txn_kv_, instance_id, db_id, label_key); + TxnErrorCode err = internal_clean_label(txn_kv_, instance_id, db_id, label_key, stats); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); msg = fmt::format("failed to clean txn label. err={}", err); @@ -3862,7 +3893,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control void MetaServiceImpl::get_txn_id(::google::protobuf::RpcController* controller, const GetTxnIdRequest* request, GetTxnIdResponse* response, ::google::protobuf::Closure* done) { - RPC_PREPROCESS(get_txn_id); + RPC_PREPROCESS(get_txn_id, get); if (!request->has_db_id()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "missing db id"; @@ -3884,7 +3915,6 @@ void MetaServiceImpl::get_txn_id(::google::protobuf::RpcController* controller, const int64_t db_id = request->db_id(); std::string label = request->label(); const std::string label_key = txn_label_key({instance_id, db_id, label}); - std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to create txn. err=" << err << " db_id=" << db_id diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-service/txn_kv.cpp index ad9b62bb0c14b7..56e5440c8fe9af 100644 --- a/cloud/src/meta-service/txn_kv.cpp +++ b/cloud/src/meta-service/txn_kv.cpp @@ -399,6 +399,7 @@ TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snaps const uint8_t* ret; int len; err = fdb_future_get_value(fut, &found, &ret, &len); + num_get_keys_++; if (err) { LOG(WARNING) << __PRETTY_FUNCTION__ @@ -438,6 +439,7 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, std::unique_ptr ret(new RangeGetIterator(fut)); RETURN_IF_ERROR(ret->init()); + num_get_keys_ += ret->size(); g_bvar_txn_kv_get_count_normalized << ret->size(); *(iter) = std::move(ret); @@ -668,6 +670,7 @@ TxnErrorCode Transaction::batch_get(std::vector>* res futures.clear(); } DCHECK_EQ(res->size(), num_keys); + num_get_keys_ += num_keys; return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-service/txn_kv.h b/cloud/src/meta-service/txn_kv.h index 645982f76164ef..56757cdb68ac9f 100644 --- a/cloud/src/meta-service/txn_kv.h +++ b/cloud/src/meta-service/txn_kv.h @@ -233,6 +233,11 @@ class Transaction { **/ virtual size_t approximate_bytes() const = 0; + /** + * @brief return the num get keys submitted to this txn. + **/ + virtual size_t num_get_keys() const = 0; + /** * @brief return the num delete keys submitted to this txn. **/ @@ -565,6 +570,8 @@ class Transaction : public cloud::Transaction { size_t approximate_bytes() const override { return approximate_bytes_; } + size_t num_get_keys() const override { return num_get_keys_; } + size_t num_del_keys() const override { return num_del_keys_; } size_t num_put_keys() const override { return num_put_keys_; } @@ -579,6 +586,7 @@ class Transaction : public cloud::Transaction { bool aborted_ = false; FDBTransaction* txn_ = nullptr; + size_t num_get_keys_ {0}; size_t num_del_keys_ {0}; size_t num_put_keys_ {0}; size_t delete_bytes_ {0}; diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 2260ea1ff23b7b..99aea5fed56fff 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -20,6 +20,7 @@ #include #include "common/logging.h" +#include "common/stats.h" #include "common/util.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" @@ -33,7 +34,8 @@ namespace doris::cloud { void scan_tmp_rowset( const std::string& instance_id, int64_t txn_id, std::shared_ptr txn_kv, MetaServiceCode& code, std::string& msg, int64_t* db_id, - std::vector>* tmp_rowsets_meta); + std::vector>* tmp_rowsets_meta, + KVStats* stats); void update_tablet_stats(const StatsTabletKeyInfo& info, const TabletStats& stats, std::unique_ptr& txn, MetaServiceCode& code, @@ -308,7 +310,7 @@ void TxnLazyCommitTask::commit() { int64_t db_id; std::vector> all_tmp_rowset_metas; scan_tmp_rowset(instance_id_, txn_id_, txn_kv_, code_, msg_, &db_id, - &all_tmp_rowset_metas); + &all_tmp_rowset_metas, nullptr); if (code_ != MetaServiceCode::OK) { LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id_ << " code=" << code_; break; diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt index 8378a33ddfd2dd..e939b21d8bf69b 100644 --- a/cloud/test/CMakeLists.txt +++ b/cloud/test/CMakeLists.txt @@ -45,6 +45,8 @@ add_executable(resource_test resource_test.cpp) add_executable(http_encode_key_test http_encode_key_test.cpp) +add_executable(rpc_kv_bvar_test rpc_kv_bvar_test.cpp) + add_executable(fdb_injection_test fdb_injection_test.cpp) add_executable(s3_accessor_test s3_accessor_test.cpp) @@ -86,6 +88,8 @@ target_link_libraries(resource_test ${TEST_LINK_LIBS}) target_link_libraries(http_encode_key_test ${TEST_LINK_LIBS}) +target_link_libraries(rpc_kv_bvar_test ${TEST_LINK_LIBS}) + target_link_libraries(s3_accessor_test ${TEST_LINK_LIBS}) target_link_libraries(s3_accessor_mock_test ${TEST_LINK_LIBS}) diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index 8de04af4a8a19d..8cff9223498989 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -35,6 +35,7 @@ #include "common/config.h" #include "common/defer.h" #include "common/logging.h" +#include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/keys.h" #include "meta-service/mem_txn_kv.h" @@ -63,7 +64,8 @@ int main(int argc, char** argv) { } namespace doris::cloud { -void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id); +void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& instance_id, + KVStats* stats); } // namespace doris::cloud TEST(MetaServerTest, FQDNRefreshInstance) { @@ -137,7 +139,7 @@ TEST(MetaServerTest, FQDNRefreshInstance) { // Refresh instance with FQDN endpoint. config::hostname = ""; - notify_refresh_instance(txn_kv, "fqdn_instance_id"); + notify_refresh_instance(txn_kv, "fqdn_instance_id", nullptr); bool refreshed = false; for (size_t i = 0; i < 100; ++i) { diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp new file mode 100644 index 00000000000000..6f6bf9d0d201ac --- /dev/null +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -0,0 +1,3141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//#define private public +#include "common/bvars.h" +#include "meta-service/meta_service.h" +//#undef private + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "common/config.h" +#include "common/logging.h" +#include "common/util.h" +#include "cpp/sync_point.h" +#include "meta-service/keys.h" +#include "meta-service/mem_txn_kv.h" +#include "meta-service/meta_service_helper.h" +#include "meta-service/txn_kv.h" +#include "meta-service/txn_kv_error.h" +#include "mock_resource_manager.h" +#include "rate-limiter/rate_limiter.h" + +int main(int argc, char** argv) { + const std::string conf_file = "doris_cloud.conf"; + if (!doris::cloud::config::init(conf_file.c_str(), true)) { + std::cerr << "failed to init config file, conf=" << conf_file << std::endl; + return -1; + } + + if (!doris::cloud::init_glog("rpc_kv_bvar_test")) { + std::cerr << "failed to init glog" << std::endl; + return -1; + } + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +namespace doris::cloud { +using Status = MetaServiceResponseStatus; + +void start_compaction_job(MetaService* meta_service, int64_t tablet_id, const std::string& job_id, + const std::string& initiator, int base_compaction_cnt, + int cumu_compaction_cnt, TabletCompactionJobPB::CompactionType type, + StartTabletJobResponse& res, + std::pair input_version = {0, 0}) { + brpc::Controller cntl; + StartTabletJobRequest req; + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto* compaction = req.mutable_job()->add_compaction(); + compaction->set_id(job_id); + compaction->set_initiator(initiator); + compaction->set_base_compaction_cnt(base_compaction_cnt); + compaction->set_cumulative_compaction_cnt(cumu_compaction_cnt); + compaction->set_type(type); + long now = time(nullptr); + compaction->set_expiration(now + 12); + compaction->set_lease(now + 3); + if (input_version.second > 0) { + compaction->add_input_versions(input_version.first); + compaction->add_input_versions(input_version.second); + compaction->set_check_input_versions_range(true); + } + meta_service->start_tablet_job(&cntl, &req, &res, nullptr); +}; + +std::unique_ptr get_meta_service() { + int ret = 0; + // MemKv + auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); + if (txn_kv != nullptr) { + ret = txn_kv->init(); + [&] { ASSERT_EQ(ret, 0); }(); + } + [&] { ASSERT_NE(txn_kv.get(), nullptr); }(); + + std::unique_ptr txn; + EXPECT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->remove("\x00", "\xfe"); // This is dangerous if the fdb is not correctly set + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + auto rs = std::make_shared(txn_kv); + auto rl = std::make_shared(); + auto meta_service = std::make_unique(txn_kv, rs, rl); + return std::make_unique(std::move(meta_service)); +} + +std::unique_ptr get_fdb_meta_service() { + config::fdb_cluster_file_path = "fdb.cluster"; + static auto txn_kv = std::dynamic_pointer_cast(std::make_shared()); + static std::atomic init {false}; + bool tmp = false; + if (init.compare_exchange_strong(tmp, true)) { + int ret = txn_kv->init(); + [&] { + ASSERT_EQ(ret, 0); + ASSERT_NE(txn_kv.get(), nullptr); + }(); + } + auto rs = std::make_shared(txn_kv); + auto rl = std::make_shared(); + auto meta_service = std::make_unique(txn_kv, rs, rl); + return std::make_unique(std::move(meta_service)); +} + +static void commit_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset, + CreateRowsetResponse& res) { + brpc::Controller cntl; + auto* arena = res.GetArena(); + auto* req = google::protobuf::Arena::CreateMessage(arena); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->commit_rowset(&cntl, req, &res, nullptr); + if (!arena) { + delete req; + } +} + +static std::string next_rowset_id() { + static int cnt = 0; + return std::to_string(++cnt); +} + +static void fill_schema(doris::TabletSchemaCloudPB* schema, int32_t schema_version) { + schema->set_schema_version(schema_version); + for (int i = 0; i < 10; ++i) { + auto* column = schema->add_column(); + column->set_unique_id(20000 + i); + column->set_type("INT"); + } +} + +static void add_tablet(CreateTabletsRequest& req, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, const std::string& rowset_id, + int32_t schema_version) { + auto* tablet = req.add_tablet_metas(); + tablet->set_table_id(table_id); + tablet->set_index_id(index_id); + tablet->set_partition_id(partition_id); + tablet->set_tablet_id(tablet_id); + auto* schema = tablet->mutable_schema(); + fill_schema(schema, schema_version); + auto* first_rowset = tablet->add_rs_metas(); + first_rowset->set_rowset_id(0); // required + first_rowset->set_rowset_id_v2(rowset_id); + first_rowset->set_start_version(0); + first_rowset->set_end_version(1); + first_rowset->mutable_tablet_schema()->CopyFrom(*schema); +} + +static void add_tablet(CreateTabletsRequest& req, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id) { + auto* tablet = req.add_tablet_metas(); + tablet->set_table_id(table_id); + tablet->set_index_id(index_id); + tablet->set_partition_id(partition_id); + tablet->set_tablet_id(tablet_id); + auto* schema = tablet->mutable_schema(); + schema->set_schema_version(0); + auto* first_rowset = tablet->add_rs_metas(); + first_rowset->set_rowset_id(0); // required + first_rowset->set_rowset_id_v2(next_rowset_id()); + first_rowset->set_start_version(0); + first_rowset->set_end_version(1); + first_rowset->mutable_tablet_schema()->CopyFrom(*schema); +} + +static void create_tablet_with_db_id(MetaServiceProxy* meta_service, int64_t db_id, + int64_t table_id, int64_t index_id, int64_t partition_id, + int64_t tablet_id) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + req.set_db_id(db_id); + add_tablet(req, table_id, index_id, partition_id, tablet_id); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +void create_tablet(MetaService* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, bool enable_mow, + bool not_ready = false) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + auto* tablet = req.add_tablet_metas(); + tablet->set_tablet_state(not_ready ? doris::TabletStatePB::PB_NOTREADY + : doris::TabletStatePB::PB_RUNNING); + tablet->set_table_id(table_id); + tablet->set_index_id(index_id); + tablet->set_partition_id(partition_id); + tablet->set_tablet_id(tablet_id); + tablet->set_enable_unique_key_merge_on_write(enable_mow); + auto* schema = tablet->mutable_schema(); + schema->set_schema_version(0); + auto* first_rowset = tablet->add_rs_metas(); + first_rowset->set_rowset_id(0); // required + first_rowset->set_rowset_id_v2(next_rowset_id()); + first_rowset->set_start_version(0); + first_rowset->set_end_version(1); + first_rowset->mutable_tablet_schema()->CopyFrom(*schema); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void create_tablet(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, const std::string& rowset_id, + int32_t schema_version) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + add_tablet(req, table_id, index_id, partition_id, tablet_id, rowset_id, schema_version); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void create_tablet(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id) { + brpc::Controller cntl; + CreateTabletsRequest req; + CreateTabletsResponse res; + add_tablet(req, table_id, index_id, partition_id, tablet_id); + meta_service->create_tablets(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void begin_txn(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label, + int64_t table_id, int64_t& txn_id) { + brpc::Controller cntl; + BeginTxnRequest req; + BeginTxnResponse res; + auto* txn_info = req.mutable_txn_info(); + txn_info->set_db_id(db_id); + txn_info->set_label(label); + txn_info->add_table_ids(table_id); + txn_info->set_timeout_ms(36000); + meta_service->begin_txn(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; + ASSERT_TRUE(res.has_txn_id()) << label; + txn_id = res.txn_id(); +} + +static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, + int partition_id = 10, int64_t version = -1, + int num_rows = 100) { + doris::RowsetMetaCloudPB rowset; + rowset.set_rowset_id(0); // required + rowset.set_rowset_id_v2(next_rowset_id()); + rowset.set_tablet_id(tablet_id); + rowset.set_partition_id(partition_id); + rowset.set_txn_id(txn_id); + if (version > 0) { + rowset.set_start_version(version); + rowset.set_end_version(version); + } + rowset.set_num_segments(1); + rowset.set_num_rows(num_rows); + rowset.set_data_disk_size(num_rows * 100); + rowset.set_index_disk_size(num_rows * 10); + rowset.set_total_disk_size(num_rows * 110); + rowset.mutable_tablet_schema()->set_schema_version(0); + rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK + return rowset; +} + +static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, int index_id, + int partition_id, int64_t version = -1, + int num_rows = 100) { + doris::RowsetMetaCloudPB rowset; + rowset.set_rowset_id(0); // required + rowset.set_rowset_id_v2(next_rowset_id()); + rowset.set_tablet_id(tablet_id); + rowset.set_partition_id(partition_id); + rowset.set_index_id(index_id); + rowset.set_txn_id(txn_id); + if (version > 0) { + rowset.set_start_version(version); + rowset.set_end_version(version); + } + rowset.set_num_segments(0); + rowset.set_num_rows(0); + rowset.set_data_disk_size(0); + rowset.mutable_tablet_schema()->set_schema_version(0); + rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK + return rowset; +} + +static void check_get_tablet(MetaServiceProxy* meta_service, int64_t tablet_id, + int32_t schema_version) { + brpc::Controller cntl; + GetTabletRequest req; + GetTabletResponse res; + req.set_tablet_id(tablet_id); + meta_service->get_tablet(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; + ASSERT_TRUE(res.has_tablet_meta()) << tablet_id; + EXPECT_TRUE(res.tablet_meta().has_schema()) << tablet_id; + EXPECT_EQ(res.tablet_meta().schema_version(), schema_version) << tablet_id; +}; + +static void create_and_commit_rowset(MetaServiceProxy* meta_service, int64_t table_id, + int64_t index_id, int64_t partition_id, int64_t tablet_id, + int64_t txn_id) { + create_tablet(meta_service, table_id, index_id, partition_id, tablet_id); + auto tmp_rowset = create_rowset(txn_id, tablet_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service, tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); +} + +static void prepare_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset, + CreateRowsetResponse& res) { + brpc::Controller cntl; + auto* arena = res.GetArena(); + auto* req = google::protobuf::Arena::CreateMessage(arena); + req->mutable_rowset_meta()->CopyFrom(rowset); + meta_service->prepare_rowset(&cntl, req, &res, nullptr); + if (!arena) { + delete req; + } +} + +static void get_tablet_stats(MetaService* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, GetTabletStatsResponse& res) { + brpc::Controller cntl; + GetTabletStatsRequest req; + auto* idx = req.add_tablet_idx(); + idx->set_table_id(table_id); + idx->set_index_id(index_id); + idx->set_partition_id(partition_id); + idx->set_tablet_id(tablet_id); + meta_service->get_tablet_stats(&cntl, &req, &res, nullptr); +} + +static void commit_txn(MetaServiceProxy* meta_service, int64_t db_id, int64_t txn_id, + const std::string& label) { + brpc::Controller cntl; + CommitTxnRequest req; + CommitTxnResponse res; + req.set_db_id(db_id); + req.set_txn_id(txn_id); + meta_service->commit_txn(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; +} + +static void begin_txn_and_commit_rowset(MetaServiceProxy* meta_service, const std::string& label, + int64_t db_id, int64_t table_id, int64_t partition_id, + int64_t tablet_id, int64_t* txn_id) { + begin_txn(meta_service, db_id, label, table_id, *txn_id); + CreateRowsetResponse res; + auto rowset = create_rowset(*txn_id, tablet_id, partition_id); + prepare_rowset(meta_service, rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + res.Clear(); + commit_rowset(meta_service, rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); +} + +static void insert_rowset(MetaServiceProxy* meta_service, int64_t db_id, const std::string& label, + int64_t table_id, int64_t partition_id, int64_t tablet_id) { + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service, db_id, label, table_id, txn_id)); + CreateRowsetResponse res; + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + prepare_rowset(meta_service, rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; + res.Clear(); + ASSERT_NO_FATAL_FAILURE(commit_rowset(meta_service, rowset, res)); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << label; + commit_txn(meta_service, db_id, txn_id, label); +} + +static void get_rowset(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, GetRowsetResponse& res) { + brpc::Controller cntl; + GetRowsetRequest req; + auto* tablet_idx = req.mutable_idx(); + tablet_idx->set_table_id(table_id); + tablet_idx->set_index_id(index_id); + tablet_idx->set_partition_id(partition_id); + tablet_idx->set_tablet_id(tablet_id); + req.set_start_version(0); + req.set_end_version(-1); + req.set_cumulative_compaction_cnt(0); + req.set_base_compaction_cnt(0); + req.set_cumulative_point(2); + meta_service->get_rowset(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; +} + +static void get_delete_bitmap_update_lock(MetaServiceProxy* meta_service, int64_t table_id, + int64_t partition_id, int64_t lock_id, + int64_t initiator) { + brpc::Controller cntl; + GetDeleteBitmapUpdateLockRequest get_lock_req; + GetDeleteBitmapUpdateLockResponse get_lock_res; + get_lock_req.set_cloud_unique_id("test_cloud_unique_id"); + get_lock_req.set_table_id(table_id); + get_lock_req.add_partition_ids(partition_id); + get_lock_req.set_expiration(5); + get_lock_req.set_lock_id(lock_id); + get_lock_req.set_initiator(initiator); + meta_service->get_delete_bitmap_update_lock( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &get_lock_req, + &get_lock_res, nullptr); + ASSERT_EQ(get_lock_res.status().code(), MetaServiceCode::OK); +} + +static void update_delete_bitmap(MetaServiceProxy* meta_service, + UpdateDeleteBitmapRequest& update_delete_bitmap_req, + UpdateDeleteBitmapResponse& update_delete_bitmap_res, + int64_t table_id, int64_t partition_id, int64_t lock_id, + int64_t initiator, int64_t tablet_id, int64_t txn_id, + int64_t next_visible_version, std::string data = "1111") { + brpc::Controller cntl; + update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + update_delete_bitmap_req.set_table_id(table_id); + update_delete_bitmap_req.set_partition_id(partition_id); + update_delete_bitmap_req.set_lock_id(lock_id); + update_delete_bitmap_req.set_initiator(initiator); + update_delete_bitmap_req.set_tablet_id(tablet_id); + update_delete_bitmap_req.set_txn_id(txn_id); + update_delete_bitmap_req.set_next_visible_version(next_visible_version); + update_delete_bitmap_req.add_rowset_ids("123"); + update_delete_bitmap_req.add_segment_ids(0); + update_delete_bitmap_req.add_versions(next_visible_version); + update_delete_bitmap_req.add_segment_delete_bitmaps(data); + meta_service->update_delete_bitmap(reinterpret_cast(&cntl), + &update_delete_bitmap_req, &update_delete_bitmap_res, + nullptr); +} + +void start_schema_change_job(MetaServiceProxy* meta_service, int64_t table_id, int64_t index_id, + int64_t partition_id, int64_t tablet_id, int64_t new_tablet_id, + const std::string& job_id, const std::string& initiator, + StartTabletJobResponse& res, int64_t alter_version = -1) { + brpc::Controller cntl; + StartTabletJobRequest req; + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto* sc = req.mutable_job()->mutable_schema_change(); + sc->set_id(job_id); + sc->set_initiator(initiator); + sc->mutable_new_tablet_idx()->set_tablet_id(new_tablet_id); + if (alter_version != -1) { + sc->set_alter_version(alter_version); + } + long now = time(nullptr); + sc->set_expiration(now + 12); + meta_service->start_tablet_job(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) + << job_id << ' ' << initiator << ' ' << res.status().msg(); +}; + +void finish_schema_change_job( + MetaService* meta_service, int64_t tablet_id, int64_t new_tablet_id, + const std::string& job_id, const std::string& initiator, + const std::vector& output_rowsets, FinishTabletJobResponse& res, + FinishTabletJobRequest_Action action = FinishTabletJobRequest::COMMIT) { + brpc::Controller cntl; + FinishTabletJobRequest req; + req.set_action(action); + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + auto* sc = req.mutable_job()->mutable_schema_change(); + sc->mutable_new_tablet_idx()->set_tablet_id(new_tablet_id); + if (output_rowsets.empty()) { + sc->set_alter_version(0); + } else { + sc->set_alter_version(output_rowsets.back().end_version()); + for (const auto& rowset : output_rowsets) { + sc->add_txn_ids(rowset.txn_id()); + sc->add_output_versions(rowset.end_version()); + sc->set_num_output_rows(sc->num_output_rows() + rowset.num_rows()); + sc->set_num_output_segments(sc->num_output_segments() + rowset.num_segments()); + sc->set_size_output_rowsets(sc->size_output_rowsets() + rowset.total_disk_size()); + sc->set_index_size_output_rowsets(sc->index_size_output_rowsets() + + rowset.index_disk_size()); + sc->set_segment_size_output_rowsets(sc->segment_size_output_rowsets() + + rowset.data_disk_size()); + } + sc->set_num_output_rowsets(output_rowsets.size()); + } + sc->set_id(job_id); + sc->set_initiator(initiator); + sc->set_delete_bitmap_lock_initiator(12345); + meta_service->finish_tablet_job(&cntl, &req, &res, nullptr); +} + +// create_tablets +TEST(RpcKvBvarTest, CreateTablets) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + LOG(INFO) << "CreateTablets: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_tablets_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_tablets_put_counter.get({mock_instance})); +} + +// get_tablet +TEST(RpcKvBvarTest, GetTablet) { + std::string cloud_unique_id = "test_cloud_unique_id"; + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + brpc::Controller cntl; + GetTabletRequest req; + req.set_cloud_unique_id(cloud_unique_id); + req.set_tablet_id(tablet_id); + GetTabletResponse resp; + + meta_service->get_tablet(&cntl, &req, &resp, nullptr); + LOG(INFO) << "GetTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_get_counter.get({mock_instance})); +} + +// get_tablet_stats +TEST(RpcKvBvarTest, GetTabletStats) { + std::string cloud_unique_id = "test_cloud_unique_id"; + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetTabletStatsResponse res; + get_tablet_stats(meta_service.get(), table_id, index_id, partition_id, tablet_id, res); + + LOG(INFO) << "GetTabletStats: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_stats_get_counter.get({mock_instance})); +} + +// update_tablet +TEST(RpcKvBvarTest, UpdateTablet) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + brpc::Controller cntl; + UpdateTabletRequest req; + UpdateTabletResponse resp; + req.set_cloud_unique_id("test_cloud_unique_id"); + TabletMetaInfoPB* tablet_meta_info = req.add_tablet_meta_infos(); + tablet_meta_info->set_tablet_id(tablet_id); + tablet_meta_info->set_is_in_memory(true); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->update_tablet(&cntl, &req, &resp, nullptr); + + LOG(INFO) << "UpdateTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tablet_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tablet_put_counter.get({mock_instance})); +} + +// update_tablet_schema +// should not call update_tablet_schema +// TEST(RpcKvBvarTest, UpdateTabletSchema) { +// auto meta_service = get_meta_service(); +// auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); +// constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; +// create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + +// brpc::Controller cntl; +// UpdateTabletSchemaRequest req; +// UpdateTabletSchemaResponse resp; +// req.set_tablet_id(tablet_id); +// req.set_cloud_unique_id("test_cloud_unique_id"); + +// mem_kv->get_count_ = 0; +// mem_kv->put_count_ = 0; +// mem_kv->del_count_ = 0; + +// meta_service->update_tablet_schema(&cntl, &req, &resp, nullptr); + +// LOG(INFO) << "UpdateTabletSchema: " << mem_kv->get_count_ << ", " +// << mem_kv->put_count_ << ", " << mem_kv->del_count_; +// ASSERT_EQ(mem_kv->get_count_, +// g_bvar_rpc_kv_update_tablet_schema_get_counter.get({mock_instance})); +// ASSERT_EQ(mem_kv->put_count_, +// g_bvar_rpc_kv_update_tablet_schema_put_counter.get({mock_instance})); +// ASSERT_EQ(mem_kv->del_count_, +// g_bvar_rpc_kv_update_tablet_schema_del_counter.get({mock_instance})); +// } + +// begin_txn +TEST(RpcKvBvarTest, BeginTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + int64_t txn_id = 0; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + LOG(INFO) << "BeginTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_txn_put_counter.get({mock_instance})); +} + +// commit_txn +TEST(RpcKvBvarTest, CommitTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + commit_txn(meta_service.get(), db_id, txn_id, label); + + LOG(INFO) << "CommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_txn_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_txn_del_counter.get({mock_instance})); +} + +// precommit_txn +TEST(RpcKvBvarTest, PrecommitTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + const int64_t db_id = 563413; + const int64_t table_id = 417417878; + const std::string& label = "label_123dae121das"; + int64_t txn_id = -1; + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info; + txn_info.set_db_id(db_id); + txn_info.set_label(label); + txn_info.add_table_ids(table_id); + txn_info.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + ASSERT_GT(txn_id, -1); + } + + std::unique_ptr txn; + TxnErrorCode err = meta_service->txn_kv()->create_txn(&txn); + ASSERT_EQ(err, TxnErrorCode::TXN_OK); + + const std::string info_key = txn_info_key({mock_instance, db_id, txn_id}); + std::string info_val; + ASSERT_EQ(txn->get(info_key, &info_val), TxnErrorCode::TXN_OK); + TxnInfoPB txn_info; + txn_info.ParseFromString(info_val); + ASSERT_EQ(txn_info.status(), TxnStatusPB::TXN_STATUS_PREPARED); + + brpc::Controller cntl; + PrecommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_precommit_timeout_ms(36000); + PrecommitTxnResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->precommit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "PrecommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_precommit_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_precommit_txn_put_counter.get({mock_instance})); +} + +// abort_txn +TEST(RpcKvBvarTest, AbortTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + brpc::Controller cntl; + AbortTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_reason("test"); + AbortTxnResponse res; + meta_service->abort_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "AbortTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_del_counter.get({mock_instance})); +} + +// get_txn +TEST(RpcKvBvarTest, GetTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + brpc::Controller cntl; + GetTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_db_id(db_id); + GetTxnResponse res; + meta_service->get_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_get_counter.get({mock_instance})); +} + +// get_txn_id +TEST(RpcKvBvarTest, GetTxnId) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + int64_t txn_id = 0; + + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + + brpc::Controller cntl; + GetTxnIdRequest req; + GetTxnIdResponse res; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_label(label); + req.set_db_id(db_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_txn_id(&cntl, &req, &res, nullptr); + + LOG(INFO) << "GetTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_id_get_counter.get({mock_instance})); +} + +// prepare_rowset +TEST(RpcKvBvarTest, PrepareRowset) { + int64_t db_id = 100201; + std::string label = "test_prepare_rowset"; + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + int64_t txn_id = 0; + ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); + CreateRowsetResponse res; + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + rowset.mutable_load_id()->set_hi(123); + rowset.mutable_load_id()->set_lo(456); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + prepare_rowset(meta_service.get(), rowset, res); + + LOG(INFO) << "PrepareRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_rowset_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_rowset_put_counter.get({mock_instance})); +} + +// get_rowset +TEST(RpcKvBvarTest, GetRowset) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + config::write_schema_kv = true; + ASSERT_NO_FATAL_FAILURE(create_tablet(meta_service.get(), table_id, index_id, partition_id, + tablet_id, next_rowset_id(), 1)); + // check get tablet response + check_get_tablet(meta_service.get(), tablet_id, 1); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + // check get rowset response + GetRowsetResponse get_rowset_res; + get_rowset(meta_service.get(), table_id, index_id, partition_id, tablet_id, get_rowset_res); + + LOG(INFO) << "GetRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rowset_get_counter.get({mock_instance})); +} + +// update_tmp_rowset +TEST(RpcKvBvarTest, UpdateTmpRowset) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + int64_t txn_id = -1; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + auto rowset = create_rowset(txn_id, tablet_id, partition_id); + rowset.set_num_segments(rowset.num_segments() + 3); + rowset.set_num_rows(rowset.num_rows() + 1000); + rowset.set_total_disk_size(rowset.total_disk_size() + 11000); + rowset.set_index_disk_size(rowset.index_disk_size() + 1000); + rowset.set_data_disk_size(rowset.data_disk_size() + 10000); + + std::unique_ptr txn; + std::string update_key; + brpc::Controller cntl; + CreateRowsetResponse res; + auto* arena = res.GetArena(); + auto* req = google::protobuf::Arena::CreateMessage(arena); + std::string instance_id = get_instance_id(meta_service->resource_mgr(), req->cloud_unique_id()); + MetaRowsetTmpKeyInfo key_info {instance_id, txn_id, tablet_id}; + meta_rowset_tmp_key(key_info, &update_key); + EXPECT_EQ(mem_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(update_key, "update_tmp_rowset_val"); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + req->mutable_rowset_meta()->CopyFrom(rowset); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->update_tmp_rowset(&cntl, req, &res, nullptr); + + if (!arena) { + delete req; + } + + LOG(INFO) << "UpdateTmpRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tmp_rowset_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tmp_rowset_put_counter.get({mock_instance})); +} + +// commit_rowset +TEST(RpcKvBvarTest, CommitRowset) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + int64_t txn_id = -1; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + auto tmp_rowset = create_rowset(txn_id, tablet_id, partition_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + + LOG(INFO) << "CommitRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_rowset_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_rowset_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_rowset_del_counter.get({mock_instance})); +} + +// get_version +TEST(RpcKvBvarTest, GetVersion) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr auto table_id = 10001, partition_id = 10003, tablet_id = 10004; + create_tablet(meta_service.get(), table_id, 1, partition_id, tablet_id); + insert_rowset(meta_service.get(), 1, "get_version_label_1", table_id, partition_id, tablet_id); + + brpc::Controller ctrl; + GetVersionRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(1); + req.set_table_id(table_id); + req.set_partition_id(partition_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetVersionResponse resp; + meta_service->get_version(&ctrl, &req, &resp, nullptr); + + LOG(INFO) << "GetVersion: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_version_get_counter.get({mock_instance})); +} + +// get_schema_dict +TEST(RpcKvBvarTest, GetSchemaDict) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr auto table_id = 10001, index_id = 10002, partition_id = 10003, tablet_id = 10004; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + + brpc::Controller ctrl; + GetSchemaDictRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_index_id(index_id); + + std::unique_ptr txn; + std::string instance_id = get_instance_id(meta_service->resource_mgr(), req.cloud_unique_id()); + std::string dict_key = meta_schema_pb_dictionary_key({instance_id, req.index_id()}); + EXPECT_EQ(mem_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(dict_key, "dict_val"); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetSchemaDictResponse resp; + meta_service->get_schema_dict(&ctrl, &req, &resp, nullptr); + + LOG(INFO) << "GetSchemaDict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_schema_dict_get_counter.get({mock_instance})); +} + +// get_delete_bitmap_update_lock +TEST(RpcKvBvarTest, GetDeleteBitmapUpdateLock) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + + LOG(INFO) << "GetDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ + << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_counter.get({mock_instance})); +} + +// update_delete_bitmap +TEST(RpcKvBvarTest, UpdateDeleteBitmap) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + int64_t version = 100; + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, + table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); + + LOG(INFO) << "UpdateDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_update_delete_bitmap_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_update_delete_bitmap_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_update_delete_bitmap_del_counter.get({mock_instance})); +} + +// get_delete_bitmap +TEST(RpcKvBvarTest, GetDeleteBitmap) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + int64_t version = 100; + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, + table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); + + brpc::Controller ctrl; + GetDeleteBitmapRequest get_delete_bitmap_req; + GetDeleteBitmapResponse get_delete_bitmap_res; + get_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id"); + get_delete_bitmap_req.set_tablet_id(tablet_id); + get_delete_bitmap_req.add_rowset_ids("123"); + get_delete_bitmap_req.add_begin_versions(0); + get_delete_bitmap_req.add_end_versions(version); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_delete_bitmap(reinterpret_cast(&ctrl), + &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); + + LOG(INFO) << "GetDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_get_counter.get({mock_instance})); +} + +// remove_delete_bitmap_update_lock +TEST(RpcKvBvarTest, RemoveDeleteBitmapUpdateLock) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label11", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + brpc::Controller cntl; + RemoveDeleteBitmapUpdateLockRequest remove_req; + RemoveDeleteBitmapUpdateLockResponse remove_res; + + remove_req.set_cloud_unique_id("test_cloud_unique_id"); + remove_req.set_table_id(table_id); + remove_req.set_lock_id(lock_id); + remove_req.set_initiator(initiator); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->remove_delete_bitmap_update_lock( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &remove_req, &remove_res, + nullptr); + + LOG(INFO) << "RemoveDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " + << mem_kv->put_count_ << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_counter.get({mock_instance})); +} + +// remove_delete_bitmap +TEST(RpcKvBvarTest, RemoveDeleteBitmap) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 99999; + int64_t table_id = 1801; + int64_t index_id = 4801; + int64_t t1p1 = 2001; + int64_t tablet_id = 3001; + int64_t txn_id; + size_t split_size = 90 * 1000; // see cloud/src/common/util.h + ASSERT_NO_FATAL_FAILURE(create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + t1p1, tablet_id)); + begin_txn_and_commit_rowset(meta_service.get(), "label1", db_id, table_id, t1p1, tablet_id, + &txn_id); + int64_t lock_id = -2; + int64_t initiator = 1009; + get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); + int64_t version = 100; + UpdateDeleteBitmapRequest update_delete_bitmap_req; + UpdateDeleteBitmapResponse update_delete_bitmap_res; + // will be splited and stored in 5 KVs + std::string data1(split_size * 5, 'c'); + update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, + table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); + + brpc::Controller ctrl; + RemoveDeleteBitmapRequest req; + RemoveDeleteBitmapResponse resp; + req.add_begin_versions(version); + req.add_end_versions(version); + req.add_rowset_ids("rowset_ids"); + req.set_cloud_unique_id("test_cloud_unique_id"); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->remove_delete_bitmap(&ctrl, &req, &resp, nullptr); + + LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_remove_delete_bitmap_del_counter.get({mock_instance})); +} + +// start_tablet_job +TEST(RpcKvBvarTest, StartTabletJob) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); + StartTabletJobResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, + TabletCompactionJobPB::BASE, res); + + LOG(INFO) << "StartTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_start_tablet_job_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_start_tablet_job_put_counter.get({mock_instance})); +} + +// finish_tablet_job +TEST(RpcKvBvarTest, FinishTabletJob) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + + int64_t table_id = 1; + int64_t index_id = 2; + int64_t partition_id = 3; + int64_t tablet_id = 4; + + ASSERT_NO_FATAL_FAILURE( + create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id, false)); + + StartTabletJobResponse res; + start_compaction_job(meta_service.get(), tablet_id, "job1", "BE1", 0, 0, + TabletCompactionJobPB::CUMULATIVE, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + res.Clear(); + + int64_t new_tablet_id = 11; + ASSERT_NO_FATAL_FAILURE(create_tablet(meta_service.get(), table_id, index_id, partition_id, + new_tablet_id, false, true)); + StartTabletJobResponse sc_res; + ASSERT_NO_FATAL_FAILURE(start_schema_change_job(meta_service.get(), table_id, index_id, + partition_id, tablet_id, new_tablet_id, "job2", + "BE1", sc_res)); + + long now = time(nullptr); + FinishTabletJobRequest req; + FinishTabletJobResponse finish_res_2; + req.set_action(FinishTabletJobRequest::LEASE); + auto* compaction = req.mutable_job()->add_compaction(); + compaction->set_id("job1"); + compaction->set_initiator("BE1"); + compaction->set_lease(now + 10); + req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->finish_tablet_job(&cntl, &req, &finish_res_2, nullptr); + + LOG(INFO) << "FinishTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_tablet_job_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_tablet_job_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_tablet_job_del_counter.get({mock_instance})); +} + +// prepare_index +TEST(RpcKvBvarTest, PrepareIndex) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecycleIndexPB index_pb; + auto index_key = recycle_index_key({instance_id, index_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + + brpc::Controller ctrl; + IndexRequest req; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.set_is_new_table(true); + IndexResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->prepare_index(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "PrepareIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_index_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_index_put_counter.get({mock_instance})); +} + +// commit_index +TEST(RpcKvBvarTest, CommitIndex) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecycleIndexPB index_pb; + auto index_key = recycle_index_key({instance_id, index_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + + brpc::Controller ctrl; + IndexRequest req; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.set_is_new_table(true); + IndexResponse res; + meta_service->prepare_index(&ctrl, &req, &res, nullptr); + res.Clear(); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->commit_index(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "CommitIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_index_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_index_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_index_del_counter.get({mock_instance})); +} + +// drop_index +TEST(RpcKvBvarTest, DropIndex) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 4524364; + int64_t table_id = 65354; + int64_t index_id = 658432; + int64_t partition_id = 76553; + std::string mock_instance = "test_instance"; + const std::string label = "test_label_67a34e2q1231"; + + int64_t tablet_id_base = 2313324; + for (int i = 0; i < 10; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id, + tablet_id_base + i); + } + int txn_id {}; + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + ASSERT_GT(txn_id, 0); + } + { + for (int i = 0; i < 10; ++i) { + auto tmp_rowset = + create_rowset(txn_id, tablet_id_base + i, index_id, partition_id, -1, 100); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + } + + brpc::Controller cntl; + IndexRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + IndexResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->drop_index(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "DropIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_index_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_index_put_counter.get({mock_instance})); +} + +// prepare_partition +TEST(RpcKvBvarTest, PreparePartition) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecyclePartitionPB partition_pb; + auto partition_key = recycle_partition_key({instance_id, partition_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + brpc::Controller ctrl; + PartitionRequest req; + PartitionResponse res; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.add_partition_ids(partition_id); + res.Clear(); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->prepare_partition(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "PreparePartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_partition_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_partition_put_counter.get({mock_instance})); +} + +// commit_partition +TEST(RpcKvBvarTest, CommitPartition) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_cloud_instance_id"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + constexpr int64_t tablet_id = 10004; + + std::unique_ptr txn; + doris::TabletMetaCloudPB tablet_pb; + tablet_pb.set_table_id(table_id); + tablet_pb.set_index_id(index_id); + tablet_pb.set_partition_id(partition_id); + tablet_pb.set_tablet_id(tablet_id); + auto tablet_key = meta_tablet_key({instance_id, table_id, index_id, partition_id, tablet_id}); + auto tablet_val = tablet_pb.SerializeAsString(); + RecyclePartitionPB partition_pb; + auto partition_key = recycle_partition_key({instance_id, partition_id}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + std::string val; + brpc::Controller ctrl; + PartitionRequest req; + PartitionResponse res; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.add_partition_ids(partition_id); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + meta_service->prepare_partition(&ctrl, &req, &res, nullptr); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->commit_partition(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "CommitPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_partition_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_partition_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_partition_del_counter.get({mock_instance})); +} + +// check_kv +TEST(RpcKvBvarTest, CheckKv) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_instance"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + + std::unique_ptr txn; + RecyclePartitionPB partition_pb; + auto partition_key = recycle_partition_key({instance_id, 10004}); + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + brpc::Controller ctrl; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(partition_key, "val"); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + CheckKVRequest req_check; + CheckKVResponse res_check; + req_check.set_op(CheckKVRequest::CREATE_PARTITION_AFTER_FE_COMMIT); + CheckKeyInfos check_keys_pb; + check_keys_pb.add_table_ids(table_id + 1); + check_keys_pb.add_index_ids(index_id + 1); + check_keys_pb.add_partition_ids(partition_id + 1); + req_check.mutable_check_keys()->CopyFrom(check_keys_pb); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->check_kv(&ctrl, &req_check, &res_check, nullptr); + + LOG(INFO) << "CheckKv: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_kv_get_counter.get({mock_instance})); +} + +// drop_partition +TEST(RpcKvBvarTest, DropPartition) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + std::string instance_id = "test_instance"; + constexpr int64_t table_id = 10001; + constexpr int64_t index_id = 10002; + constexpr int64_t partition_id = 10003; + auto tbl_version_key = table_version_key({instance_id, 1, table_id}); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->atomic_add(tbl_version_key, 1); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + brpc::Controller ctrl; + PartitionRequest req; + PartitionResponse res; + req.set_db_id(1); + req.set_table_id(table_id); + req.add_index_ids(index_id); + req.add_partition_ids(partition_id); + req.set_need_update_table_version(true); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->drop_partition(&ctrl, &req, &res, nullptr); + + LOG(INFO) << "DropPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_partition_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_partition_put_counter.get({mock_instance})); +} + +// get_obj_store_info +TEST(RpcKvBvarTest, GetObjStoreInfo) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto rate_limiter = std::make_shared(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + InstanceKeyInfo key_info {"test_instance"}; + std::string key; + instance_key(key_info, &key); + txn->put(key, "val"); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + GetObjStoreInfoResponse res; + GetObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_obj_store_info(&cntl, &req, &res, nullptr); + + LOG(INFO) << "GetObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_obj_store_info_get_counter.get({mock_instance})); +} + +// alter_storage_vault +TEST(RpcKvBvarTest, AlterStorageVault) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + constexpr char vault_name[] = "test_alter_s3_vault"; + + InstanceKeyInfo key_info {"test_instance"}; + std::string key; + instance_key(key_info, &key); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, "val"); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + StorageVaultPB vault; + vault.mutable_obj_info()->set_ak("new_ak"); + vault.set_name(vault_name); + req.mutable_vault()->CopyFrom(vault); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->alter_storage_vault(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + + LOG(INFO) << "AlterStorageVault: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_alter_storage_vault_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_alter_storage_vault_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, + g_bvar_rpc_kv_alter_storage_vault_del_counter.get({mock_instance})); +} + +// alter_obj_store_info +TEST(RpcKvBvarTest, AlterObjStoreInfo) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + + ObjectStoreInfoPB obj_info; + obj_info.set_id("1"); + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + InstanceInfoPB instance; + instance.add_obj_info()->CopyFrom(obj_info); + val = instance.SerializeAsString(); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK); + req.mutable_obj()->set_id("1"); + req.mutable_obj()->set_ak("new_ak"); + req.mutable_obj()->set_sk(plain_sk); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->alter_obj_store_info(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + + LOG(INFO) << "AlterObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_alter_obj_store_info_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, + g_bvar_rpc_kv_alter_obj_store_info_put_counter.get({mock_instance})); + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} + +// update_ak_sk +TEST(RpcKvBvarTest, UpdateAkSk) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + + ObjectStoreInfoPB obj_info; + + obj_info.set_user_id("111"); + + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + InstanceInfoPB instance; + instance.add_obj_info()->CopyFrom(obj_info); + val = instance.SerializeAsString(); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + UpdateAkSkRequest req; + req.set_instance_id("test_instance"); + RamUserPB ram_user; + ram_user.set_user_id("111"); + + ram_user.set_ak("new_ak"); + ram_user.set_sk(plain_sk); + req.add_internal_bucket_user()->CopyFrom(ram_user); + + brpc::Controller cntl; + UpdateAkSkResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->update_ak_sk(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "UpdateAkSk: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_ak_sk_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_ak_sk_put_counter.get({mock_instance})); + + SyncPoint::get_instance()->disable_processing(); + SyncPoint::get_instance()->clear_all_call_backs(); +} + +// create_instance +TEST(RpcKvBvarTest, CreateInstance) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + CreateInstanceRequest req; + req.set_instance_id("test_instance"); + req.set_user_id("test_user"); + req.set_name("test_name"); + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + req.mutable_obj_info()->CopyFrom(obj); + + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->enable_processing(); + CreateInstanceResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "CreateInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_instance_get_counter.get({mock_instance})); + + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// get_instance +TEST(RpcKvBvarTest, GetInstance) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->enable_processing(); + brpc::Controller cntl; + { + CreateInstanceRequest req; + req.set_instance_id("test_instance"); + req.set_user_id("test_user"); + req.set_name("test_name"); + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + } + GetInstanceRequest req; + GetInstanceResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + req.set_cloud_unique_id("1:test_instance:m-n3qdpyal27rh8iprxx"); + meta_service->get_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "GetInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_instance_get_counter.get({mock_instance})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// alter_cluster +// alter cluster have not do kv op +// TEST(RpcKvBvarTest, AlterCluster) { +// auto meta_service = get_meta_service(); +// auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + +// brpc::Controller cntl; +// AlterClusterRequest req; +// req.set_instance_id(mock_instance); +// req.mutable_cluster()->set_cluster_name(mock_cluster_name); +// req.set_op(AlterClusterRequest::ADD_CLUSTER); +// AlterClusterResponse res; + +// mem_kv->get_count_ = 0; +// mem_kv->put_count_ = 0; +// mem_kv->del_count_ = 0; + +// meta_service->alter_cluster(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, +// &res, nullptr); + +// LOG(INFO) << "AlterCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " +// << mem_kv->del_count_; +// ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_cluster_get_counter.get({mock_instance})); +// } + +// get_cluster +TEST(RpcKvBvarTest, GetCluster) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + InstanceKeyInfo key_info {mock_instance}; + std::string key; + std::string val; + instance_key(key_info, &key); + + InstanceInfoPB instance; + instance.set_instance_id(mock_instance); + ClusterPB c1; + c1.set_cluster_name(mock_cluster_name); + c1.set_cluster_id(mock_cluster_id); + c1.add_mysql_user_name()->append("m1"); + instance.add_clusters()->CopyFrom(c1); + val = instance.SerializeAsString(); + + std::unique_ptr txn; + std::string get_val; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + GetClusterRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_cluster_id(mock_cluster_id); + req.set_cluster_name("test_cluster"); + GetClusterResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_cluster(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_put_counter.get({mock_instance})); +} + +// create_stage +TEST(RpcKvBvarTest, CreateStage) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + const auto* cloud_unique_id = "test_cloud_unique_id"; + std::string instance_id = "stage_test_instance_id"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + EncryptionInfoPB encry_info; + encry_info.set_encryption_method("encry_method_test"); + encry_info.set_key_id(1111); + ram_user.mutable_encryption_info()->CopyFrom(encry_info); + + // create instance + { + CreateInstanceRequest req; + req.set_instance_id(instance_id); + req.set_user_id("test_user"); + req.set_name("test_name"); + req.mutable_ram_user()->CopyFrom(ram_user); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + // create an internal stage + CreateStageRequest create_stage_request; + StagePB stage; + stage.set_type(StagePB::INTERNAL); + stage.add_mysql_user_name("root"); + stage.add_mysql_user_id("root_id"); + stage.set_stage_id("internal_stage_id"); + create_stage_request.set_cloud_unique_id(cloud_unique_id); + create_stage_request.mutable_stage()->CopyFrom(stage); + CreateStageResponse create_stage_response; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->create_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &create_stage_request, &create_stage_response, nullptr); + ASSERT_EQ(create_stage_response.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "CreateStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_stage_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_stage_put_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// get_stage +TEST(RpcKvBvarTest, GetStage) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + const auto* cloud_unique_id = "test_cloud_unique_id"; + std::string instance_id = "stage_test_instance_id"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + EncryptionInfoPB encry_info; + encry_info.set_encryption_method("encry_method_test"); + encry_info.set_key_id(1111); + ram_user.mutable_encryption_info()->CopyFrom(encry_info); + + // create instance + { + CreateInstanceRequest req; + req.set_instance_id(instance_id); + req.set_user_id("test_user"); + req.set_name("test_name"); + req.mutable_ram_user()->CopyFrom(ram_user); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + // create an internal stage + CreateStageRequest create_stage_request; + StagePB stage; + stage.set_type(StagePB::INTERNAL); + stage.add_mysql_user_name("root"); + stage.add_mysql_user_id("root_id"); + stage.set_stage_id("internal_stage_id"); + create_stage_request.set_cloud_unique_id(cloud_unique_id); + create_stage_request.mutable_stage()->CopyFrom(stage); + CreateStageResponse create_stage_response; + meta_service->create_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &create_stage_request, &create_stage_response, nullptr); + ASSERT_EQ(create_stage_response.status().code(), MetaServiceCode::OK); + + GetStageRequest get_stage_req; + get_stage_req.set_type(StagePB::INTERNAL); + get_stage_req.set_cloud_unique_id(cloud_unique_id); + get_stage_req.set_mysql_user_name("root"); + get_stage_req.set_mysql_user_id("root_id"); + + // get existent internal stage + GetStageResponse res2; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &get_stage_req, &res2, nullptr); + ASSERT_EQ(res2.status().code(), MetaServiceCode::OK); + ASSERT_EQ(1, res2.stage().size()); + + LOG(INFO) << "GetStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_stage_get_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// get_iam +TEST(RpcKvBvarTest, GetIam) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + std::string instance_id = "get_iam_test_instance_id"; + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + config::arn_id = "iam_arn"; + config::arn_ak = "iam_ak"; + config::arn_sk = "iam_sk"; + + // create instance + { + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + + CreateInstanceRequest req; + req.set_instance_id(instance_id); + req.set_user_id("test_user"); + req.set_name("test_name"); + req.mutable_ram_user()->CopyFrom(ram_user); + req.mutable_obj_info()->CopyFrom(obj); + + CreateInstanceResponse res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + GetIamRequest request; + request.set_cloud_unique_id(cloud_unique_id); + GetIamResponse response; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_iam(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &request, + &response, nullptr); + ASSERT_EQ(response.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_iam_get_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// alter_iam +TEST(RpcKvBvarTest, AlterIam) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "selectdbselectdbselectdbselectdb"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; + std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; + + AlterIamRequest req; + req.set_account_id("123"); + req.set_ak("ak1"); + req.set_sk(plain_sk); + + brpc::Controller cntl; + AlterIamResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->alter_iam(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "AlterIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_iam_get_counter.get({"alter_iam_instance"})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_iam_put_counter.get({"alter_iam_instance"})); +} + +// alter_ram_user +TEST(RpcKvBvarTest, AlterRamUser) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + std::string instance_id = "alter_ram_user_instance_id"; + [[maybe_unused]] auto* sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->set_call_back("encrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* ret = try_any_cast(args[0]); + *ret = 0; + auto* key = try_any_cast(args[1]); + *key = "test"; + auto* key_id = try_any_cast(args[2]); + *key_id = 1; + }); + sp->set_call_back("decrypt_ak_sk:get_encryption_key", [](auto&& args) { + auto* key = try_any_cast(args[0]); + *key = "test"; + auto* ret = try_any_cast(args[1]); + *ret = 0; + }); + sp->enable_processing(); + + config::arn_id = "iam_arn"; + config::arn_ak = "iam_ak"; + config::arn_sk = "iam_sk"; + + ObjectStoreInfoPB obj; + obj.set_ak("123"); + obj.set_sk("321"); + obj.set_bucket("456"); + obj.set_prefix("654"); + obj.set_endpoint("789"); + obj.set_region("987"); + obj.set_external_endpoint("888"); + obj.set_provider(ObjectStoreInfoPB::BOS); + + // create instance without ram user + CreateInstanceRequest create_instance_req; + create_instance_req.set_instance_id(instance_id); + create_instance_req.set_user_id("test_user"); + create_instance_req.set_name("test_name"); + create_instance_req.mutable_obj_info()->CopyFrom(obj); + CreateInstanceResponse create_instance_res; + meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &create_instance_req, &create_instance_res, nullptr); + ASSERT_EQ(create_instance_res.status().code(), MetaServiceCode::OK); + + // alter ram user + RamUserPB ram_user; + ram_user.set_user_id("test_user_id"); + ram_user.set_ak("test_ak"); + ram_user.set_sk("test_sk"); + AlterRamUserRequest alter_ram_user_request; + alter_ram_user_request.set_instance_id(instance_id); + alter_ram_user_request.mutable_ram_user()->CopyFrom(ram_user); + AlterRamUserResponse alter_ram_user_response; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + meta_service->alter_ram_user(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &alter_ram_user_request, &alter_ram_user_response, nullptr); + + LOG(INFO) << "AlterRamUser: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_ram_user_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_ram_user_put_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// begin_copy +TEST(RpcKvBvarTest, BeginCopy) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + auto stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "BeginCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_copy_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_copy_put_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// get_copy_job +TEST(RpcKvBvarTest, GetCopyJob) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + const char* cloud_unique_id = "test_cloud_unique_id"; + const char* stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + GetCopyJobRequest get_copy_job_request; + get_copy_job_request.set_cloud_unique_id(cloud_unique_id); + get_copy_job_request.set_stage_id(stage_id); + get_copy_job_request.set_table_id(table_id); + get_copy_job_request.set_copy_id("test_copy_id"); + get_copy_job_request.set_group_id(0); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetCopyJobResponse res; + meta_service->get_copy_job(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &get_copy_job_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetCopyJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_job_get_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// finish_copy +TEST(RpcKvBvarTest, FinishCopy) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + const char* cloud_unique_id = "test_cloud_unique_id"; + const char* stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + FinishCopyRequest finish_copy_request; + finish_copy_request.set_cloud_unique_id(cloud_unique_id); + finish_copy_request.set_stage_id(stage_id); + finish_copy_request.set_stage_type(StagePB::EXTERNAL); + finish_copy_request.set_table_id(table_id); + finish_copy_request.set_copy_id("test_copy_id"); + finish_copy_request.set_group_id(0); + finish_copy_request.set_action(FinishCopyRequest::COMMIT); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + FinishCopyResponse res; + meta_service->finish_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &finish_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "FinishCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_copy_get_counter.get({instance_id})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_copy_put_counter.get({instance_id})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_copy_del_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// get_copy_files +TEST(RpcKvBvarTest, GetCopyFiles) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + auto stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + GetCopyFilesRequest get_copy_file_req; + get_copy_file_req.set_cloud_unique_id(cloud_unique_id); + get_copy_file_req.set_stage_id(stage_id); + get_copy_file_req.set_table_id(table_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + GetCopyFilesResponse res; + meta_service->get_copy_files(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &get_copy_file_req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "GetCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_files_get_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// filter_copy_files +TEST(RpcKvBvarTest, FilterCopyFiles) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + brpc::Controller cntl; + auto cloud_unique_id = "test_cloud_unique_id"; + auto stage_id = "test_stage_id"; + int64_t table_id = 100; + std::string instance_id = "copy_job_test_instance_id"; + + [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + sp->set_call_back("get_instance_id", [&](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = instance_id; + ret->second = true; + }); + sp->enable_processing(); + { + // generate a begin copy request + BeginCopyRequest begin_copy_request; + begin_copy_request.set_cloud_unique_id(cloud_unique_id); + begin_copy_request.set_stage_id(stage_id); + begin_copy_request.set_stage_type(StagePB::EXTERNAL); + begin_copy_request.set_table_id(table_id); + begin_copy_request.set_copy_id("test_copy_id"); + begin_copy_request.set_group_id(0); + begin_copy_request.set_start_time_ms(200); + begin_copy_request.set_timeout_time_ms(300); + for (int i = 0; i < 20; ++i) { + ObjectFilePB object_file_pb; + object_file_pb.set_relative_path("obj_" + std::to_string(i)); + object_file_pb.set_etag("obj_" + std::to_string(i) + "_etag"); + begin_copy_request.add_object_files()->CopyFrom(object_file_pb); + } + BeginCopyResponse res; + meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &begin_copy_request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + FilterCopyFilesRequest request; + request.set_cloud_unique_id(cloud_unique_id); + request.set_stage_id(stage_id); + request.set_table_id(table_id); + for (int i = 0; i < 10; ++i) { + ObjectFilePB object_file; + object_file.set_relative_path("file" + std::to_string(i)); + object_file.set_etag("etag" + std::to_string(i)); + request.add_object_files()->CopyFrom(object_file); + } + FilterCopyFilesResponse res; + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->filter_copy_files(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &request, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + + LOG(INFO) << "FilterCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_filter_copy_files_get_counter.get({instance_id})); + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + +// get_cluster_status +TEST(RpcKvBvarTest, GetClusterStatus) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + InstanceKeyInfo key_info {mock_instance}; + std::string key; + std::string val; + instance_key(key_info, &key); + + InstanceInfoPB instance; + instance.set_instance_id(mock_instance); + ClusterPB c1; + c1.set_type(ClusterPB::COMPUTE); + c1.set_cluster_name(mock_cluster_name); + c1.set_cluster_id(mock_cluster_id); + c1.add_mysql_user_name()->append("m1"); + c1.set_cluster_status(ClusterStatus::NORMAL); + ClusterPB c2; + c2.set_type(ClusterPB::COMPUTE); + c2.set_cluster_name(mock_cluster_name + "2"); + c2.set_cluster_id(mock_cluster_id + "2"); + c2.add_mysql_user_name()->append("m2"); + c2.set_cluster_status(ClusterStatus::SUSPENDED); + ClusterPB c3; + c3.set_type(ClusterPB::COMPUTE); + c3.set_cluster_name(mock_cluster_name + "3"); + c3.set_cluster_id(mock_cluster_id + "3"); + c3.add_mysql_user_name()->append("m3"); + c3.set_cluster_status(ClusterStatus::TO_RESUME); + instance.add_clusters()->CopyFrom(c1); + instance.add_clusters()->CopyFrom(c2); + instance.add_clusters()->CopyFrom(c3); + val = instance.SerializeAsString(); + + std::unique_ptr txn; + std::string get_val; + TxnErrorCode err = meta_service->txn_kv()->create_txn(&txn); + ASSERT_EQ(err, TxnErrorCode::TXN_OK); + txn->put(key, val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + brpc::Controller cntl; + GetClusterStatusRequest req; + req.add_instance_ids(mock_instance); + GetClusterStatusResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_cluster_status(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(res.details().at(0).clusters().size(), 3); + + LOG(INFO) << "GetClusterStatus: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_cluster_status_get_counter.get({mock_instance})); +} + +// get_current_max_txn_id +TEST(RpcKvBvarTest, GetCurrentMaxTxnId) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + const int64_t db_id = 123; + const std::string label = "test_label123"; + const std::string cloud_unique_id = "test_cloud_unique_id"; + + brpc::Controller begin_txn_cntl; + BeginTxnRequest begin_txn_req; + BeginTxnResponse begin_txn_res; + TxnInfoPB txn_info_pb; + + begin_txn_req.set_cloud_unique_id(cloud_unique_id); + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(12345); + txn_info_pb.set_timeout_ms(36000); + begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); + + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &begin_txn_req, &begin_txn_res, nullptr); + ASSERT_EQ(begin_txn_res.status().code(), MetaServiceCode::OK); + + brpc::Controller max_txn_id_cntl; + GetCurrentMaxTxnRequest max_txn_id_req; + GetCurrentMaxTxnResponse max_txn_id_res; + + std::unique_ptr txn; + EXPECT_EQ(mem_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put("schema change", "val"); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + max_txn_id_req.set_cloud_unique_id(cloud_unique_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->get_current_max_txn_id( + reinterpret_cast<::google::protobuf::RpcController*>(&max_txn_id_cntl), &max_txn_id_req, + &max_txn_id_res, nullptr); + + LOG(INFO) << "GetCurrentMaxTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_get_current_max_txn_id_get_counter.get({mock_instance})); +} + +// begin_sub_txn +TEST(RpcKvBvarTest, BeginSubTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 98131; + int64_t txn_id = -1; + int64_t t1 = 10; + int64_t t1_index = 100; + int64_t t1_p1 = 11; + int64_t t1_p1_t1 = 12; + int64_t t1_p1_t2 = 13; + int64_t t1_p2 = 14; + int64_t t1_p2_t1 = 15; + int64_t t2 = 16; + std::string label = "test_label"; + std::string label2 = "test_label_0"; + // begin txn + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(t1); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + } + + // mock rowset and tablet: for sub_txn1 + int64_t sub_txn_id1 = txn_id; + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t1, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t2, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p2, t1_p2_t1, sub_txn_id1); + + brpc::Controller cntl; + BeginSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_sub_txn_num(0); + req.set_db_id(db_id); + req.set_label(label2); + req.mutable_table_ids()->Add(t1); + req.mutable_table_ids()->Add(t2); + BeginSubTxnResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->begin_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "BeginSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_sub_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_sub_txn_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_sub_txn_del_counter.get({mock_instance})); +} + +// abort_sub_txn +TEST(RpcKvBvarTest, AbortSubTxn) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 98131; + int64_t txn_id = -1; + int64_t t1 = 10; + int64_t t1_index = 100; + int64_t t1_p1 = 11; + int64_t t1_p1_t1 = 12; + int64_t t1_p1_t2 = 13; + int64_t t1_p2 = 14; + int64_t t1_p2_t1 = 15; + int64_t t2 = 16; + std::string label = "test_label"; + std::string label2 = "test_label_0"; + // begin txn + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(t1); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id = res.txn_id(); + } + + // mock rowset and tablet: for sub_txn1 + int64_t sub_txn_id1 = txn_id; + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t1, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p1, t1_p1_t2, sub_txn_id1); + create_and_commit_rowset(meta_service.get(), t1, t1_index, t1_p2, t1_p2_t1, sub_txn_id1); + brpc::Controller cntl; + { + BeginSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_sub_txn_num(0); + req.set_db_id(db_id); + req.set_label(label2); + req.mutable_table_ids()->Add(t1); + req.mutable_table_ids()->Add(t2); + BeginSubTxnResponse res; + + meta_service->begin_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + } + + AbortSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id); + req.set_sub_txn_num(2); + req.set_sub_txn_id(sub_txn_id1); + req.set_db_id(db_id); + req.mutable_table_ids()->Add(t1); + req.mutable_table_ids()->Add(t2); + AbortSubTxnResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->abort_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "AbortSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_sub_txn_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_sub_txn_put_counter.get({mock_instance})); +} + +// abort_txn_with_coordinator +TEST(RpcKvBvarTest, AbortTxnWithCoordinator) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + const int64_t db_id = 666; + const int64_t table_id = 777; + const std::string label = "test_label"; + const std::string cloud_unique_id = "test_cloud_unique_id"; + const int64_t coordinator_id = 15623; + int64_t cur_time = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + std::string host = "127.0.0.1:15586"; + int64_t txn_id = -1; + + brpc::Controller begin_txn_cntl; + BeginTxnRequest begin_txn_req; + BeginTxnResponse begin_txn_res; + TxnInfoPB txn_info_pb; + TxnCoordinatorPB coordinator; + + begin_txn_req.set_cloud_unique_id(cloud_unique_id); + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + coordinator.set_id(coordinator_id); + coordinator.set_ip(host); + coordinator.set_sourcetype(::doris::cloud::TxnSourceTypePB::TXN_SOURCE_TYPE_BE); + coordinator.set_start_time(cur_time); + txn_info_pb.mutable_coordinator()->CopyFrom(coordinator); + begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); + + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &begin_txn_req, &begin_txn_res, nullptr); + ASSERT_EQ(begin_txn_res.status().code(), MetaServiceCode::OK); + txn_id = begin_txn_res.txn_id(); + ASSERT_GT(txn_id, -1); + + brpc::Controller abort_txn_cntl; + AbortTxnWithCoordinatorRequest abort_txn_req; + AbortTxnWithCoordinatorResponse abort_txn_resp; + + abort_txn_req.set_id(coordinator_id); + abort_txn_req.set_ip(host); + abort_txn_req.set_start_time(cur_time + 3600); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->abort_txn_with_coordinator( + reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), &abort_txn_req, + &abort_txn_resp, nullptr); + + LOG(INFO) << "AbortTxnWithCoordinator: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ + << ", " << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter.get({mock_instance})); +} + +// check_txn_conflict +TEST(RpcKvBvarTest, CheckTxnConflict) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + + const int64_t db_id = 666; + const int64_t table_id = 777; + const std::string label = "test_label"; + const std::string cloud_unique_id = "test_cloud_unique_id"; + int64_t txn_id = -1; + + brpc::Controller begin_txn_cntl; + BeginTxnRequest begin_txn_req; + BeginTxnResponse begin_txn_res; + TxnInfoPB txn_info_pb; + + begin_txn_req.set_cloud_unique_id(cloud_unique_id); + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + begin_txn_req.mutable_txn_info()->CopyFrom(txn_info_pb); + + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &begin_txn_req, &begin_txn_res, nullptr); + ASSERT_EQ(begin_txn_res.status().code(), MetaServiceCode::OK); + txn_id = begin_txn_res.txn_id(); + ASSERT_GT(txn_id, -1); + + brpc::Controller check_txn_conflict_cntl; + CheckTxnConflictRequest check_txn_conflict_req; + CheckTxnConflictResponse check_txn_conflict_res; + + check_txn_conflict_req.set_cloud_unique_id(cloud_unique_id); + check_txn_conflict_req.set_db_id(db_id); + check_txn_conflict_req.set_end_txn_id(txn_id + 1); + check_txn_conflict_req.add_table_ids(table_id); + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->check_txn_conflict( + reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), + &check_txn_conflict_req, &check_txn_conflict_res, nullptr); + + LOG(INFO) << "CheckTxnConflict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, + g_bvar_rpc_kv_check_txn_conflict_get_counter.get({mock_instance})); +} + +// clean_txn_label +TEST(RpcKvBvarTest, CleanTxnLabel) { + auto meta_service = get_meta_service(); + auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); + int64_t db_id = 1987211; + const std::string& label = "test_clean_label"; + brpc::Controller cntl; + { + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label(label); + txn_info_pb.add_table_ids(1234); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + } + CleanTxnLabelRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.add_labels(label); + CleanTxnLabelResponse res; + + mem_kv->get_count_ = 0; + mem_kv->put_count_ = 0; + mem_kv->del_count_ = 0; + + meta_service->clean_txn_label(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + + LOG(INFO) << "CleanTxnLabel: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " + << mem_kv->del_count_; + ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_clean_txn_label_get_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_clean_txn_label_put_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_clean_txn_label_del_counter.get({mock_instance})); +} +} // namespace doris::cloud From 17d6f1c354cfe2e590077e9f50def55fd9f40fc3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:35:38 +0800 Subject: [PATCH 166/572] branch-3.0: [fix](be) fix parquet file reader not updating page index when processing it #52228 (#52781) Cherry-picked from #52228 Co-authored-by: SWEI Co-authored-by: zengsiwei Co-authored-by: suxiaogang223 --- be/src/common/config.cpp | 2 +- .../vec/exec/format/parquet/vparquet_reader.cpp | 2 +- .../parquet/small_2rowgroup.parquet | Bin 0 -> 13059 bytes .../hive/test_hive_parquet_skip_page.out | 4 ++++ .../hive/test_hive_parquet_skip_page.groovy | 12 +++++++++++- 5 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet/small_2rowgroup.parquet diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9f14823e97523c..9d4e578de6fd42 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1464,7 +1464,7 @@ DEFINE_mInt64(compaction_batch_size, "-1"); // If set to false, the parquet reader will not use page index to filter data. // This is only for debug purpose, in case sometimes the page index // filter wrong data. -DEFINE_mBool(enable_parquet_page_index, "false"); +DEFINE_mBool(enable_parquet_page_index, "true"); DEFINE_mBool(ignore_not_found_file_in_external_table, "true"); diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp index d669a57c609cd3..7e0012486d63b5 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp @@ -859,7 +859,7 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group, // use the union row range skipped_row_ranges.emplace_back(skipped_row_range); } - _col_offsets.emplace(parquet_col_id, offset_index); + _col_offsets[parquet_col_id] = offset_index; } if (skipped_row_ranges.empty()) { read_whole_row_group(); diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet/small_2rowgroup.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet/small_2rowgroup.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dcd05f5e28e08e28d472b9682cf5fed1a87d0b14 GIT binary patch literal 13059 zcmaLe1(X#>+aTb*|wIbvYMfPioT0Re+zhLHz^W?>1f z3Qq)_5fF*UL?J5Ch|V{BOAKNXi`c{=E`bCQkN5F-b^DGLrKhDM(2wQj>Yl_l%Xu;C{G0{ zQi;k`p(@p=P7P{Oi`vwoF7>ES0~*qZ#x$WR&1g;wTGEQvwBZNZ(vJ3Ypd+2=Oc%P+ zjqdcIC%x!RANtad{tRFsgBZ*ZhBA!dj9?_A7|oB2VJzbq&reKXB9oZR6s9tb>C9jz zvzW~s<}#1@EMOsvSj-ZZvW(@dU?r;3J>-lfU@P-+bXK{}6ifhaoKC2u}ne5{bw} zAu7>`&NqBZ3_{;QViB7-#3hg*;t`)<5|EHYBqj+-Nk($MBLyi*MQYNJmUN^i0~yIg zX0ni#Y-A?~ImtzC@{pH&MQr5Vj>K}%ZEnl}7ETiVf{4s@gwo#{eXy3w5;^rRQP z=|f-o(VqbfWDtWH!cc}WoDqy<6r=f(F^pv#H!(R5Wp937^5QjO! z&-}tsj&Yn5oa7XzIm21bah?lYZgHDC+~pqkdB8&+@t7w( zF-b^DGLrKhDM(2wQj>Yl_l%Xu;C{G0{Qi;k`p(@p=P7P{Oi`vwo zF7>ES0~*qZ#x$WR&1g;wTGEQvwBZNZ(vJ3Ypd+2=Oc%P+jqdcIC%x!RANtad{tRFs zgBZ*ZhBA!dj9?_A7|oB2VJzbq&reKXB9oZR6s9tb>C9jzvzW~s<}#1@EMOsvSj-ZZ zvW(@dU?r;3J>-lfU@P-+bXK{}B35e;C3Nj_^bvB9Vwp6rvK1=zPPs#2_ZIh)o>g z5=ap7h)*yHNJt_QlZ2!sBRSuZf|R5pHEBpoI?|JYjASA+S;$H@vXg_HHNAm8eV=s#1;W)SxD{s7)Q}QjhvHpdpQD zOcR>YjOMhUC9P;p8-AcI?PyO2I?{>GbfGKV=uQuM(u>~op)dXD&j1E8h`|hDD8m@e z2u3oB(fr65#xjoa{KNz%GKtAdVJg#@&J1QUi`mR!F7uer0v57}#Vlbd%UI3|RMQr5Vj>K}%ZEnl}7ETiVf{ z4s@gwo#{eXy3w5;^rRQP=|f-o(VqbfWDtWH!cc}WoDqy<6r=f(F^pv#H!(R5Wp937^5QjO!&-}tsj&Yn5oa7XzIm21bah?lY zZgHDC+~pqkdB8&+@t7w(< zaD*oU5s5@(q7ap6MCTj6B?d8xMQq{_mq3DuM|^@wKtd9cm?R`68OiyM6r>~-sYydx z(vhAFWF!-r$wF4Lk)0gmBp12KLtgTcp8^!55QQm1Q9>w2alWSnB`HN|%21Yal&1m} zsYGR}P?c&_rv^2tMQ!R(mwMEv0S#$HW17&EW;CY-Eont-+VBHyX-9iH(2-7brVCx^ zMt6G9lV0?u4}IxJe+Dp+K@4UHLm9?!Mlh05jOItiFqUzQ=O-pGkx5Ku3R9WJbY?J< zS-EM^HyS;lf!u##1*W({ju$9gufkxgu73tQR7c6P9nUF>ELd)dc+ z4seh|9Oei=^9x5g#&J$?l2e@K3}-pVc`k5~OI+qxu5gv#_?>H9=LR>q#cl3zmwVjj z0S|e^W1jGoXFTTxfAEr5yygvWdB=M`@R3ja$zOctZ@%!Ae+d0JBMe~)M|dI-kw`=) z3Q>thbiUzRVi1#9#3l}L2_%Sk#3z^pBqR}uNkUSRk(}>HK}u4Qnlz*(9qGwHMlz9^ zEMz4c*~vjpa*>-nru_RjEdG zYEY9})TRz~sYiVp(2zznrU^}HMsr%wl2){)4L{J9cC@Dh9qB}8y3mzwbf*VB=|yk) z(3gJnX8;2k#9)Rnlwk~K1S1*6Xntf2V;RSIeqsU>nZ#tKFqLUcX9hEw#cbveG#ASZv3Rn4!-?_$hZg7)Z+~y8htQ8Q!w{BmgeL+Ki9}?g5S3^|=NrBy z1~G|6Y~m1?K!S)ze1b_pLK2afBqSvn$@z{Hq$CxoNkdxFk)8}>Bomp*LRPYoogCyO z7rDtpUh+=@B?jWM|(QZkxq1`3tj0(cY4s1Ui799ed$Mk1~8C8 z3}y&J8OCr%Fp^P>=10abmT`>dCnhkFNla!6Q<=teW-yak%w`UAna6wAZhTiM2TcCeFO>}C&p*~fkkaF9bB<_JIY3r9J|aZYfO zQ=H}uXZimuC^)D2JQujgB`)(TSGdY={LVG5bAy}Q;x>1<%RTP%fQLNdF;95PGoJH; zKX}P2Uh{^xyyHC|_{b;z@d+jY2}wj^l8}^SBQSEt zG^7!YX+l$)(VP~vq!q1c!w^4&E_9_E-RVJ3deNIc^ravD8NfgWF_<9? zWf;R5!AM3injaa%SjI7)pP0ZzCNY^QOl2C=nZZnEF`GHeWghccz(N+Wm?bP_8OvG0 zN>;I&HLPVF>)F6YHnEv4Y-JnU*}+bBv70^YWgq)Fz(Edim?QkmFC66<$2q}CPH~zu zoaG$nxxhs(ahYGa!c~6bcdl`r8{Fg;x4FYz?s1<7Jme9NdBRhk@tha@!AoB8nm4@V z9q;+TM?Uct(vyLVWFj+J$VxV{lY^Y(A~$)+OFr^bfPxgF zFhwXz2*oJQ_mrR{r6^4q%2JN>6Q1&n=e*z#Uh<09yx}eHc+Uqu@`*qB zi_iSc7rycjp(`815SDO+Cjt?PL}a26m1sog8@?q5F^NTN;t-cWf`~_af=NI^5|NlB zBqbTi`HmE%Bo(PiLt4_2o(yCp6Pd|ERP^DMC>~C`NI< zrvxP_MQO@VmU5J*0u`x5WvWn>YE-8NHK|2y>QI+@)TaRrX+&e1(3EC0rv)u(MQhsd z18r$XdpgjOPIRUVUFk-5deD)oEPH>V_oaPK?ImdY}aFI(~=2xz8mEZWC zYh33BH@U@a?r@iT+~)xgdBkI$@RVmf=LLW8l2^Ru4R3kJdp_`yPyESWeCBVy@RffE zUDX_hu!JK#5r{}6A`^wEL?b%i@GUWjNi1R$hqwe1L_FdXOac;;h{PlzDalCAccdUC zsYp#4(vpt!WFRA%$V?Wpl8x--ASb!VO&;=+kNgy%AcZJQ5sDH*F^cm&B`8TLN>hfi zl%qTqs7NI$Q-!KjqdGOHNiAwqhq~0GJ`HF{BO23$rZl5DEoezATGNIfXiGcV(}9k3 zqBC9SN;kUGgP!!FH+|?!Kl(F(fed0WLm0|1hBJbZjAArDGKR5?V>~}Gfr(6FGEEMhTBSjsY%vx1eZVl``6%R1JxfsJfpGh5ioHny{ao$O*a zd)Ui9_H%%P9O5uX_?cfg$}x^}f|H!$G-o)Asewd+i82O;|34QLgb#C0(|LlEhYcH? z*6IRxmPdec=rV+V%Yk8oQ~TvE>3azQVFQA*g>`D)@S$rB!iUZ(gb$6OB6I>Gd}vqU zLj$@fw_xd7hF)b09}t`|D01MoqCxJteMJLv2dNAQE*SK#=zl9$4Gt4pQXO>rV3@$d zL4%SB6FxX&u7LOvyEbdxqJ5LVu;Qi)8$V2wfS?gsgEIyvN*$0$PpKjX1gi`T2pSSI z=y_3F>c|D-|NG?SQc!@gDgEatAUKg%LF|Bn=^`525YHs&{^2lzX@lk+3KLj8AT-91 zn5iS%JNN(YV_32v&oxx)cSUs*I^6jGz8<07{O_ax=_cZTyBTIYL1T}E2`m=$MmJpo z{tYuMnJA$pr^k;TQnhNOTKWGo+W2e2WC=|^VDgJL$ zNU>reAr(R!i-&}i3EeyXpS`0Jp;a|PGbQ_0q1TrVjU!bFJ^4SWMDc&mEcS2xzs3L7 z+ZAgaS{7eFuJ8Gu`2V(7`!_EX5I V*|TQ$2u_s94o))Z literal 0 HcmV?d00001 diff --git a/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out b/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out index 6c869dbc789466..a8973479e4289b 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out +++ b/regression-test/data/external_table_p0/hive/test_hive_parquet_skip_page.out @@ -1,4 +1,8 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !small_2rowgroup -- +1024 +4049 + -- !q01 -- 1 2132 4633 4 28.00 28955.64 0.09 0.06 N O 1996-04-21 1996-03-30 1996-05-16 NONE AIR lites. fluffily even de 1 15635 638 6 32.00 49620.16 0.07 0.02 N O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON MAIL arefully slyly ex diff --git a/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy b/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy index ebdbedf139d37c..0bded7d820a711 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_parquet_skip_page.groovy @@ -94,11 +94,21 @@ suite("test_hive_parquet_skip_page", "p0,external,hive,external_docker,external_ return; } + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def hdfs_port = context.config.otherConfigs.get("hive2HdfsPort") + def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" + def hdfsUserName = "doris" + def uri = "${defaultFS}" + "/user/doris/preinstalled_data/parquet/small_2rowgroup.parquet" + qt_small_2rowgroup """ select * from HDFS( + "uri" = "${uri}", + "hadoop.username" = "${hdfsUserName}", + "format" = "parquet") where a = 1024 or a = 4049 + order by a;""" + for (String hivePrefix : ["hive2", "hive3"]) { try { String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String catalog_name = "${hivePrefix}_test_parquet_skip_page" - String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( From a26558865567f4c64097b63acc9fdb0ca63dbddb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:36:30 +0800 Subject: [PATCH 167/572] branch-3.0: [opt](cloud) Optimize balance speed by reducing the complexity of the rebalance algorithm #51733 (#52813) Cherry-picked from #51733 Co-authored-by: deardeng --- .../cloud/catalog/CloudTabletRebalancer.java | 510 ++++++++++-------- 1 file changed, 288 insertions(+), 222 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index 1b014caea82779..17d72dd7446a44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -54,6 +54,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -69,31 +70,31 @@ public class CloudTabletRebalancer extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(CloudTabletRebalancer.class); - private volatile ConcurrentHashMap> beToTabletsGlobal = - new ConcurrentHashMap>(); + private volatile ConcurrentHashMap> beToTabletsGlobal = + new ConcurrentHashMap>(); - private volatile ConcurrentHashMap> beToColocateTabletsGlobal = - new ConcurrentHashMap>(); + private volatile ConcurrentHashMap> beToColocateTabletsGlobal = + new ConcurrentHashMap>(); // used for cloud tablet report - private volatile ConcurrentHashMap> beToTabletsGlobalInSecondary = - new ConcurrentHashMap>(); + private volatile ConcurrentHashMap> beToTabletsGlobalInSecondary = + new ConcurrentHashMap>(); - private Map> futureBeToTabletsGlobal; + private Map> futureBeToTabletsGlobal; private Map> clusterToBes; private Set allBes; // partitionId -> indexId -> be -> tablet - private Map>>> partitionToTablets; + private Map>>> partitionToTablets; - private Map>>> futurePartitionToTablets; + private Map>>> futurePartitionToTablets; // tableId -> be -> tablet - private Map>> beToTabletsInTable; + private Map>> beToTabletsInTable; - private Map>> futureBeToTabletsInTable; + private Map>> futureBeToTabletsInTable; private Map beToDecommissionedTime = new HashMap(); @@ -157,7 +158,7 @@ private class InfightTask { public long srcBe; public long destBe; public boolean isGlobal; - public Map> beToTablets; + public Map> beToTablets; public long startTimestamp; BalanceType balanceType; } @@ -172,7 +173,7 @@ private class TransferPairInfo { public Set getSnapshotTabletsInPrimaryByBeId(Long beId) { Set tabletIds = Sets.newHashSet(); - List tablets = beToTabletsGlobal.get(beId); + Set tablets = beToTabletsGlobal.get(beId); if (tablets != null) { for (Tablet tablet : tablets) { tabletIds.add(tablet.getId()); @@ -191,7 +192,7 @@ public Set getSnapshotTabletsInPrimaryByBeId(Long beId) { public Set getSnapshotTabletsInSecondaryByBeId(Long beId) { Set tabletIds = Sets.newHashSet(); - List tablets = beToTabletsGlobalInSecondary.get(beId); + Set tablets = beToTabletsGlobalInSecondary.get(beId); if (tablets != null) { for (Tablet tablet : tablets) { tabletIds.add(tablet.getId()); @@ -208,8 +209,8 @@ public Set getSnapshotTabletsInPrimaryAndSecondaryByBeId(Long beId) { } public int getTabletNumByBackendId(long beId) { - List tablets = beToTabletsGlobal.get(beId); - List colocateTablets = beToColocateTabletsGlobal.get(beId); + Set tablets = beToTabletsGlobal.get(beId); + Set colocateTablets = beToColocateTabletsGlobal.get(beId); return (tablets == null ? 0 : tablets.size()) + (colocateTablets == null ? 0 : colocateTablets.size()); @@ -232,80 +233,80 @@ protected void runAfterCatalogReady() { } LOG.info("cloud tablet rebalance begin"); - - clusterToBes = new HashMap>(); - allBes = new HashSet(); long start = System.currentTimeMillis(); - // 1 build cluster to backend info + buildClusterToBackendMap(); + if (!completeRouteInfo()) { + return; + } + + checkInflightWarmUpCacheAsync(); + statRouteInfo(); + migrateTabletsForSmoothUpgrade(); + statRouteInfo(); + + indexBalanced = true; + tableBalanced = true; + + performBalancing(); + + checkDecommissionState(clusterToBes); + LOG.info("finished to rebalancer. cost: {} ms", (System.currentTimeMillis() - start)); + } + + private void buildClusterToBackendMap() { + clusterToBes = new HashMap<>(); + allBes = new HashSet<>(); for (Long beId : cloudSystemInfoService.getAllBackendIds()) { Backend be = cloudSystemInfoService.getBackend(beId); if (be == null) { LOG.info("backend {} not found", beId); continue; } - clusterToBes.putIfAbsent(be.getCloudClusterId(), new ArrayList()); + clusterToBes.putIfAbsent(be.getCloudClusterId(), new ArrayList<>()); clusterToBes.get(be.getCloudClusterId()).add(beId); allBes.add(beId); } LOG.info("cluster to backends {}", clusterToBes); + } - // 2 complete route info - if (!completeRouteInfo()) { - return; - } - - // 3 check whether the inflight preheating task has been completed - checkInflghtWarmUpCacheAsync(); - - // 4 migrate tablet for smooth upgrade + private void migrateTabletsForSmoothUpgrade() { Pair pair; - statRouteInfo(); while (!tabletsMigrateTasks.isEmpty()) { try { pair = tabletsMigrateTasks.take(); + LOG.debug("begin tablets migration from be {} to be {}", pair.first, pair.second); + migrateTablets(pair.first, pair.second); } catch (InterruptedException e) { + LOG.warn("migrate tablets failed", e); throw new RuntimeException(e); } - if (LOG.isDebugEnabled()) { - LOG.debug("begin tablets migration from be {} to be {}", pair.first, pair.second); - } - migrateTablets(pair.first, pair.second); } + } - // 5 statistics be to tablets mapping information - statRouteInfo(); - - indexBalanced = true; - tableBalanced = true; - - // 6 partition-level balance + private void performBalancing() { + // ATTN: In general, the order of `balance` should follow `partition`, `table`, and `global`. + // This is because performing `global` scheduling first and then `partition` scheduling may + // lead to ineffective scheduling. Specifically, `global` scheduling might place multiple tablets belonging + // to the same table or partition onto the same BE, while `partition` scheduling later requires these tablets + // to be dispersed across different BEs, resulting in unnecessary scheduling. if (Config.enable_cloud_partition_balance) { balanceAllPartitions(); } - - // 7 if tablets in partition-level already balanced, perform table balance if (Config.enable_cloud_table_balance && indexBalanced) { balanceAllTables(); } - - // 8 if tablets in partition-level and table-level already balanced, perform global balance if (Config.enable_cloud_global_balance && indexBalanced && tableBalanced) { globalBalance(); } - - // 9 check whether all tablets of decomission have been migrated - checkDecommissionState(clusterToBes); - - LOG.info("finished to rebalancer. cost: {} ms", (System.currentTimeMillis() - start)); } public void balanceAllPartitions() { - for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { LOG.info("before partition balance be {} tablet num {}", entry.getKey(), entry.getValue().size()); } - for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { LOG.info("before partition balance be {} tablet num(current + pre heating inflight) {}", entry.getKey(), entry.getValue().size()); } @@ -326,22 +327,22 @@ public void balanceAllPartitions() { return; } - for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { LOG.info("after partition balance be {} tablet num {}", entry.getKey(), entry.getValue().size()); } - for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { LOG.info("after partition balance be {} tablet num(current + pre heating inflight) {}", entry.getKey(), entry.getValue().size()); } } public void balanceAllTables() { - for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { LOG.info("before table balance be {} tablet num {}", entry.getKey(), entry.getValue().size()); } - for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { LOG.info("before table balance be {} tablet num(current + pre heating inflight) {}", entry.getKey(), entry.getValue().size()); } @@ -362,22 +363,22 @@ public void balanceAllTables() { return; } - for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { LOG.info("after table balance be {} tablet num {}", entry.getKey(), entry.getValue().size()); } - for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { LOG.info("after table balance be {} tablet num(current + pre heating inflight) {}", entry.getKey(), entry.getValue().size()); } } public void globalBalance() { - for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { LOG.info("before global balance be {} tablet num {}", entry.getKey(), entry.getValue().size()); } - for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { LOG.info("before global balance be {} tablet num(current + pre heating inflight) {}", entry.getKey(), entry.getValue().size()); } @@ -397,17 +398,17 @@ public void globalBalance() { return; } - for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : beToTabletsGlobal.entrySet()) { LOG.info("after global balance be {} tablet num {}", entry.getKey(), entry.getValue().size()); } - for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { + for (Map.Entry> entry : futureBeToTabletsGlobal.entrySet()) { LOG.info("after global balance be {} tablet num(current + pre heating inflight) {}", entry.getKey(), entry.getValue().size()); } } - public void checkInflghtWarmUpCacheAsync() { + public void checkInflightWarmUpCacheAsync() { Map> beToInfightTasks = new HashMap>(); for (Map.Entry entry : tabletToInfightTask.entrySet()) { @@ -494,7 +495,7 @@ public void checkDecommissionState(Map> clusterToBes) { for (Map.Entry> entry : clusterToBes.entrySet()) { List beList = entry.getValue(); for (long beId : beList) { - List tablets = beToTabletsGlobal.get(beId); + Set tablets = beToTabletsGlobal.get(beId); int tabletNum = tablets == null ? 0 : tablets.size(); Backend backend = cloudSystemInfoService.getBackend(beId); if (backend == null) { @@ -644,42 +645,42 @@ private boolean completeRouteInfo() { } public void fillBeToTablets(long be, long tableId, long partId, long indexId, Tablet tablet, - Map> globalBeToTablets, - Map>> beToTabletsInTable, - Map>>> partToTablets) { + Map> globalBeToTablets, + Map>> beToTabletsInTable, + Map>>> partToTablets) { // global - globalBeToTablets.putIfAbsent(be, new ArrayList()); + globalBeToTablets.putIfAbsent(be, new HashSet()); globalBeToTablets.get(be).add(tablet); // table - beToTabletsInTable.putIfAbsent(tableId, new HashMap>()); - Map> beToTabletsOfTable = beToTabletsInTable.get(tableId); - beToTabletsOfTable.putIfAbsent(be, new ArrayList()); + beToTabletsInTable.putIfAbsent(tableId, new HashMap>()); + Map> beToTabletsOfTable = beToTabletsInTable.get(tableId); + beToTabletsOfTable.putIfAbsent(be, new HashSet()); beToTabletsOfTable.get(be).add(tablet); // partition - partToTablets.putIfAbsent(partId, new HashMap>>()); - Map>> indexToTablets = partToTablets.get(partId); - indexToTablets.putIfAbsent(indexId, new HashMap>()); - Map> beToTabletsOfIndex = indexToTablets.get(indexId); - beToTabletsOfIndex.putIfAbsent(be, new ArrayList()); + partToTablets.putIfAbsent(partId, new HashMap>>()); + Map>> indexToTablets = partToTablets.get(partId); + indexToTablets.putIfAbsent(indexId, new HashMap>()); + Map> beToTabletsOfIndex = indexToTablets.get(indexId); + beToTabletsOfIndex.putIfAbsent(be, new HashSet()); beToTabletsOfIndex.get(be).add(tablet); } public void statRouteInfo() { - ConcurrentHashMap> tmpBeToTabletsGlobal = new ConcurrentHashMap>(); - ConcurrentHashMap> tmpBeToTabletsGlobalInSecondary - = new ConcurrentHashMap>(); - ConcurrentHashMap> tmpBeToColocateTabletsGlobal - = new ConcurrentHashMap>(); + ConcurrentHashMap> tmpBeToTabletsGlobal = new ConcurrentHashMap>(); + ConcurrentHashMap> tmpBeToTabletsGlobalInSecondary + = new ConcurrentHashMap>(); + ConcurrentHashMap> tmpBeToColocateTabletsGlobal + = new ConcurrentHashMap>(); - futureBeToTabletsGlobal = new HashMap>(); + futureBeToTabletsGlobal = new HashMap>(); - partitionToTablets = new HashMap>>>(); - futurePartitionToTablets = new HashMap>>>(); + partitionToTablets = new HashMap>>>(); + futurePartitionToTablets = new HashMap>>>(); - beToTabletsInTable = new HashMap>>(); - futureBeToTabletsInTable = new HashMap>>(); + beToTabletsInTable = new HashMap>>(); + futureBeToTabletsInTable = new HashMap>>(); loopCloudReplica((Database db, Table table, Partition partition, MaterializedIndex index, String cluster) -> { boolean isColocated = Env.getCurrentColocateIndex().isColocateTable(table.getId()); @@ -694,8 +695,8 @@ public void statRouteInfo() { continue; } if (allBes.contains(beId)) { - List colocateTablets = - tmpBeToColocateTabletsGlobal.computeIfAbsent(beId, k -> new ArrayList<>()); + Set colocateTablets = + tmpBeToColocateTabletsGlobal.computeIfAbsent(beId, k -> new HashSet<>()); colocateTablets.add(tablet); } continue; @@ -710,8 +711,8 @@ public void statRouteInfo() { Backend secondaryBe = replica.getSecondaryBackend(cluster); long secondaryBeId = secondaryBe == null ? -1L : secondaryBe.getId(); if (allBes.contains(secondaryBeId)) { - List tablets = tmpBeToTabletsGlobalInSecondary - .computeIfAbsent(secondaryBeId, k -> new ArrayList<>()); + Set tablets = tmpBeToTabletsGlobalInSecondary + .computeIfAbsent(secondaryBeId, k -> new HashSet<>()); tablets.add(tablet); } @@ -764,10 +765,10 @@ public void loopCloudReplica(Operator operator) { public void balanceInPartition(List bes, String clusterId, List infos) { // balance all partition - for (Map.Entry>>> partitionEntry : futurePartitionToTablets.entrySet()) { - Map>> indexToTablets = partitionEntry.getValue(); + for (Map.Entry>>> partitionEntry : futurePartitionToTablets.entrySet()) { + Map>> indexToTablets = partitionEntry.getValue(); // balance all index of a partition - for (Map.Entry>> entry : indexToTablets.entrySet()) { + for (Map.Entry>> entry : indexToTablets.entrySet()) { // balance a index balanceImpl(bes, clusterId, entry.getValue(), BalanceType.PARTITION, infos); } @@ -776,7 +777,7 @@ public void balanceInPartition(List bes, String clusterId, List bes, String clusterId, List infos) { // balance all tables - for (Map.Entry>> entry : futureBeToTabletsInTable.entrySet()) { + for (Map.Entry>> entry : futureBeToTabletsInTable.entrySet()) { balanceImpl(bes, clusterId, entry.getValue(), BalanceType.TABLE, infos); } } @@ -846,10 +847,10 @@ private Map sendCheckWarmUpCacheAsyncRpc(List tabletIds, lo return null; } - private void updateBeToTablets(Tablet pickedTablet, long srcBe, long destBe, BalanceType balanceType, - Map> globalBeToTablets, - Map>> beToTabletsInTable, - Map>>> partToTablets) { + private void updateBeToTablets(Tablet pickedTablet, long srcBe, long destBe, + Map> globalBeToTablets, + Map>> beToTabletsInTable, + Map>>> partToTablets) { CloudReplica replica = (CloudReplica) pickedTablet.getReplicas().get(0); long tableId = replica.getTableId(); long partId = replica.getPartitionId(); @@ -892,47 +893,73 @@ private void updateClusterToBeMap(Tablet pickedTablet, long destBe, String clust } } - private boolean getTransferPair(List bes, Map> beToTablets, long avgNum, + private boolean getTransferPair(List bes, Map> beToTablets, long avgNum, TransferPairInfo pairInfo) { - long destBe = bes.get(0); - long srcBe = bes.get(0); + long srcBe = findSourceBackend(bes, beToTablets); + long destBe = findDestinationBackend(bes, beToTablets, srcBe); - long minTabletsNum = Long.MAX_VALUE; + if (srcBe == -1 || destBe == -1) { + return false; // No valid backend found + } + + long minTabletsNum = beToTablets.get(destBe) == null ? 0 : beToTablets.get(destBe).size(); + long maxTabletsNum = beToTablets.get(srcBe) == null ? 0 : beToTablets.get(srcBe).size(); + + if (!isTransferValid(srcBe, minTabletsNum, maxTabletsNum, avgNum)) { + return false; // Transfer conditions not met + } + + pairInfo.srcBe = srcBe; + pairInfo.destBe = destBe; + pairInfo.minTabletsNum = minTabletsNum; + pairInfo.maxTabletsNum = maxTabletsNum; + return true; + } + + private long findSourceBackend(List bes, Map> beToTablets) { + long srcBe = -1; long maxTabletsNum = 0; - boolean srcDecommissioned = false; for (Long be : bes) { - long tabletNum = beToTablets.get(be) == null ? 0 : beToTablets.get(be).size(); - if (tabletNum > maxTabletsNum) { - srcBe = be; - maxTabletsNum = tabletNum; - } - + long tabletNum = beToTablets.getOrDefault(be, Collections.emptySet()).size(); Backend backend = cloudSystemInfoService.getBackend(be); - if (backend == null) { + + // Check if the backend is decommissioned + if (backend != null) { + if (backend.isDecommissioning() && tabletNum > 0) { + srcBe = be; // Mark as source if decommissioned and has tablets + break; // Exit early if we found a decommissioned backend + } + if (!backend.isDecommissioning() && tabletNum > maxTabletsNum) { + srcBe = be; + maxTabletsNum = tabletNum; + } + } else { LOG.info("backend {} not found", be); - continue; - } - if (tabletNum < minTabletsNum && backend.isAlive() && !backend.isDecommissioning() - && !backend.isSmoothUpgradeSrc()) { - destBe = be; - minTabletsNum = tabletNum; } } + return srcBe; + } + + private long findDestinationBackend(List bes, Map> beToTablets, long srcBe) { + long destBe = -1; + long minTabletsNum = Long.MAX_VALUE; for (Long be : bes) { - long tabletNum = beToTablets.get(be) == null ? 0 : beToTablets.get(be).size(); + long tabletNum = beToTablets.getOrDefault(be, Collections.emptySet()).size(); Backend backend = cloudSystemInfoService.getBackend(be); - if (backend == null) { - LOG.info("backend {} not found", be); - continue; - } - if (backend.isDecommissioning() && tabletNum > 0) { - srcBe = be; - srcDecommissioned = true; - break; + if (backend != null && backend.isAlive() && !backend.isDecommissioning() && !backend.isSmoothUpgradeSrc()) { + if (tabletNum < minTabletsNum) { + destBe = be; + minTabletsNum = tabletNum; + } } } + return destBe; + } + + private boolean isTransferValid(long srcBe, long minTabletsNum, long maxTabletsNum, long avgNum) { + boolean srcDecommissioned = cloudSystemInfoService.getBackend(srcBe).isDecommissioning(); if (!srcDecommissioned) { if ((maxTabletsNum < avgNum * (1 + Config.cloud_rebalance_percent_threshold) @@ -941,142 +968,183 @@ private boolean getTransferPair(List bes, Map> beToTabl return false; } } - - pairInfo.srcBe = srcBe; - pairInfo.destBe = destBe; - pairInfo.minTabletsNum = minTabletsNum; - pairInfo.maxTabletsNum = maxTabletsNum; return true; } private boolean isConflict(long srcBe, long destBe, CloudReplica cloudReplica, BalanceType balanceType, - Map>>> beToTabletsInParts, - Map>> beToTabletsInTables) { - if (balanceType == balanceType.GLOBAL) { - // check is conflict with partition balance - long maxBeSize = beToTabletsInParts.get(cloudReplica.getPartitionId()) - .get(cloudReplica.getIndexId()).get(srcBe).size(); - List destBeTablets = beToTabletsInParts.get(cloudReplica.getPartitionId()) - .get(cloudReplica.getIndexId()).get(destBe); - long minBeSize = destBeTablets == null ? 0 : destBeTablets.size(); - if (minBeSize >= maxBeSize) { - return true; - } - - // check is conflict with table balance - maxBeSize = beToTabletsInTables.get(cloudReplica.getTableId()).get(srcBe).size(); - destBeTablets = beToTabletsInTables.get(cloudReplica.getTableId()).get(destBe); - minBeSize = destBeTablets == null ? 0 : destBeTablets.size(); - if (minBeSize >= maxBeSize) { - return true; - } + Map>>> beToTabletsInParts, + Map>> beToTabletsInTables) { + if (cloudSystemInfoService.getBackend(srcBe).isDecommissioning()) { + return false; // If source BE is decommissioned, no conflict } - if (balanceType == balanceType.TABLE) { - // check is conflict with partition balance - long maxBeSize = beToTabletsInParts.get(cloudReplica.getPartitionId()) - .get(cloudReplica.getIndexId()).get(srcBe).size(); - List destBeTablets = beToTabletsInParts.get(cloudReplica.getPartitionId()) - .get(cloudReplica.getIndexId()).get(destBe); - long minBeSize = destBeTablets == null ? 0 : destBeTablets.size(); - return minBeSize >= maxBeSize; + if (balanceType == BalanceType.GLOBAL) { + return checkGlobalBalanceConflict(srcBe, destBe, cloudReplica, beToTabletsInParts, beToTabletsInTables); + } else if (balanceType == BalanceType.TABLE) { + return checkTableBalanceConflict(srcBe, destBe, cloudReplica, beToTabletsInParts); } return false; } - private void balanceImpl(List bes, String clusterId, Map> beToTablets, + private boolean checkGlobalBalanceConflict(long srcBe, long destBe, CloudReplica cloudReplica, + Map>>> beToTabletsInParts, + Map>> beToTabletsInTables) { + long maxBeSize = getTabletSizeInParts(srcBe, cloudReplica, beToTabletsInParts); + long minBeSize = getTabletSizeInParts(destBe, cloudReplica, beToTabletsInParts); + + if (minBeSize >= maxBeSize) { + return true; // Conflict detected + } + + maxBeSize = getTabletSizeInBes(srcBe, cloudReplica, beToTabletsInTables); + minBeSize = getTabletSizeInBes(destBe, cloudReplica, beToTabletsInTables); + + return minBeSize >= maxBeSize; // Conflict detected + } + + private boolean checkTableBalanceConflict(long srcBe, long destBe, CloudReplica cloudReplica, + Map>>> beToTabletsInParts) { + long maxBeSize = getTabletSizeInParts(srcBe, cloudReplica, beToTabletsInParts); + long minBeSize = getTabletSizeInParts(destBe, cloudReplica, beToTabletsInParts); + + return minBeSize >= maxBeSize; // Conflict detected + } + + private long getTabletSizeInParts(long beId, CloudReplica cloudReplica, + Map>>> beToTabletsInParts) { + Set tablets = beToTabletsInParts.get(cloudReplica.getPartitionId()) + .get(cloudReplica.getIndexId()).get(beId); + return tablets == null ? 0 : tablets.size(); + } + + private long getTabletSizeInBes(long beId, CloudReplica cloudReplica, + Map>> beToTabletsInTables) { + Set tablets = beToTabletsInTables.get(cloudReplica.getTableId()).get(beId); + return tablets == null ? 0 : tablets.size(); + } + + + private void balanceImpl(List bes, String clusterId, Map> beToTablets, BalanceType balanceType, List infos) { if (bes == null || bes.isEmpty() || beToTablets == null || beToTablets.isEmpty()) { return; } - long totalTabletsNum = 0; - long beNum = 0; - for (Long be : bes) { - long tabletNum = beToTablets.get(be) == null ? 0 : beToTablets.get(be).size(); - Backend backend = cloudSystemInfoService.getBackend(be); - if (backend != null && !backend.isDecommissioning()) { - beNum++; - } - totalTabletsNum += tabletNum; - } + long totalTabletsNum = calculateTotalTablets(bes, beToTablets); + long beNum = countActiveBackends(bes); + if (beNum == 0) { LOG.warn("zero be, but want balance, skip"); return; } + long avgNum = totalTabletsNum / beNum; - long transferNum = Math.max(Math.round(avgNum * Config.cloud_balance_tablet_percent_per_run), - Config.cloud_min_balance_tablet_num_per_run); + long transferNum = calculateTransferNum(avgNum); for (int i = 0; i < transferNum; i++) { TransferPairInfo pairInfo = new TransferPairInfo(); if (!getTransferPair(bes, beToTablets, avgNum, pairInfo)) { - // no need balance; - break; - } - - if (balanceType == balanceType.PARTITION) { - indexBalanced = false; + break; // no need balance } - if (balanceType == balanceType.TABLE) { - tableBalanced = false; - } + updateBalanceStatus(balanceType); long srcBe = pairInfo.srcBe; long destBe = pairInfo.destBe; - long minTabletsNum = pairInfo.minTabletsNum; - long maxTabletsNum = pairInfo.maxTabletsNum; - int randomIndex = rand.nextInt(beToTablets.get(srcBe).size()); - Tablet pickedTablet = beToTablets.get(srcBe).get(randomIndex); - CloudReplica cloudReplica = (CloudReplica) pickedTablet.getReplicas().get(0); + Tablet pickedTablet = pickRandomTablet(beToTablets.get(srcBe)); + if (pickedTablet == null) { + continue; // No tablet to pick + } + CloudReplica cloudReplica = (CloudReplica) pickedTablet.getReplicas().get(0); Backend srcBackend = Env.getCurrentSystemInfo().getBackend(srcBe); - // if srcBe is dead, destBe cann't download cache from it, preheating will failed + if (Config.enable_cloud_warm_up_for_rebalance && srcBackend != null && srcBackend.isAlive()) { - if (isConflict(srcBe, destBe, cloudReplica, balanceType, futurePartitionToTablets, - futureBeToTabletsInTable)) { + if (isConflict(srcBe, destBe, cloudReplica, balanceType, + futurePartitionToTablets, futureBeToTabletsInTable)) { continue; } - - try { - sendPreHeatingRpc(pickedTablet, srcBe, destBe); - } catch (Exception e) { - break; - } - - InfightTask task = new InfightTask(); - task.pickedTablet = pickedTablet; - task.srcBe = srcBe; - task.destBe = destBe; - task.balanceType = balanceType; - task.beToTablets = beToTablets; - task.startTimestamp = System.currentTimeMillis() / 1000; - tabletToInfightTask.put(new InfightTablet(pickedTablet.getId(), clusterId), task); - - LOG.info("pre cache {} from {} to {}, cluster {} minNum {} maxNum {} beNum {} tabletsNum {}, part {}", - pickedTablet.getId(), srcBe, destBe, clusterId, - minTabletsNum, maxTabletsNum, beNum, totalTabletsNum, cloudReplica.getPartitionId()); - updateBeToTablets(pickedTablet, srcBe, destBe, balanceType, - futureBeToTabletsGlobal, futureBeToTabletsInTable, futurePartitionToTablets); + preheatAndUpdateTablet(pickedTablet, srcBe, destBe, clusterId, balanceType, beToTablets); } else { if (isConflict(srcBe, destBe, cloudReplica, balanceType, partitionToTablets, beToTabletsInTable)) { continue; } + transferTablet(pickedTablet, srcBe, destBe, clusterId, balanceType, infos); + } + } + } - LOG.info("transfer {} from {} to {}, cluster {} minNum {} maxNum {} beNum {} tabletsNum {}, part {}", - pickedTablet.getId(), srcBe, destBe, clusterId, - minTabletsNum, maxTabletsNum, beNum, totalTabletsNum, cloudReplica.getPartitionId()); + private long calculateTotalTablets(List bes, Map> beToTablets) { + return bes.stream() + .mapToLong(be -> beToTablets.getOrDefault(be, Collections.emptySet()).size()) + .sum(); + } - updateBeToTablets(pickedTablet, srcBe, destBe, balanceType, beToTabletsGlobal, - beToTabletsInTable, partitionToTablets); - updateBeToTablets(pickedTablet, srcBe, destBe, balanceType, - futureBeToTabletsGlobal, futureBeToTabletsInTable, futurePartitionToTablets); - updateClusterToBeMap(pickedTablet, destBe, clusterId, infos); - } + private long countActiveBackends(List bes) { + return bes.stream() + .filter(be -> { + Backend backend = cloudSystemInfoService.getBackend(be); + return backend != null && !backend.isDecommissioning(); + }) + .count(); + } + + private long calculateTransferNum(long avgNum) { + return Math.max(Math.round(avgNum * Config.cloud_balance_tablet_percent_per_run), + Config.cloud_min_balance_tablet_num_per_run); + } + + private void updateBalanceStatus(BalanceType balanceType) { + if (balanceType == BalanceType.PARTITION) { + indexBalanced = false; + } else if (balanceType == BalanceType.TABLE) { + tableBalanced = false; + } + } + + private Tablet pickRandomTablet(Set tablets) { + if (tablets.isEmpty()) { + return null; + } + int randomIndex = rand.nextInt(tablets.size()); + return tablets.stream().skip(randomIndex).findFirst().orElse(null); + } + + private void preheatAndUpdateTablet(Tablet pickedTablet, long srcBe, long destBe, String clusterId, + BalanceType balanceType, Map> beToTablets) { + try { + sendPreHeatingRpc(pickedTablet, srcBe, destBe); + } catch (Exception e) { + LOG.warn("Failed to preheat tablet {} from {} to {}, " + + "help msg turn off fe config enable_cloud_warm_up_for_rebalance", + pickedTablet.getId(), srcBe, destBe, e); + return; } + + InfightTask task = new InfightTask(); + task.pickedTablet = pickedTablet; + task.srcBe = srcBe; + task.destBe = destBe; + task.balanceType = balanceType; + task.beToTablets = beToTablets; + task.startTimestamp = System.currentTimeMillis() / 1000; + tabletToInfightTask.put(new InfightTablet(pickedTablet.getId(), clusterId), task); + + LOG.info("pre cache {} from {} to {}, cluster {}", pickedTablet.getId(), srcBe, destBe, clusterId); + updateBeToTablets(pickedTablet, srcBe, destBe, + futureBeToTabletsGlobal, futureBeToTabletsInTable, futurePartitionToTablets); + } + + private void transferTablet(Tablet pickedTablet, long srcBe, long destBe, String clusterId, + BalanceType balanceType, List infos) { + LOG.info("transfer {} from {} to {}, cluster {}", pickedTablet.getId(), srcBe, destBe, clusterId); + updateBeToTablets(pickedTablet, srcBe, destBe, + beToTabletsGlobal, beToTabletsInTable, partitionToTablets); + updateBeToTablets(pickedTablet, srcBe, destBe, + futureBeToTabletsGlobal, futureBeToTabletsInTable, futurePartitionToTablets); + updateClusterToBeMap(pickedTablet, destBe, clusterId, infos); } public void addTabletMigrationTask(Long srcBe, Long dstBe) { @@ -1088,7 +1156,7 @@ public void addTabletMigrationTask(Long srcBe, Long dstBe) { */ private void migrateTablets(Long srcBe, Long dstBe) { // get tablets - List tablets = beToTabletsGlobal.get(srcBe); + Set tablets = beToTabletsGlobal.get(srcBe); if (tablets == null || tablets.isEmpty()) { LOG.info("smooth upgrade srcBe={} does not have any tablets, set inactive", srcBe); ((CloudEnv) Env.getCurrentEnv()).getCloudUpgradeMgr().setBeStateInactive(srcBe); @@ -1224,6 +1292,4 @@ private List batchUpdateCloudReplicaInfoEditlogs(List Date: Tue, 8 Jul 2025 09:37:03 +0800 Subject: [PATCH 168/572] branch-3.0: [fix](be) fix macos report wrong cpu cores #52726 (#52857) Cherry-picked from #52726 Co-authored-by: 924060929 --- be/src/util/cpu_info.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index b49985cdc06830..6b8680c3a4d11e 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -161,6 +161,11 @@ void CpuInfo::init() { } } +#ifdef __APPLE__ + size_t len = sizeof(max_num_cores_); + sysctlbyname("hw.physicalcpu", &physical_num_cores, &len, nullptr, 0); +#endif + int num_cores = CGroupUtil::get_cgroup_limited_cpu_number(physical_num_cores); if (max_mhz != 0) { cycles_per_ms_ = int64_t(max_mhz) * 1000; @@ -179,7 +184,6 @@ void CpuInfo::init() { } #ifdef __APPLE__ - size_t len = sizeof(max_num_cores_); sysctlbyname("hw.logicalcpu", &max_num_cores_, &len, nullptr, 0); #else max_num_cores_ = get_nprocs_conf(); From fd15f769d71e0d339bab5dcbb9b66dc319c76da9 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 8 Jul 2025 09:37:35 +0800 Subject: [PATCH 169/572] branch-3.0: [fix](inverted index) catch IO exception to avoid coredump in inverted index string reader #51844 (#52864) cherry pick from #51844 --- .../segment_v2/inverted_index_reader.cpp | 74 ++++---- .../segment_v2/inverted_index_reader_test.cpp | 167 +++++++++++++++++- 2 files changed, 203 insertions(+), 38 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp index 662c930ead2a93..dfd128b4cae0d0 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp @@ -411,32 +411,31 @@ Status StringTypeInvertedIndexReader::query(const io::IOContext* io_ctx, std::string search_str(search_query->data, act_len); VLOG_DEBUG << "begin to query the inverted index from clucene" << ", column_name: " << column_name << ", search_str: " << search_str; + try { + auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta); + // try to get query bitmap result from cache and return immediately on cache hit + InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, + search_str}; + auto* cache = InvertedIndexQueryCache::instance(); + InvertedIndexQueryCacheHandle cache_handler; + auto cache_status = handle_query_cache(cache, cache_key, &cache_handler, stats, bit_map); + if (cache_status.ok()) { + return Status::OK(); + } - auto index_file_key = _inverted_index_file_reader->get_index_file_cache_key(&_index_meta); - // try to get query bitmap result from cache and return immediately on cache hit - InvertedIndexQueryCache::CacheKey cache_key {index_file_key, column_name, query_type, - search_str}; - auto* cache = InvertedIndexQueryCache::instance(); - InvertedIndexQueryCacheHandle cache_handler; - auto cache_status = handle_query_cache(cache, cache_key, &cache_handler, stats, bit_map); - if (cache_status.ok()) { - return Status::OK(); - } - - std::wstring column_name_ws = StringUtil::string_to_wstring(column_name); + std::wstring column_name_ws = StringUtil::string_to_wstring(column_name); - InvertedIndexQueryInfo query_info; - query_info.field_name = column_name_ws; - query_info.terms.emplace_back(search_str); + InvertedIndexQueryInfo query_info; + query_info.field_name = column_name_ws; + query_info.terms.emplace_back(search_str); - auto result = std::make_shared(); - FulltextIndexSearcherPtr* searcher_ptr = nullptr; - InvertedIndexCacheHandle inverted_index_cache_handle; - RETURN_IF_ERROR(handle_searcher_cache(&inverted_index_cache_handle, io_ctx, stats)); - auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); - searcher_ptr = std::get_if(&searcher_variant); - if (searcher_ptr != nullptr) { - try { + auto result = std::make_shared(); + FulltextIndexSearcherPtr* searcher_ptr = nullptr; + InvertedIndexCacheHandle inverted_index_cache_handle; + RETURN_IF_ERROR(handle_searcher_cache(&inverted_index_cache_handle, io_ctx, stats)); + auto searcher_variant = inverted_index_cache_handle.get_index_searcher(); + searcher_ptr = std::get_if(&searcher_variant); + if (searcher_ptr != nullptr) { switch (query_type) { case InvertedIndexQueryType::MATCH_ANY_QUERY: case InvertedIndexQueryType::MATCH_ALL_QUERY: @@ -488,27 +487,28 @@ Status StringTypeInvertedIndexReader::query(const io::IOContext* io_ctx, return Status::Error( "invalid query type when query untokenized inverted index"); } - } catch (const CLuceneError& e) { - if (is_range_query(query_type) && e.number() == CL_ERR_TooManyClauses) { - return Status::Error( - "range query term exceeds limits, try to downgrade from inverted index, " - "column " - "name:{}, search_str:{}", - column_name, search_str); - } else { - return Status::Error( - "CLuceneError occured, error msg: {}, column name: {}, search_str: {}", - e.what(), column_name, search_str); - } } - // add to cache result->runOptimize(); cache->insert(cache_key, result, &cache_handler); bit_map = result; + return Status::OK(); + } catch (const CLuceneError& e) { + if (is_range_query(query_type) && e.number() == CL_ERR_TooManyClauses) { + return Status::Error( + "range query term exceeds limits, try to downgrade from inverted index, " + "column " + "name:{}, search_str:{}", + column_name, search_str); + } else { + LOG(ERROR) << "CLuceneError occurred, error msg: " << e.what() + << ", column name: " << column_name << ", search_str: " << search_str; + return Status::Error( + "CLuceneError occurred, error msg: {}, column name: {}, search_str: {}", + e.what(), column_name, search_str); + } } - return Status::OK(); } InvertedIndexReaderType StringTypeInvertedIndexReader::type() { diff --git a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp index 524936829177e8..2e27cef2539198 100644 --- a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp +++ b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp @@ -1531,6 +1531,161 @@ class InvertedIndexReaderTest : public testing::Test { } } + class MockStringTypeInvertedIndexReader final : public StringTypeInvertedIndexReader { + public: + static std::shared_ptr create_shared( + const TabletIndex* idx_meta, + std::shared_ptr& file_reader) { + return std::shared_ptr( + new MockStringTypeInvertedIndexReader(idx_meta, file_reader)); + } + + protected: + Status handle_searcher_cache(InvertedIndexCacheHandle*, const io::IOContext*, + OlapReaderStatistics*) override { + CLuceneError err; + err.set(CL_ERR_IO, "mock handle_searcher_cache failure"); + throw err; + } + + private: + MockStringTypeInvertedIndexReader(const TabletIndex* idx_meta, + std::shared_ptr& file_reader) + : StringTypeInvertedIndexReader(idx_meta, file_reader) {} + }; + + // Mock class for testing tokenized index query exceptions + class MockTokenizedStringTypeInvertedIndexReader final : public FullTextIndexReader { + public: + static std::shared_ptr create_shared( + const TabletIndex* idx_meta, + std::shared_ptr& file_reader) { + return std::shared_ptr( + new MockTokenizedStringTypeInvertedIndexReader(idx_meta, file_reader)); + } + + protected: + Status handle_searcher_cache(InvertedIndexCacheHandle*, const io::IOContext*, + OlapReaderStatistics*) override { + CLuceneError err; + err.set(CL_ERR_IO, "mock tokenized index searcher cache failure"); + throw err; + } + + private: + MockTokenizedStringTypeInvertedIndexReader( + const TabletIndex* idx_meta, std::shared_ptr& file_reader) + : FullTextIndexReader(idx_meta, file_reader) {} + }; + + void test_cache_error_scenarios() { + std::string_view rowset_id = "test_handle_searcher_cache_exception"; + int seg_id = 0; + std::vector values = {Slice("apple"), Slice("banana")}; + + TabletIndex idx_meta; + { + auto index_meta_pb = std::make_unique(); + index_meta_pb->set_index_type(IndexType::INVERTED); + index_meta_pb->set_index_id(1); + index_meta_pb->set_index_name("test_mock_cache"); + index_meta_pb->add_col_unique_id(1); // c2 + idx_meta.init_from_pb(*index_meta_pb); + } + + std::string index_path_prefix; + prepare_string_index(rowset_id, seg_id, values, &idx_meta, &index_path_prefix); + + auto file_reader = std::make_shared( + io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2); + ASSERT_TRUE(file_reader->init().ok()); + + auto mock_reader = MockStringTypeInvertedIndexReader::create_shared(&idx_meta, file_reader); + ASSERT_NE(mock_reader, nullptr); + + io::IOContext io_ctx; + OlapReaderStatistics stats; + RuntimeState runtime_state; + TQueryOptions opts; + runtime_state.set_query_options(opts); + + std::shared_ptr bitmap = std::make_shared(); + std::string field_name = "1"; // c2 unique_id + StringRef query_val(values[0].data, values[0].size); + + Status st = mock_reader->query(&io_ctx, &stats, &runtime_state, field_name, &query_val, + InvertedIndexQueryType::EQUAL_QUERY, bitmap); + + EXPECT_FALSE(st.ok()); + EXPECT_EQ(st.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR); + } + + void test_tokenized_index_query_error_scenarios() { + std::string_view rowset_id = "test_tokenized_index_query_exception"; + int seg_id = 0; + std::vector values = {Slice("Hello world this is a test"), + Slice("Apache Doris is a modern analytics database"), + Slice("Inverted index provides fast text search")}; + + TabletIndex idx_meta; + { + auto index_meta_pb = std::make_unique(); + index_meta_pb->set_index_type(IndexType::INVERTED); + index_meta_pb->set_index_id(2); + index_meta_pb->set_index_name("test_tokenized_mock_cache"); + index_meta_pb->add_col_unique_id(1); // c2 + + // Set tokenized index properties + auto* properties = index_meta_pb->mutable_properties(); + (*properties)[INVERTED_INDEX_PARSER_KEY] = INVERTED_INDEX_PARSER_ENGLISH; + (*properties)[INVERTED_INDEX_PARSER_PHRASE_SUPPORT_KEY] = + INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES; + (*properties)[INVERTED_INDEX_PARSER_LOWERCASE_KEY] = INVERTED_INDEX_PARSER_TRUE; + + idx_meta.init_from_pb(*index_meta_pb); + } + + std::string index_path_prefix; + prepare_string_index(rowset_id, seg_id, values, &idx_meta, &index_path_prefix); + + auto file_reader = std::make_shared( + io::global_local_filesystem(), index_path_prefix, InvertedIndexStorageFormatPB::V2); + ASSERT_TRUE(file_reader->init().ok()); + + auto mock_reader = + MockTokenizedStringTypeInvertedIndexReader::create_shared(&idx_meta, file_reader); + ASSERT_NE(mock_reader, nullptr); + + io::IOContext io_ctx; + OlapReaderStatistics stats; + RuntimeState runtime_state; + TQueryOptions opts; + runtime_state.set_query_options(opts); + + std::shared_ptr bitmap = std::make_shared(); + std::string field_name = "1"; // c2 unique_id + + // Test tokenized query with "world" which should be found in "Hello world this is a test" + std::string query_term = "world"; + StringRef query_val(query_term.data(), query_term.size()); + + Status st = mock_reader->query(&io_ctx, &stats, &runtime_state, field_name, &query_val, + InvertedIndexQueryType::MATCH_ANY_QUERY, bitmap); + + EXPECT_FALSE(st.ok()); + EXPECT_EQ(st.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR); + + // Test phrase query + std::string phrase_query = "Apache Doris"; + StringRef phrase_query_val(phrase_query.data(), phrase_query.size()); + + st = mock_reader->query(&io_ctx, &stats, &runtime_state, field_name, &phrase_query_val, + InvertedIndexQueryType::MATCH_PHRASE_QUERY, bitmap); + + EXPECT_FALSE(st.ok()); + EXPECT_EQ(st.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR); + } + private: std::unique_ptr _inverted_index_searcher_cache; std::unique_ptr _inverted_index_query_cache; @@ -1561,4 +1716,14 @@ TEST_F(InvertedIndexReaderTest, CompatibleTest) { test_compatible_read_cross_platform(); } -} // namespace doris::segment_v2 \ No newline at end of file +// Test cache error scenarios that could crash BE +TEST_F(InvertedIndexReaderTest, CacheErrorScenarios) { + test_cache_error_scenarios(); +} + +// Test tokenized index query error scenarios +TEST_F(InvertedIndexReaderTest, TokenizedIndexQueryErrorScenarios) { + test_tokenized_index_query_error_scenarios(); +} + +} // namespace doris::segment_v2 From d6ee283cb16347f0e744219b0610416f8d89aa3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:38:25 +0800 Subject: [PATCH 170/572] branch-3.0: [chore](cloud) Add async wrap for `bthread_fork_join` with promise-future #52816 (#52844) Cherry-picked from #52816 Co-authored-by: Gavin Chou --- be/src/cloud/cloud_meta_mgr.cpp | 32 +++++++--- be/src/cloud/cloud_meta_mgr.h | 9 +++ be/test/cloud/cloud_meta_mgr_test.cpp | 89 +++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 be/test/cloud/cloud_meta_mgr_test.cpp diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 684d89cb01bb44..9baf5db3709a32 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -69,6 +69,13 @@ namespace doris::cloud { #include "common/compile_check_begin.h" using namespace ErrorCode; +void* run_bthread_work(void* arg) { + auto* f = reinterpret_cast*>(arg); + (*f)(); + delete f; + return nullptr; +} + Status bthread_fork_join(const std::vector>& tasks, int concurrency) { if (tasks.empty()) { return Status::OK(); @@ -79,13 +86,6 @@ Status bthread_fork_join(const std::vector>& tasks, int Status status; // Guard by lock int count = 0; // Guard by lock - auto* run_bthread_work = +[](void* arg) -> void* { - auto* f = reinterpret_cast*>(arg); - (*f)(); - delete f; - return nullptr; - }; - for (const auto& task : tasks) { { std::unique_lock lk(lock); @@ -131,6 +131,24 @@ Status bthread_fork_join(const std::vector>& tasks, int return status; } +Status bthread_fork_join(const std::vector>& tasks, int concurrency, + std::future* fut) { + // std::function will cause `copy`, we need to use heap memory to avoid copy ctor called + auto prom = std::make_shared>(); + *fut = prom->get_future(); + std::function* fn = + new std::function([&tasks, concurrency, p = std::move(prom)]() mutable { + p->set_value(bthread_fork_join(tasks, concurrency)); + }); + + bthread_t bthread_id; + if (bthread_start_background(&bthread_id, nullptr, run_bthread_work, fn) != 0) { + delete fn; + return Status::InternalError("failed to create bthread"); + } + return Status::OK(); +} + namespace { constexpr int kBrpcRetryTimes = 3; diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index 0cc58e48166963..494f491de14831 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -18,6 +18,7 @@ #include +#include #include #include #include @@ -49,8 +50,16 @@ class TabletIndexPB; using StorageVaultInfos = std::vector< std::tuple, StorageVaultPB_PathFormat>>; +// run tasks in bthread with concurrency and wait until all tasks done +// it stops running tasks if there are any tasks return !ok, leaving some tasks untouched +// return OK if all tasks successfully done, otherwise return the result of the failed task Status bthread_fork_join(const std::vector>& tasks, int concurrency); +// An async wrap of `bthread_fork_join` declared previously using promise-future +// return OK if fut successfully created, otherwise return error +Status bthread_fork_join(const std::vector>& tasks, int concurrency, + std::future* fut); + class CloudMetaMgr { public: CloudMetaMgr() = default; diff --git a/be/test/cloud/cloud_meta_mgr_test.cpp b/be/test/cloud/cloud_meta_mgr_test.cpp new file mode 100644 index 00000000000000..43611c6e4e4f82 --- /dev/null +++ b/be/test/cloud/cloud_meta_mgr_test.cpp @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cloud/cloud_meta_mgr.h" + +#include + +#include +#include + +namespace doris { +using namespace cloud; +using namespace std::chrono; + +class CloudMetaMgrTest : public testing::Test { + void SetUp() override {} + void TearDown() override {} +}; + +TEST_F(CloudMetaMgrTest, bthread_fork_join_test) { + // clang-format off + std::vector> tasks { + []{ bthread_usleep(20000); return Status::OK(); }, + []{ bthread_usleep(20000); return Status::OK(); }, + []{ bthread_usleep(20000); return Status::OK(); }, + []{ bthread_usleep(20000); return Status::OK(); }, + []{ bthread_usleep(20000); return Status::OK(); }, + []{ bthread_usleep(20000); return Status::OK(); }, + []{ bthread_usleep(20000); return Status::OK(); }, + }; + { + auto start = steady_clock::now(); + EXPECT_TRUE(bthread_fork_join(tasks, 3).ok()); + auto end = steady_clock::now(); + auto elapsed = duration_cast(end - start).count(); + EXPECT_GT(elapsed, 40); // at least 2 rounds running for 7 tasks + } + { + std::future fut; + auto start = steady_clock::now(); + EXPECT_TRUE(bthread_fork_join(tasks, 3, &fut).ok()); // return immediately + auto end = steady_clock::now(); + auto elapsed = duration_cast(end - start).count(); + EXPECT_LE(elapsed, 40); // async + EXPECT_TRUE(fut.get().ok()); + end = steady_clock::now(); + elapsed = duration_cast(end - start).count(); + EXPECT_GT(elapsed, 40); // at least 2 rounds running for 7 tasks + } + + // make the first batch fail fast + tasks.insert(tasks.begin(), []{ bthread_usleep(20000); return Status::InternalError("error"); }); + { + auto start = steady_clock::now(); + EXPECT_FALSE(bthread_fork_join(tasks, 3).ok()); + auto end = steady_clock::now(); + auto elapsed = duration_cast(end - start).count(); + EXPECT_LE(elapsed, 40); // at most 1 round running for 7 tasks + } + { + std::future fut; + auto start = steady_clock::now(); + EXPECT_TRUE(bthread_fork_join(tasks, 3, &fut).ok()); // return immediately + auto end = steady_clock::now(); + auto elapsed = duration_cast(end - start).count(); + EXPECT_LE(elapsed, 40); // async + EXPECT_FALSE(fut.get().ok()); + end = steady_clock::now(); + elapsed = duration_cast(end - start).count(); + EXPECT_LE(elapsed, 40); // at most 1 round running for 7 tasks + } + // clang-format on +} + +} // namespace doris From 02fc60c10b14e05d92fe6526fbf68172b2ae6f55 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Tue, 8 Jul 2025 18:02:10 +0800 Subject: [PATCH 171/572] [fix](ut) Fix the compatibility of JSON unit test cases (#52933) --- be/test/util/jsonb_parser_simd_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/util/jsonb_parser_simd_test.cpp b/be/test/util/jsonb_parser_simd_test.cpp index dca45a460dec8f..3e7e70cf6664f3 100644 --- a/be/test/util/jsonb_parser_simd_test.cpp +++ b/be/test/util/jsonb_parser_simd_test.cpp @@ -232,7 +232,7 @@ TEST_F(JsonbParserTest, ParseJsonWithLongInt) { TEST_F(JsonbParserTest, ParseInvalidJsonFormat) { std::string_view invalid_json = R"({"key": "value")"; - EXPECT_EQ(parse_json_and_check(invalid_json, invalid_json), JsonbErrType::E_EXCEPTION); + EXPECT_NE(parse_json_and_check(invalid_json, invalid_json), JsonbErrType::E_NONE); } TEST_F(JsonbParserTest, ParseJsonWithInvalidKeyType) { From 4fffbebc87ef582405e5e385c91657206d9e22d6 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 8 Jul 2025 21:11:16 +0800 Subject: [PATCH 172/572] [Opt](multi-catalog)Disable dict filter in parquet/orc reader if have non-single conjuncts. (#52536) ### What problem does this PR solve? Problem Summary: ### Release note Cherry-pick #44777 --- be/src/vec/exec/format/orc/vorc_reader.cpp | 57 +++++------ be/src/vec/exec/format/orc/vorc_reader.h | 1 + .../format/parquet/vparquet_group_reader.cpp | 98 ++++++++----------- .../format/parquet/vparquet_group_reader.h | 1 - 4 files changed, 64 insertions(+), 93 deletions(-) diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index c7b4467f679ee4..2e7047aecf9188 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -956,18 +956,22 @@ Status OrcReader::set_fill_columns( } } - if (!_slot_id_to_filter_conjuncts) { - return Status::OK(); - } - - // Add predicate_partition_columns in _slot_id_to_filter_conjuncts(single slot conjuncts) - // to _filter_conjuncts, others should be added from not_single_slot_filter_conjuncts. - for (auto& kv : _lazy_read_ctx.predicate_partition_columns) { - auto& [value, slot_desc] = kv.second; - auto iter = _slot_id_to_filter_conjuncts->find(slot_desc->id()); - if (iter != _slot_id_to_filter_conjuncts->end()) { - for (auto& ctx : iter->second) { - _filter_conjuncts.push_back(ctx); + if (!_not_single_slot_filter_conjuncts.empty()) { + _filter_conjuncts.insert(_filter_conjuncts.end(), _not_single_slot_filter_conjuncts.begin(), + _not_single_slot_filter_conjuncts.end()); + _disable_dict_filter = true; + } + + if (_slot_id_to_filter_conjuncts && !_slot_id_to_filter_conjuncts->empty()) { + // Add predicate_partition_columns in _slot_id_to_filter_conjuncts(single slot conjuncts) + // to _filter_conjuncts, others should be added from not_single_slot_filter_conjuncts. + for (auto& kv : _lazy_read_ctx.predicate_partition_columns) { + auto& [value, slot_desc] = kv.second; + auto iter = _slot_id_to_filter_conjuncts->find(slot_desc->id()); + if (iter != _slot_id_to_filter_conjuncts->end()) { + for (const auto& ctx : iter->second) { + _filter_conjuncts.push_back(ctx); + } } } } @@ -1716,16 +1720,8 @@ Status OrcReader::get_next_block_impl(Block* block, size_t* read_rows, bool* eof RETURN_IF_CATCH_EXCEPTION( Block::filter_block_internal(block, columns_to_filter, *_filter)); } - if (!_not_single_slot_filter_conjuncts.empty()) { - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); - RETURN_IF_CATCH_EXCEPTION( - RETURN_IF_ERROR(VExprContext::execute_conjuncts_and_filter_block( - _not_single_slot_filter_conjuncts, block, columns_to_filter, - column_to_keep))); - } else { - Block::erase_useless_column(block, column_to_keep); - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); - } + Block::erase_useless_column(block, column_to_keep); + RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); *read_rows = block->rows(); } else { uint64_t rr; @@ -1842,17 +1838,8 @@ Status OrcReader::get_next_block_impl(Block* block, size_t* read_rows, bool* eof RETURN_IF_CATCH_EXCEPTION( Block::filter_block_internal(block, columns_to_filter, result_filter)); } - //_not_single_slot_filter_conjuncts check : missing column1 == missing column2 , missing column == exists column ... - if (!_not_single_slot_filter_conjuncts.empty()) { - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); - RETURN_IF_CATCH_EXCEPTION( - RETURN_IF_ERROR(VExprContext::execute_conjuncts_and_filter_block( - _not_single_slot_filter_conjuncts, block, columns_to_filter, - column_to_keep))); - } else { - Block::erase_useless_column(block, column_to_keep); - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); - } + Block::erase_useless_column(block, column_to_keep); + RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); } else { if (_delete_rows_filter_ptr) { _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr); @@ -2033,8 +2020,8 @@ Status OrcReader::fill_dict_filter_column_names( int i = 0; for (auto& predicate_col_name : predicate_col_names) { int slot_id = predicate_col_slot_ids[i]; - if (_can_filter_by_dict(slot_id)) { - _dict_filter_cols.emplace_back(std::make_pair(predicate_col_name, slot_id)); + if (!_disable_dict_filter && _can_filter_by_dict(slot_id)) { + _dict_filter_cols.emplace_back(predicate_col_name, slot_id); column_names.emplace_back(_col_name_to_file_col_name[predicate_col_name]); } else { if (_slot_id_to_filter_conjuncts->find(slot_id) != diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h index dc9d565e8031c3..866d22ff56cc20 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.h +++ b/be/src/vec/exec/format/orc/vorc_reader.h @@ -612,6 +612,7 @@ class OrcReader : public GenericReader { VExprContextSPtrs _dict_filter_conjuncts; VExprContextSPtrs _non_dict_filter_conjuncts; VExprContextSPtrs _filter_conjuncts; + bool _disable_dict_filter = false; // std::pair std::vector> _dict_filter_cols; std::shared_ptr _obj_pool; diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 5c5489d3f86e0e..00beb59e7dcb64 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -109,11 +109,6 @@ Status RowGroupReader::init( _tuple_descriptor = tuple_descriptor; _row_descriptor = row_descriptor; _col_name_to_slot_id = colname_to_slot_id; - if (not_single_slot_filter_conjuncts != nullptr && !not_single_slot_filter_conjuncts->empty()) { - _not_single_slot_filter_conjuncts.insert(_not_single_slot_filter_conjuncts.end(), - not_single_slot_filter_conjuncts->begin(), - not_single_slot_filter_conjuncts->end()); - } _slot_id_to_filter_conjuncts = slot_id_to_filter_conjuncts; _merge_read_ranges(row_ranges); if (_read_columns.empty()) { @@ -140,45 +135,52 @@ Status RowGroupReader::init( } _column_readers[read_col] = std::move(reader); } - // Check if single slot can be filtered by dict. - if (!_slot_id_to_filter_conjuncts) { - return Status::OK(); + + bool disable_dict_filter = false; + if (not_single_slot_filter_conjuncts != nullptr && !not_single_slot_filter_conjuncts->empty()) { + disable_dict_filter = true; + _filter_conjuncts.insert(_filter_conjuncts.end(), not_single_slot_filter_conjuncts->begin(), + not_single_slot_filter_conjuncts->end()); } - const std::vector& predicate_col_names = _lazy_read_ctx.predicate_columns.first; - const std::vector& predicate_col_slot_ids = _lazy_read_ctx.predicate_columns.second; - for (size_t i = 0; i < predicate_col_names.size(); ++i) { - const string& predicate_col_name = predicate_col_names[i]; - int slot_id = predicate_col_slot_ids[i]; - auto field = const_cast(schema.get_column(predicate_col_name)); - if (!_lazy_read_ctx.has_complex_type && - _can_filter_by_dict(slot_id, - _row_group_meta.columns[field->physical_column_index].meta_data)) { - _dict_filter_cols.emplace_back(std::make_pair(predicate_col_name, slot_id)); - } else { - if (_slot_id_to_filter_conjuncts->find(slot_id) != - _slot_id_to_filter_conjuncts->end()) { - for (auto& ctx : _slot_id_to_filter_conjuncts->at(slot_id)) { - _filter_conjuncts.push_back(ctx); + + // Check if single slot can be filtered by dict. + if (_slot_id_to_filter_conjuncts && !_slot_id_to_filter_conjuncts->empty()) { + const std::vector& predicate_col_names = _lazy_read_ctx.predicate_columns.first; + const std::vector& predicate_col_slot_ids = _lazy_read_ctx.predicate_columns.second; + for (size_t i = 0; i < predicate_col_names.size(); ++i) { + const string& predicate_col_name = predicate_col_names[i]; + int slot_id = predicate_col_slot_ids[i]; + auto field = const_cast(schema.get_column(predicate_col_name)); + if (!disable_dict_filter && !_lazy_read_ctx.has_complex_type && + _can_filter_by_dict( + slot_id, _row_group_meta.columns[field->physical_column_index].meta_data)) { + _dict_filter_cols.emplace_back(std::make_pair(predicate_col_name, slot_id)); + } else { + if (_slot_id_to_filter_conjuncts->find(slot_id) != + _slot_id_to_filter_conjuncts->end()) { + for (auto& ctx : _slot_id_to_filter_conjuncts->at(slot_id)) { + _filter_conjuncts.push_back(ctx); + } } } } - } - // Add predicate_partition_columns in _slot_id_to_filter_conjuncts(single slot conjuncts) - // to _filter_conjuncts, others should be added from not_single_slot_filter_conjuncts. - for (auto& kv : _lazy_read_ctx.predicate_partition_columns) { - auto& [value, slot_desc] = kv.second; - auto iter = _slot_id_to_filter_conjuncts->find(slot_desc->id()); - if (iter != _slot_id_to_filter_conjuncts->end()) { - for (auto& ctx : iter->second) { - _filter_conjuncts.push_back(ctx); + // Add predicate_partition_columns in _slot_id_to_filter_conjuncts(single slot conjuncts) + // to _filter_conjuncts, others should be added from not_single_slot_filter_conjuncts. + for (auto& kv : _lazy_read_ctx.predicate_partition_columns) { + auto& [value, slot_desc] = kv.second; + auto iter = _slot_id_to_filter_conjuncts->find(slot_desc->id()); + if (iter != _slot_id_to_filter_conjuncts->end()) { + for (auto& ctx : iter->second) { + _filter_conjuncts.push_back(ctx); + } } } + //For check missing column : missing column == xx, missing column is null,missing column is not null. + _filter_conjuncts.insert(_filter_conjuncts.end(), + _lazy_read_ctx.missing_columns_conjuncts.begin(), + _lazy_read_ctx.missing_columns_conjuncts.end()); + RETURN_IF_ERROR(_rewrite_dict_predicates()); } - //For check missing column : missing column == xx, missing column is null,missing column is not null. - _filter_conjuncts.insert(_filter_conjuncts.end(), - _lazy_read_ctx.missing_columns_conjuncts.begin(), - _lazy_read_ctx.missing_columns_conjuncts.end()); - RETURN_IF_ERROR(_rewrite_dict_predicates()); return Status::OK(); } @@ -351,17 +353,8 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_ RETURN_IF_CATCH_EXCEPTION( Block::filter_block_internal(block, columns_to_filter, result_filter)); - if (!_not_single_slot_filter_conjuncts.empty()) { - _convert_dict_cols_to_string_cols(block); - SCOPED_RAW_TIMER(&_predicate_filter_time); - RETURN_IF_CATCH_EXCEPTION( - RETURN_IF_ERROR(VExprContext::execute_conjuncts_and_filter_block( - _not_single_slot_filter_conjuncts, block, columns_to_filter, - column_to_keep))); - } else { - Block::erase_useless_column(block, column_to_keep); - _convert_dict_cols_to_string_cols(block); - } + Block::erase_useless_column(block, column_to_keep); + _convert_dict_cols_to_string_cols(block); } else { RETURN_IF_CATCH_EXCEPTION( RETURN_IF_ERROR(_filter_block(block, column_to_keep, columns_to_filter))); @@ -594,15 +587,6 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re *batch_eof = pre_eof; RETURN_IF_ERROR(_fill_partition_columns(block, column_size, _lazy_read_ctx.partition_columns)); RETURN_IF_ERROR(_fill_missing_columns(block, column_size, _lazy_read_ctx.missing_columns)); - if (!_not_single_slot_filter_conjuncts.empty()) { - { - SCOPED_RAW_TIMER(&_predicate_filter_time); - RETURN_IF_CATCH_EXCEPTION( - RETURN_IF_ERROR(VExprContext::execute_conjuncts_and_filter_block( - _not_single_slot_filter_conjuncts, block, columns_to_filter, - origin_column_num))); - } - } return Status::OK(); } diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h index f73e9ebe09eee6..8106241014ba63 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h @@ -220,7 +220,6 @@ class RowGroupReader : public ProfileCollector { const TupleDescriptor* _tuple_descriptor = nullptr; const RowDescriptor* _row_descriptor = nullptr; const std::unordered_map* _col_name_to_slot_id = nullptr; - VExprContextSPtrs _not_single_slot_filter_conjuncts; const std::unordered_map* _slot_id_to_filter_conjuncts = nullptr; VExprContextSPtrs _dict_filter_conjuncts; VExprContextSPtrs _filter_conjuncts; From fc3e8f84f4531b6f53d0230e62950150642da5f6 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 8 Jul 2025 21:12:18 +0800 Subject: [PATCH 173/572] [opt](multi-catalog) Optimize remote scan concurrency. (#52516) Cherry-pick #51415 ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/pipeline/exec/file_scan_operator.cpp | 11 +++++------ be/src/vec/exec/scan/scanner_scheduler.cpp | 11 ++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/be/src/pipeline/exec/file_scan_operator.cpp b/be/src/pipeline/exec/file_scan_operator.cpp index 62994bc6db4cb8..00ebfe83535e0a 100644 --- a/be/src/pipeline/exec/file_scan_operator.cpp +++ b/be/src/pipeline/exec/file_scan_operator.cpp @@ -39,9 +39,9 @@ Status FileScanLocalState::_init_scanners(std::list* s auto& p = _parent->cast(); // There's only one scan range for each backend in batch split mode. Each backend only starts up one ScanNode instance. - size_t shard_num = std::min( - config::doris_scanner_thread_pool_thread_num / p.query_parallel_instance_num(), - _max_scanners); + size_t shard_num = std::min(vectorized::ScannerScheduler::get_remote_scan_thread_num() / + p.query_parallel_instance_num(), + _max_scanners); shard_num = std::max(shard_num, (size_t)1); _kv_cache.reset(new vectorized::ShardedKVCache(shard_num)); for (int i = 0; i < _max_scanners; ++i) { @@ -65,9 +65,8 @@ void FileScanLocalState::set_scan_ranges(RuntimeState* state, auto& p = _parent->cast(); auto calc_max_scanners = [&](int parallel_instance_num) -> int { - int max_scanners = config::doris_scanner_thread_pool_thread_num / parallel_instance_num; - max_scanners = - std::max(std::max(max_scanners, state->parallel_scan_max_scanners_count()), 1); + int max_scanners = + vectorized::ScannerScheduler::get_remote_scan_thread_num() / parallel_instance_num; if (should_run_serial()) { max_scanners = 1; } diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp index 1b14d172790671..b518056c897c1a 100644 --- a/be/src/vec/exec/scan/scanner_scheduler.cpp +++ b/be/src/vec/exec/scan/scanner_scheduler.cpp @@ -323,11 +323,12 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, } int ScannerScheduler::get_remote_scan_thread_num() { - int remote_max_thread_num = config::doris_max_remote_scanner_thread_pool_thread_num != -1 - ? config::doris_max_remote_scanner_thread_pool_thread_num - : std::max(512, CpuInfo::num_cores() * 10); - remote_max_thread_num = - std::max(remote_max_thread_num, config::doris_scanner_thread_pool_thread_num); + static int remote_max_thread_num = []() { + int num = config::doris_max_remote_scanner_thread_pool_thread_num != -1 + ? config::doris_max_remote_scanner_thread_pool_thread_num + : std::max(512, CpuInfo::num_cores() * 10); + return std::max(num, config::doris_scanner_thread_pool_thread_num); + }(); return remote_max_thread_num; } From c8da1b0a647fd3514de2037c32850fac075c3dc7 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 9 Jul 2025 09:49:16 +0800 Subject: [PATCH 174/572] branch-3.0: [chore](cloud) Move txn kv related files to meta-store/ #52638 (#52956) cherry pick from #52638 --- cloud/CMakeLists.txt | 2 + cloud/src/common/encryption_util.cpp | 6 +- cloud/src/common/metric.cpp | 4 +- cloud/src/common/metric.h | 2 +- cloud/src/common/util.cpp | 140 +-------------- cloud/src/common/util.h | 68 +------- cloud/src/main.cpp | 4 +- cloud/src/meta-service/CMakeLists.txt | 4 - cloud/src/meta-service/http_encode_key.cpp | 9 +- .../src/meta-service/injection_point_http.cpp | 6 +- cloud/src/meta-service/meta_server.cpp | 6 +- cloud/src/meta-service/meta_server.h | 2 +- cloud/src/meta-service/meta_service.cpp | 10 +- cloud/src/meta-service/meta_service.h | 2 +- cloud/src/meta-service/meta_service_helper.h | 6 +- cloud/src/meta-service/meta_service_http.cpp | 6 +- cloud/src/meta-service/meta_service_job.cpp | 7 +- .../meta-service/meta_service_partition.cpp | 4 +- .../meta-service/meta_service_resource.cpp | 6 +- .../src/meta-service/meta_service_schema.cpp | 7 +- .../meta_service_tablet_stats.cpp | 6 +- cloud/src/meta-service/meta_service_txn.cpp | 6 +- cloud/src/meta-service/txn_lazy_committer.cpp | 2 +- cloud/src/meta-service/txn_lazy_committer.h | 2 +- cloud/src/meta-store/CMakeLists.txt | 13 ++ cloud/src/meta-store/blob_message.cpp | 161 ++++++++++++++++++ cloud/src/meta-store/blob_message.h | 97 +++++++++++ .../{meta-service => meta-store}/codec.cpp | 0 .../src/{meta-service => meta-store}/codec.h | 0 .../src/{meta-service => meta-store}/keys.cpp | 4 +- cloud/src/{meta-service => meta-store}/keys.h | 0 .../mem_txn_kv.cpp | 2 +- .../{meta-service => meta-store}/mem_txn_kv.h | 2 +- .../{meta-service => meta-store}/txn_kv.cpp | 2 +- .../src/{meta-service => meta-store}/txn_kv.h | 0 .../txn_kv_error.h | 0 cloud/src/recycler/checker.cpp | 6 +- cloud/src/recycler/meta_checker.cpp | 4 +- cloud/src/recycler/recycler.cpp | 7 +- cloud/src/recycler/recycler_service.cpp | 4 +- cloud/src/recycler/recycler_service.h | 2 +- cloud/src/recycler/util.cpp | 6 +- .../src/resource-manager/resource_manager.cpp | 4 +- cloud/src/resource-manager/resource_manager.h | 4 +- cloud/test/codec_test.cpp | 2 +- cloud/test/doris_txn_test.cpp | 2 +- cloud/test/encryption_test.cpp | 8 +- cloud/test/fdb_injection_test.cpp | 2 +- cloud/test/http_encode_key_test.cpp | 4 +- cloud/test/keys_test.cpp | 2 +- cloud/test/mem_txn_kv_test.cpp | 6 +- cloud/test/meta_server_test.cpp | 8 +- cloud/test/meta_service_http_test.cpp | 8 +- cloud/test/meta_service_job_test.cpp | 6 +- cloud/test/meta_service_tablet_stats_test.cpp | 4 +- cloud/test/meta_service_test.cpp | 6 +- cloud/test/metric_test.cpp | 6 +- cloud/test/mock_resource_manager.h | 4 +- cloud/test/rate_limiter_test.cpp | 6 +- cloud/test/recycler_test.cpp | 8 +- cloud/test/resource_test.cpp | 6 +- cloud/test/rpc_kv_bvar_test.cpp | 8 +- cloud/test/schema_kv_test.cpp | 6 +- cloud/test/txn_kv_test.cpp | 13 +- cloud/test/txn_lazy_commit_test.cpp | 6 +- 65 files changed, 416 insertions(+), 340 deletions(-) create mode 100644 cloud/src/meta-store/CMakeLists.txt create mode 100644 cloud/src/meta-store/blob_message.cpp create mode 100644 cloud/src/meta-store/blob_message.h rename cloud/src/{meta-service => meta-store}/codec.cpp (100%) rename cloud/src/{meta-service => meta-store}/codec.h (100%) rename cloud/src/{meta-service => meta-store}/keys.cpp (99%) rename cloud/src/{meta-service => meta-store}/keys.h (100%) rename cloud/src/{meta-service => meta-store}/mem_txn_kv.cpp (99%) rename cloud/src/{meta-service => meta-store}/mem_txn_kv.h (99%) rename cloud/src/{meta-service => meta-store}/txn_kv.cpp (99%) rename cloud/src/{meta-service => meta-store}/txn_kv.h (100%) rename cloud/src/{meta-service => meta-store}/txn_kv_error.h (100%) diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index 627e6f283b91d5..59f77c4927e166 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -327,6 +327,7 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} CloudGen Common MetaService + MetaStore ResourceManager Recycler RateLimiter @@ -481,6 +482,7 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfdb_c -L${THIRDPARTY_DIR add_subdirectory(${SRC_DIR}/common) add_subdirectory(${SRC_DIR}/gen-cpp) add_subdirectory(${SRC_DIR}/meta-service) +add_subdirectory(${SRC_DIR}/meta-store) add_subdirectory(${SRC_DIR}/recycler) add_subdirectory(${SRC_DIR}/rate-limiter) add_subdirectory(${SRC_DIR}/resource-manager) diff --git a/cloud/src/common/encryption_util.cpp b/cloud/src/common/encryption_util.cpp index 84cbfc13054ed8..2d8b842bcd36fc 100644 --- a/cloud/src/common/encryption_util.cpp +++ b/cloud/src/common/encryption_util.cpp @@ -38,9 +38,9 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { namespace config { diff --git a/cloud/src/common/metric.cpp b/cloud/src/common/metric.cpp index 2425ae0e48bbc2..a9b91c6c853ccd 100644 --- a/cloud/src/common/metric.cpp +++ b/cloud/src/common/metric.cpp @@ -28,8 +28,8 @@ #include #include "common/bvars.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { diff --git a/cloud/src/common/metric.h b/cloud/src/common/metric.h index 33429a276c6034..ad918482daa460 100644 --- a/cloud/src/common/metric.h +++ b/cloud/src/common/metric.h @@ -25,7 +25,7 @@ #include #include "common/logging.h" -#include "meta-service/txn_kv.h" +#include "meta-store/txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/common/util.cpp b/cloud/src/common/util.cpp index 377f83674ce4b2..84363741db31cc 100644 --- a/cloud/src/common/util.cpp +++ b/cloud/src/common/util.cpp @@ -24,10 +24,11 @@ // FIXME: we should not rely other modules that may rely on this common module #include "common/logging.h" -#include "meta-service/keys.h" -#include "meta-service/codec.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "common/config.h" +#include "meta-store/keys.h" +#include "meta-store/codec.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include #include @@ -224,117 +225,6 @@ std::string proto_to_json(const ::google::protobuf::Message& msg, bool add_white return json; } -std::vector split_string(const std::string_view& str, int n) { - std::vector substrings; - - for (size_t i = 0; i < str.size(); i += n) { - substrings.push_back(str.substr(i, n)); - } - - return substrings; -} - -bool ValueBuf::to_pb(google::protobuf::Message* pb) const { - butil::IOBuf merge; - for (auto&& it : iters) { - it->reset(); - while (it->has_next()) { - auto [k, v] = it->next(); - merge.append_user_data((void*)v.data(), v.size(), +[](void*) {}); - } - } - butil::IOBufAsZeroCopyInputStream merge_stream(merge); - return pb->ParseFromZeroCopyStream(&merge_stream); -} - -std::string ValueBuf::value() const { - butil::IOBuf merge; - for (auto&& it : iters) { - it->reset(); - while (it->has_next()) { - auto [k, v] = it->next(); - merge.append_user_data((void*)v.data(), v.size(), +[](void*) {}); - } - } - return merge.to_string(); -} - -std::vector ValueBuf::keys() const { - std::vector ret; - for (auto&& it : iters) { - it->reset(); - while (it->has_next()) { - auto [k, _] = it->next(); - ret.push_back({k.data(), k.size()}); - } - } - return ret; -} - -void ValueBuf::remove(Transaction* txn) const { - for (auto&& it : iters) { - it->reset(); - while (it->has_next()) { - txn->remove(it->next().first); - } - } -} - -TxnErrorCode ValueBuf::get(Transaction* txn, std::string_view key, bool snapshot) { - iters.clear(); - ver = -1; - - std::string begin_key {key}; - std::string end_key {key}; - encode_int64(INT64_MAX, &end_key); - std::unique_ptr it; - TxnErrorCode err = txn->get(begin_key, end_key, &it, snapshot); - if (err != TxnErrorCode::TXN_OK) { - return err; - } - if (!it->has_next()) { - return TxnErrorCode::TXN_KEY_NOT_FOUND; - } - // Extract version - auto [k, _] = it->next(); - if (k.size() == key.size()) { // Old version KV - DCHECK(k == key) << hex(k) << ' ' << hex(key); - DCHECK_EQ(it->size(), 1) << hex(k) << ' ' << hex(key); - ver = 0; - } else { - k.remove_prefix(key.size()); - int64_t suffix; - if (decode_int64(&k, &suffix) != 0) [[unlikely]] { - LOG_WARNING("failed to decode key").tag("key", hex(k)); - return TxnErrorCode::TXN_UNIDENTIFIED_ERROR; - } - ver = suffix >> 56 & 0xff; - } - bool more = it->more(); - if (!more) { - iters.push_back(std::move(it)); - return TxnErrorCode::TXN_OK; - } - begin_key = it->next_begin_key(); - iters.push_back(std::move(it)); - do { - err = txn->get(begin_key, end_key, &it, snapshot); - if (err != TxnErrorCode::TXN_OK) { - return err; - } - more = it->more(); - if (more) { - begin_key = it->next_begin_key(); - } - iters.push_back(std::move(it)); - } while (more); - return TxnErrorCode::TXN_OK; -} - -TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot) { - return val->get(txn, key, snapshot); -} - TxnErrorCode key_exists(Transaction* txn, std::string_view key, bool snapshot) { std::string end_key {key}; encode_int64(INT64_MAX, &end_key); @@ -346,24 +236,4 @@ TxnErrorCode key_exists(Transaction* txn, std::string_view key, bool snapshot) { return it->has_next() ? TxnErrorCode::TXN_OK : TxnErrorCode::TXN_KEY_NOT_FOUND; } -void blob_put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, - uint8_t ver, size_t split_size) { - std::string value; - bool ret = pb.SerializeToString(&value); // Always success - DCHECK(ret) << hex(key) << ' ' << pb.ShortDebugString(); - blob_put(txn, key, value, ver, split_size); -} - -void blob_put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, - size_t split_size) { - auto split_vec = split_string(value, split_size); - int64_t suffix_base = ver; - suffix_base <<= 56; - for (size_t i = 0; i < split_vec.size(); ++i) { - std::string k(key); - encode_int64(suffix_base + i, &k); - txn->put(k, split_vec[i]); - } -} - } // namespace doris::cloud diff --git a/cloud/src/common/util.h b/cloud/src/common/util.h index 838c683125aded..2b7fa694a77e6f 100644 --- a/cloud/src/common/util.h +++ b/cloud/src/common/util.h @@ -22,7 +22,7 @@ #include #include -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv_error.h" namespace google::protobuf { class Message; @@ -66,49 +66,6 @@ std::string prettify_key(std::string_view key_hex, bool unicode = false); */ std::string proto_to_json(const ::google::protobuf::Message& msg, bool add_whitespace = false); -/** - * Supports splitting large values (>100KB) into multiple KVs, with a logical value size of up to the fdb transaction limit (<10MB). - * Supports multi version format parsing of values (which can be any byte sequence format), and can recognize the version of values - * that are forward compatible with older versions of values. - * Key format: - * {origin_key}{suffix: i64} - * suffix (big-endian): - * |Bytes 0 |Bytes 1-5 |Bytes 6-7 | - * |-------------|-------------|-------------| - * |version |dummy |sequence | - */ -struct ValueBuf { - // TODO(plat1ko): Support decompression - [[nodiscard]] bool to_pb(google::protobuf::Message* pb) const; - // TODO: More bool to_xxx(Xxx* xxx) const; - - // Remove all splitted KV in `iters_` via `txn` - void remove(Transaction* txn) const; - - // Get a key, save raw splitted values of the key to `this`, value length may be bigger than 100k - // Return TXN_OK for success get a key, TXN_KEY_NOT_FOUND for key not found, otherwise for error. - TxnErrorCode get(Transaction* txn, std::string_view key, bool snapshot = false); - - // return the merged value in ValueBuf - std::string value() const; - - // return all keys in ValueBuf, if the value is not splitted, size of keys is 1 - std::vector keys() const; - - std::vector> iters; - int8_t ver {-1}; -}; - -/** - * Get a key, return key's value, value length may be bigger than 100k - * @param txn fdb txn handler - * @param key encode key - * @param val return wrapped raw splitted values of the key - * @param snapshot if true, `key` will not be included in txn conflict detection this time - * @return return TXN_OK for success get a key, TXN_KEY_NOT_FOUND for key not found, otherwise for error. - */ -TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot = false); - /** * Test whether key exists * @param txn fdb txn handler @@ -118,27 +75,4 @@ TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, boo */ TxnErrorCode key_exists(Transaction* txn, std::string_view key, bool snapshot = false); -/** - * Put a KV, it's value may be bigger than 100k - * TODO(plat1ko): Support compression - * @param txn fdb txn handler - * @param key encode key - * @param pb value to save - * @param ver value version - * @param split_size how many byte sized fragments are the value split into - */ -void blob_put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, - uint8_t ver, size_t split_size = 90 * 1000); - -/** - * Put a KV, it's value may be bigger than 100k - * @param txn fdb txn handler - * @param key encode key - * @param value value to save - * @param ver value version - * @param split_size how many byte sized fragments are the value split into - */ -void blob_put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, - size_t split_size = 90 * 1000); - } // namespace doris::cloud diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 0aad97aab4d4c3..6d0c22f3204e80 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -36,9 +36,9 @@ #include "common/configbase.h" #include "common/encryption_util.h" #include "common/logging.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_server.h" -#include "meta-service/txn_kv.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" #include "recycler/recycler.h" using namespace doris::cloud; diff --git a/cloud/src/meta-service/CMakeLists.txt b/cloud/src/meta-service/CMakeLists.txt index d11f87e7fa23d4..d55d9d19499e96 100644 --- a/cloud/src/meta-service/CMakeLists.txt +++ b/cloud/src/meta-service/CMakeLists.txt @@ -19,11 +19,7 @@ add_library(MetaService meta_service_tablet_stats.cpp meta_service_partition.cpp meta_service_txn.cpp - txn_kv.cpp - codec.cpp - keys.cpp doris_txn.cpp - mem_txn_kv.cpp http_encode_key.cpp txn_lazy_committer.cpp ) diff --git a/cloud/src/meta-service/http_encode_key.cpp b/cloud/src/meta-service/http_encode_key.cpp index fcb2b81a6a8f00..cae6fb322f7fb4 100644 --- a/cloud/src/meta-service/http_encode_key.cpp +++ b/cloud/src/meta-service/http_encode_key.cpp @@ -37,14 +37,15 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/codec.h" #include "meta-service/doris_txn.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_http.h" #include "meta-service/meta_service_schema.h" #include "meta-service/meta_service_tablet_stats.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/blob_message.h" +#include "meta-store/codec.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/injection_point_http.cpp b/cloud/src/meta-service/injection_point_http.cpp index 910b709ca89d1a..20e35b07f8a29c 100644 --- a/cloud/src/meta-service/injection_point_http.cpp +++ b/cloud/src/meta-service/injection_point_http.cpp @@ -22,10 +22,10 @@ #include "common/config.h" #include "common/logging.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "meta_service.h" #include "meta_service_http.h" diff --git a/cloud/src/meta-service/meta_server.cpp b/cloud/src/meta-service/meta_server.cpp index 4762c55d812221..9677d70b8c042e 100644 --- a/cloud/src/meta-service/meta_server.cpp +++ b/cloud/src/meta-service/meta_server.cpp @@ -33,12 +33,12 @@ #include "common/network_util.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "rate-limiter/rate_limiter.h" #include "resource-manager/resource_manager.h" -#include "txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_server.h b/cloud/src/meta-service/meta_server.h index ee44e377cb3ebd..556da7739185a1 100644 --- a/cloud/src/meta-service/meta_server.h +++ b/cloud/src/meta-service/meta_server.h @@ -25,7 +25,7 @@ #include #include "common/metric.h" -#include "txn_kv.h" +#include "meta-store/txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 9b131c58334c5b..7e0d3a267b85c6 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -55,15 +55,15 @@ #include "common/string_util.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "keys.h" -#include "meta-service/codec.h" #include "meta-service/doris_txn.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_schema.h" #include "meta-service/meta_service_tablet_stats.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/blob_message.h" +#include "meta-store/codec.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "rate-limiter/rate_limiter.h" using namespace std::chrono; diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h index ce8e04fed48f19..355bdc7a982608 100644 --- a/cloud/src/meta-service/meta_service.h +++ b/cloud/src/meta-service/meta_service.h @@ -30,8 +30,8 @@ #include "common/config.h" #include "common/stats.h" #include "cpp/sync_point.h" -#include "meta-service/txn_kv.h" #include "meta-service/txn_lazy_committer.h" +#include "meta-store/txn_kv.h" #include "rate-limiter/rate_limiter.h" #include "resource-manager/resource_manager.h" diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 256816b68bb3cf..7ecdf856659db0 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -34,9 +34,9 @@ #include "common/stopwatch.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "resource-manager/resource_manager.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp index d12ff59ffe094c..21db2ec0357b64 100644 --- a/cloud/src/meta-service/meta_service_http.cpp +++ b/cloud/src/meta-service/meta_service_http.cpp @@ -48,9 +48,9 @@ #include "common/configbase.h" #include "common/logging.h" #include "common/string_util.h" -#include "meta-service/keys.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "meta_service.h" #include "rate-limiter/rate_limiter.h" diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 218742713680f6..7c1f13aa8b6cd7 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -29,12 +29,11 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "keys.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_tablet_stats.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "meta_service.h" // Empty string not is not processed diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index 379a7d72eaa340..109b86813f39f6 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -19,9 +19,9 @@ #include #include "common/logging.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv_error.h" #include "meta_service.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 8b3c922b87633c..db8d76eab8d1ac 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -34,11 +34,11 @@ #include "common/stats.h" #include "common/string_util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" using namespace std::chrono; diff --git a/cloud/src/meta-service/meta_service_schema.cpp b/cloud/src/meta-service/meta_service_schema.cpp index 31c067faea4991..bb82e049427aaa 100644 --- a/cloud/src/meta-service/meta_service_schema.cpp +++ b/cloud/src/meta-service/meta_service_schema.cpp @@ -32,10 +32,11 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/blob_message.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { namespace config { diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp index da556b6c574557..c8bb315f60ebf3 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.cpp +++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp @@ -28,11 +28,11 @@ #include "common/logging.h" #include "common/util.h" -#include "meta-service/keys.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 55e201d43d2bc9..4f8fc6f7299de7 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -26,12 +26,12 @@ #include "common/stats.h" #include "cpp/sync_point.h" #include "meta-service/doris_txn.h" -#include "meta-service/keys.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_tablet_stats.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" using namespace std::chrono; diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 99aea5fed56fff..3ad57fb0b6a78a 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -23,9 +23,9 @@ #include "common/stats.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_tablet_stats.h" +#include "meta-store/keys.h" using namespace std::chrono; diff --git a/cloud/src/meta-service/txn_lazy_committer.h b/cloud/src/meta-service/txn_lazy_committer.h index dca618739ec8ba..1bcfa317a5d849 100644 --- a/cloud/src/meta-service/txn_lazy_committer.h +++ b/cloud/src/meta-service/txn_lazy_committer.h @@ -22,7 +22,7 @@ #include #include "common/simple_thread_pool.h" -#include "meta-service/txn_kv.h" +#include "meta-store/txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/meta-store/CMakeLists.txt b/cloud/src/meta-store/CMakeLists.txt new file mode 100644 index 00000000000000..07349f7921ce9d --- /dev/null +++ b/cloud/src/meta-store/CMakeLists.txt @@ -0,0 +1,13 @@ +# where to put generated libraries +set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/meta-store") + +# where to put generated binaries +set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/meta-store") + +add_library(MetaStore STATIC + blob_message.cpp + codec.cpp + keys.cpp + mem_txn_kv.cpp + txn_kv.cpp +) diff --git a/cloud/src/meta-store/blob_message.cpp b/cloud/src/meta-store/blob_message.cpp new file mode 100644 index 00000000000000..edc2c2e96d5b9b --- /dev/null +++ b/cloud/src/meta-store/blob_message.cpp @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "blob_message.h" + +#include +#include + +#include "common/logging.h" +#include "common/util.h" +#include "meta-store/codec.h" +#include "meta-store/txn_kv.h" + +namespace doris::cloud { + +static std::vector split_string(const std::string_view& str, int n) { + std::vector substrings; + + for (size_t i = 0; i < str.size(); i += n) { + substrings.push_back(str.substr(i, n)); + } + + return substrings; +} + +bool ValueBuf::to_pb(google::protobuf::Message* pb) const { + butil::IOBuf merge; + for (auto&& it : iters) { + it->reset(); + while (it->has_next()) { + auto [k, v] = it->next(); + merge.append_user_data((void*)v.data(), v.size(), +[](void*) {}); + } + } + butil::IOBufAsZeroCopyInputStream merge_stream(merge); + return pb->ParseFromZeroCopyStream(&merge_stream); +} + +std::string ValueBuf::value() const { + butil::IOBuf merge; + for (auto&& it : iters) { + it->reset(); + while (it->has_next()) { + auto [k, v] = it->next(); + merge.append_user_data((void*)v.data(), v.size(), +[](void*) {}); + } + } + return merge.to_string(); +} + +std::vector ValueBuf::keys() const { + std::vector ret; + for (auto&& it : iters) { + it->reset(); + while (it->has_next()) { + auto [k, _] = it->next(); + ret.emplace_back(k.data(), k.size()); + } + } + return ret; +} + +void ValueBuf::remove(Transaction* txn) const { + for (auto&& it : iters) { + it->reset(); + while (it->has_next()) { + txn->remove(it->next().first); + } + } +} + +TxnErrorCode ValueBuf::get(Transaction* txn, std::string_view key, bool snapshot) { + iters.clear(); + ver = -1; + + std::string begin_key {key}; + std::string end_key {key}; + encode_int64(INT64_MAX, &end_key); + std::unique_ptr it; + TxnErrorCode err = txn->get(begin_key, end_key, &it, snapshot); + if (err != TxnErrorCode::TXN_OK) { + return err; + } + if (!it->has_next()) { + return TxnErrorCode::TXN_KEY_NOT_FOUND; + } + // Extract version + auto [k, _] = it->next(); + if (k.size() == key.size()) { // Old version KV + DCHECK(k == key) << hex(k) << ' ' << hex(key); + DCHECK_EQ(it->size(), 1) << hex(k) << ' ' << hex(key); + ver = 0; + } else { + k.remove_prefix(key.size()); + int64_t suffix; + if (decode_int64(&k, &suffix) != 0) [[unlikely]] { + LOG_WARNING("failed to decode key").tag("key", hex(k)); + return TxnErrorCode::TXN_UNIDENTIFIED_ERROR; + } + ver = suffix >> 56 & 0xff; + } + bool more = it->more(); + if (!more) { + iters.push_back(std::move(it)); + return TxnErrorCode::TXN_OK; + } + begin_key = it->next_begin_key(); + iters.push_back(std::move(it)); + do { + err = txn->get(begin_key, end_key, &it, snapshot); + if (err != TxnErrorCode::TXN_OK) { + return err; + } + more = it->more(); + if (more) { + begin_key = it->next_begin_key(); + } + iters.push_back(std::move(it)); + } while (more); + return TxnErrorCode::TXN_OK; +} + +TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot) { + return val->get(txn, key, snapshot); +} + +void blob_put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, + uint8_t ver, size_t split_size) { + std::string value; + bool ret = pb.SerializeToString(&value); // Always success + DCHECK(ret) << hex(key) << ' ' << pb.ShortDebugString(); + blob_put(txn, key, value, ver, split_size); +} + +void blob_put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, + size_t split_size) { + auto split_vec = split_string(value, split_size); + int64_t suffix_base = ver; + suffix_base <<= 56; + for (size_t i = 0; i < split_vec.size(); ++i) { + std::string k(key); + encode_int64(suffix_base + i, &k); + txn->put(k, split_vec[i]); + } +} + +} // namespace doris::cloud diff --git a/cloud/src/meta-store/blob_message.h b/cloud/src/meta-store/blob_message.h new file mode 100644 index 00000000000000..4bba034bf0e167 --- /dev/null +++ b/cloud/src/meta-store/blob_message.h @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" + +namespace google::protobuf { +class Message; +} + +namespace doris::cloud { + +/** + * Supports splitting large values (>100KB) into multiple KVs, with a logical value size of up to the fdb transaction limit (<10MB). + * Supports multi version format parsing of values (which can be any byte sequence format), and can recognize the version of values + * that are forward compatible with older versions of values. + * Key format: + * {origin_key}{suffix: i64} + * suffix (big-endian): + * |Bytes 0 |Bytes 1-5 |Bytes 6-7 | + * |-------------|-------------|-------------| + * |version |dummy |sequence | + */ +struct ValueBuf { + // TODO(plat1ko): Support decompression + [[nodiscard]] bool to_pb(google::protobuf::Message* pb) const; + // TODO: More bool to_xxx(Xxx* xxx) const; + + // Remove all splitted KV in `iters_` via `txn` + void remove(Transaction* txn) const; + + // Get a key, save raw splitted values of the key to `this`, value length may be bigger than 100k + // Return TXN_OK for success get a key, TXN_KEY_NOT_FOUND for key not found, otherwise for error. + TxnErrorCode get(Transaction* txn, std::string_view key, bool snapshot = false); + + // return the merged value in ValueBuf + std::string value() const; + + // return all keys in ValueBuf, if the value is not splitted, size of keys is 1 + std::vector keys() const; + + std::vector> iters; + int8_t ver {-1}; +}; + +/** + * Get a key, return key's value, value length may be bigger than 100k + * @param txn fdb txn handler + * @param key encode key + * @param val return wrapped raw splitted values of the key + * @param snapshot if true, `key` will not be included in txn conflict detection this time + * @return return TXN_OK for success get a key, TXN_KEY_NOT_FOUND for key not found, otherwise for error. + */ +TxnErrorCode blob_get(Transaction* txn, std::string_view key, ValueBuf* val, bool snapshot = false); + +/** + * Put a KV, it's value may be bigger than 100k + * TODO(plat1ko): Support compression + * @param txn fdb txn handler + * @param key encode key + * @param pb value to save + * @param ver value version + * @param split_size how many byte sized fragments are the value split into + */ +void blob_put(Transaction* txn, std::string_view key, const google::protobuf::Message& pb, + uint8_t ver, size_t split_size = 90 * 1000); + +/** + * Put a KV, it's value may be bigger than 100k + * @param txn fdb txn handler + * @param key encode key + * @param value value to save + * @param ver value version + * @param split_size how many byte sized fragments are the value split into + */ +void blob_put(Transaction* txn, std::string_view key, std::string_view value, uint8_t ver, + size_t split_size = 90 * 1000); + +} // namespace doris::cloud diff --git a/cloud/src/meta-service/codec.cpp b/cloud/src/meta-store/codec.cpp similarity index 100% rename from cloud/src/meta-service/codec.cpp rename to cloud/src/meta-store/codec.cpp diff --git a/cloud/src/meta-service/codec.h b/cloud/src/meta-store/codec.h similarity index 100% rename from cloud/src/meta-service/codec.h rename to cloud/src/meta-store/codec.h diff --git a/cloud/src/meta-service/keys.cpp b/cloud/src/meta-store/keys.cpp similarity index 99% rename from cloud/src/meta-service/keys.cpp rename to cloud/src/meta-store/keys.cpp index a518b6e264d20d..dff688c22d75d5 100644 --- a/cloud/src/meta-service/keys.cpp +++ b/cloud/src/meta-store/keys.cpp @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -#include "keys.h" +#include "meta-store/keys.h" -#include "codec.h" +#include "meta-store/codec.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/keys.h b/cloud/src/meta-store/keys.h similarity index 100% rename from cloud/src/meta-service/keys.h rename to cloud/src/meta-store/keys.h diff --git a/cloud/src/meta-service/mem_txn_kv.cpp b/cloud/src/meta-store/mem_txn_kv.cpp similarity index 99% rename from cloud/src/meta-service/mem_txn_kv.cpp rename to cloud/src/meta-store/mem_txn_kv.cpp index 72ecbda3254c2d..5e049c6fe411f5 100644 --- a/cloud/src/meta-service/mem_txn_kv.cpp +++ b/cloud/src/meta-store/mem_txn_kv.cpp @@ -28,7 +28,7 @@ #include #include "cpp/sync_point.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv_error.h" #include "txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/mem_txn_kv.h b/cloud/src/meta-store/mem_txn_kv.h similarity index 99% rename from cloud/src/meta-service/mem_txn_kv.h rename to cloud/src/meta-store/mem_txn_kv.h index 9be6a2b5573c49..337f23a0361971 100644 --- a/cloud/src/meta-service/mem_txn_kv.h +++ b/cloud/src/meta-store/mem_txn_kv.h @@ -29,8 +29,8 @@ #include #include -#include "meta-service/txn_kv_error.h" #include "txn_kv.h" +#include "txn_kv_error.h" namespace doris::cloud { diff --git a/cloud/src/meta-service/txn_kv.cpp b/cloud/src/meta-store/txn_kv.cpp similarity index 99% rename from cloud/src/meta-service/txn_kv.cpp rename to cloud/src/meta-store/txn_kv.cpp index 56e5440c8fe9af..76f3c724958067 100644 --- a/cloud/src/meta-service/txn_kv.cpp +++ b/cloud/src/meta-store/txn_kv.cpp @@ -39,7 +39,7 @@ #include "common/stopwatch.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv_error.h" // ============================================================================= // FoundationDB implementation of TxnKv diff --git a/cloud/src/meta-service/txn_kv.h b/cloud/src/meta-store/txn_kv.h similarity index 100% rename from cloud/src/meta-service/txn_kv.h rename to cloud/src/meta-store/txn_kv.h diff --git a/cloud/src/meta-service/txn_kv_error.h b/cloud/src/meta-store/txn_kv_error.h similarity index 100% rename from cloud/src/meta-service/txn_kv_error.h rename to cloud/src/meta-store/txn_kv_error.h diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 3fbd862ae1a97c..36eb4e325ee866 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -43,9 +43,9 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "recycler/hdfs_accessor.h" #include "recycler/s3_accessor.h" #include "recycler/storage_vault_accessor.h" diff --git a/cloud/src/recycler/meta_checker.cpp b/cloud/src/recycler/meta_checker.cpp index 3718c2f2099951..a299c2839df882 100644 --- a/cloud/src/recycler/meta_checker.cpp +++ b/cloud/src/recycler/meta_checker.cpp @@ -27,8 +27,8 @@ #include "common/logging.h" #include "common/util.h" -#include "meta-service/keys.h" -#include "meta-service/txn_kv.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index ed1028ca699f9f..e285886acfb66f 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -42,8 +42,9 @@ #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_schema.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/blob_message.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "recycler/checker.h" #include "recycler/hdfs_accessor.h" #include "recycler/s3_accessor.h" @@ -58,7 +59,7 @@ #include "common/simple_thread_pool.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" +#include "meta-store/keys.h" #include "recycler/recycler_service.h" #include "recycler/sync_executor.h" #include "recycler/util.h" diff --git a/cloud/src/recycler/recycler_service.cpp b/cloud/src/recycler/recycler_service.cpp index c77357c764ebae..b812fbd27eec62 100644 --- a/cloud/src/recycler/recycler_service.cpp +++ b/cloud/src/recycler/recycler_service.cpp @@ -28,8 +28,8 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/s3_rate_limiter.h" -#include "meta-service/keys.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv_error.h" #include "recycler/checker.h" #include "recycler/meta_checker.h" #include "recycler/recycler.h" diff --git a/cloud/src/recycler/recycler_service.h b/cloud/src/recycler/recycler_service.h index 5ece69c86ba0da..67eb9d21333075 100644 --- a/cloud/src/recycler/recycler_service.h +++ b/cloud/src/recycler/recycler_service.h @@ -19,8 +19,8 @@ #include -#include "meta-service/txn_kv.h" #include "meta-service/txn_lazy_committer.h" +#include "meta-store/txn_kv.h" namespace doris::cloud { diff --git a/cloud/src/recycler/util.cpp b/cloud/src/recycler/util.cpp index 6797782d5d665c..3666b6836e6c9c 100644 --- a/cloud/src/recycler/util.cpp +++ b/cloud/src/recycler/util.cpp @@ -22,10 +22,10 @@ #include #include "common/util.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_schema.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { namespace config { diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp index 99296d025cc3ce..548478a85395d5 100644 --- a/cloud/src/resource-manager/resource_manager.cpp +++ b/cloud/src/resource-manager/resource_manager.cpp @@ -26,9 +26,9 @@ #include "common/string_util.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { diff --git a/cloud/src/resource-manager/resource_manager.h b/cloud/src/resource-manager/resource_manager.h index 7d411e9059f6dc..0c0bc461c2d1f6 100644 --- a/cloud/src/resource-manager/resource_manager.h +++ b/cloud/src/resource-manager/resource_manager.h @@ -23,8 +23,8 @@ #include #include -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { diff --git a/cloud/test/codec_test.cpp b/cloud/test/codec_test.cpp index 94c7a9912d7133..dd9065d8a3d237 100644 --- a/cloud/test/codec_test.cpp +++ b/cloud/test/codec_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "meta-service/codec.h" +#include "meta-store/codec.h" #include diff --git a/cloud/test/doris_txn_test.cpp b/cloud/test/doris_txn_test.cpp index ca2fde09d6cfef..af386763e501ae 100644 --- a/cloud/test/doris_txn_test.cpp +++ b/cloud/test/doris_txn_test.cpp @@ -24,7 +24,7 @@ #include "common/config.h" #include "common/util.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" +#include "meta-store/txn_kv.h" int main(int argc, char** argv) { doris::cloud::config::init(nullptr, true); diff --git a/cloud/test/encryption_test.cpp b/cloud/test/encryption_test.cpp index 072e6a820cfd05..55f6ee1adce416 100644 --- a/cloud/test/encryption_test.cpp +++ b/cloud/test/encryption_test.cpp @@ -24,10 +24,10 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" using namespace doris; diff --git a/cloud/test/fdb_injection_test.cpp b/cloud/test/fdb_injection_test.cpp index 60226e7f9521a2..2331dc41dd40eb 100644 --- a/cloud/test/fdb_injection_test.cpp +++ b/cloud/test/fdb_injection_test.cpp @@ -33,7 +33,7 @@ #include "common/logging.h" #include "cpp/sync_point.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" +#include "meta-store/txn_kv.h" using namespace doris; diff --git a/cloud/test/http_encode_key_test.cpp b/cloud/test/http_encode_key_test.cpp index 58fdb3a4c97ef1..4ceace3f6ebd13 100644 --- a/cloud/test/http_encode_key_test.cpp +++ b/cloud/test/http_encode_key_test.cpp @@ -22,9 +22,9 @@ #include "common/defer.h" #include "common/logging.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service_http.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" using namespace doris::cloud; diff --git a/cloud/test/keys_test.cpp b/cloud/test/keys_test.cpp index 52d840e1f5ce6b..b028160cf7f652 100644 --- a/cloud/test/keys_test.cpp +++ b/cloud/test/keys_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "meta-service/keys.h" +#include "meta-store/keys.h" #include #include diff --git a/cloud/test/mem_txn_kv_test.cpp b/cloud/test/mem_txn_kv_test.cpp index 9db71ba96c8b5b..d86fbfe135cf3d 100644 --- a/cloud/test/mem_txn_kv_test.cpp +++ b/cloud/test/mem_txn_kv_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "meta-service/mem_txn_kv.h" +#include "meta-store/mem_txn_kv.h" #include #include @@ -25,8 +25,8 @@ #include "common/config.h" #include "common/util.h" #include "meta-service/doris_txn.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" using namespace doris; diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index 8cff9223498989..a6a44851f791e4 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -37,11 +37,11 @@ #include "common/logging.h" #include "common/stats.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" #include "resource-manager/resource_manager.h" diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp index 325770b658b0b4..855ffbb33cb990 100644 --- a/cloud/test/meta_service_http_test.cpp +++ b/cloud/test/meta_service_http_test.cpp @@ -45,11 +45,11 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_resource_manager.h" #include "resource-manager/resource_manager.h" diff --git a/cloud/test/meta_service_job_test.cpp b/cloud/test/meta_service_job_test.cpp index 21ef2e41349449..194413307cbc3f 100644 --- a/cloud/test/meta_service_job_test.cpp +++ b/cloud/test/meta_service_job_test.cpp @@ -33,10 +33,10 @@ #include "common/defer.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" namespace doris::cloud { extern std::unique_ptr get_meta_service(); diff --git a/cloud/test/meta_service_tablet_stats_test.cpp b/cloud/test/meta_service_tablet_stats_test.cpp index 000d33602e5dc2..804cf165439baa 100644 --- a/cloud/test/meta_service_tablet_stats_test.cpp +++ b/cloud/test/meta_service_tablet_stats_test.cpp @@ -20,8 +20,8 @@ #include #include -#include "meta-service/codec.h" -#include "meta-service/keys.h" +#include "meta-store/codec.h" +#include "meta-store/keys.h" namespace doris::cloud { diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 994086703c8854..ad1c054e639352 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -37,10 +37,10 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" #include "resource-manager/resource_manager.h" diff --git a/cloud/test/metric_test.cpp b/cloud/test/metric_test.cpp index f46d512effffff..31a2b7b3c5821f 100644 --- a/cloud/test/metric_test.cpp +++ b/cloud/test/metric_test.cpp @@ -26,9 +26,9 @@ #include "common/bvars.h" #include "common/config.h" -#include "meta-service/mem_txn_kv.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); diff --git a/cloud/test/mock_resource_manager.h b/cloud/test/mock_resource_manager.h index 25b0d5fbb4bc71..eccb6ee93e90a1 100644 --- a/cloud/test/mock_resource_manager.h +++ b/cloud/test/mock_resource_manager.h @@ -17,8 +17,8 @@ #pragma once -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "resource-manager/resource_manager.h" using namespace doris::cloud; diff --git a/cloud/test/rate_limiter_test.cpp b/cloud/test/rate_limiter_test.cpp index a406fbb4932dbd..3f853cec50727d 100644 --- a/cloud/test/rate_limiter_test.cpp +++ b/cloud/test/rate_limiter_test.cpp @@ -27,10 +27,10 @@ #include "common/config.h" #include "common/util.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_resource_manager.h" #include "resource-manager/resource_manager.h" diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index bb6ba0c4b7f462..bc920c78d9a6bb 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -35,11 +35,11 @@ #include "common/simple_thread_pool.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_accessor.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" diff --git a/cloud/test/resource_test.cpp b/cloud/test/resource_test.cpp index e2aa09514360e1..e473594c8a07d1 100644 --- a/cloud/test/resource_test.cpp +++ b/cloud/test/resource_test.cpp @@ -36,9 +36,9 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "rate-limiter/rate_limiter.h" #include "resource-manager/resource_manager.h" diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 6f6bf9d0d201ac..9ff4aac0c20749 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -35,11 +35,11 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service_helper.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" diff --git a/cloud/test/schema_kv_test.cpp b/cloud/test/schema_kv_test.cpp index 1840598b40e5da..bab9739ff29d4d 100644 --- a/cloud/test/schema_kv_test.cpp +++ b/cloud/test/schema_kv_test.cpp @@ -26,10 +26,10 @@ #include "common/config.h" #include "common/defer.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" #include "meta-service/meta_service.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" static std::string instance_id = "schema_kv_test"; diff --git a/cloud/test/txn_kv_test.cpp b/cloud/test/txn_kv_test.cpp index 78c8c18da4272b..b63773f38c4cb3 100644 --- a/cloud/test/txn_kv_test.cpp +++ b/cloud/test/txn_kv_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "meta-service/txn_kv.h" +#include "meta-store/txn_kv.h" #include #include @@ -34,12 +34,13 @@ #include "common/stopwatch.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/codec.h" #include "meta-service/doris_txn.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" -#include "meta-service/txn_kv.h" -#include "meta-service/txn_kv_error.h" +#include "meta-store/blob_message.h" +#include "meta-store/codec.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv.h" +#include "meta-store/txn_kv_error.h" using namespace doris::cloud; diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 81a5c8a2d9902c..d2bf82435d8adc 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -37,12 +37,12 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" -#include "meta-service/keys.h" -#include "meta-service/mem_txn_kv.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" #include "meta-service/meta_service_txn.cpp" -#include "meta-service/txn_kv_error.h" +#include "meta-store/keys.h" +#include "meta-store/mem_txn_kv.h" +#include "meta-store/txn_kv_error.h" #include "mock_resource_manager.h" #include "rate-limiter/rate_limiter.h" #include "recycler/recycler.h" From 6bb784abbcca1d2f1745c98eb9f9cbf6859b475e Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 9 Jul 2025 09:49:58 +0800 Subject: [PATCH 175/572] branch-3.0: [fix](coordinator) fix cte with local shuffle throw exception #52870 (#52875) cherry pick from #52870 --- .../java/org/apache/doris/qe/Coordinator.java | 4 +- .../org/apache/doris/qe/LocalShuffleTest.java | 221 ++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index d05fb4e36940c0..cc6dd424ecbad8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -1573,7 +1573,7 @@ private void computeMultiCastFragmentParams() throws Exception { } // process bucket shuffle join on fragment without scan node while (bucketSeq < bucketNum) { - TPlanFragmentDestination dest = setDestination(destParams, params.destinations.size(), + TPlanFragmentDestination dest = setDestination(destParams, destinations.size(), bucketSeq); bucketSeq++; destinations.add(dest); @@ -1611,7 +1611,7 @@ private void computeMultiCastFragmentParams() throws Exception { dest.fragment_instance_id = destParams.instanceExecParams.get(j).instanceId; dest.server = toRpcHost(destParams.instanceExecParams.get(j).host); dest.brpc_server = toBrpcHost(destParams.instanceExecParams.get(j).host); - destParams.instanceExecParams.get(j).recvrId = params.destinations.size(); + destParams.instanceExecParams.get(j).recvrId = destinations.size(); destinations.add(dest); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleTest.java new file mode 100644 index 00000000000000..597b7984cb0322 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/LocalShuffleTest.java @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.planner.PlanFragmentId; +import org.apache.doris.qe.Coordinator.FInstanceExecParam; +import org.apache.doris.qe.Coordinator.FragmentExecParams; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.SetMultimap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; + +public class LocalShuffleTest extends TestWithFeService { + @Override + protected int backendNum() { + return 2; + } + + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + useDatabase("test"); + createTable("CREATE TABLE `oooo_oneid` (\n" + + " `identity_code` varchar(128) NULL COMMENT \"\",\n" + + " `identity_value` varchar(128) NULL COMMENT \"\",\n" + + " `oneid` bigint NULL COMMENT \"\",\n" + + " INDEX idx_oneid (`oneid`) USING INVERTED COMMENT ''\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`identity_code`, `identity_value`)\n" + + "COMMENT 'oooo'\n" + + "DISTRIBUTED BY HASH(`identity_code`, `identity_value`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 2\"\n" + + ")"); + + createTable("CREATE TABLE `aaaa_target` (\n" + + " `base_id` varchar(128) NULL COMMENT \"\",\n" + + " `hash_partition_id` bigint NOT NULL COMMENT \"\",\n" + + " `j_bbbb` text NULL COMMENT \"j_bbbb\"\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`base_id`, `hash_partition_id`)\n" + + "COMMENT 'aaaa'\n" + + "PARTITION BY LIST (`hash_partition_id`)\n" + + "(PARTITION p0 VALUES IN (\"0\"),\n" + + "PARTITION p1 VALUES IN (\"1\"),\n" + + "PARTITION p2 VALUES IN (\"2\"),\n" + + "PARTITION p3 VALUES IN (\"3\"),\n" + + "PARTITION p4 VALUES IN (\"4\"),\n" + + "PARTITION p5 VALUES IN (\"5\"),\n" + + "PARTITION p6 VALUES IN (\"6\"),\n" + + "PARTITION p7 VALUES IN (\"7\"),\n" + + "PARTITION p8 VALUES IN (\"8\"),\n" + + "PARTITION p9 VALUES IN (\"9\"),\n" + + "PARTITION p10 VALUES IN (\"10\"),\n" + + "PARTITION p11 VALUES IN (\"11\"),\n" + + "PARTITION p12 VALUES IN (\"12\"),\n" + + "PARTITION p13 VALUES IN (\"13\"),\n" + + "PARTITION p14 VALUES IN (\"14\"),\n" + + "PARTITION p15 VALUES IN (\"15\"),\n" + + "PARTITION p16 VALUES IN (\"16\"),\n" + + "PARTITION p17 VALUES IN (\"17\"),\n" + + "PARTITION p18 VALUES IN (\"18\"),\n" + + "PARTITION p19 VALUES IN (\"19\"))\n" + + "DISTRIBUTED BY HASH(`base_id`, `hash_partition_id`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 2\"\n" + + ");"); + + createTable("CREATE TABLE `bbbb_target` (\n" + + " `base_id` varchar(128) NULL COMMENT \"\",\n" + + " `hash_partition_id` bigint NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`base_id`, `hash_partition_id`)\n" + + "COMMENT 'bbbb'\n" + + "PARTITION BY LIST (`hash_partition_id`)\n" + + "(PARTITION p0 VALUES IN (\"0\"),\n" + + "PARTITION p1 VALUES IN (\"1\"),\n" + + "PARTITION p2 VALUES IN (\"2\"),\n" + + "PARTITION p3 VALUES IN (\"3\"),\n" + + "PARTITION p4 VALUES IN (\"4\"),\n" + + "PARTITION p5 VALUES IN (\"5\"),\n" + + "PARTITION p6 VALUES IN (\"6\"),\n" + + "PARTITION p7 VALUES IN (\"7\"),\n" + + "PARTITION p8 VALUES IN (\"8\"),\n" + + "PARTITION p9 VALUES IN (\"9\"),\n" + + "PARTITION p10 VALUES IN (\"10\"),\n" + + "PARTITION p11 VALUES IN (\"11\"),\n" + + "PARTITION p12 VALUES IN (\"12\"),\n" + + "PARTITION p13 VALUES IN (\"13\"),\n" + + "PARTITION p14 VALUES IN (\"14\"),\n" + + "PARTITION p15 VALUES IN (\"15\"),\n" + + "PARTITION p16 VALUES IN (\"16\"),\n" + + "PARTITION p17 VALUES IN (\"17\"),\n" + + "PARTITION p18 VALUES IN (\"18\"),\n" + + "PARTITION p19 VALUES IN (\"19\"))\n" + + "DISTRIBUTED BY HASH(`base_id`, `hash_partition_id`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 2\"\n" + + ");"); + + createTable("CREATE TABLE `cccc_target` (\n" + + " `base_id` varchar(128) NULL COMMENT \"\",\n" + + " `hash_partition_id` bigint NOT NULL COMMENT \"\"\n" + + ") ENGINE=OLAP\n" + + "UNIQUE KEY(`base_id`, `hash_partition_id`)\n" + + "COMMENT 'bbbb'\n" + + "PARTITION BY LIST (`hash_partition_id`)\n" + + "(PARTITION p0 VALUES IN (\"0\"),\n" + + "PARTITION p1 VALUES IN (\"1\"),\n" + + "PARTITION p2 VALUES IN (\"2\"),\n" + + "PARTITION p3 VALUES IN (\"3\"),\n" + + "PARTITION p4 VALUES IN (\"4\"),\n" + + "PARTITION p5 VALUES IN (\"5\"),\n" + + "PARTITION p6 VALUES IN (\"6\"),\n" + + "PARTITION p7 VALUES IN (\"7\"),\n" + + "PARTITION p8 VALUES IN (\"8\"),\n" + + "PARTITION p9 VALUES IN (\"9\"),\n" + + "PARTITION p10 VALUES IN (\"10\"),\n" + + "PARTITION p11 VALUES IN (\"11\"),\n" + + "PARTITION p12 VALUES IN (\"12\"),\n" + + "PARTITION p13 VALUES IN (\"13\"),\n" + + "PARTITION p14 VALUES IN (\"14\"),\n" + + "PARTITION p15 VALUES IN (\"15\"),\n" + + "PARTITION p16 VALUES IN (\"16\"),\n" + + "PARTITION p17 VALUES IN (\"17\"),\n" + + "PARTITION p18 VALUES IN (\"18\"),\n" + + "PARTITION p19 VALUES IN (\"19\"))\n" + + "DISTRIBUTED BY HASH(`base_id`, `hash_partition_id`) BUCKETS 1\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 2\"\n" + + ");"); + } + + @Test + public void testCteWithLocalShuffle() throws Exception { + connectContext.getState().reset(); + connectContext.getSessionVariable().parallelPipelineTaskNum = 2; + connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name()); + connectContext.getSessionVariable().setQueryTimeoutS(10); + connectContext.getSessionVariable().setDisableJoinReorder(true); + + String sql = "with oooo_oneid as (select identity_value from test.oooo_oneid),\n" + + " aaaa_frm_$oooo as (select j_bbbb,base_id from test.aaaa_target),\n" + + " bbbb_frm_$aaaa_target as (select * from bbbb_target where base_id in (select 1 from aaaa_frm_$oooo)),\n" + + " bbbb_cdn_1602527468_lnkb_bbbb_target$aaaa_target as (select base_id from bbbb_frm_$aaaa_target),\n" + + " bbbb_frm_$oooo as (select *\n" + + " from bbbb_target\n" + + " where base_id in (select identity_value from oooo_oneid)\n" + + " ),\n" + + " cccc_frm_$bbbb_target as (select * from cccc_target where base_id in (select 1 from bbbb_frm_$oooo))\n" + + "select\n" + + " j_bbbb in (select base_id from bbbb_cdn_1602527468_lnkb_bbbb_target$aaaa_target),\n" + + " j_bbbb in (select base_id from bbbb_cdn_1602527468_lnkb_bbbb_target$aaaa_target)\n" + + "from (select j_bbbb\n" + + " from oooo_oneid\n" + + " left outer join aaaa_frm_$oooo on (oooo_oneid.identity_value = aaaa_frm_$oooo.base_id)\n" + + " left outer join bbbb_frm_$oooo on (oooo_oneid.identity_value = bbbb_frm_$oooo.base_id)\n" + + ")final_plain_table"; + StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql); + try { + stmtExecutor.execute(); + } catch (Throwable t) { + // ignore + } + Coordinator coord = stmtExecutor.getCoord(); + Map fragmentExecParamsMap = coord.getFragmentExecParamsMap(); + + for (FragmentExecParams fragmentExecParams : fragmentExecParamsMap.values()) { + // skip check root fragment + if (fragmentExecParams.fragment.getChildren().isEmpty()) { + continue; + } + SetMultimap receiverIds = LinkedHashMultimap.create(); + ArrayListMultimap hostToInstances = ArrayListMultimap.create(); + boolean setRecvrId = false; + for (FInstanceExecParam instanceExecParam : fragmentExecParams.instanceExecParams) { + if (instanceExecParam.recvrId != -1) { + setRecvrId = true; + } + receiverIds.put(instanceExecParam.host, instanceExecParam.recvrId); + hostToInstances.put(instanceExecParam.host, instanceExecParam); + } + if (!setRecvrId) { + // skip check when share broadcast hash table + continue; + } + for (Entry> hostToReceiverIds : receiverIds.asMap() + .entrySet()) { + // if this host has 2 instances, it should contain 2 receiverId + Assertions.assertEquals( + hostToInstances.get(hostToReceiverIds.getKey()).size(), + hostToReceiverIds.getValue().size() + ); + } + } + } +} From e04a4aac216b94c7ef6fa0a274ad9e5fb8fcf77b Mon Sep 17 00:00:00 2001 From: amory Date: Wed, 9 Jul 2025 09:52:08 +0800 Subject: [PATCH 176/572] [fix](serde)fix write to orc file (#52884) ### What problem does this PR solve? backport https://github.com/apache/doris/pull/52676 Issue Number: close #xxx --- .../serde/data_type_bitmap_serde.cpp | 34 +++++++++--- .../serde/data_type_date64_serde.cpp | 46 ++++++++++++---- .../data_types/serde/data_type_hll_serde.cpp | 34 +++++++++--- .../data_types/serde/data_type_ipv6_serde.cpp | 47 ++++++++++++---- .../serde/data_type_jsonb_serde.cpp | 51 +++++++++++++----- .../serde/data_type_number_serde.cpp | 37 ++++++++++--- .../serde/data_type_object_serde.cpp | 54 +++++++++++++------ .../serde/data_type_quantilestate_serde.h | 34 +++++++++--- be/src/vec/data_types/serde/data_type_serde.h | 25 --------- 9 files changed, 257 insertions(+), 105 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp b/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp index f60d054df31893..9697a3b71bbd34 100644 --- a/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp @@ -191,23 +191,43 @@ Status DataTypeBitMapSerDe::write_column_to_orc(const std::string& timezone, con std::vector& buffer_list) const { auto& col_data = assert_cast(column); orc::StringVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { auto bitmap_value = const_cast(col_data.get_element(row_id)); size_t len = bitmap_value.getSizeInBytes(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - + total_size += len; + } + } + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t row_id = start; row_id < end; row_id++) { + if (cur_batch->notNull[row_id] == 1) { + auto bitmap_value = const_cast(col_data.get_element(row_id)); + size_t len = bitmap_value.getSizeInBytes(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with len " + "{} exceed total_size {} . ", + offset, len, total_size); + } bitmap_value.write_to(const_cast(bufferRef.data) + offset); cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; cur_batch->length[row_id] = len; offset += len; } } - cur_batch->numElements = end - start; return Status::OK(); } diff --git a/be/src/vec/data_types/serde/data_type_date64_serde.cpp b/be/src/vec/data_types/serde/data_type_date64_serde.cpp index 1ce0ff54ac8506..0aa81907180d60 100644 --- a/be/src/vec/data_types/serde/data_type_date64_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_date64_serde.cpp @@ -324,23 +324,47 @@ Status DataTypeDate64SerDe::write_column_to_orc(const std::string& timezone, con auto& col_data = static_cast&>(column).get_data(); orc::StringVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + std::vector serialized_values; + std::vector valid_row_indices; + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { - if (cur_batch->notNull[row_id] == 0) { - continue; + if (cur_batch->notNull[row_id] == 1) { + char buf[64]; + size_t len = binary_cast(col_data[row_id]).to_buffer(buf); + total_size += len; + // avoid copy + serialized_values.emplace_back(buf, len); + valid_row_indices.push_back(row_id); } - - int len = binary_cast(col_data[row_id]) - .to_buffer(const_cast(bufferRef.data) + offset); - - REALLOC_MEMORY_FOR_ORC_WRITER() - + } + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t i = 0; i < serialized_values.size(); i++) { + const auto& serialized_value = serialized_values[i]; + size_t row_id = valid_row_indices[i]; + size_t len = serialized_value.length(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with len {} " + "exceed total_size {} . ", + offset, len, total_size); + } + memcpy(const_cast(bufferRef.data) + offset, serialized_value.data(), len); cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; cur_batch->length[row_id] = len; offset += len; } - cur_batch->numElements = end - start; return Status::OK(); } diff --git a/be/src/vec/data_types/serde/data_type_hll_serde.cpp b/be/src/vec/data_types/serde/data_type_hll_serde.cpp index aba3a9d0619a71..156e1dce7b62a9 100644 --- a/be/src/vec/data_types/serde/data_type_hll_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_hll_serde.cpp @@ -184,23 +184,43 @@ Status DataTypeHLLSerDe::write_column_to_orc(const std::string& timezone, const int end, std::vector& buffer_list) const { auto& col_data = assert_cast(column); orc::StringVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { auto hll_value = const_cast(col_data.get_element(row_id)); size_t len = hll_value.max_serialized_size(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - + total_size += len; + } + } + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t row_id = start; row_id < end; row_id++) { + if (cur_batch->notNull[row_id] == 1) { + auto hll_value = const_cast(col_data.get_element(row_id)); + size_t len = hll_value.max_serialized_size(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with len " + "{} exceed total_size {} ", + offset, len, total_size); + } hll_value.serialize((uint8_t*)(bufferRef.data) + offset); cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; cur_batch->length[row_id] = len; offset += len; } } - cur_batch->numElements = end - start; return Status::OK(); } diff --git a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp index 83acbf9cda130e..a25628f62b5e33 100644 --- a/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_ipv6_serde.cpp @@ -198,20 +198,45 @@ Status DataTypeIPv6SerDe::write_column_to_orc(const std::string& timezone, const const auto& col_data = assert_cast(column).get_data(); orc::StringVectorBatch* cur_batch = assert_cast(orc_col_batch); - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + std::vector serialized_values; + std::vector valid_row_indices; + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { - std::string ipv6_str = IPv6Value::to_string(col_data[row_id]); - size_t len = ipv6_str.size(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - - strcpy(const_cast(bufferRef.data) + offset, ipv6_str.c_str()); - cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; - cur_batch->length[row_id] = len; - offset += len; + auto serialized_value = IPv6Value::to_string(col_data[row_id]); + serialized_values.push_back(std::move(serialized_value)); + size_t len = serialized_values.back().length(); + total_size += len; + valid_row_indices.push_back(row_id); + } + } + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t i = 0; i < serialized_values.size(); i++) { + const auto& serialized_value = serialized_values[i]; + size_t row_id = valid_row_indices[i]; + size_t len = serialized_value.length(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with len {} " + "exceed total_size {} . ", + offset, len, total_size); } + memcpy(const_cast(bufferRef.data) + offset, serialized_value.data(), len); + cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; + cur_batch->length[row_id] = len; + offset += len; } cur_batch->numElements = end - start; diff --git a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp index eb6c783cf28579..181669555d6a35 100644 --- a/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_jsonb_serde.cpp @@ -142,25 +142,48 @@ Status DataTypeJsonbSerDe::write_column_to_orc(const std::string& timezone, cons int end, std::vector& buffer_list) const { auto* cur_batch = dynamic_cast(orc_col_batch); const auto& string_column = assert_cast(column); - - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + std::vector serialized_values; + std::vector valid_row_indices; + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { std::string_view string_ref = string_column.get_data_at(row_id).to_string_view(); - auto serialized_value = std::make_unique( - JsonbToJson::jsonb_to_json_string(string_ref.data(), string_ref.size())); - auto len = serialized_value->size(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - - memcpy(const_cast(bufferRef.data) + offset, serialized_value->data(), len); - cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; - cur_batch->length[row_id] = len; - offset += len; + auto serialized_value = + JsonbToJson::jsonb_to_json_string(string_ref.data(), string_ref.size()); + serialized_values.push_back(std::move(serialized_value)); + size_t len = serialized_values.back().length(); + total_size += len; + valid_row_indices.push_back(row_id); } } - + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t i = 0; i < serialized_values.size(); i++) { + const auto& serialized_value = serialized_values[i]; + size_t row_id = valid_row_indices[i]; + size_t len = serialized_value.length(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with len {} " + "exceed total_size {} . ", + offset, len, total_size); + } + memcpy(const_cast(bufferRef.data) + offset, serialized_value.data(), len); + cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; + cur_batch->length[row_id] = len; + offset += len; + } cur_batch->numElements = end - start; return Status::OK(); } diff --git a/be/src/vec/data_types/serde/data_type_number_serde.cpp b/be/src/vec/data_types/serde/data_type_number_serde.cpp index 20ce78bf9c2db0..c691c55d8d2013 100644 --- a/be/src/vec/data_types/serde/data_type_number_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_number_serde.cpp @@ -351,17 +351,40 @@ Status DataTypeNumberSerDe::write_column_to_orc(const std::string& timezone, if constexpr (std::is_same_v) { // largeint orc::StringVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { std::string value_str = fmt::format("{}", col_data[row_id]); size_t len = value_str.size(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - - strcpy(const_cast(bufferRef.data) + offset, value_str.c_str()); + total_size += len; + } + } + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", + total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: fill the data and update the batch + size_t offset = 0; + for (size_t row_id = start; row_id < end; row_id++) { + if (cur_batch->notNull[row_id] == 1) { + std::string value_str = fmt::format("{}", col_data[row_id]); + size_t len = value_str.size(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with " + "len {} exceed total_size {} . ", + offset, len, total_size); + } + // do not use strcpy here, because this buffer is not null-terminated + memcpy(const_cast(bufferRef.data) + offset, value_str.c_str(), len); cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; cur_batch->length[row_id] = len; offset += len; diff --git a/be/src/vec/data_types/serde/data_type_object_serde.cpp b/be/src/vec/data_types/serde/data_type_object_serde.cpp index 31d91444823ff4..3a944904e54f87 100644 --- a/be/src/vec/data_types/serde/data_type_object_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_object_serde.cpp @@ -208,27 +208,47 @@ Status DataTypeObjectSerDe::write_column_to_orc(const std::string& timezone, con std::vector& buffer_list) const { const auto* var = check_and_get_column(column); orc::StringVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + std::vector serialized_values; + std::vector valid_row_indices; + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { - auto serialized_value = std::make_unique(); - if (!var->serialize_one_row_to_string(row_id, serialized_value.get())) { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, "Failed to serialize variant {}", - var->dump_structure()); - } - auto len = serialized_value->length(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - - memcpy(const_cast(bufferRef.data) + offset, serialized_value->data(), len); - cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; - cur_batch->length[row_id] = len; - offset += len; + // avoid move the string data, use emplace_back to construct in place + serialized_values.emplace_back(); + RETURN_IF_ERROR(var->serialize_one_row_to_string(row_id, &serialized_values.back())); + size_t len = serialized_values.back().length(); + total_size += len; + valid_row_indices.push_back(row_id); } } - + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.emplace_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t i = 0; i < serialized_values.size(); i++) { + const auto& serialized_value = serialized_values[i]; + size_t row_id = valid_row_indices[i]; + size_t len = serialized_value.length(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with len {} " + "exceed total_size {} . ", + offset, len, total_size); + } + memcpy(const_cast(bufferRef.data) + offset, serialized_value.data(), len); + cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; + cur_batch->length[row_id] = len; + offset += len; + } cur_batch->numElements = end - start; return Status::OK(); } diff --git a/be/src/vec/data_types/serde/data_type_quantilestate_serde.h b/be/src/vec/data_types/serde/data_type_quantilestate_serde.h index e24a3a295439ee..675300008106c3 100644 --- a/be/src/vec/data_types/serde/data_type_quantilestate_serde.h +++ b/be/src/vec/data_types/serde/data_type_quantilestate_serde.h @@ -154,16 +154,38 @@ class DataTypeQuantileStateSerDe : public DataTypeSerDe { std::vector& buffer_list) const override { auto& col_data = assert_cast(column); orc::StringVectorBatch* cur_batch = dynamic_cast(orc_col_batch); - - INIT_MEMORY_FOR_ORC_WRITER() - + // First pass: calculate total memory needed and collect serialized values + size_t total_size = 0; for (size_t row_id = start; row_id < end; row_id++) { if (cur_batch->notNull[row_id] == 1) { auto quantilestate_value = const_cast(col_data.get_element(row_id)); size_t len = quantilestate_value.get_serialized_size(); - - REALLOC_MEMORY_FOR_ORC_WRITER() - + total_size += len; + } + } + // Allocate continues memory based on calculated size + char* ptr = (char*)malloc(total_size); + if (!ptr) { + return Status::InternalError( + "malloc memory {} error when write variant column data to orc file.", + total_size); + } + StringRef bufferRef; + bufferRef.data = ptr; + bufferRef.size = total_size; + buffer_list.push_back(bufferRef); + // Second pass: copy data to allocated memory + size_t offset = 0; + for (size_t row_id = start; row_id < end; row_id++) { + if (cur_batch->notNull[row_id] == 1) { + auto quantilestate_value = const_cast(col_data.get_element(row_id)); + size_t len = quantilestate_value.get_serialized_size(); + if (offset + len > total_size) { + return Status::InternalError( + "Buffer overflow when writing column data to ORC file. offset {} with " + "len {} exceed total_size {} . ", + offset, len, total_size); + } quantilestate_value.serialize((uint8_t*)(bufferRef.data) + offset); cur_batch->data[row_id] = const_cast(bufferRef.data) + offset; cur_batch->length[row_id] = len; diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h index f7446d5c52a7aa..d9da4dd872cbf3 100644 --- a/be/src/vec/data_types/serde/data_type_serde.h +++ b/be/src/vec/data_types/serde/data_type_serde.h @@ -75,31 +75,6 @@ struct ColumnVectorBatch; ++*num_deserialized; \ } -#define INIT_MEMORY_FOR_ORC_WRITER() \ - char* ptr = (char*)malloc(BUFFER_UNIT_SIZE); \ - if (!ptr) { \ - return Status::InternalError( \ - "malloc memory error when write largeint column data to orc file."); \ - } \ - StringRef bufferRef; \ - bufferRef.data = ptr; \ - bufferRef.size = BUFFER_UNIT_SIZE; \ - size_t offset = 0; \ - buffer_list.emplace_back(bufferRef); - -#define REALLOC_MEMORY_FOR_ORC_WRITER() \ - while (bufferRef.size - BUFFER_RESERVED_SIZE < offset + len) { \ - char* new_ptr = (char*)malloc(bufferRef.size + BUFFER_UNIT_SIZE); \ - if (!new_ptr) { \ - return Status::InternalError( \ - "malloc memory error when write largeint column data to orc file."); \ - } \ - memcpy(new_ptr, bufferRef.data, bufferRef.size); \ - free(const_cast(bufferRef.data)); \ - bufferRef.data = new_ptr; \ - bufferRef.size = bufferRef.size + BUFFER_UNIT_SIZE; \ - } - namespace doris { class PValues; class JsonbValue; From 9c5e721bbf3cf9f747bd84afb5c936b433bfbc30 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Wed, 9 Jul 2025 09:58:56 +0800 Subject: [PATCH 177/572] branch-3.0; [Opt](cloud-mow) Do fast retry when commit compaction job for mow tablet (#52476) (#52952) pick https://github.com/apache/doris/pull/52476 --- cloud/src/meta-service/meta_service_job.cpp | 134 +++++++++++--------- 1 file changed, 73 insertions(+), 61 deletions(-) diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 7c1f13aa8b6cd7..000edc76a30395 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -1407,77 +1407,89 @@ void MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr return; } - bool need_commit = false; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - msg = "failed to create txn"; - return; - } + for (int retry = 0; retry <= 1; retry++) { + bool need_commit = false; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + msg = "failed to create txn"; + return; + } - int64_t tablet_id = request->job().idx().tablet_id(); - if (tablet_id <= 0) { - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "no valid tablet_id given"; - return; - } - auto& tablet_idx = const_cast(request->job().idx()); - if (!tablet_idx.has_table_id() || !tablet_idx.has_index_id() || - !tablet_idx.has_partition_id()) { - get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, tablet_idx); - if (code != MetaServiceCode::OK) return; - } - // Check if tablet has been dropped - if (is_dropped_tablet(txn.get(), instance_id, tablet_idx.index_id(), - tablet_idx.partition_id())) { - code = MetaServiceCode::TABLET_NOT_FOUND; - msg = fmt::format("tablet {} has been dropped", tablet_id); - return; - } + int64_t tablet_id = request->job().idx().tablet_id(); + if (tablet_id <= 0) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "no valid tablet_id given"; + return; + } + auto& tablet_idx = const_cast(request->job().idx()); + if (!tablet_idx.has_table_id() || !tablet_idx.has_index_id() || + !tablet_idx.has_partition_id()) { + get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, tablet_idx); + if (code != MetaServiceCode::OK) return; + } + // Check if tablet has been dropped + if (is_dropped_tablet(txn.get(), instance_id, tablet_idx.index_id(), + tablet_idx.partition_id())) { + code = MetaServiceCode::TABLET_NOT_FOUND; + msg = fmt::format("tablet {} has been dropped", tablet_id); + return; + } - // TODO(gavin): remove duplicated code with start_tablet_job() - // Begin to process finish tablet job - std::string job_key = job_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), - tablet_idx.partition_id(), tablet_id}); - std::string job_val; - err = txn->get(job_key, &job_val); - if (err != TxnErrorCode::TXN_OK) { - SS << (err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "job not found," : "internal error,") - << " instance_id=" << instance_id << " tablet_id=" << tablet_id - << " job=" << proto_to_json(request->job()) << " err=" << err; - msg = ss.str(); - code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::INVALID_ARGUMENT - : cast_as(err); - return; - } - TabletJobInfoPB recorded_job; - recorded_job.ParseFromString(job_val); - VLOG_DEBUG << "get tablet job, tablet_id=" << tablet_id - << " job=" << proto_to_json(recorded_job); + // TODO(gavin): remove duplicated code with start_tablet_job() + // Begin to process finish tablet job + std::string job_key = + job_tablet_key({instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_id}); + std::string job_val; + err = txn->get(job_key, &job_val); + if (err != TxnErrorCode::TXN_OK) { + SS << (err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "job not found," : "internal error,") + << " instance_id=" << instance_id << " tablet_id=" << tablet_id + << " job=" << proto_to_json(request->job()) << " err=" << err; + msg = ss.str(); + code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::INVALID_ARGUMENT + : cast_as(err); + return; + } + TabletJobInfoPB recorded_job; + recorded_job.ParseFromString(job_val); + VLOG_DEBUG << "get tablet job, tablet_id=" << tablet_id + << " job=" << proto_to_json(recorded_job); + + if (!request->job().compaction().empty()) { + // Process compaction commit + process_compaction_job(code, msg, ss, txn, request, response, recorded_job, instance_id, + job_key, need_commit); + } else if (request->job().has_schema_change()) { + // Process schema change commit + process_schema_change_job(code, msg, ss, txn, request, response, recorded_job, + instance_id, job_key, need_commit); + } - DORIS_CLOUD_DEFER { if (!need_commit) return; - TxnErrorCode err = txn->commit(); + err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { + if (err == TxnErrorCode::TXN_CONFLICT) { + if (retry == 0 && !request->job().compaction().empty() && + request->job().compaction(0).has_delete_bitmap_lock_initiator()) { + // Do a fast retry for mow when commit compaction job. + // The only fdb txn conflict here is that during the compaction job commit, + // a compaction lease RPC comes and finishes before the commit, + // so we retry to commit the compaction job again. + response->Clear(); + code = MetaServiceCode::OK; + msg.clear(); + continue; + } + } + code = cast_as(err); ss << "failed to commit job kv, err=" << err; msg = ss.str(); return; } - }; - - // Process compaction commit - if (!request->job().compaction().empty()) { - process_compaction_job(code, msg, ss, txn, request, response, recorded_job, instance_id, - job_key, need_commit); - return; - } - - // Process schema change commit - if (request->job().has_schema_change()) { - process_schema_change_job(code, msg, ss, txn, request, response, recorded_job, instance_id, - job_key, need_commit); - return; + break; } } From 08b741371c0be299683896c9fe236f88e19cd790 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:22:34 +0800 Subject: [PATCH 178/572] branch-3.0: [fix](iceberg)Ensure proper authentication context before accessing Iceberg Catalog #52149 (#52233) Cherry-picked from #52149 Co-authored-by: Calvin Kirs Co-authored-by: Tiewei Fang --- .../datasource/iceberg/IcebergExternalCatalog.java | 12 ++++++++++++ .../datasource/iceberg/IcebergExternalDatabase.java | 12 +++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 7282de52aa0d22..30590f5af26070 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -68,6 +68,18 @@ protected void initLocalObjectsImpl() { metadataOps = ops; } + /** + * Returns the underlying {@link Catalog} instance used by this external catalog. + * + *

Warning: This method does not handle any authentication logic. If the + * returned catalog implementation relies on external systems + * that require authentication — especially in environments where Kerberos is enabled — the caller is + * fully responsible for ensuring the appropriate authentication has been performed before + * invoking this method. + *

Failing to authenticate beforehand may result in authorization errors or IO failures. + * + * @return the underlying catalog instance + */ public Catalog getCatalog() { makeSureInitialized(); return ((IcebergMetadataOps) metadataOps).getCatalog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index 7a1a53825a15d3..2e4e45f062e7be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -41,8 +41,14 @@ public IcebergExternalTable buildTableInternal(String remoteTableName, String lo } public String getLocation() { - Map props = ((SupportsNamespaces) ((IcebergExternalCatalog) getCatalog()).getCatalog()) - .loadNamespaceMetadata(Namespace.of(name)); - return props.getOrDefault("location", ""); + try { + return extCatalog.getPreExecutionAuthenticator().execute(() -> { + Map props = ((SupportsNamespaces) ((IcebergExternalCatalog) getCatalog()).getCatalog()) + .loadNamespaceMetadata(Namespace.of(name)); + return props.getOrDefault("location", ""); + }); + } catch (Exception e) { + throw new RuntimeException("Failed to get location for Iceberg database: " + name, e); + } } } From 8b62d349fca5116899669ab2fe780787872a152f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:23:20 +0800 Subject: [PATCH 179/572] branch-3.0: [fix](job) remove can not transform RUNNING to NEED_SCHEDULE limit #52887 (#52908) Cherry-picked from #52887 Co-authored-by: hui lai --- .../org/apache/doris/load/routineload/RoutineLoadJob.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 5cb1b6147a9ae7..d7e3c6fc49450f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -873,11 +873,6 @@ public boolean isAbnormalPause() { // All of private method could not be call without lock private void checkStateTransform(RoutineLoadJob.JobState desireState) throws UserException { switch (state) { - case RUNNING: - if (desireState == JobState.NEED_SCHEDULE) { - throw new DdlException("Could not transform " + state + " to " + desireState); - } - break; case PAUSED: if (desireState == JobState.PAUSED) { throw new DdlException("Could not transform " + state + " to " + desireState); From 177e4dcc8562c9a5f0acc18590118235da03d352 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:23:56 +0800 Subject: [PATCH 180/572] branch-3.0: [opt](create table) Fixed table creation becomes slower as the number of tablets increases #52688 (#52918) Cherry-picked from #52688 Co-authored-by: deardeng --- .../apache/doris/catalog/TabletInvertedIndex.java | 14 ++++++++++++++ .../org/apache/doris/system/SystemInfoService.java | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java index 0362205dbdef94..a1faada3fea669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java @@ -718,6 +718,20 @@ public List getReplicasByTabletId(long tabletId) { } } + public Long getTabletSizeByBackendId(long backendId) { + Long ret = 0L; + long stamp = readLock(); + try { + Map replicaMetaWithBackend = backingReplicaMetaTable.row(backendId); + if (replicaMetaWithBackend != null) { + ret += replicaMetaWithBackend.size(); + } + } finally { + readUnlock(stamp); + } + return ret; + } + public List getTabletIdsByBackendId(long backendId) { List tabletIds = Lists.newArrayList(); long stamp = readLock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java index ca67dff585b193..660f256a592476 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java @@ -470,7 +470,7 @@ public int getStartPosOfRoundRobin(Tag tag, TStorageMedium storageMedium, boolea long minBeTabletsNum = Long.MAX_VALUE; int minIndex = -1; for (int i = 0; i < beIds.size(); ++i) { - long tabletsNum = Env.getCurrentInvertedIndex().getTabletIdsByBackendId(beIds.get(i)).size(); + long tabletsNum = Env.getCurrentInvertedIndex().getTabletSizeByBackendId(beIds.get(i)); if (tabletsNum < minBeTabletsNum) { minBeTabletsNum = tabletsNum; minIndex = i; From d51643fa0d3571324071f6abe955f7f7ba7b7da8 Mon Sep 17 00:00:00 2001 From: Xin Liao Date: Wed, 9 Jul 2025 10:28:49 +0800 Subject: [PATCH 181/572] branch-3.0: [fix](load) Convert RowInBlock* to shared_ptr to fix potential memory leaks in MemTable (#52902) (#52965) pick from: #52902 --- be/src/olap/memtable.cpp | 62 ++++++++++++++++++---------------------- be/src/olap/memtable.h | 6 ++-- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index b0d930e6d09eaf..1bc3960dd6dfcc 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -156,8 +156,6 @@ MemTable::~MemTable() { } } } - std::for_each(_row_in_blocks.begin(), _row_in_blocks.end(), - std::default_delete()); // Arena has to be destroyed after agg state, because some agg state's memory may be // allocated in arena. _arena.reset(); @@ -216,7 +214,7 @@ Status MemTable::insert(const vectorized::Block* input_block, RETURN_IF_ERROR(_input_mutable_block.add_rows(input_block, row_idxs.data(), row_idxs.data() + num_rows, &_column_offset)); for (int i = 0; i < num_rows; i++) { - _row_in_blocks.emplace_back(new RowInBlock {cursor_in_mutableblock + i}); + _row_in_blocks.emplace_back(std::make_shared(cursor_in_mutableblock + i)); } _stat.raw_rows += num_rows; @@ -264,7 +262,7 @@ size_t MemTable::_sort() { // sort new rows Tie tie = Tie(_last_sorted_pos, _row_in_blocks.size()); for (size_t i = 0; i < _tablet_schema->num_key_columns(); i++) { - auto cmp = [&](const RowInBlock* lhs, const RowInBlock* rhs) -> int { + auto cmp = [&](RowInBlock* lhs, RowInBlock* rhs) -> int { return _input_mutable_block.compare_one_column(lhs->_row_pos, rhs->_row_pos, i, -1); }; _sort_one_column(_row_in_blocks, tie, cmp); @@ -275,16 +273,17 @@ size_t MemTable::_sort() { while (iter.next()) { pdqsort(std::next(_row_in_blocks.begin(), iter.left()), std::next(_row_in_blocks.begin(), iter.right()), - [&is_dup](const RowInBlock* lhs, const RowInBlock* rhs) -> bool { + [&is_dup](const std::shared_ptr& lhs, + const std::shared_ptr& rhs) -> bool { return is_dup ? lhs->_row_pos > rhs->_row_pos : lhs->_row_pos < rhs->_row_pos; }); same_keys_num += iter.right() - iter.left(); } // merge new rows and old rows _vec_row_comparator->set_block(&_input_mutable_block); - auto cmp_func = [this, is_dup, &same_keys_num](const RowInBlock* l, - const RowInBlock* r) -> bool { - auto value = (*(this->_vec_row_comparator))(l, r); + auto cmp_func = [this, is_dup, &same_keys_num](const std::shared_ptr& l, + const std::shared_ptr& r) -> bool { + auto value = (*(this->_vec_row_comparator))(l.get(), r.get()); if (value == 0) { same_keys_num++; return is_dup ? l->_row_pos > r->_row_pos : l->_row_pos < r->_row_pos; @@ -308,14 +307,10 @@ Status MemTable::_sort_by_cluster_keys() { auto clone_block = in_block.clone_without_columns(); _output_mutable_block = vectorized::MutableBlock::build_mutable_block(&clone_block); - std::vector row_in_blocks; - std::unique_ptr> row_in_blocks_deleter((int*)0x01, [&](int*) { - std::for_each(row_in_blocks.begin(), row_in_blocks.end(), - std::default_delete()); - }); + std::vector> row_in_blocks; row_in_blocks.reserve(mutable_block.rows()); for (size_t i = 0; i < mutable_block.rows(); i++) { - row_in_blocks.emplace_back(new RowInBlock {i}); + row_in_blocks.emplace_back(std::make_shared(i)); } Tie tie = Tie(0, mutable_block.rows()); @@ -336,9 +331,8 @@ Status MemTable::_sort_by_cluster_keys() { while (iter.next()) { pdqsort(std::next(row_in_blocks.begin(), iter.left()), std::next(row_in_blocks.begin(), iter.right()), - [](const RowInBlock* lhs, const RowInBlock* rhs) -> bool { - return lhs->_row_pos < rhs->_row_pos; - }); + [](const std::shared_ptr& lhs, const std::shared_ptr& rhs) + -> bool { return lhs->_row_pos < rhs->_row_pos; }); } in_block = mutable_block.to_block(); @@ -353,16 +347,16 @@ Status MemTable::_sort_by_cluster_keys() { row_pos_vec.data() + in_block.rows(), &_column_offset); } -void MemTable::_sort_one_column(std::vector& row_in_blocks, Tie& tie, - std::function cmp) { +void MemTable::_sort_one_column(std::vector>& row_in_blocks, Tie& tie, + std::function cmp) { auto iter = tie.iter(); while (iter.next()) { - pdqsort(std::next(row_in_blocks.begin(), iter.left()), - std::next(row_in_blocks.begin(), iter.right()), - [&cmp](auto lhs, auto rhs) -> bool { return cmp(lhs, rhs) < 0; }); + pdqsort(std::next(row_in_blocks.begin(), static_cast(iter.left())), + std::next(row_in_blocks.begin(), static_cast(iter.right())), + [&cmp](auto lhs, auto rhs) -> bool { return cmp(lhs.get(), rhs.get()) < 0; }); tie[iter.left()] = 0; - for (int i = iter.left() + 1; i < iter.right(); i++) { - tie[i] = (cmp(row_in_blocks[i - 1], row_in_blocks[i]) == 0); + for (auto i = iter.left() + 1; i < iter.right(); i++) { + tie[i] = (cmp(row_in_blocks[i - 1].get(), row_in_blocks[i].get()) == 0); } } } @@ -423,14 +417,14 @@ void MemTable::_aggregate() { vectorized::MutableBlock::build_mutable_block(&in_block); _vec_row_comparator->set_block(&mutable_block); auto& block_data = in_block.get_columns_with_type_and_name(); - std::vector temp_row_in_blocks; + std::vector> temp_row_in_blocks; temp_row_in_blocks.reserve(_last_sorted_pos); RowInBlock* prev_row = nullptr; int row_pos = -1; //only init agg if needed - for (int i = 0; i < _row_in_blocks.size(); i++) { - if (!temp_row_in_blocks.empty() && - (*_vec_row_comparator)(prev_row, _row_in_blocks[i]) == 0) { + for (const auto& row_ptr : _row_in_blocks) { + RowInBlock* current_row = row_ptr.get(); + if (!temp_row_in_blocks.empty() && (*_vec_row_comparator)(prev_row, current_row) == 0) { if (!prev_row->has_init_agg()) { prev_row->init_agg_places( _arena->aligned_alloc(_total_size_of_aggregate_states, 16), @@ -445,20 +439,20 @@ void MemTable::_aggregate() { } } _stat.merged_rows++; - _aggregate_two_row_in_block(mutable_block, _row_in_blocks[i], prev_row); + _aggregate_two_row_in_block(mutable_block, current_row, prev_row); } else { - prev_row = _row_in_blocks[i]; + prev_row = current_row; if (!temp_row_in_blocks.empty()) { // no more rows to merge for prev row, finalize it - _finalize_one_row(temp_row_in_blocks.back(), block_data, row_pos); + _finalize_one_row(temp_row_in_blocks.back().get(), block_data, row_pos); } - temp_row_in_blocks.push_back(prev_row); + temp_row_in_blocks.push_back(row_ptr); row_pos++; } } if (!temp_row_in_blocks.empty()) { - // finalize the last low - _finalize_one_row(temp_row_in_blocks.back(), block_data, row_pos); + // finalize the last row + _finalize_one_row(temp_row_in_blocks.back().get(), block_data, row_pos); } if constexpr (!is_final) { // if is not final, we collect the agg results to input_block and then continue to insert diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 4ae92c2d2d8949..1e08891f3f8fe7 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -250,8 +250,8 @@ class MemTable { //return number of same keys size_t _sort(); Status _sort_by_cluster_keys(); - void _sort_one_column(std::vector& row_in_blocks, Tie& tie, - std::function cmp); + void _sort_one_column(std::vector>& row_in_blocks, Tie& tie, + std::function cmp); template void _finalize_one_row(RowInBlock* row, const vectorized::ColumnsWithTypeAndName& block_data, int row_pos); @@ -264,7 +264,7 @@ class MemTable { std::vector _agg_functions; std::vector _offsets_of_aggregate_states; size_t _total_size_of_aggregate_states; - std::vector _row_in_blocks; + std::vector> _row_in_blocks; size_t _num_columns; int32_t _seq_col_idx_in_block = -1; From 1bd6984a1cf4fb3af4c0c50bb35f22005b0f5280 Mon Sep 17 00:00:00 2001 From: zzzxl Date: Wed, 9 Jul 2025 10:41:26 +0800 Subject: [PATCH 182/572] [fix](test) fix nonConcurrent case failure (#52769) https://github.com/apache/doris/pull/52681 --- .../data/inverted_index_p0/load/test_stream_load.out | 4 ---- .../suites/inverted_index_p0/load/test_stream_load.groovy | 6 ------ 2 files changed, 10 deletions(-) diff --git a/regression-test/data/inverted_index_p0/load/test_stream_load.out b/regression-test/data/inverted_index_p0/load/test_stream_load.out index c09ee5451bc991..ac6411d73fa0b2 100644 --- a/regression-test/data/inverted_index_p0/load/test_stream_load.out +++ b/regression-test/data/inverted_index_p0/load/test_stream_load.out @@ -11,7 +11,3 @@ davidjhulse/davesbingrewardsbot davidjhulse/davesbingrewardsbot davidjhulse/davesbingrewardsbot --- !sql_1 -- -davidjhulse/davesbingrewardsbot -davidjhulse/davesbingrewardsbot - diff --git a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy index d9268eec9d1e07..73a68455cede15 100644 --- a/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy +++ b/regression-test/suites/inverted_index_p0/load/test_stream_load.groovy @@ -113,12 +113,6 @@ suite("test_stream_load_with_inverted_index_p0", "p0, nonConcurrent") { } test.call("V2") set_be_config("inverted_index_ram_dir_enable", "false") - setFeConfigTemporary([enable_inverted_index_v1_for_variant: false]) { - if (isCloudMode()) { - return; - } - test.call("V1") - } test.call("V2") set_be_config("inverted_index_ram_dir_enable", "true") } \ No newline at end of file From cfa4f6701cb9398854ee0a87cc1d0e3aa07b0696 Mon Sep 17 00:00:00 2001 From: zhannngchen Date: Wed, 9 Jul 2025 11:38:02 +0800 Subject: [PATCH 183/572] branch-3.0:[fix](mow) fix potential mem leak for DeleteBitmap::get_agg (#52718, #52756) (#52931) cherry-pick #52718, #52756 --- be/src/olap/tablet_meta.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index d2fc09455143d5..4a86bbbb0132cf 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1472,13 +1472,15 @@ std::shared_ptr DeleteBitmap::get_agg(const BitmapKey& bmk) co val->bitmap |= reinterpret_cast( DeleteBitmapAggCache::instance()->value(handle2)) ->bitmap; - DeleteBitmapAggCache::instance()->release(handle2); VLOG_DEBUG << "get agg cache version=" << start_version << " for tablet=" << _tablet_id << ", rowset=" << std::get<0>(bmk).to_string() << ", segment=" << std::get<1>(bmk); start_version += 1; } + if (handle2 != nullptr) { + DeleteBitmapAggCache::instance()->release(handle2); + } } { std::shared_lock l(lock); From 36ea0a5155344f816079573e08bd6f21ddc92d6b Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Wed, 9 Jul 2025 14:54:14 +0800 Subject: [PATCH 184/572] [opt](auditlog) Use varchar(1024) for column frontend_ip of audit log table #52762 (#52984) pick #52762 --- .../src/main/java/org/apache/doris/catalog/InternalSchema.java | 2 +- regression-test/data/audit/test_audit_log_behavior.out | 2 +- regression-test/suites/manager/test_manager_interface_1.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java index a571334660a5fa..6ae362012f0906 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java @@ -156,7 +156,7 @@ public class InternalSchema { AUDIT_SCHEMA.add(new ColumnDef("is_query", TypeDef.create(PrimitiveType.TINYINT), ColumnNullableType.NULLABLE)); AUDIT_SCHEMA.add( new ColumnDef("is_nereids", TypeDef.create(PrimitiveType.TINYINT), ColumnNullableType.NULLABLE)); - AUDIT_SCHEMA.add(new ColumnDef("frontend_ip", TypeDef.createVarchar(128), ColumnNullableType.NULLABLE)); + AUDIT_SCHEMA.add(new ColumnDef("frontend_ip", TypeDef.createVarchar(1024), ColumnNullableType.NULLABLE)); AUDIT_SCHEMA .add(new ColumnDef("cpu_time_ms", TypeDef.create(PrimitiveType.BIGINT), ColumnNullableType.NULLABLE)); AUDIT_SCHEMA.add(new ColumnDef("sql_hash", TypeDef.createVarchar(128), ColumnNullableType.NULLABLE)); diff --git a/regression-test/data/audit/test_audit_log_behavior.out b/regression-test/data/audit/test_audit_log_behavior.out index 904fa6299e3194..7c0d0e3026db70 100644 --- a/regression-test/data/audit/test_audit_log_behavior.out +++ b/regression-test/data/audit/test_audit_log_behavior.out @@ -21,7 +21,7 @@ stmt_id bigint Yes false \N NONE stmt_type varchar(48) Yes false \N NONE is_query tinyint Yes false \N NONE is_nereids tinyint Yes false \N NONE -frontend_ip varchar(128) Yes false \N NONE +frontend_ip varchar(1024) Yes false \N NONE cpu_time_ms bigint Yes false \N NONE sql_hash varchar(128) Yes false \N NONE sql_digest varchar(128) Yes false \N NONE diff --git a/regression-test/suites/manager/test_manager_interface_1.groovy b/regression-test/suites/manager/test_manager_interface_1.groovy index ec57261b147dcb..929d3b33db446a 100644 --- a/regression-test/suites/manager/test_manager_interface_1.groovy +++ b/regression-test/suites/manager/test_manager_interface_1.groovy @@ -568,7 +568,7 @@ suite('test_manager_interface_1',"p0") { assertTrue(result[0][1].contains("`return_rows` bigint NULL,")) assertTrue(result[0][1].contains("`stmt_id` bigint NULL,")) assertTrue(result[0][1].contains("`is_query` tinyint NULL,")) - assertTrue(result[0][1].contains("`frontend_ip` varchar(128) NULL,")) + assertTrue(result[0][1].contains("`frontend_ip` varchar(1024) NULL,")) assertTrue(result[0][1].contains("`cpu_time_ms` bigint NULL,")) assertTrue(result[0][1].contains("`sql_hash` varchar(128) NULL,")) assertTrue(result[0][1].contains("`sql_digest` varchar(128) NULL,")) From 632b033ecd8d7a0c4e6e429e968f3f5925a1f730 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 15:43:15 +0800 Subject: [PATCH 185/572] branch-3.0: [Fix](mow) Fix `DeleteBitmap`'s assignment operator and constructor #52582 (#52974) Cherry-picked from #52582 Co-authored-by: bobhan1 --- be/src/olap/tablet_meta.cpp | 30 +++-- be/src/olap/tablet_meta.h | 4 +- .../test_cloud_mow_sc_inc_rowsets_dup.out | 13 +++ .../test_cloud_mow_sc_inc_rowsets_dup.groovy | 103 ++++++++++++++++++ 4 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.out create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.groovy diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 4a86bbbb0132cf..19250c292027bc 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1101,24 +1101,40 @@ DeleteBitmapAggCache* DeleteBitmapAggCache::create_instance(size_t capacity) { DeleteBitmap::DeleteBitmap(int64_t tablet_id) : _tablet_id(tablet_id) {} DeleteBitmap::DeleteBitmap(const DeleteBitmap& o) { - delete_bitmap = o.delete_bitmap; // just copy data + std::shared_lock l1(o.lock); + delete_bitmap = o.delete_bitmap; _tablet_id = o._tablet_id; } DeleteBitmap& DeleteBitmap::operator=(const DeleteBitmap& o) { - delete_bitmap = o.delete_bitmap; // just copy data - _tablet_id = o._tablet_id; + if (this == &o) return *this; + if (this < &o) { + std::unique_lock l1(lock); + std::shared_lock l2(o.lock); + delete_bitmap = o.delete_bitmap; + _tablet_id = o._tablet_id; + } else { + std::shared_lock l2(o.lock); + std::unique_lock l1(lock); + delete_bitmap = o.delete_bitmap; + _tablet_id = o._tablet_id; + } return *this; } -DeleteBitmap::DeleteBitmap(DeleteBitmap&& o) { +DeleteBitmap::DeleteBitmap(DeleteBitmap&& o) noexcept { + std::scoped_lock l(o.lock, o._rowset_cache_version_lock); delete_bitmap = std::move(o.delete_bitmap); - _tablet_id = o._tablet_id; + _tablet_id = std::move(o._tablet_id); + o._rowset_cache_version.clear(); } -DeleteBitmap& DeleteBitmap::operator=(DeleteBitmap&& o) { +DeleteBitmap& DeleteBitmap::operator=(DeleteBitmap&& o) noexcept { + if (this == &o) return *this; + std::scoped_lock l(lock, o.lock, o._rowset_cache_version_lock); delete_bitmap = std::move(o.delete_bitmap); - _tablet_id = o._tablet_id; + _tablet_id = std::move(o._tablet_id); + o._rowset_cache_version.clear(); return *this; } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 949ce05fd74881..39dba82a17e856 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -428,8 +428,8 @@ class DeleteBitmap { /** * Move c-tor for making delete bitmap snapshot on read path */ - DeleteBitmap(DeleteBitmap&& r); - DeleteBitmap& operator=(DeleteBitmap&& r); + DeleteBitmap(DeleteBitmap&& r) noexcept; + DeleteBitmap& operator=(DeleteBitmap&& r) noexcept; static DeleteBitmap from_pb(const DeleteBitmapPB& pb, int64_t tablet_id); diff --git a/regression-test/data/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.out b/regression-test/data/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.out new file mode 100644 index 00000000000000..545f2c2edbbe5a --- /dev/null +++ b/regression-test/data/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 10 +2 2 2 20 +3 3 3 30 + +-- !dup_key_count -- + +-- !sql -- +1 99 99 99 +2 2 2 20 +3 3 3 30 + diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.groovy new file mode 100644 index 00000000000000..f57a1b4de1f71b --- /dev/null +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_sc_inc_rowsets_dup.groovy @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite("test_cloud_mow_sc_inc_rowsets_dup", "nonConcurrent") { + if (!isCloudMode()) { + return + } + + GetDebugPoint().clearDebugPointsForAllFEs() + GetDebugPoint().clearDebugPointsForAllBEs() + + def table1 = "test_cloud_mow_sc_inc_rowsets_dup" + sql "DROP TABLE IF EXISTS ${table1} FORCE;" + sql """ CREATE TABLE IF NOT EXISTS ${table1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int, + `c3` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1"); """ + + sql "insert into ${table1} values(1,1,1,10);" // 2 + sql "insert into ${table1} values(2,2,2,20);" // 3 + sql "insert into ${table1} values(3,3,3,30);" // 4 + sql "sync;" + qt_sql "select * from ${table1} order by k1;" + + def backends = sql_return_maparray('show backends') + def tabletStats = sql_return_maparray("show tablets from ${table1};") + assert tabletStats.size() == 1 + def tabletId = tabletStats[0].TabletId + def tabletBackendId = tabletStats[0].BackendId + def tabletBackend + for (def be : backends) { + if (be.BackendId == tabletBackendId) { + tabletBackend = be + break; + } + } + logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); + + try { + GetDebugPoint().enableDebugPointForAllBEs("CloudSchemaChangeJob::_process_delete_bitmap.before_new_inc.block") + sql "alter table ${table1} modify column c2 varchar(100);" + + Thread.sleep(1000) + + tabletStats = sql_return_maparray("show tablets from ${table1};") + def newTabletId = "-1" + for (def stat : tabletStats) { + if (stat.TabletId != tabletId) { + newTabletId = stat.TabletId + break + } + } + logger.info("new_tablet_id: ${newTabletId}") + + sql "insert into ${table1} values(1,99,99,99);" + sql "insert into ${table1} values(1,99,99,99);" + sql "insert into ${table1} values(1,99,99,99);" + sql "insert into ${table1} values(1,99,99,99);" + + Thread.sleep(1000) + + GetDebugPoint().disableDebugPointForAllBEs("CloudSchemaChangeJob::_process_delete_bitmap.before_new_inc.block") + + waitForSchemaChangeDone { + sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${table1}' ORDER BY createtime DESC LIMIT 1 """ + time 1000 + } + + qt_dup_key_count "select k1,count() as cnt from ${table1} group by k1 having cnt>1;" + order_qt_sql "select * from ${table1};" + + } catch(Exception e) { + logger.info(e.getMessage()) + throw e + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + } +} From 916da87fc053a4ef958521fff951fd45fcaee0b1 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei Date: Wed, 9 Jul 2025 16:07:11 +0800 Subject: [PATCH 186/572] branch-3.0: [fix](nereids) fix sum0 cannot pass multi distinct check #51234 (#52950) cherry-pick #51234 to branch-3.0 --- .../nereids/rules/rewrite/CheckMultiDistinct.java | 3 ++- .../data/nereids_syntax_p0/analyze_agg.out | 12 ++++++++++++ .../suites/nereids_syntax_p0/analyze_agg.groovy | 11 +++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java index 4488a94b8d14c0..588d11a41df271 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/CheckMultiDistinct.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum0; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -41,7 +42,7 @@ */ public class CheckMultiDistinct extends OneRewriteRuleFactory { private final ImmutableSet> supportedFunctions = - ImmutableSet.of(Count.class, Sum.class, Avg.class, GroupConcat.class); + ImmutableSet.of(Count.class, Sum.class, Avg.class, GroupConcat.class, Sum0.class); @Override public Rule build() { diff --git a/regression-test/data/nereids_syntax_p0/analyze_agg.out b/regression-test/data/nereids_syntax_p0/analyze_agg.out index 9c9c4c6c8a2be1..8316c4aefe20c8 100644 --- a/regression-test/data/nereids_syntax_p0/analyze_agg.out +++ b/regression-test/data/nereids_syntax_p0/analyze_agg.out @@ -1,3 +1,15 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- +-- !test_sum0 -- +0 0 +0 3 +0 5 +0 7 +5 21 + +-- !test_sum0_all_null -- +0 3 +0 5 +0 7 + diff --git a/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy b/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy index 9a79df6bad524b..cf93cad471ca4b 100644 --- a/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy +++ b/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy @@ -88,4 +88,15 @@ suite("analyze_agg") { 1, x """ + + sql "drop table if exists test_sum0_multi_distinct_with_group_by" + sql "create table test_sum0_multi_distinct_with_group_by (a int, b int, c int) distributed by hash(a) properties('replication_num'='1');" + sql """ + INSERT INTO test_sum0_multi_distinct_with_group_by VALUES + (1, NULL, 3), (2, NULL, 5), (3, NULL, 7), + (4,5,6),(4,5,7),(4,5,8), + (5,0,0),(5,0,0),(5,0,0); + """ + qt_test_sum0 "select sum0(distinct b),sum(distinct c) from test_sum0_multi_distinct_with_group_by group by a order by 1,2" + qt_test_sum0_all_null "select sum0(distinct b),sum(distinct c) from test_sum0_multi_distinct_with_group_by where a in (1,2,3) group by a order by 1,2" } \ No newline at end of file From b537621c09442495ec1775d713acd3a482a9c02d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:17:28 +0800 Subject: [PATCH 187/572] branch-3.0: [opt](cloud) add Sanitizer options #52883 (#52978) Cherry-picked from #52883 Co-authored-by: Dongyang Li --- cloud/CMakeLists.txt | 3 +++ cloud/conf/asan_suppr.conf | 17 +++++++++++++++++ .../{lsan_suppression.conf => lsan_suppr.conf} | 1 + cloud/conf/ubsan_suppr.conf | 17 +++++++++++++++++ cloud/script/run_all_tests.sh | 2 +- cloud/script/start.sh | 10 ++++++++++ cloud/test/CMakeLists.txt | 2 +- 7 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 cloud/conf/asan_suppr.conf rename cloud/conf/{lsan_suppression.conf => lsan_suppr.conf} (96%) create mode 100644 cloud/conf/ubsan_suppr.conf diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index 59f77c4927e166..575e55f297f34a 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -532,6 +532,9 @@ install(FILES install(FILES ${BASE_DIR}/conf/doris_cloud.conf + ${BASE_DIR}/conf/asan_suppr.conf + ${BASE_DIR}/conf/lsan_suppr.conf + ${BASE_DIR}/conf/ubsan_suppr.conf DESTINATION ${OUTPUT_DIR}/conf) install(FILES diff --git a/cloud/conf/asan_suppr.conf b/cloud/conf/asan_suppr.conf new file mode 100644 index 00000000000000..245692337bc3fa --- /dev/null +++ b/cloud/conf/asan_suppr.conf @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/cloud/conf/lsan_suppression.conf b/cloud/conf/lsan_suppr.conf similarity index 96% rename from cloud/conf/lsan_suppression.conf rename to cloud/conf/lsan_suppr.conf index 4b6fc3127a9d35..6b9ee40e23f93e 100644 --- a/cloud/conf/lsan_suppression.conf +++ b/cloud/conf/lsan_suppr.conf @@ -18,3 +18,4 @@ # This is a known leak. leak:libfdb_c.so leak:brpc::Server::StartInternal +leak:aws_mqtt_library_init diff --git a/cloud/conf/ubsan_suppr.conf b/cloud/conf/ubsan_suppr.conf new file mode 100644 index 00000000000000..245692337bc3fa --- /dev/null +++ b/cloud/conf/ubsan_suppr.conf @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + diff --git a/cloud/script/run_all_tests.sh b/cloud/script/run_all_tests.sh index f2b5f4dd9b0b06..003510f91d473c 100644 --- a/cloud/script/run_all_tests.sh +++ b/cloud/script/run_all_tests.sh @@ -127,7 +127,7 @@ function report_coverage() { ${binary_objects_options[*]} } -export LSAN_OPTIONS=suppressions=./lsan_suppression.conf +export LSAN_OPTIONS=suppressions=./lsan_suppr.conf unittest_files=() for i in *_test; do [[ -e "${i}" ]] || break diff --git a/cloud/script/start.sh b/cloud/script/start.sh index abb944f2412692..506a279ad78250 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -126,6 +126,16 @@ export CLASSPATH="${DORIS_CLASSPATH}" export LD_LIBRARY_PATH="${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}" +# filter known leak +export LSAN_OPTIONS=suppressions=${DORIS_HOME}/conf/lsan_suppr.conf +export ASAN_OPTIONS=suppressions=${DORIS_HOME}/conf/asan_suppr.conf +export UBSAN_OPTIONS=suppressions=${DORIS_HOME}/conf/ubsan_suppr.conf + +## set asan and ubsan env to generate core file +## detect_container_overflow=0, https://github.com/google/sanitizers/issues/193 +export ASAN_OPTIONS=symbolize=1:abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1:detect_container_overflow=0:check_malloc_usable_size=0:${ASAN_OPTIONS} +export UBSAN_OPTIONS=print_stacktrace=1:${UBSAN_OPTIONS} + ## set libhdfs3 conf if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml" diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt index e939b21d8bf69b..76729aca47fc66 100644 --- a/cloud/test/CMakeLists.txt +++ b/cloud/test/CMakeLists.txt @@ -135,7 +135,7 @@ install(FILES ${BASE_DIR}/script/run_all_tests.sh ${BASE_DIR}/conf/fdb.cluster ${BASE_DIR}/conf/doris_cloud.conf - ${BASE_DIR}/conf/lsan_suppression.conf + ${BASE_DIR}/conf/lsan_suppr.conf ${BASE_DIR}/test/fdb_metric_example.json PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_WRITE GROUP_EXECUTE From f2e411314353556d64c1fbdd3f23f04e5e7c74f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 9 Jul 2025 19:04:12 +0800 Subject: [PATCH 188/572] branch-3.0: [Chore](status) show inner error msg when write_file_cache_version failed #52953 (#52980) Cherry-picked from #52953 Co-authored-by: Pxl --- be/src/io/cache/fs_file_cache_storage.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/be/src/io/cache/fs_file_cache_storage.cpp b/be/src/io/cache/fs_file_cache_storage.cpp index ecb594e14a207c..6af81ce78b7864 100644 --- a/be/src/io/cache/fs_file_cache_storage.cpp +++ b/be/src/io/cache/fs_file_cache_storage.cpp @@ -536,8 +536,9 @@ Status FSFileCacheStorage::upgrade_cache_dir_if_necessary() const { ++failure_count; } } - if (!write_file_cache_version().ok()) { - return Status::InternalError("Failed to write version hints for file cache"); + if (auto st = write_file_cache_version(); !st.ok()) { + return Status::InternalError("Failed to write version hints for file cache, err={}", + st.to_string()); } } From e95458bceddf6173d2679af01495496d8286d21c Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 9 Jul 2025 19:04:53 +0800 Subject: [PATCH 189/572] =?UTF-8?q?branch-3.0:[fix](auth)fix=20when=20auth?= =?UTF-8?q?entication,=20the=20permissions=20of=20multiple=20roles?= =?UTF-8?q?=E2=80=A6=20(#52948)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … should be merged (#52349) pick: https://github.com/apache/doris/pull/52349 --- .../apache/doris/mysql/privilege/Auth.java | 49 +++++++------------ .../apache/doris/mysql/privilege/Role.java | 46 +++++------------ .../doris/mysql/privilege/RoleManager.java | 2 +- .../suites/account_p0/test_grant_priv.groovy | 38 ++++++++++++++ 4 files changed, 70 insertions(+), 65 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index 457d7066445462..6075821f04e1fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -270,8 +270,9 @@ protected boolean checkGlobalPriv(UserIdentity currentUser, PrivPredicate wanted readLock(); try { Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkGlobalPriv(wanted)) { + if (role.checkGlobalPriv(wanted, savedPrivs)) { return true; } } @@ -293,8 +294,9 @@ protected boolean checkCtlPriv(UserIdentity currentUser, String ctl, PrivPredica readLock(); try { Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkCtlPriv(ctl, wanted)) { + if (role.checkCtlPriv(ctl, wanted, savedPrivs)) { return true; } } @@ -316,8 +318,9 @@ protected boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, P readLock(); try { Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkDbPriv(ctl, db, wanted)) { + if (role.checkDbPriv(ctl, db, wanted, savedPrivs)) { return true; } } @@ -339,8 +342,9 @@ protected boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, readLock(); try { Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkTblPriv(ctl, db, tbl, wanted)) { + if (role.checkTblPriv(ctl, db, tbl, wanted, savedPrivs)) { return true; } } @@ -367,8 +371,9 @@ protected void checkColsPriv(UserIdentity currentUser, String ctl, String db, St private boolean checkColPriv(String ctl, String db, String tbl, String col, PrivPredicate wanted, Set roles) { + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkColPriv(ctl, db, tbl, col, wanted)) { + if (role.checkColPriv(ctl, db, tbl, col, wanted, savedPrivs)) { return true; } } @@ -380,8 +385,9 @@ protected boolean checkResourcePriv(UserIdentity currentUser, String resourceNam readLock(); try { Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkResourcePriv(resourceName, wanted)) { + if (role.checkResourcePriv(resourceName, wanted, savedPrivs)) { return true; } } @@ -396,8 +402,9 @@ protected boolean checkStorageVaultPriv(UserIdentity currentUser, String storage readLock(); try { Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkStorageVaultPriv(storageVaultName, wanted)) { + if (role.checkStorageVaultPriv(storageVaultName, wanted, savedPrivs)) { return true; } } @@ -418,8 +425,9 @@ protected boolean checkWorkloadGroupPriv(UserIdentity currentUser, String worklo } Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkWorkloadGroupPriv(workloadGroupName, wanted)) { + if (role.checkWorkloadGroupPriv(workloadGroupName, wanted, savedPrivs)) { return true; } } @@ -440,29 +448,10 @@ protected boolean checkCloudPriv(UserIdentity currentUser, String cloudName, return true; } } - Set roles = userRoleManager.getRolesByUser(currentUser); - for (String roleName : roles) { - if (roleManager.getRole(roleName).checkCloudPriv(cloudName, wanted, type)) { - return true; - } - } - return false; - } finally { - readUnlock(); - } - } - - // ==== Other ==== - /* - * Check if current user has certain privilege. - * This method will check the given privilege levels - */ - public boolean checkHasPriv(ConnectContext ctx, PrivPredicate priv, PrivLevel... levels) { - readLock(); - try { - Set roles = getRolesByUserWithLdap(ctx.getCurrentUserIdentity()); + Set roles = getRolesByUserWithLdap(currentUser); + PrivBitSet savedPrivs = PrivBitSet.of(); for (Role role : roles) { - if (role.checkHasPriv(priv, levels)) { + if (role.checkCloudPriv(cloudName, wanted, type, savedPrivs)) { return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java index 56e6b09641f48b..4f87574924b00e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java @@ -312,13 +312,11 @@ public void merge(Role other) throws DdlException { mergeNotCheck(other); } - public boolean checkGlobalPriv(PrivPredicate wanted) { - PrivBitSet savedPrivs = PrivBitSet.of(); + public boolean checkGlobalPriv(PrivPredicate wanted, PrivBitSet savedPrivs) { return checkGlobalInternal(wanted, savedPrivs); } - public boolean checkCtlPriv(String ctl, PrivPredicate wanted) { - PrivBitSet savedPrivs = PrivBitSet.of(); + public boolean checkCtlPriv(String ctl, PrivPredicate wanted, PrivBitSet savedPrivs) { if (checkGlobalInternal(wanted, savedPrivs) || checkCatalogInternal(ctl, wanted, savedPrivs)) { return true; @@ -362,8 +360,7 @@ private boolean checkCatalogInternal(String ctl, PrivPredicate wanted, PrivBitSe return false; } - public boolean checkDbPriv(String ctl, String db, PrivPredicate wanted) { - PrivBitSet savedPrivs = PrivBitSet.of(); + public boolean checkDbPriv(String ctl, String db, PrivPredicate wanted, PrivBitSet savedPrivs) { if (checkGlobalInternal(wanted, savedPrivs) || checkCatalogInternal(ctl, wanted, savedPrivs) || checkDbInternal(ctl, db, wanted, savedPrivs)) { @@ -434,8 +431,7 @@ private boolean checkDbInternal(String ctl, String db, PrivPredicate wanted, return false; } - public boolean checkTblPriv(String ctl, String db, String tbl, PrivPredicate wanted) { - PrivBitSet savedPrivs = PrivBitSet.of(); + public boolean checkTblPriv(String ctl, String db, String tbl, PrivPredicate wanted, PrivBitSet savedPrivs) { if (checkGlobalInternal(wanted, savedPrivs) || checkCatalogInternal(ctl, wanted, savedPrivs) || checkDbInternal(ctl, db, wanted, savedPrivs) @@ -456,14 +452,13 @@ public boolean checkTblPriv(String ctl, String db, String tbl, PrivPredicate wan } public boolean checkCloudPriv(String cloudName, - PrivPredicate wanted, ResourceTypeEnum type) { + PrivPredicate wanted, ResourceTypeEnum type, PrivBitSet savedPrivs) { ResourcePrivTable cloudPrivTable = getCloudPrivTable(type); if (cloudPrivTable == null) { LOG.warn("cloud resource type err: {}", type); return false; } - PrivBitSet savedPrivs = PrivBitSet.of(); if (checkGlobalInternal(wanted, savedPrivs) || checkCloudInternal(cloudName, wanted, savedPrivs, cloudPrivTable, type)) { return true; @@ -473,12 +468,14 @@ public boolean checkCloudPriv(String cloudName, return false; } - public boolean checkColPriv(String ctl, String db, String tbl, String col, PrivPredicate wanted) { + public boolean checkColPriv(String ctl, String db, String tbl, String col, PrivPredicate wanted, + PrivBitSet savedPrivs) { Optional colPrivilege = wanted.getColPrivilege(); if (!colPrivilege.isPresent()) { throw new IllegalStateException("this privPredicate should not use checkColPriv:" + wanted); } - return checkTblPriv(ctl, db, tbl, wanted) || onlyCheckColPriv(ctl, db, tbl, col, colPrivilege.get()); + return checkTblPriv(ctl, db, tbl, wanted, savedPrivs) || onlyCheckColPriv(ctl, db, tbl, col, + colPrivilege.get()); } private boolean onlyCheckColPriv(String ctl, String db, String tbl, String col, @@ -495,8 +492,7 @@ private boolean checkTblInternal(String ctl, String db, String tbl, PrivPredicat return Privilege.satisfy(savedPrivs, wanted); } - public boolean checkResourcePriv(String resourceName, PrivPredicate wanted) { - PrivBitSet savedPrivs = PrivBitSet.of(); + public boolean checkResourcePriv(String resourceName, PrivPredicate wanted, PrivBitSet savedPrivs) { if (checkGlobalInternal(wanted, savedPrivs) || checkResourceInternal(resourceName, wanted, savedPrivs)) { return true; @@ -513,8 +509,7 @@ private boolean checkResourceInternal(String resourceName, PrivPredicate wanted, return Privilege.satisfy(savedPrivs, wanted); } - public boolean checkStorageVaultPriv(String storageVaultName, PrivPredicate wanted) { - PrivBitSet savedPrivs = PrivBitSet.of(); + public boolean checkStorageVaultPriv(String storageVaultName, PrivPredicate wanted, PrivBitSet savedPrivs) { if (checkGlobalInternal(wanted, savedPrivs) || checkStorageVaultInternal(storageVaultName, wanted, savedPrivs)) { return true; @@ -537,12 +532,11 @@ private boolean checkCloudInternal(String cloudName, PrivPredicate wanted, return Privilege.satisfy(savedPrivs, wanted); } - public boolean checkWorkloadGroupPriv(String workloadGroupName, PrivPredicate wanted) { + public boolean checkWorkloadGroupPriv(String workloadGroupName, PrivPredicate wanted, PrivBitSet savedPrivs) { // For compatibility with older versions, it is not needed to check the privileges of the default group. if (WorkloadGroupMgr.DEFAULT_GROUP_NAME.equals(workloadGroupName)) { return true; } - PrivBitSet savedPrivs = PrivBitSet.of(); // usage priv not in global, but grant_priv may in global if (checkGlobalInternal(wanted, savedPrivs) || checkWorkloadGroupInternal(workloadGroupName, wanted, savedPrivs)) { @@ -633,22 +627,6 @@ public void setComment(String comment) { this.comment = comment; } - public boolean checkCanEnterCluster(String clusterName) { - if (checkGlobalPriv(PrivPredicate.ALL)) { - return true; - } - - if (dbPrivTable.hasClusterPriv(clusterName)) { - return true; - } - - if (tablePrivTable.hasClusterPriv(clusterName)) { - return true; - } - return false; - } - - private void grantPrivs(ResourcePattern resourcePattern, PrivBitSet privs) throws DdlException { if (privs.isEmpty()) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java index 46949ef473953a..eba297bfd37148 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RoleManager.java @@ -224,7 +224,7 @@ public void getRoleWorkloadGroupPrivs(List> result, Set lim if (limitedRole != null && !limitedRole.contains(role.getRoleName())) { continue; } - String isGrantable = role.checkGlobalPriv(PrivPredicate.ADMIN) ? "YES" : "NO"; + String isGrantable = role.checkGlobalPriv(PrivPredicate.ADMIN, PrivBitSet.of()) ? "YES" : "NO"; for (Map.Entry entry : role.getWorkloadGroupPatternToPrivs().entrySet()) { List row = Lists.newArrayList(); diff --git a/regression-test/suites/account_p0/test_grant_priv.groovy b/regression-test/suites/account_p0/test_grant_priv.groovy index d2e04945905bee..ef0dda3a07ad42 100644 --- a/regression-test/suites/account_p0/test_grant_priv.groovy +++ b/regression-test/suites/account_p0/test_grant_priv.groovy @@ -21,6 +21,7 @@ suite("test_grant_priv") { def user1 = 'test_grant_priv_user1' def user2 = 'test_grant_priv_user2' def role1 = 'test_grant_priv_role1' + def role2 = 'test_grant_priv_role2' def pwd = '123456' def dbName = 'test_grant_priv_db' def tokens = context.config.jdbcUrl.split('/') @@ -29,6 +30,7 @@ suite("test_grant_priv") { sql """drop user if exists ${user1}""" sql """drop user if exists ${user2}""" sql """drop role if exists ${role1}""" + sql """drop role if exists ${role2}""" sql """DROP DATABASE IF EXISTS ${dbName}""" sql """CREATE DATABASE ${dbName}""" @@ -82,5 +84,41 @@ suite("test_grant_priv") { sql """drop user if exists ${user1}""" sql """drop user if exists ${user2}""" sql """drop role if exists ${role1}""" + sql """drop role if exists ${role2}""" + + sql """CREATE ROLE ${role1}""" + sql """CREATE ROLE ${role2}""" + sql """CREATE USER '${user1}' IDENTIFIED BY '${pwd}'""" + // for login + sql """grant select_priv on ${dbName}.* to ${user1}""" + sql """CREATE USER '${user2}' IDENTIFIED BY '${pwd}'""" + sql """grant grant_priv on *.*.* to role '${role1}'""" + sql """grant select_priv on *.*.* to role '${role2}'""" + sql """grant '${role1}' to ${user1}""" + // test only have role1 can not grant + connect(user1, "${pwd}", url) { + test { + sql """grant select_priv on *.*.* to ${user2}""" + exception "denied" + } + } + sql """revoke '${role1}' from ${user1}""" + sql """grant '${role2}' to ${user1}""" + // test only have role2 can not grant + connect(user1, "${pwd}", url) { + test { + sql """grant select_priv on *.*.* to ${user2}""" + exception "denied" + } + } + // test both have role1 and role2 can grant to other + sql """grant '${role1}' to ${user1}""" + connect(user1, "${pwd}", url) { + sql """grant select_priv on *.*.* to ${user2}""" + } + sql """drop user if exists ${user1}""" + sql """drop user if exists ${user2}""" + sql """drop role if exists ${role1}""" + sql """drop role if exists ${role2}""" sql """DROP DATABASE IF EXISTS ${dbName}""" } From e7cd9fdbfe9f2a2ec29bb044ef914950b46d193a Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 9 Jul 2025 19:47:08 +0800 Subject: [PATCH 190/572] [cherry-pick](branch-30) fix bitmap_union return error result in query sql (#52033) (#52999) ### What problem does this PR solve? Problem Summary: cherry-pick from master (#52033) ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/aggregate_functions/aggregate_function_bitmap.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h index b0619a63e1ffe8..6784d78a1aa743 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_bitmap.h +++ b/be/src/vec/aggregate_functions/aggregate_function_bitmap.h @@ -54,6 +54,7 @@ struct AggregateFunctionBitmapUnionOp { template static void add(BitmapValue& res, const T& data, bool& is_first) { res.add(data); + is_first = false; } static void add(BitmapValue& res, const BitmapValue& data, bool& is_first) { @@ -67,6 +68,9 @@ struct AggregateFunctionBitmapUnionOp { static void add_batch(BitmapValue& res, std::vector& data, bool& is_first) { res.fastunion(data); + // after fastunion, res myabe have many datas, so is_first should be false + // then call add function will not reset res + is_first = false; } static void merge(BitmapValue& res, const BitmapValue& data, bool& is_first) { From 1357b68ecaa8a2fae4404714abad2eb3ad16b2ff Mon Sep 17 00:00:00 2001 From: morrySnow Date: Thu, 10 Jul 2025 10:29:25 +0800 Subject: [PATCH 191/572] branch-3.0: [opt](Nereids) support defer materialization with project (#52522) pick from #47661 #47975 #48747 --- .../org/apache/doris/catalog/OlapTable.java | 7 +- .../translator/PhysicalPlanTranslator.java | 49 ++-- .../rewrite/DeferMaterializeTopNResult.java | 251 ++++++++++++++++-- .../limit_push_down/order_push_down.out | 32 +-- .../lazy_materialize_topn.groovy | 73 +++++ .../limit_push_down/order_push_down.groovy | 2 +- .../single_table_without_aggregate.groovy | 3 + .../suites/query_p0/sort/topn_2pr_rule.groovy | 2 +- 8 files changed, 352 insertions(+), 67 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 7e6457f29409af..a77b082d1033c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -92,6 +92,7 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; import lombok.Setter; +import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -2879,11 +2880,13 @@ public AutoIncrementGenerator getAutoIncrementGenerator() { * @param selectedIndexId the index want to scan */ public TFetchOption generateTwoPhaseReadOption(long selectedIndexId) { + boolean useStoreRow = this.storeRowColumn() + && CollectionUtils.isEmpty(getTableProperty().getCopiedRowStoreColumns()); TFetchOption fetchOption = new TFetchOption(); - fetchOption.setFetchRowStore(this.storeRowColumn()); + fetchOption.setFetchRowStore(useStoreRow); fetchOption.setUseTwoPhaseFetch(true); fetchOption.setNodesInfo(SystemInfoService.createAliveNodesInfo()); - if (!this.storeRowColumn()) { + if (!useStoreRow) { List columnsDesc = Lists.newArrayList(); getColumnDesc(selectedIndexId, columnsDesc, null, null); fetchOption.setColumnDesc(columnsDesc); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 82d3f36ea6b86b..2c7e1988c48e14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1935,7 +1935,13 @@ public PlanFragment visitPhysicalProject(PhysicalProject project List allProjectionExprs = Lists.newArrayList(); List slots = null; // TODO FE/BE do not support multi-layer-project on MultiDataSink now. - if (project.hasMultiLayerProjection() && !(inputFragment instanceof MultiCastPlanFragment)) { + if (project.hasMultiLayerProjection() + && !(inputFragment instanceof MultiCastPlanFragment) + // TODO support for two phase read with project, remove it after refactor + && !(project.child() instanceof PhysicalDeferMaterializeTopN) + && !(project.child() instanceof PhysicalDeferMaterializeOlapScan + || (project.child() instanceof PhysicalFilter + && ((PhysicalFilter) project.child()).child() instanceof PhysicalDeferMaterializeOlapScan))) { int layerCount = project.getMultiLayerProjects().size(); for (int i = 0; i < layerCount; i++) { List layer = project.getMultiLayerProjects().get(i); @@ -2043,37 +2049,28 @@ public PlanFragment visitPhysicalProject(PhysicalProject project } if (inputPlanNode instanceof ScanNode) { - TupleDescriptor projectionTuple = null; - // slotIdsByOrder is used to ensure the ScanNode's output order is same with current Project - // if we change the output order in translate project, the upper node will receive wrong order - // tuple, since they get the order from project.getOutput() not scan.getOutput()./ - projectionTuple = generateTupleDesc(slots, - ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); - inputPlanNode.setProjectList(projectionExprs); - inputPlanNode.setOutputTupleDesc(projectionTuple); - - // TODO: this is a temporary scheme to support two phase read when has project. - // we need to refactor all topn opt into rbo stage. + // TODO support for two phase read with project, remove this if after refactor + if (!(project.child() instanceof PhysicalDeferMaterializeOlapScan + || (project.child() instanceof PhysicalFilter + && ((PhysicalFilter) project.child()).child() instanceof PhysicalDeferMaterializeOlapScan))) { + TupleDescriptor projectionTuple = generateTupleDesc(slots, + ((ScanNode) inputPlanNode).getTupleDesc().getTable(), context); + inputPlanNode.setProjectList(projectionExprs); + inputPlanNode.setOutputTupleDesc(projectionTuple); + } if (inputPlanNode instanceof OlapScanNode) { - ArrayList olapScanSlots = - context.getTupleDesc(inputPlanNode.getTupleIds().get(0)).getSlots(); - SlotDescriptor lastSlot = olapScanSlots.get(olapScanSlots.size() - 1); - if (lastSlot.getColumn() != null - && lastSlot.getColumn().getName().equals(Column.ROWID_COL)) { - injectRowIdColumnSlot(projectionTuple); - SlotRef slotRef = new SlotRef(lastSlot); - inputPlanNode.getProjectList().add(slotRef); - requiredByProjectSlotIdSet.add(lastSlot.getId()); - requiredSlotIdSet.add(lastSlot.getId()); - } ((OlapScanNode) inputPlanNode).updateRequiredSlots(context, requiredByProjectSlotIdSet); } updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet, requiredByProjectSlotIdSet, context); } else { - TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context); - inputPlanNode.setProjectList(projectionExprs); - inputPlanNode.setOutputTupleDesc(tupleDescriptor); + if (project.child() instanceof PhysicalDeferMaterializeTopN) { + inputFragment.setOutputExprs(allProjectionExprs); + } else { + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, null, context); + inputPlanNode.setProjectList(projectionExprs); + inputPlanNode.setOutputTupleDesc(tupleDescriptor); + } } return inputFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java index 6d90d81349d08f..765081ae016afe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java @@ -33,17 +33,21 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * rewrite simple top n query to defer materialize slot not use for sort or predicate @@ -54,51 +58,256 @@ public class DeferMaterializeTopNResult implements RewriteRuleFactory { public List buildRules() { return ImmutableList.of( RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalTopN(logicalOlapScan())) - .when(r -> r.child().getLimit() < getTopNOptLimitThreshold()) - .whenNot(r -> r.child().getOrderKeys().isEmpty()) - .when(r -> r.child().getOrderKeys().stream().map(OrderKey::getExpr) - .allMatch(Expression::isColumnFromTable)) - .when(r -> r.child().child().getTable().getEnableLightSchemaChange()) - .when(r -> r.child().child().getTable().isDupKeysOrMergeOnWrite()) - .then(r -> deferMaterialize(r, r.child(), Optional.empty(), r.child().child())) + logicalResultSink( + logicalTopN( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> t.getOrderKeys().stream() + .map(OrderKey::getExpr) + .allMatch(Expression::isColumnFromTable)) + ).then(r -> deferMaterialize(r, r.child(), + Optional.empty(), Optional.empty(), r.child().child())) ), RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( - logicalResultSink(logicalTopN(logicalFilter(logicalOlapScan()))) - .when(r -> r.child().getLimit() < getTopNOptLimitThreshold()) - .whenNot(r -> r.child().getOrderKeys().isEmpty()) - .when(r -> r.child().getOrderKeys().stream().map(OrderKey::getExpr) - .allMatch(Expression::isColumnFromTable)) - .when(r -> r.child().child().child().getTable().getEnableLightSchemaChange()) - .when(r -> r.child().child().child().getTable().isDupKeysOrMergeOnWrite()) - .then(r -> { - LogicalFilter filter = r.child().child(); - return deferMaterialize(r, r.child(), Optional.of(filter), filter.child()); - }) + logicalResultSink( + logicalTopN( + logicalProject( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).then(r -> { + LogicalProject project = r.child().child(); + return deferMaterialize(r, r.child(), Optional.of(project), + Optional.empty(), project.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink( + logicalTopN( + logicalFilter( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> t.getOrderKeys().stream() + .map(OrderKey::getExpr) + .allMatch(Expression::isColumnFromTable)) + ).then(r -> { + LogicalFilter filter = r.child().child(); + return deferMaterialize(r, r.child(), Optional.empty(), + Optional.of(filter), filter.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink( + logicalTopN( + logicalProject( + logicalFilter( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).then(r -> { + LogicalProject> project = r.child().child(); + LogicalFilter filter = project.child(); + return deferMaterialize(r, r.child(), Optional.of(project), + Optional.of(filter), filter.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink(logicalProject( + logicalTopN( + logicalProject( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { + LogicalProject upperProject = r.child(); + LogicalProject bottomProject = r.child().child().child(); + List projections = upperProject.mergeProjections(bottomProject); + LogicalProject project = upperProject.withProjects(projections); + return deferMaterialize(r, r.child().child(), Optional.of(project), + Optional.empty(), bottomProject.child()); + }) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink(logicalProject( + logicalTopN( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + } + return true; + }) + )).then(r -> deferMaterialize(r, r.child().child(), Optional.of(r.child()), + Optional.empty(), r.child().child().child())) + ), + RuleType.DEFER_MATERIALIZE_TOP_N_RESULT.build( + logicalResultSink(logicalProject( + logicalTopN( + logicalProject(logicalFilter( + logicalOlapScan() + .when(s -> s.getTable().getEnableLightSchemaChange()) + .when(s -> s.getTable().isDupKeysOrMergeOnWrite()) + ) + ) + ).when(t -> t.getLimit() < getTopNOptLimitThreshold()) + .whenNot(t -> t.getOrderKeys().isEmpty()) + .when(t -> { + for (OrderKey orderKey : t.getOrderKeys()) { + if (!orderKey.getExpr().isColumnFromTable()) { + return false; + } + if (!(orderKey.getExpr() instanceof SlotReference)) { + return false; + } + SlotReference slotRef = (SlotReference) orderKey.getExpr(); + // do not support alias in project now + if (!t.child().getProjects().contains(slotRef)) { + return false; + } + } + return true; + }) + ).when(project -> project.canMergeProjections(project.child().child()))).then(r -> { + LogicalProject upperProject = r.child(); + LogicalProject> bottomProject = r.child().child().child(); + List projections = upperProject.mergeProjections(bottomProject); + LogicalProject project = upperProject.withProjects(projections); + LogicalFilter filter = bottomProject.child(); + return deferMaterialize(r, r.child().child(), Optional.of(project), + Optional.of(filter), filter.child()); + }) ) ); } private Plan deferMaterialize(LogicalResultSink logicalResultSink, - LogicalTopN logicalTopN, Optional> logicalFilter, - LogicalOlapScan logicalOlapScan) { + LogicalTopN logicalTopN, Optional> logicalProject, + Optional> logicalFilter, LogicalOlapScan logicalOlapScan) { Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column"); SlotReference columnId = SlotReference.fromColumn( logicalOlapScan.getTable(), rowId, logicalOlapScan.getQualifier()); + Set orderKeys = Sets.newHashSet(); Set deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet()); logicalFilter.ifPresent(filter -> filter.getConjuncts() .forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds()))); logicalTopN.getOrderKeys().stream() .map(OrderKey::getExpr) .map(Slot.class::cast) + .peek(orderKeys::add) .map(NamedExpression::getExprId) .filter(Objects::nonNull) .forEach(deferredMaterializedExprIds::remove); + if (logicalProject.isPresent()) { + deferredMaterializedExprIds.retainAll(logicalProject.get().getInputSlots().stream() + .map(NamedExpression::getExprId).collect(Collectors.toSet())); + } + if (deferredMaterializedExprIds.isEmpty()) { + // nothing to deferred materialize + return null; + } LogicalDeferMaterializeOlapScan deferOlapScan = new LogicalDeferMaterializeOlapScan( logicalOlapScan, deferredMaterializedExprIds, columnId); Plan root = logicalFilter.map(f -> f.withChildren(deferOlapScan)).orElse(deferOlapScan); + Set inputSlots = Sets.newHashSet(); + logicalFilter.ifPresent(filter -> inputSlots.addAll(filter.getInputSlots())); + if (logicalProject.isPresent()) { + ImmutableList.Builder requiredSlots = ImmutableList.builder(); + inputSlots.addAll(logicalProject.get().getInputSlots()); + for (Slot output : root.getOutput()) { + if (inputSlots.contains(output) || orderKeys.contains(output)) { + requiredSlots.add(output); + } + } + requiredSlots.add(columnId); + root = new LogicalProject<>(requiredSlots.build(), root); + } root = new LogicalDeferMaterializeTopN<>((LogicalTopN) logicalTopN.withChildren(root), deferredMaterializedExprIds, columnId); + if (logicalProject.isPresent()) { + // generate projections with the order exactly same as result output's + Map projectsMap = Maps.newHashMap(); + logicalProject.get().getProjects().forEach(p -> projectsMap.put(p.toSlot(), p)); + List outputProjects = logicalResultSink.getOutput().stream() + .map(projectsMap::get) + .collect(ImmutableList.toImmutableList()); + root = logicalProject.get().withProjectsAndChild(outputProjects, root); + } root = logicalResultSink.withChildren(root); return new LogicalDeferMaterializeResultSink<>((LogicalResultSink) root, logicalOlapScan.getTable(), logicalOlapScan.getSelectedIndexId()); diff --git a/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out b/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out index 694e609eaeb2af..5f0e3fa80998db 100644 --- a/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out +++ b/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out @@ -195,18 +195,18 @@ PhysicalResultSink ----------PhysicalOlapScan[t1] -- !limit_sort_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_offset_sort_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_subquery_order_by_inside_limit_outside -- PhysicalResultSink @@ -275,18 +275,18 @@ PhysicalResultSink ----------PhysicalOlapScan[t1] -- !limit_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_offset_filter -- -PhysicalResultSink ---PhysicalTopN[MERGE_SORT] -----PhysicalTopN[LOCAL_SORT] +PhysicalDeferMaterializeResultSink +--PhysicalDeferMaterializeTopN +----PhysicalDeferMaterializeTopN ------filter((t1.id = 1)) ---------PhysicalOlapScan[t1] +--------PhysicalDeferMaterializeOlapScan[t1] -- !limit_project_filter -- PhysicalResultSink diff --git a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy new file mode 100644 index 00000000000000..cd683b08f280f2 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("lazy_materialize_topn") { + sql """ + set enable_two_phase_read_opt = true + """ + + sql """ + drop table if exists lazy_materialize_topn; + """ + + sql """ + CREATE TABLE `lazy_materialize_topn` ( + `c1` int NULL, + `c2` int NULL, + `c3` int NULL, + `c4` array NULL + ) + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "light_schema_change" = "true" + ); + """ + + sql """ + insert into lazy_materialize_topn values (1, 1, 1, [1]), (2, 2, 2, [2]), (3, 3, 3, [3]); + """ + + sql """ + sync + """ + + List sqls = [ + // TopN(Scan) + """select * from lazy_materialize_topn order by c1 limit 10""", + // TopN(Project(Scan)) + """select c1, c2 from lazy_materialize_topn order by c1 limit 10""", + // Project(TopN(Scan)) + """select c1, c2, c3, c4 from lazy_materialize_topn order by c1 limit 10""", + // Project(TopN(Project(Scan))) + """select c1 + 1, c2 + 1 from (select c1, c2 from lazy_materialize_topn order by c1 limit 10) t""", + // TopN(Filter(Scan)) + """select * from lazy_materialize_topn where c2 < 5 order by c1 limit 10;""", + // TopN(Project(Filter(Scan))) + """select c1, c2, c3 from lazy_materialize_topn where c2 < 5 order by c1 limit 10;""", + // Project(TopN(Project(Filter(Scan)))) + """select c1 + 1, c2 + 1, c3 + 1 from ( select c1, c2, c3 from lazy_materialize_topn where c2 < 5 order by c1 limit 10) t""", + // project set is diff with output list + """select c1, c1, c2 from (select c1, c2 from lazy_materialize_topn where c3 < 1 order by c2 limit 1)t;""" + ] + + for (sqlStr in sqls) { + explain { + sql """${sqlStr}""" + contains """OPT TWO PHASE""" + } + sql """${sqlStr}""" + } +} diff --git a/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy b/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy index 1ae008df588a82..bbf4a64f1180a7 100644 --- a/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy @@ -26,7 +26,7 @@ suite("order_push_down") { sql 'set be_number_for_test=3' sql "set disable_nereids_rules='push_down_top_n_distinct_through_union'" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - + sql 'set enable_two_phase_read_opt = true' //`limit 1 offset 1 + sort, project`: qt_limit_offset_sort_project """ explain shape plan SELECT t1.id FROM t1 ORDER BY id LIMIT 1 OFFSET 1; """ diff --git a/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy b/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy index 450fb9c0ea3187..3a07a959511390 100644 --- a/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/single_table_without_agg/single_table_without_aggregate.groovy @@ -21,6 +21,9 @@ suite("single_table_without_aggregate") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" sql "set enable_materialized_view_rewrite=true" + // TODO remove this variable after mv rewrite support defer materialized nodes + sql 'set enable_two_phase_read_opt = false' + sql """ drop table if exists orders diff --git a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy index 4b74cdad8538e4..0df8ab260309c9 100644 --- a/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy +++ b/regression-test/suites/query_p0/sort/topn_2pr_rule.groovy @@ -52,7 +52,7 @@ suite("topn_2pr_rule") { } else if("${key_type}" == "UNIQUE") { explain { sql("select * from ${table_name} order by k limit 1;") - notContains "OPT TWO PHASE" + contains "OPT TWO PHASE" } } else if("${key_type}" == "AGGREGATE") { explain { From d9a4e62369175d208e31a7a4a64b22cc1ae30549 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Thu, 10 Jul 2025 21:08:11 +0800 Subject: [PATCH 192/572] [fix](test) Avoid the configuration item disable_datev1 affecting other test cases (#52915) --- regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy | 2 -- .../conditional_functions/test_coalesce_new.groovy | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy index ff8710c5998fac..41aefe51d21bc6 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter.groovy @@ -181,6 +181,4 @@ suite("test_bloom_filter") { qt_select_datetime_v2 """SELECT * FROM ${test_datetime_tb} WHERE d2 IN ("2024-12-17 20:00:00", "2024-12-18 20:00:00") order by a""" qt_select_date_v1 """SELECT * FROM ${test_datetime_tb} WHERE da IN ("2024-12-17", "2024-12-18") order by a""" qt_select_date_v2 """SELECT * FROM ${test_datetime_tb} WHERE dav2 IN ("2024-12-17", "2024-12-18") order by a""" - sql """ADMIN SET FRONTEND CONFIG ('disable_decimalv2' = 'true')""" - sql """ADMIN SET FRONTEND CONFIG ('disable_datev1' = 'true')""" } diff --git a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_coalesce_new.groovy b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_coalesce_new.groovy index 44a17d66b65258..df113b7595f6d0 100644 --- a/regression-test/suites/query_p0/sql_functions/conditional_functions/test_coalesce_new.groovy +++ b/regression-test/suites/query_p0/sql_functions/conditional_functions/test_coalesce_new.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_coalesce_new") { +suite("test_coalesce_new", 'nonConcurrent') { // test parameter:datetime, datev2 sql """ admin set frontend config ("enable_date_conversion"="false") From 1af1dcbde9d97f936530747920e9348ddd0c3404 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 09:38:08 +0800 Subject: [PATCH 193/572] branch-3.0: [Bug](function) fix bitmap_from_base64 function cause heap-buffer-overflow error #53018 (#53043) Cherry-picked from #53018 Co-authored-by: zhangstar333 --- be/src/vec/functions/function_bitmap.cpp | 4 ++-- .../bitmap_functions/test_bitmap_function.groovy | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/be/src/vec/functions/function_bitmap.cpp b/be/src/vec/functions/function_bitmap.cpp index fb0e0471caac06..7f8ca009f991cb 100644 --- a/be/src/vec/functions/function_bitmap.cpp +++ b/be/src/vec/functions/function_bitmap.cpp @@ -289,8 +289,8 @@ struct BitmapFromBase64 { } else { BitmapValue bitmap_val; if (!bitmap_val.deserialize(decode_buff.data())) { - return Status::RuntimeError( - fmt::format("bitmap_from_base64 decode failed: base64: {}", src_str)); + return Status::RuntimeError("bitmap_from_base64 decode failed: base64: {}", + std::string(src_str, src_size)); } res.emplace_back(std::move(bitmap_val)); } diff --git a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy index 523cfc0a0cd8a0..5be83b29aa385a 100644 --- a/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy @@ -927,4 +927,9 @@ suite("test_bitmap_function") { // BITMAP_FROM_ARRAY sql """ set experimental_enable_nereids_planner=true; """ qt_sql """ select bitmap_to_string(BITMAP_FROM_ARRAY([]));""" + + test { + sql """ SELECT bitmap_from_base64('CQoL') AS result; """ + exception "bitmap_from_base64 decode failed" + } } From e0d56af430eec0a6da8cb7f406df0477496cc4c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:58:15 +0800 Subject: [PATCH 194/572] branch-3.0: [Enhancement](sql-dialect) Support multiple sql-converter service urls #52636 (#53026) Cherry-picked from #52636 Co-authored-by: zy-kkk --- .../plugin/dialect/HttpDialectUtils.java | 268 ++++++++++++- .../doris/plugin/HttpDialectUtilsTest.java | 360 +++++++++++++++++- 2 files changed, 599 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java index 89acd66658d6f0..5131cf82bf66fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java @@ -17,6 +17,9 @@ package org.apache.doris.plugin.dialect; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.collect.Lists; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import lombok.Data; @@ -30,35 +33,113 @@ import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; /** * This class is used to convert sql with different dialects using sql convertor service. * The sql convertor service is a http service which is used to convert sql. + *

+ * Features: + * - Support multiple URLs (comma separated) + * - Blacklist mechanism for failed URLs + * - Automatic failover and retry + * - URL caching and smart selection */ public class HttpDialectUtils { private static final Logger LOG = LogManager.getLogger(HttpDialectUtils.class); - public static String convertSql(String targetURL, String originStmt, String dialect, + // Cache URL manager instances to avoid duplicate parsing with automatic expiration + private static final Cache urlManagerCache = Caffeine.newBuilder() + .maximumSize(10) + .expireAfterAccess(8, TimeUnit.HOURS) + .build(); + + // Blacklist recovery time (ms): 1 minute + private static final long BLACKLIST_RECOVERY_TIME_MS = 60 * 1000; + // Connection timeout period (ms): 3 seconds + private static final int CONNECTION_TIMEOUT_MS = 3000; + // Read timeout period (ms): 10 seconds + private static final int READ_TIMEOUT_MS = 10000; + + public static String convertSql(String targetURLs, String originStmt, String dialect, String[] features, String config) { + UrlManager urlManager = getOrCreateUrlManager(targetURLs); ConvertRequest convertRequest = new ConvertRequest(originStmt, dialect, features, config); + String requestStr = convertRequest.toJson(); + + // Try to convert SQL using intelligent URL selection strategy + return tryConvertWithIntelligentSelection(urlManager, requestStr, originStmt); + } + + /** + * Try to convert SQL using intelligent URL selection strategy + * CRITICAL: This method ensures 100% success rate when ANY service is available + */ + private static String tryConvertWithIntelligentSelection( + UrlManager urlManager, String requestStr, String originStmt) { + // Strategy: Try ALL URLs in intelligent order, regardless of blacklist status + // This ensures 100% success rate when any service is actually available + List allUrls = urlManager.getAllUrlsInPriorityOrder(); + + for (String url : allUrls) { + try { + String result = doConvertSql(url, requestStr); + // If no exception thrown, HTTP response was successful (200) + // Mark URL as healthy and return result (even if empty) + urlManager.markUrlAsHealthy(url); + if (LOG.isDebugEnabled()) { + LOG.debug("Successfully converted SQL using URL: {}", url); + } + return result; + } catch (Exception e) { + LOG.warn("Failed to convert SQL using URL: {}, error: {}", url, e.getMessage()); + // Add failed URL to blacklist for future optimization + urlManager.markUrlAsBlacklisted(url); + // Continue trying next URL - this is CRITICAL for 100% success rate + } + } + + return originStmt; + } + + /** + * Get or create a URL manager + */ + private static UrlManager getOrCreateUrlManager(String targetURLs) { + return urlManagerCache.get(targetURLs, UrlManager::new); + } + + /** + * Perform SQL conversion for individual URL + */ + private static String doConvertSql(String targetURL, String requestStr) throws Exception { HttpURLConnection connection = null; try { - URL url = new URL(targetURL); + if (targetURL == null || targetURL.trim().isEmpty()) { + throw new Exception("Target URL is null or empty"); + } + URL url = new URL(targetURL.trim()); connection = (HttpURLConnection) url.openConnection(); connection.setRequestMethod("POST"); connection.setRequestProperty("Content-Type", "application/json"); connection.setUseCaches(false); connection.setDoOutput(true); + connection.setConnectTimeout(CONNECTION_TIMEOUT_MS); + connection.setReadTimeout(READ_TIMEOUT_MS); - String requestStr = convertRequest.toJson(); try (OutputStream outputStream = connection.getOutputStream()) { outputStream.write(requestStr.getBytes(StandardCharsets.UTF_8)); } int responseCode = connection.getResponseCode(); if (LOG.isDebugEnabled()) { - LOG.debug("POST Response Code: {}, post data: {}", responseCode, requestStr); + LOG.debug("POST Response Code: {}, URL: {}, post data: {}", responseCode, targetURL, requestStr); } if (responseCode == HttpURLConnection.HTTP_OK) { @@ -76,26 +157,20 @@ public static String convertSql(String targetURL, String originStmt, String dial }.getType(); ConvertResponse result = new Gson().fromJson(response.toString(), type); if (LOG.isDebugEnabled()) { - LOG.debug("convert response: {}", result); + LOG.debug("Convert response: {}, URL: {}", result, targetURL); } if (result.code == 0) { if (!"v1".equals(result.version)) { - LOG.warn("failed to convert sql, response version is not v1: {}", result.version); - return originStmt; + throw new Exception("Unsupported version: " + result.version); } return result.data; } else { - LOG.warn("failed to convert sql, response: {}", result); - return originStmt; + throw new Exception("Conversion failed: " + result.message); } } } else { - LOG.warn("failed to convert sql, response code: {}", responseCode); - return originStmt; + throw new Exception("HTTP response code: " + responseCode); } - } catch (Exception e) { - LOG.warn("failed to convert sql", e); - return originStmt; } finally { if (connection != null) { connection.disconnect(); @@ -103,6 +178,171 @@ public static String convertSql(String targetURL, String originStmt, String dial } } + /** + * URL Manager - Responsible for URL parsing, caching, blacklist management, and smart selection + */ + private static class UrlManager { + private final List parsedUrls; + private final ConcurrentHashMap blacklist; + + public UrlManager(String urls) { + this.parsedUrls = parseUrls(urls); + this.blacklist = new ConcurrentHashMap<>(); + if (LOG.isDebugEnabled()) { + LOG.debug("Created UrlManager with URLs: {}, parsed: {}", urls, parsedUrls); + } + } + + /** + * Parse comma separated URL strings + */ + private List parseUrls(String urls) { + List result = Lists.newArrayList(); + if (urls != null && !urls.trim().isEmpty()) { + String[] urlArray = urls.split(","); + for (String url : urlArray) { + String trimmedUrl = url.trim(); + if (!trimmedUrl.isEmpty()) { + result.add(trimmedUrl); + } + } + } + return result; + } + + /** + * Mark URL as healthy (remove from blacklist) + */ + public void markUrlAsHealthy(String url) { + if (blacklist.remove(url) != null) { + LOG.info("Removed URL from blacklist due to successful request: {}", url); + } + } + + /** + * Add URL to blacklist + */ + public void markUrlAsBlacklisted(String url) { + // If URL is already in blacklist, just return + if (blacklist.containsKey(url)) { + return; + } + + long currentTime = System.currentTimeMillis(); + long recoverTime = currentTime + BLACKLIST_RECOVERY_TIME_MS; + blacklist.put(url, new BlacklistEntry(currentTime, recoverTime)); + LOG.warn("Added URL to blacklist: {}, will recover at: {}", url, new Date(recoverTime)); + } + + /** + * Check if URL is localhost (127.0.0.1 or localhost) + */ + private boolean isLocalhost(String url) { + return url.contains("127.0.0.1") || url.contains("localhost"); + } + + /** + * Get ALL URLs in priority order for 100% success guarantee + * CRITICAL: This method ensures we try every URL when any service might be available + *

+ * Priority order: + * 1. Localhost URLs (127.0.0.1 or localhost) that are healthy + * 2. Other healthy URLs (randomly selected) + * 3. Localhost URLs in blacklist + * 4. Other blacklisted URLs (sorted by recovery time) + */ + public List getAllUrlsInPriorityOrder() { + List prioritizedUrls = Lists.newArrayList(); + List healthyLocalhost = Lists.newArrayList(); + List healthyOthers = Lists.newArrayList(); + List blacklistedLocalhost = Lists.newArrayList(); + List blacklistedOthers = Lists.newArrayList(); + + long currentTime = System.currentTimeMillis(); + + // Single traversal to categorize all URLs + for (String url : parsedUrls) { + BlacklistEntry entry = blacklist.get(url); + boolean isHealthy = false; + + if (entry == null) { + // URL is not in blacklist, consider it healthy + isHealthy = true; + } else if (currentTime >= entry.recoverTime) { + // URL has reached recovery time, remove from blacklist and consider healthy + blacklist.remove(url); + isHealthy = true; + if (LOG.isDebugEnabled()) { + LOG.debug("URL recovered from blacklist: {}", url); + } + } + + boolean isLocal = isLocalhost(url); + + if (isHealthy) { + if (isLocal) { + healthyLocalhost.add(url); + } else { + healthyOthers.add(url); + } + } else { + if (isLocal) { + blacklistedLocalhost.add(url); + } else { + blacklistedOthers.add(url); + } + } + } + + // Add URLs in priority order + // 1. Healthy localhost URLs first + prioritizedUrls.addAll(healthyLocalhost); + + // 2. Other healthy URLs (randomly shuffled for load balancing) + Collections.shuffle(healthyOthers, ThreadLocalRandom.current()); + prioritizedUrls.addAll(healthyOthers); + + // 3. Blacklisted localhost URLs + prioritizedUrls.addAll(blacklistedLocalhost); + + // 4. Other blacklisted URLs (sorted by recovery time) + blacklistedOthers.sort((url1, url2) -> { + BlacklistEntry entry1 = blacklist.get(url1); + BlacklistEntry entry2 = blacklist.get(url2); + if (entry1 == null && entry2 == null) { + return 0; + } + if (entry1 == null) { + return -1; + } + if (entry2 == null) { + return 1; + } + return Long.compare(entry1.recoverTime, entry2.recoverTime); + }); + prioritizedUrls.addAll(blacklistedOthers); + + if (LOG.isDebugEnabled()) { + LOG.debug("All URLs in priority order: {}", prioritizedUrls); + } + + return prioritizedUrls; + } + } + + /** + * Blacklist entry + */ + private static class BlacklistEntry { + final long blacklistedTime; + final long recoverTime; + + BlacklistEntry(long blacklistedTime, long recoverTime) { + this.blacklistedTime = blacklistedTime; + this.recoverTime = recoverTime; + } + } + @Data private static class ConvertRequest { private String version; // CHECKSTYLE IGNORE THIS LINE diff --git a/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java index de359f79475cc4..4ce71e196a33e1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java @@ -28,49 +28,379 @@ import java.net.DatagramSocket; import java.net.ServerSocket; import java.net.SocketException; +import java.util.ArrayList; +import java.util.List; public class HttpDialectUtilsTest { - private int port; - private SimpleHttpServer server; + private List ports = new ArrayList<>(); + private List servers = new ArrayList<>(); @Before public void setUp() throws Exception { - port = findValidPort(); - server = new SimpleHttpServer(port); - server.start("/api/v1/convert"); + // Create three test servers + for (int i = 0; i < 3; i++) { + int port = findValidPort(); + ports.add(port); + SimpleHttpServer server = new SimpleHttpServer(port); + server.start("/api/v1/convert"); + servers.add(server); + } } @After public void tearDown() { - if (server != null) { - server.stop(); + for (SimpleHttpServer server : servers) { + if (server != null) { + server.stop(); + } } + servers.clear(); + ports.clear(); } @Test - public void testSqlConvert() { + public void testSingleUrlConvert() { String originSql = "select * from t1 where \"k1\" = 1"; String expectedSql = "select * from t1 where `k1` = 1"; String[] features = new String[] {"ctas"}; - String targetURL = "http://127.0.0.1:" + port + "/api/v1/convert"; + String targetURL = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert"; + + // Test with no response (should return original SQL) String res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features, "{}"); Assert.assertEquals(originSql, res); - // test presto - server.setResponse("{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + // Test successful conversion + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features, "{}"); Assert.assertEquals(expectedSql, res); - // test response version error - server.setResponse("{\"version\": \"v2\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + // Test version error + servers.get(0).setResponse( + "{\"version\": \"v2\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features, "{}"); Assert.assertEquals(originSql, res); - // test response code error - server.setResponse( + + // Test code error + servers.get(0).setResponse( "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 400, \"message\": \"\"}"); res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features, "{}"); Assert.assertEquals(originSql, res); } + @Test + public void testMultipleUrlsConvert() { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(2) + "/api/v1/convert"; + + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + String res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + } + + @Test + public void testFailoverMechanism() throws InterruptedException { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert"; + + // First server returns error, second server succeeds + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 500, \"message\": \"error\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + String res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + } + + @Test + public void testBlacklistMechanism() throws InterruptedException { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert"; + + // Stop first server, set second server to work + servers.get(0).stop(); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + // First call should succeed via second server + String res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + + // Restart first server + servers.set(0, new SimpleHttpServer(ports.get(0))); + try { + servers.get(0).start("/api/v1/convert"); + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } catch (IOException e) { + return; // Skip test if port is occupied + } + + // Should still work with blacklist recovery + res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + } + + @Test + public void testAllUrlsFailure() { + String originSql = "select * from t1 where \"k1\" = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert"; + + // All servers return error + servers.get(0).setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(1).setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + + String res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(originSql, res); + } + + @Test + public void testUrlParsing() { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + // Test URL parsing with spaces and empty items + String targetURLs = " http://127.0.0.1:" + ports.get(0) + "/api/v1/convert , ," + + " http://127.0.0.1:" + ports.get(1) + "/api/v1/convert "; + + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + String res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + } + + @Test + public void testSeamlessFailover() throws IOException { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert"; + + // Both servers start healthy + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + String res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + + // Stop first server + servers.get(0).stop(); + res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + + // Restart first server, stop second + servers.set(0, new SimpleHttpServer(ports.get(0))); + servers.get(0).start("/api/v1/convert"); + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(1).stop(); + + // Should seamlessly switch to first server + res = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + Assert.assertEquals(expectedSql, res); + } + + @Test + public void testConcurrentRequests() throws InterruptedException { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert"; + + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + + // Test with multiple concurrent threads + Thread[] threads = new Thread[10]; + String[] results = new String[10]; + + for (int i = 0; i < 10; i++) { + final int index = i; + threads[i] = new Thread(() -> { + results[index] = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + }); + threads[i].start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + // Verify all results + for (String result : results) { + Assert.assertEquals(expectedSql, result); + } + } + + @Test + public void testZeroFailureGuarantee() throws InterruptedException { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(2) + "/api/v1/convert"; + + int totalRequests = 30; // Reduced for faster testing with production timeouts + int successCount = 0; + + // Test various failure scenarios while ensuring at least one service is always available + for (int i = 0; i < totalRequests; i++) { + if (i < 6) { + // All servers healthy + setAllServersHealthy(expectedSql); + } else if (i < 12) { + // Server 0 fails, others healthy + servers.get(0) + .setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(2).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } else if (i < 18) { + // Servers 0,1 fail, server 2 healthy + servers.get(0) + .setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(1) + .setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 503, \"message\": \"error\"}"); + servers.get(2).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } else if (i < 24) { + // Only server 1 healthy + servers.get(0) + .setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(2) + .setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + } else { + // Alternating recovery + if (i % 2 == 0) { + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(2).setResponse( + "{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + } else { + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"error\"}"); + servers.get(2).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } + } + + String result = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + if (expectedSql.equals(result)) { + successCount++; + } + + Thread.sleep(50); // Small delay between requests + } + + System.out.println("Zero Failure Guarantee Test Results:"); + System.out.println("Total requests: " + totalRequests); + System.out.println("Successful: " + successCount); + System.out.println("Success rate: " + (successCount * 100.0 / totalRequests) + "%"); + + // Must achieve 100% success rate when at least one service is available + Assert.assertEquals("Must achieve 100% success rate when service is available", + totalRequests, successCount); + } + + @Test + public void testNetworkJitterStress() throws InterruptedException { + String originSql = "select * from t1 where \"k1\" = 1"; + String expectedSql = "select * from t1 where `k1` = 1"; + String[] features = new String[] {"ctas"}; + + String targetURLs = "http://127.0.0.1:" + ports.get(0) + "/api/v1/convert," + + "http://127.0.0.1:" + ports.get(1) + "/api/v1/convert"; + + int totalRequests = 15; // Reduced for faster testing with production timeouts + int successCount = 0; + + // Simulate network jitter while ensuring at least one server is always available + for (int i = 0; i < totalRequests; i++) { + double random = Math.random(); + if (random < 0.3) { + // Server 0 fails, Server 1 works + servers.get(0) + .setResponse("{\"version\": \"v1\", \"data\": \"\", \"code\": 500, \"message\": \"timeout\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } else if (random < 0.5) { + // Server 1 fails, Server 0 works + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"\", \"code\": 503, \"message\": \"service unavailable\"}"); + } else { + // Both servers work + servers.get(0).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + servers.get(1).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } + + String result = HttpDialectUtils.convertSql(targetURLs, originSql, "presto", features, "{}"); + if (expectedSql.equals(result)) { + successCount++; + } + + Thread.sleep(100); // Delay between requests for production timeouts + } + + System.out.println("Network Jitter Test Results:"); + System.out.println("Total requests: " + totalRequests); + System.out.println("Successful: " + successCount); + System.out.println("Success rate: " + (successCount * 100.0 / totalRequests) + "%"); + + // Must achieve 100% success rate since we ensure at least one server is always available + Assert.assertEquals("Must handle network jitter with 100% success when service is available", + totalRequests, successCount); + } + + private void setAllServersHealthy(String expectedSql) { + for (int i = 0; i < 3; i++) { + servers.get(i).setResponse( + "{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}"); + } + } + private static int findValidPort() { int port; while (true) { From c4bb5a79d77b5e15842ab13a2ec6c2bc06177a97 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:58:47 +0800 Subject: [PATCH 195/572] branch-3.0: [fix](TabletSched)Prioritize non-drop replicas as src replicas #53006 (#53033) Cherry-picked from #53006 Co-authored-by: deardeng --- .../apache/doris/clone/TabletSchedCtx.java | 28 +++++++++++-------- .../doris/clone/TabletSchedCtxTest.java | 27 +++++++++++++++--- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 610695900806f5..9e63a4fa1c4835 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -86,7 +86,8 @@ public class TabletSchedCtx implements Comparable { public static final int FINISHED_COUNTER_THRESHOLD = 4; - private static VersionCountComparator VERSION_COUNTER_COMPARATOR = new VersionCountComparator(); + private static CloneSrcComparator CLONE_SRC_COMPARATOR + = new CloneSrcComparator(); public enum Type { BALANCE, REPAIR @@ -638,9 +639,12 @@ public void chooseSrcReplica(Map backendsWorkingSlots, long exce throw new SchedException(Status.UNRECOVERABLE, "unable to find copy source replica"); } + // make candidates more random + Collections.shuffle(candidates); + // choose a replica which slot is available from candidates. - // sort replica by version count asc, so that we prefer to choose replicas with fewer versions - Collections.sort(candidates, VERSION_COUNTER_COMPARATOR); + // sort replica by version count asc and isUserDrop, so that we prefer to choose replicas with fewer versions + Collections.sort(candidates, CLONE_SRC_COMPARATOR); for (Replica srcReplica : candidates) { long replicaBeId = srcReplica.getBackendIdWithoutException(); PathSlot slot = backendsWorkingSlots.get(replicaBeId); @@ -1370,19 +1374,21 @@ public String toString() { return sb.toString(); } - // Comparator to sort the replica with version count, asc - public static class VersionCountComparator implements Comparator { + // Comparator to sort the replica with version count and isUserDrop, if isUserDrop true, put it to end. + // In same isUserDrop, version count asc + // such as [(100, true), (50, false), (-1, false), (200, false), (-1, true) + // after sort [(50, false), (200, false), (-1, false), (100, ture), (-1, true)] + public static class CloneSrcComparator implements Comparator { @Override public int compare(Replica r1, Replica r2) { + boolean isUserDrop1 = r1.isUserDrop(); + boolean isUserDrop2 = r2.isUserDrop(); long verCount1 = r1.getVisibleVersionCount() == -1 ? Long.MAX_VALUE : r1.getVisibleVersionCount(); long verCount2 = r2.getVisibleVersionCount() == -1 ? Long.MAX_VALUE : r2.getVisibleVersionCount(); - if (verCount1 < verCount2) { - return -1; - } else if (verCount1 > verCount2) { - return 1; - } else { - return 0; + if (isUserDrop1 == isUserDrop2) { + return Long.compare(verCount1, verCount2); } + return Boolean.compare(isUserDrop1, isUserDrop2); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletSchedCtxTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletSchedCtxTest.java index bc9d7f25d8a43a..8b316c474a867e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletSchedCtxTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletSchedCtxTest.java @@ -160,34 +160,53 @@ public void testPriorityCompare() { @Test public void testVersionCountComparator() { - TabletSchedCtx.VersionCountComparator countComparator = new TabletSchedCtx.VersionCountComparator(); + TabletSchedCtx.CloneSrcComparator countComparator + = new TabletSchedCtx.CloneSrcComparator(); List replicaList = Lists.newArrayList(); Replica replica1 = new Replica(); replica1.setVisibleVersionCount(100); replica1.setState(Replica.ReplicaState.NORMAL); + // user drop true + replica1.setUserDropTime(System.currentTimeMillis()); Replica replica2 = new Replica(); replica2.setVisibleVersionCount(50); replica2.setState(Replica.ReplicaState.NORMAL); + // user drop false + replica2.setUserDropTime(-1); Replica replica3 = new Replica(); replica3.setVisibleVersionCount(-1); replica3.setState(Replica.ReplicaState.NORMAL); + // user drop false + replica3.setUserDropTime(-1); Replica replica4 = new Replica(); replica4.setVisibleVersionCount(200); replica4.setState(Replica.ReplicaState.NORMAL); + // user drop false + replica4.setUserDropTime(-1); + + Replica replica5 = new Replica(); + replica5.setVisibleVersionCount(-1); + replica5.setState(Replica.ReplicaState.NORMAL); + // user drop true + replica5.setUserDropTime(System.currentTimeMillis()); replicaList.add(replica1); replicaList.add(replica2); replicaList.add(replica3); replicaList.add(replica4); + replicaList.add(replica5); Collections.sort(replicaList, countComparator); + // user drop false Assert.assertEquals(50, replicaList.get(0).getVisibleVersionCount()); - Assert.assertEquals(100, replicaList.get(1).getVisibleVersionCount()); - Assert.assertEquals(200, replicaList.get(2).getVisibleVersionCount()); - Assert.assertEquals(-1, replicaList.get(3).getVisibleVersionCount()); + Assert.assertEquals(200, replicaList.get(1).getVisibleVersionCount()); + Assert.assertEquals(-1, replicaList.get(2).getVisibleVersionCount()); + // user drop true + Assert.assertEquals(100, replicaList.get(3).getVisibleVersionCount()); + Assert.assertEquals(-1, replicaList.get(4).getVisibleVersionCount()); } @Test From 32b7f1fe175384657bc02c6c55f92c38eb3858f2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Jul 2025 15:07:16 +0800 Subject: [PATCH 196/572] branch-3.0: [enhance](meta-service)add bvar for fdb process status #52882 (#53100) Cherry-picked from #52882 Co-authored-by: koarz --- cloud/src/common/bvars.cpp | 2 + cloud/src/common/bvars.h | 2 + cloud/src/common/metric.cpp | 70 +++++++++++++++++++++ cloud/test/metric_test.cpp | 117 ++++++++++++++++++++++++++++++++++++ 4 files changed, 191 insertions(+) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 9f7753b4063a52..4485fb9f5d6022 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -199,6 +199,8 @@ bvar::Status g_bvar_fdb_workload_transactions_started_hz("fdb_workload_ bvar::Status g_bvar_fdb_workload_transactions_committed_hz("fdb_workload_transactions_committed_hz", BVAR_FDB_INVALID_VALUE); bvar::Status g_bvar_fdb_workload_transactions_rejected_hz("fdb_workload_transactions_rejected_hz", BVAR_FDB_INVALID_VALUE); bvar::Status g_bvar_fdb_client_thread_busyness_percent("fdb_client_thread_busyness_percent", BVAR_FDB_INVALID_VALUE); +mBvarStatus g_bvar_fdb_process_status_int("fdb_process_status_int", {"process_id", "component", "metric"}); +mBvarStatus g_bvar_fdb_process_status_float("fdb_process_status_float", {"process_id", "component", "metric"}); // checker's bvars BvarStatusWithTag g_bvar_checker_num_scanned("checker", "num_scanned"); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 2288209531dce6..83ab481764f926 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -338,6 +338,8 @@ extern bvar::Status g_bvar_fdb_workload_transactions_started_hz; extern bvar::Status g_bvar_fdb_workload_transactions_committed_hz; extern bvar::Status g_bvar_fdb_workload_transactions_rejected_hz; extern bvar::Status g_bvar_fdb_client_thread_busyness_percent; +extern mBvarStatus g_bvar_fdb_process_status_int; +extern mBvarStatus g_bvar_fdb_process_status_float; // checker extern BvarStatusWithTag g_bvar_checker_num_scanned; diff --git a/cloud/src/common/metric.cpp b/cloud/src/common/metric.cpp index a9b91c6c853ccd..124a5f26a063af 100644 --- a/cloud/src/common/metric.cpp +++ b/cloud/src/common/metric.cpp @@ -17,10 +17,12 @@ #include "metric.h" +#include #include #include #include +#include #include #include #include @@ -28,6 +30,7 @@ #include #include "common/bvars.h" +#include "common/logging.h" #include "meta-store/txn_kv.h" #include "meta-store/txn_kv_error.h" @@ -134,6 +137,68 @@ static void export_fdb_status_details(const std::string& status_str) { DCHECK(node->value.IsDouble()); return static_cast(node->value.GetDouble() * NANOSECONDS); }; + auto get_process_metric = [&](std::string component) { + auto node = document.FindMember("cluster"); + if (!node->value.HasMember("processes")) return; + node = node->value.FindMember("processes"); + // process + for (auto process_node = node->value.MemberBegin(); process_node != node->value.MemberEnd(); + process_node++) { + const char* process_id = process_node->name.GetString(); + decltype(process_node) component_node; + // get component iter + if (!process_node->value.HasMember(component.data())) return; + component_node = process_node->value.FindMember(component.data()); + // There are three cases here: int64, double, and object. + // If it is double or int64, put it directly into the bvar. + // If it is an object, recursively obtain the full name and corresponding value. + // such as: {"disk": {"reads": {"counter": 123, "hz": 0}}} + // component is "disk", the names of these two values should be "reads_counter" and "reads_hz" + auto recursive_name_helper = [](std::string& origin_name, + const char* next_level_name) -> std::string { + return origin_name + '_' + next_level_name; + }; + // proved two type lambda func to handle object and other type + + // set_bvar_value is responsible for setting integer and float values to the corresponding bvar. + auto set_bvar_value = [&process_id, &component]( + std::string& name, + decltype(process_node)& temp_node) -> void { + if (temp_node->value.IsInt64()) { + g_bvar_fdb_process_status_int.put({process_id, component, name}, + temp_node->value.GetInt64()); + return; + } + if (temp_node->value.IsDouble()) { + g_bvar_fdb_process_status_float.put({process_id, component, name}, + temp_node->value.GetDouble()); + return; + } + LOG(WARNING) << fmt::format( + "Get process metrics set_bvar_value input a wrong type node {}", name); + }; + auto object_recursive = [&set_bvar_value, &recursive_name_helper]( + auto&& self, std::string name, + decltype(process_node) temp_node) -> void { + // if the node is an object, then get Member(iter) and recursive with iter as arg + if (temp_node->value.IsObject()) { + for (auto iter = temp_node->value.MemberBegin(); + iter != temp_node->value.MemberEnd(); iter++) { + self(self, recursive_name_helper(name, iter->name.GetString()), iter); + } + return; + } + // if not object, set bvar value + set_bvar_value(name, temp_node); + }; + // Note that the parameter passed to set_bvar_value here is the current node, not its Member + // so we can directly call object_recursive in the loop + for (auto metric_node = component_node->value.MemberBegin(); + metric_node != component_node->value.MemberEnd(); metric_node++) { + object_recursive(object_recursive, metric_node->name.GetString(), metric_node); + } + } + }; // Configuration g_bvar_fdb_configuration_coordinators_count.set_value( get_value({"configuration", "coordinators_count"})); @@ -226,6 +291,11 @@ static void export_fdb_status_details(const std::string& status_str) { } } } + + // Process Status + get_process_metric("cpu"); + get_process_metric("disk"); + get_process_metric("memory"); } void FdbMetricExporter::export_fdb_metrics(TxnKv* txn_kv) { diff --git a/cloud/test/metric_test.cpp b/cloud/test/metric_test.cpp index 31a2b7b3c5821f..81174c73924de9 100644 --- a/cloud/test/metric_test.cpp +++ b/cloud/test/metric_test.cpp @@ -172,4 +172,121 @@ TEST(MetricTest, FdbMetricExporterTest) { ASSERT_EQ(g_bvar_fdb_machines_count.get_value(), BVAR_FDB_INVALID_VALUE); ASSERT_EQ(g_bvar_fdb_client_count.get_value(), BVAR_FDB_INVALID_VALUE); } + + // process status + { + g_bvar_fdb_machines_count.set_value(BVAR_FDB_INVALID_VALUE); + g_bvar_fdb_client_count.set_value(BVAR_FDB_INVALID_VALUE); + + std::string fdb_metric_example = "./fdb_metric_example.json"; + std::ifstream inFile(fdb_metric_example); + + ASSERT_TRUE(inFile.is_open()); + std::string fileContent((std::istreambuf_iterator(inFile)), + std::istreambuf_iterator()); + + std::shared_ptr txn_kv = std::make_shared(); + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put("\xff\xff/status/json", fileContent); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + FdbMetricExporter fdb_metric_exporter(txn_kv); + fdb_metric_exporter.sleep_interval_ms_ = 1; + fdb_metric_exporter.start(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + fdb_metric_exporter.stop(); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "cpu", "usage_cores"}), + 0.0012292); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "busy"}), + 0.0085999800000000001); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "free_bytes"}), + 490412584960); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "reads_counter"}), + 854857); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "reads_hz"}), + 0); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "reads_sectors"}), + 0); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "total_bytes"}), + 527295578112); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "writes_counter"}), + 73765457); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "writes_hz"}), + 26.1999); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "disk", "writes_sectors"}), + 1336); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "memory", "available_bytes"}), + 3065090867); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "memory", "limit_bytes"}), + 8589934592); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "memory", "rss_bytes"}), + 46551040); + ASSERT_EQ(g_bvar_fdb_process_status_int.get({"09ca90b9f3f413e5816b2610ed8b465d", "memory", + "unused_allocated_memory"}), + 655360); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"09ca90b9f3f413e5816b2610ed8b465d", "memory", "used_bytes"}), + 122974208); + + // test second process + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "cpu", "usage_cores"}), + 0.0049765900000000004); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "busy"}), + 0.012200000000000001); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "free_bytes"}), + 489160159232); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "reads_counter"}), + 877107); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "reads_hz"}), + 0); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "reads_sectors"}), + 0); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "total_bytes"}), + 527295578112); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "writes_counter"}), + 79316112); + ASSERT_EQ(g_bvar_fdb_process_status_float.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "writes_hz"}), + 30.9999); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "disk", "writes_sectors"}), + 744); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "memory", "available_bytes"}), + 3076787404); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "memory", "limit_bytes"}), + 8589934592); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "memory", "rss_bytes"}), + 72359936); + ASSERT_EQ(g_bvar_fdb_process_status_int.get({"0a456165f04e1ec1a2ade0ce523d54a8", "memory", + "unused_allocated_memory"}), + 393216); + ASSERT_EQ(g_bvar_fdb_process_status_int.get( + {"0a456165f04e1ec1a2ade0ce523d54a8", "memory", "used_bytes"}), + 157978624); + } } \ No newline at end of file From 19b08a068c09fd8569a3fa030617cd67455fc5e1 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 11 Jul 2025 15:38:31 +0800 Subject: [PATCH 197/572] branch-3.0: [feature](cloud) support event driven or periodic warm up #52370 (#52514) ### What problem does this PR solve? backport #52370 --- be/src/cloud/cloud_backend_service.cpp | 22 +- be/src/cloud/cloud_internal_service.cpp | 266 +++++++++++ be/src/cloud/cloud_internal_service.h | 8 + be/src/cloud/cloud_meta_mgr.cpp | 5 +- be/src/cloud/cloud_meta_mgr.h | 2 +- be/src/cloud/cloud_tablet.cpp | 153 +++++-- be/src/cloud/cloud_tablet.h | 4 +- be/src/cloud/cloud_warm_up_manager.cpp | 317 +++++++++++++- be/src/cloud/cloud_warm_up_manager.h | 20 +- be/src/cloud/config.cpp | 4 + be/src/cloud/config.h | 4 + be/src/common/config.h | 2 + .../io/cache/block_file_cache_downloader.cpp | 20 + be/src/olap/rowset/rowset_meta.cpp | 2 +- be/src/olap/rowset/rowset_meta.h | 2 +- .../java/org/apache/doris/common/Config.java | 3 + .../org/apache/doris/nereids/DorisParser.g4 | 3 +- fe/fe-core/src/main/cup/sql_parser.cup | 8 +- .../doris/analysis/ShowCloudWarmUpStmt.java | 5 +- .../doris/analysis/WarmUpClusterStmt.java | 11 +- .../doris/cloud/CacheHotspotManager.java | 260 +++++++++-- .../apache/doris/cloud/CloudWarmUpJob.java | 413 +++++++++++++++--- .../org/apache/doris/metric/CloudMetrics.java | 24 + .../doris/metric/DorisMetricRegistry.java | 6 +- .../org/apache/doris/metric/MetricRepo.java | 98 +++++ .../doris/service/FrontendServiceImpl.java | 45 +- .../persist/ModifyCloudWarmUpJobTest.java | 5 +- gensrc/proto/internal_service.proto | 24 + gensrc/thrift/BackendService.thrift | 8 +- gensrc/thrift/FrontendService.thrift | 1 + .../cluster/test_warm_up_cluster.groovy | 2 +- .../cluster/test_warm_up_cluster_batch.groovy | 2 +- .../test_warm_up_cluster_bigsize.groovy | 2 +- .../cluster/test_warm_up_cluster_empty.groovy | 2 +- .../cluster/test_warm_up_cluster_event.groovy | 200 +++++++++ ...st_warm_up_cluster_event_add_new_be.groovy | 203 +++++++++ ...warm_up_cluster_event_cancel_active.groovy | 206 +++++++++ ...arm_up_cluster_event_cancel_passive.groovy | 248 +++++++++++ ...st_warm_up_cluster_event_compaction.groovy | 241 ++++++++++ .../test_warm_up_cluster_event_rename.groovy | 251 +++++++++++ ...arm_up_cluster_event_restart_all_be.groovy | 226 ++++++++++ ..._up_cluster_event_restart_master_fe.groovy | 218 +++++++++ ...warm_up_cluster_event_schema_change.groovy | 266 +++++++++++ .../test_warm_up_cluster_periodic.groovy | 192 ++++++++ ...warm_up_cluster_periodic_add_new_be.groovy | 190 ++++++++ ..._warm_up_cluster_periodic_and_event.groovy | 237 ++++++++++ ...est_warm_up_cluster_periodic_rename.groovy | 219 ++++++++++ ..._cluster_periodic_restart_master_fe.groovy | 226 ++++++++++ ...t_warm_up_cluster_periodic_slow_job.groovy | 288 ++++++++++++ .../test_warm_up_cluster_repeat_jobs.groovy | 138 ++++++ .../cluster/test_warm_up_compute_group.groovy | 2 +- .../table/test_warm_up_partition.groovy | 2 +- .../warm_up/table/test_warm_up_table.groovy | 2 +- .../warm_up/table/test_warm_up_tables.groovy | 2 +- 54 files changed, 5130 insertions(+), 180 deletions(-) create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy create mode 100644 regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy diff --git a/be/src/cloud/cloud_backend_service.cpp b/be/src/cloud/cloud_backend_service.cpp index f94807282b380c..a50d0e36419f0c 100644 --- a/be/src/cloud/cloud_backend_service.cpp +++ b/be/src/cloud/cloud_backend_service.cpp @@ -36,6 +36,9 @@ namespace doris { +bvar::Adder g_file_cache_warm_up_cache_async_submitted_segment_num( + "file_cache_warm_up_cache_async_submitted_segment_num"); + CloudBackendService::CloudBackendService(CloudStorageEngine& engine, ExecEnv* exec_env) : BaseBackendService(exec_env), _engine(engine) {} @@ -93,8 +96,15 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response, LOG_INFO("receive the warm up request.") .tag("request_type", "SET_JOB") .tag("job_id", request.job_id); - st = manager.check_and_set_job_id(request.job_id); - if (!st) { + if (request.__isset.event) { + st = manager.set_event(request.job_id, request.event); + if (st.ok()) { + break; + } + } else { + st = manager.check_and_set_job_id(request.job_id); + } + if (!st.ok()) { LOG_WARNING("SET_JOB failed.").error(st); break; } @@ -142,7 +152,11 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response, LOG_INFO("receive the warm up request.") .tag("request_type", "CLEAR_JOB") .tag("job_id", request.job_id); - st = manager.clear_job(request.job_id); + if (request.__isset.event) { + st = manager.set_event(request.job_id, request.event, /* clear: */ true); + } else { + st = manager.clear_job(request.job_id); + } break; } default: @@ -181,6 +195,8 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons PGetFileCacheMetaResponse brpc_response; brpc_stub->get_file_cache_meta_by_tablet_id(&cntl, &brpc_request, &brpc_response, nullptr); if (!cntl.Failed()) { + g_file_cache_warm_up_cache_async_submitted_segment_num + << brpc_response.file_cache_block_metas().size(); _engine.file_cache_block_downloader().submit_download_task( std::move(*brpc_response.mutable_file_cache_block_metas())); } else { diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 66e089c22e9d53..72267252aa63b8 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -19,7 +19,9 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_downloader.h" #include "io/cache/block_file_cache_factory.h" namespace doris { @@ -105,4 +107,268 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id( } } +bvar::Adder g_file_cache_event_driven_warm_up_submitted_segment_num( + "file_cache_event_driven_warm_up_submitted_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_segment_num( + "file_cache_event_driven_warm_up_finished_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_segment_num( + "file_cache_event_driven_warm_up_failed_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_submitted_segment_size( + "file_cache_event_driven_warm_up_submitted_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_segment_size( + "file_cache_event_driven_warm_up_finished_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_segment_size( + "file_cache_event_driven_warm_up_failed_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_submitted_index_num( + "file_cache_event_driven_warm_up_submitted_index_num"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_index_num( + "file_cache_event_driven_warm_up_finished_index_num"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_index_num( + "file_cache_event_driven_warm_up_failed_index_num"); +bvar::Adder g_file_cache_event_driven_warm_up_submitted_index_size( + "file_cache_event_driven_warm_up_submitted_index_size"); +bvar::Adder g_file_cache_event_driven_warm_up_finished_index_size( + "file_cache_event_driven_warm_up_finished_index_size"); +bvar::Adder g_file_cache_event_driven_warm_up_failed_index_size( + "file_cache_event_driven_warm_up_failed_index_size"); +bvar::Status g_file_cache_warm_up_rowset_last_handle_unix_ts( + "file_cache_warm_up_rowset_last_handle_unix_ts", 0); +bvar::Status g_file_cache_warm_up_rowset_last_finish_unix_ts( + "file_cache_warm_up_rowset_last_finish_unix_ts", 0); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_latency("file_cache_warm_up_rowset_latency"); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_request_to_handle_latency( + "file_cache_warm_up_rowset_request_to_handle_latency"); +bvar::LatencyRecorder g_file_cache_warm_up_rowset_handle_to_finish_latency( + "file_cache_warm_up_rowset_handle_to_finish_latency"); +bvar::Adder g_file_cache_warm_up_rowset_slow_count( + "file_cache_warm_up_rowset_slow_count"); +bvar::Adder g_file_cache_warm_up_rowset_request_to_handle_slow_count( + "file_cache_warm_up_rowset_request_to_handle_slow_count"); +bvar::Adder g_file_cache_warm_up_rowset_handle_to_finish_slow_count( + "file_cache_warm_up_rowset_handle_to_finish_slow_count"); + +void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* controller + [[maybe_unused]], + const PWarmUpRowsetRequest* request, + PWarmUpRowsetResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard closure_guard(done); + for (auto& rs_meta_pb : request->rowset_metas()) { + RowsetMeta rs_meta; + rs_meta.init_from_pb(rs_meta_pb); + auto storage_resource = rs_meta.remote_storage_resource(); + if (!storage_resource) { + LOG(WARNING) << storage_resource.error(); + continue; + } + int64_t tablet_id = rs_meta.tablet_id(); + auto res = _engine.tablet_mgr().get_tablet(tablet_id); + if (!res.has_value()) { + LOG_WARNING("Warm up error ").tag("tablet_id", tablet_id).error(res.error()); + continue; + } + auto tablet = res.value(); + auto tablet_meta = tablet->tablet_meta(); + + int64_t handle_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_handle_unix_ts.set_value(handle_ts); + int64_t request_ts = request->has_unix_ts_us() ? request->unix_ts_us() : 0; + g_file_cache_warm_up_rowset_request_to_handle_latency << (handle_ts - request_ts); + if (request_ts > 0 && handle_ts - request_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_request_to_handle_slow_count << 1; + LOG(INFO) << "warm up rowset (request to handle) took " << handle_ts - request_ts + << " us, tablet_id: " << rs_meta.tablet_id() + << ", rowset_id: " << rs_meta.rowset_id().to_string(); + } + int64_t expiration_time = + tablet_meta->ttl_seconds() == 0 || rs_meta.newest_write_timestamp() <= 0 + ? 0 + : rs_meta.newest_write_timestamp() + tablet_meta->ttl_seconds(); + if (expiration_time <= UnixSeconds()) { + expiration_time = 0; + } + + for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { + auto download_done = [=, tablet_id = rs_meta.tablet_id(), + rowset_id = rs_meta.rowset_id().to_string(), + segment_size = rs_meta.segment_file_size(segment_id)](Status st) { + if (st.ok()) { + g_file_cache_event_driven_warm_up_finished_segment_num << 1; + g_file_cache_event_driven_warm_up_finished_segment_size << segment_size; + int64_t now_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_finish_unix_ts.set_value(now_ts); + g_file_cache_warm_up_rowset_latency << (now_ts - request_ts); + g_file_cache_warm_up_rowset_handle_to_finish_latency << (now_ts - handle_ts); + if (request_ts > 0 && + now_ts - request_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_slow_count << 1; + LOG(INFO) << "warm up rowset took " << now_ts - request_ts + << " us, tablet_id: " << tablet_id << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + if (now_ts - handle_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_handle_to_finish_slow_count << 1; + LOG(INFO) << "warm up rowset (handle to finish) took " << now_ts - handle_ts + << " us, tablet_id: " << tablet_id << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + } else { + g_file_cache_event_driven_warm_up_failed_segment_num << 1; + g_file_cache_event_driven_warm_up_failed_segment_size << segment_size; + LOG(WARNING) << "download segment failed, tablet_id: " << tablet_id + << " rowset_id: " << rowset_id << ", error: " << st; + } + }; + + io::DownloadFileMeta download_meta { + .path = storage_resource.value()->remote_segment_path(rs_meta, segment_id), + .file_size = rs_meta.segment_file_size(segment_id), + .offset = 0, + .download_size = rs_meta.segment_file_size(segment_id), + .file_system = storage_resource.value()->fs, + .ctx = + { + .is_index_data = false, + .expiration_time = expiration_time, + .is_dryrun = + config::enable_reader_dryrun_when_download_file_cache, + }, + .download_done = std::move(download_done), + }; + g_file_cache_event_driven_warm_up_submitted_segment_num << 1; + g_file_cache_event_driven_warm_up_submitted_segment_size + << rs_meta.segment_file_size(segment_id); + _engine.file_cache_block_downloader().submit_download_task(download_meta); + + auto download_inverted_index = [&](std::string index_path, uint64_t idx_size) { + auto storage_resource = rs_meta.remote_storage_resource(); + auto download_done = [=, tablet_id = rs_meta.tablet_id(), + rowset_id = rs_meta.rowset_id().to_string()](Status st) { + if (st.ok()) { + g_file_cache_event_driven_warm_up_finished_index_num << 1; + g_file_cache_event_driven_warm_up_finished_index_size << idx_size; + int64_t now_ts = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_finish_unix_ts.set_value(now_ts); + g_file_cache_warm_up_rowset_latency << (now_ts - request_ts); + g_file_cache_warm_up_rowset_handle_to_finish_latency + << (now_ts - handle_ts); + if (request_ts > 0 && + now_ts - request_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_slow_count << 1; + LOG(INFO) << "warm up rowset took " << now_ts - request_ts + << " us, tablet_id: " << tablet_id + << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + if (now_ts - handle_ts > config::warm_up_rowset_slow_log_ms * 1000) { + g_file_cache_warm_up_rowset_handle_to_finish_slow_count << 1; + LOG(INFO) << "warm up rowset (handle to finish) took " + << now_ts - handle_ts << " us, tablet_id: " << tablet_id + << ", rowset_id: " << rowset_id + << ", segment_id: " << segment_id; + } + } else { + g_file_cache_event_driven_warm_up_failed_index_num << 1; + g_file_cache_event_driven_warm_up_failed_index_size << idx_size; + LOG(WARNING) << "download inverted index failed, tablet_id: " << tablet_id + << " rowset_id: " << rowset_id << ", error: " << st; + } + }; + io::DownloadFileMeta download_meta { + .path = io::Path(index_path), + .file_size = static_cast(idx_size), + .file_system = storage_resource.value()->fs, + .ctx = + { + .is_index_data = false, // DORIS-20877 + .expiration_time = expiration_time, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, + }, + .download_done = std::move(download_done), + }; + g_file_cache_event_driven_warm_up_submitted_index_num << 1; + g_file_cache_event_driven_warm_up_submitted_index_size << idx_size; + _engine.file_cache_block_downloader().submit_download_task(download_meta); + }; + + // inverted index + auto schema_ptr = rs_meta.tablet_schema(); + auto idx_version = schema_ptr->get_inverted_index_storage_format(); + bool has_inverted_index = schema_ptr->has_inverted_index(); + + if (has_inverted_index) { + if (idx_version == InvertedIndexStorageFormatPB::V1) { + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + std::unordered_map index_size_map; + for (const auto& info : inverted_index_info.index_info()) { + if (info.index_file_size() != -1) { + index_size_map[info.index_id()] = info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << segment_id + << ", index_id " << info.index_id(); + } + } + for (const auto& index : schema_ptr->inverted_indexes()) { + auto idx_path = storage_resource.value()->remote_idx_v1_path( + rs_meta, segment_id, index->index_id(), index->get_index_suffix()); + download_inverted_index(idx_path, index_size_map[index->index_id()]); + } + } else { // InvertedIndexStorageFormatPB::V2 + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + int64_t idx_size = 0; + if (inverted_index_info.has_index_size()) { + idx_size = inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << segment_id; + } + auto idx_path = + storage_resource.value()->remote_idx_v2_path(rs_meta, segment_id); + download_inverted_index(idx_path, idx_size); + } + } + } + } +} + +bvar::Adder g_file_cache_recycle_cache_finished_segment_num( + "file_cache_recycle_cache_finished_segment_num"); +bvar::Adder g_file_cache_recycle_cache_finished_index_num( + "file_cache_recycle_cache_finished_index_num"); + +void CloudInternalServiceImpl::recycle_cache(google::protobuf::RpcController* controller + [[maybe_unused]], + const PRecycleCacheRequest* request, + PRecycleCacheResponse* response, + google::protobuf::Closure* done) { + brpc::ClosureGuard closure_guard(done); + + if (!config::enable_file_cache) { + return; + } + for (const auto& meta : request->cache_metas()) { + for (int64_t segment_id = 0; segment_id < meta.num_segments(); segment_id++) { + auto file_key = Segment::file_cache_key(meta.rowset_id(), segment_id); + auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key); + file_cache->remove_if_cached_async(file_key); + g_file_cache_recycle_cache_finished_segment_num << 1; + } + + // inverted index + for (const auto& file_name : meta.index_file_names()) { + auto file_key = io::BlockFileCache::hash(file_name); + auto* file_cache = io::FileCacheFactory::instance()->get_by_path(file_key); + file_cache->remove_if_cached_async(file_key); + g_file_cache_recycle_cache_finished_index_num << 1; + } + } +} + } // namespace doris diff --git a/be/src/cloud/cloud_internal_service.h b/be/src/cloud/cloud_internal_service.h index db93a82a719e37..59d8739cbf46d6 100644 --- a/be/src/cloud/cloud_internal_service.h +++ b/be/src/cloud/cloud_internal_service.h @@ -40,6 +40,14 @@ class CloudInternalServiceImpl final : public PInternalService { PGetFileCacheMetaResponse* response, google::protobuf::Closure* done) override; + void warm_up_rowset(google::protobuf::RpcController* controller, + const PWarmUpRowsetRequest* request, PWarmUpRowsetResponse* response, + google::protobuf::Closure* done) override; + + void recycle_cache(google::protobuf::RpcController* controller, + const PRecycleCacheRequest* request, PRecycleCacheResponse* response, + google::protobuf::Closure* done) override; + private: CloudStorageEngine& _engine; }; diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 9baf5db3709a32..21a91d7a64e6c7 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -39,6 +39,7 @@ #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" +#include "cloud/cloud_warm_up_manager.h" #include "cloud/config.h" #include "cloud/pb_convert.h" #include "cloud/schema_cloud_dictionary_cache.h" @@ -984,7 +985,7 @@ Status CloudMetaMgr::prepare_rowset(const RowsetMeta& rs_meta, const std::string return st; } -Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id, +Status CloudMetaMgr::commit_rowset(RowsetMeta& rs_meta, const std::string& job_id, RowsetMetaSharedPtr* existed_rs_meta) { VLOG_DEBUG << "commit rowset, tablet_id: " << rs_meta.tablet_id() << ", rowset_id: " << rs_meta.rowset_id() << " txn_id: " << rs_meta.txn_id(); @@ -1026,6 +1027,8 @@ Status CloudMetaMgr::commit_rowset(const RowsetMeta& rs_meta, const std::string& RETURN_IF_ERROR( engine.get_schema_cloud_dictionary_cache().refresh_dict(rs_meta_pb.index_id())); } + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.warm_up_rowset(rs_meta); return st; } diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h index 494f491de14831..00e86e26d43ef2 100644 --- a/be/src/cloud/cloud_meta_mgr.h +++ b/be/src/cloud/cloud_meta_mgr.h @@ -82,7 +82,7 @@ class CloudMetaMgr { Status prepare_rowset(const RowsetMeta& rs_meta, const std::string& job_id, std::shared_ptr* existed_rs_meta = nullptr); - Status commit_rowset(const RowsetMeta& rs_meta, const std::string& job_id, + Status commit_rowset(RowsetMeta& rs_meta, const std::string& job_id, std::shared_ptr* existed_rs_meta = nullptr); Status update_tmp_rowset(const RowsetMeta& rs_meta); diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index 85b60e1588afaf..e4c62c592f0821 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -33,6 +33,7 @@ #include "cloud/cloud_meta_mgr.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/cloud_warm_up_manager.h" #include "common/config.h" #include "common/logging.h" #include "io/cache/block_file_cache_downloader.h" @@ -61,6 +62,30 @@ bvar::Adder g_unused_rowsets_count("unused_rowsets_count"); static constexpr int LOAD_INITIATOR_ID = -1; +bvar::Adder g_file_cache_cloud_tablet_submitted_segment_size( + "file_cache_cloud_tablet_submitted_segment_size"); +bvar::Adder g_file_cache_cloud_tablet_submitted_segment_num( + "file_cache_cloud_tablet_submitted_segment_num"); +bvar::Adder g_file_cache_cloud_tablet_submitted_index_size( + "file_cache_cloud_tablet_submitted_index_size"); +bvar::Adder g_file_cache_cloud_tablet_submitted_index_num( + "file_cache_cloud_tablet_submitted_index_num"); +bvar::Adder g_file_cache_cloud_tablet_finished_segment_size( + "file_cache_cloud_tablet_finished_segment_size"); +bvar::Adder g_file_cache_cloud_tablet_finished_segment_num( + "file_cache_cloud_tablet_finished_segment_num"); +bvar::Adder g_file_cache_cloud_tablet_finished_index_size( + "file_cache_cloud_tablet_finished_index_size"); +bvar::Adder g_file_cache_cloud_tablet_finished_index_num( + "file_cache_cloud_tablet_finished_index_num"); + +bvar::Adder g_file_cache_recycle_cached_data_segment_num( + "file_cache_recycle_cached_data_segment_num"); +bvar::Adder g_file_cache_recycle_cached_data_segment_size( + "file_cache_recycle_cached_data_segment_size"); +bvar::Adder g_file_cache_recycle_cached_data_index_num( + "file_cache_recycle_cached_data_index_num"); + CloudTablet::CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta) : BaseTablet(std::move(tablet_meta)), _engine(engine) {} @@ -238,15 +263,21 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ ? 0 : rowset_meta->newest_write_timestamp() + _tablet_meta->ttl_seconds(); - // clang-format off + g_file_cache_cloud_tablet_submitted_segment_num << 1; + if (rs->rowset_meta()->segment_file_size(seg_id) > 0) { + g_file_cache_cloud_tablet_submitted_segment_size + << rs->rowset_meta()->segment_file_size(seg_id); + } _engine.file_cache_block_downloader().submit_download_task(io::DownloadFileMeta { - .path = storage_resource.value()->remote_segment_path(*rowset_meta, seg_id), + .path = storage_resource.value()->remote_segment_path(*rowset_meta, + seg_id), .file_size = rs->rowset_meta()->segment_file_size(seg_id), .file_system = storage_resource.value()->fs, .ctx = { .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, }, .download_done {[](Status st) { if (!st) { @@ -255,15 +286,16 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ }}, }); - auto download_idx_file = [&](const io::Path& idx_path) { + auto download_idx_file = [&](const io::Path& idx_path, int64_t idx_size) { io::DownloadFileMeta meta { .path = idx_path, - .file_size = -1, + .file_size = idx_size, .file_system = storage_resource.value()->fs, .ctx = { .expiration_time = expiration_time, - .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, + .is_dryrun = config:: + enable_reader_dryrun_when_download_file_cache, }, .download_done {[](Status st) { if (!st) { @@ -272,22 +304,42 @@ void CloudTablet::add_rowsets(std::vector to_add, bool version_ }}, }; _engine.file_cache_block_downloader().submit_download_task(std::move(meta)); + g_file_cache_cloud_tablet_submitted_index_num << 1; + g_file_cache_cloud_tablet_submitted_index_size << idx_size; }; // clang-format on auto schema_ptr = rowset_meta->tablet_schema(); auto idx_version = schema_ptr->get_inverted_index_storage_format(); if (idx_version == InvertedIndexStorageFormatPB::V1) { + std::unordered_map index_size_map; + auto&& inverted_index_info = rowset_meta->inverted_index_file_info(seg_id); + for (const auto& info : inverted_index_info.index_info()) { + if (info.index_file_size() != -1) { + index_size_map[info.index_id()] = info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << seg_id + << ", index_id " << info.index_id(); + } + } for (const auto& index : schema_ptr->inverted_indexes()) { auto idx_path = storage_resource.value()->remote_idx_v1_path( *rowset_meta, seg_id, index->index_id(), index->get_index_suffix()); - download_idx_file(idx_path); + download_idx_file(idx_path, index_size_map[index->index_id()]); } } else { if (schema_ptr->has_inverted_index()) { + auto&& inverted_index_info = + rowset_meta->inverted_index_file_info(seg_id); + int64_t idx_size = 0; + if (inverted_index_info.has_index_size()) { + idx_size = inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << seg_id; + } auto idx_path = storage_resource.value()->remote_idx_v2_path( *rowset_meta, seg_id); - download_idx_file(idx_path); + download_idx_file(idx_path, idx_size); } } } @@ -461,29 +513,55 @@ void CloudTablet::add_unused_rowsets(const std::vector& rowsets } void CloudTablet::remove_unused_rowsets() { - int64_t removed_rowsets_num = 0; + std::vector> removed_rowsets; OlapStopWatch watch; - std::lock_guard lock(_gc_mutex); - // 1. remove unused rowsets's cache data and delete bitmap - for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) { - // it->second is std::shared_ptr - auto&& rs = it->second; - if (rs.use_count() > 1) { - LOG(WARNING) << "tablet_id:" << tablet_id() << " rowset: " << rs->rowset_id() << " has " - << rs.use_count() << " references, it cannot be removed"; - ++it; - continue; + + { + std::lock_guard lock(_gc_mutex); + // 1. remove unused rowsets's cache data and delete bitmap + for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) { + auto& rs = it->second; + if (rs.use_count() > 1) { + LOG(WARNING) << "tablet_id:" << tablet_id() << " rowset: " << rs->rowset_id() + << " has " << rs.use_count() << " references, it cannot be removed"; + ++it; + continue; + } + tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); + rs->clear_cache(); + removed_rowsets.push_back(std::move(rs)); + g_unused_rowsets_count << -1; + it = _unused_rowsets.erase(it); } - tablet_meta()->remove_rowset_delete_bitmap(rs->rowset_id(), rs->version()); - rs->clear_cache(); - it = _unused_rowsets.erase(it); - g_unused_rowsets_count << -1; - removed_rowsets_num++; } - LOG(INFO) << "tablet_id=" << tablet_id() << ", unused_rowset size=" << _unused_rowsets.size() - << ", removed_rowsets_num=" << removed_rowsets_num - << ", cost(us)=" << watch.get_elapse_time_us(); + std::vector rowset_ids; + std::vector num_segments; + std::vector> index_file_names; + + for (auto& rs : removed_rowsets) { + rowset_ids.push_back(rs->rowset_id()); + num_segments.push_back(rs->num_segments()); + auto index_names = rs->get_index_file_names(); + index_file_names.push_back(index_names); + int64_t segment_size_sum = 0; + for (int32_t i = 0; i < rs->num_segments(); i++) { + segment_size_sum += rs->rowset_meta()->segment_file_size(i); + } + g_file_cache_recycle_cached_data_segment_num << rs->num_segments(); + g_file_cache_recycle_cached_data_segment_size << segment_size_sum; + g_file_cache_recycle_cached_data_index_num << index_names.size(); + } + + if (removed_rowsets.size() > 0) { + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(tablet_id(), rowset_ids, num_segments, index_file_names); + + LOG(INFO) << "tablet_id=" << tablet_id() + << ", unused_rowset size=" << _unused_rowsets.size() + << ", removed_rowsets_num=" << removed_rowsets.size() + << ", cost(us)=" << watch.get_elapse_time_us(); + } } void CloudTablet::update_base_size(const Rowset& rs) { @@ -499,14 +577,33 @@ void CloudTablet::clear_cache() { } void CloudTablet::recycle_cached_data(const std::vector& rowsets) { + std::vector rowset_ids; + std::vector num_segments; + std::vector> index_file_names; for (const auto& rs : rowsets) { // rowsets and tablet._rs_version_map each hold a rowset shared_ptr, so at this point, the reference count of the shared_ptr is at least 2. if (rs.use_count() > 2) { LOG(WARNING) << "Rowset " << rs->rowset_id().to_string() << " has " << rs.use_count() << " references. File Cache won't be recycled when query is using it."; - return; + continue; } rs->clear_cache(); + rowset_ids.push_back(rs->rowset_id()); + num_segments.push_back(rs->num_segments()); + auto index_names = rs->get_index_file_names(); + index_file_names.push_back(index_names); + int64_t segment_size_sum = 0; + for (int32_t i = 0; i < rs->num_segments(); i++) { + segment_size_sum += rs->rowset_meta()->segment_file_size(i); + } + g_file_cache_recycle_cached_data_segment_num << rs->num_segments(); + g_file_cache_recycle_cached_data_segment_size << segment_size_sum; + g_file_cache_recycle_cached_data_index_num << index_names.size(); + } + if (!rowsets.empty()) { + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(rowsets.front()->rowset_meta()->tablet_id(), rowset_ids, num_segments, + index_file_names); } } diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index dc357eb7249c95..df573314938339 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -259,8 +259,6 @@ class CloudTablet final : public BaseTablet { void build_tablet_report_info(TTabletInfo* tablet_info); - static void recycle_cached_data(const std::vector& rowsets); - // check that if the delete bitmap in delete bitmap cache has the same cardinality with the expected_delete_bitmap's Status check_delete_bitmap_cache(int64_t txn_id, DeleteBitmap* expected_delete_bitmap) override; @@ -269,6 +267,8 @@ class CloudTablet final : public BaseTablet { void add_unused_rowsets(const std::vector& rowsets); void remove_unused_rowsets(); + static void recycle_cached_data(const std::vector& rowsets); + private: // FIXME(plat1ko): No need to record base size if rowsets are ordered by version void update_base_size(const Rowset& rs); diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 510c677f06f34c..f3084555ecd73e 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -24,17 +24,52 @@ #include #include +#include "bvar/bvar.h" #include "cloud/cloud_tablet_mgr.h" +#include "cloud/config.h" #include "common/logging.h" #include "io/cache/block_file_cache_downloader.h" #include "olap/rowset/beta_rowset.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/tablet.h" +#include "runtime/client_cache.h" #include "runtime/exec_env.h" +#include "util/brpc_client_cache.h" // BrpcClientCache +#include "util/thrift_rpc_helper.h" #include "util/time.h" namespace doris { +bvar::Adder g_file_cache_event_driven_warm_up_requested_segment_size( + "file_cache_event_driven_warm_up_requested_segment_size"); +bvar::Adder g_file_cache_event_driven_warm_up_requested_segment_num( + "file_cache_event_driven_warm_up_requested_segment_num"); +bvar::Adder g_file_cache_event_driven_warm_up_requested_index_size( + "file_cache_event_driven_warm_up_requested_index_size"); +bvar::Adder g_file_cache_event_driven_warm_up_requested_index_num( + "file_cache_event_driven_warm_up_requested_index_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_segment_size( + "file_cache_once_or_periodic_warm_up_submitted_segment_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_segment_num( + "file_cache_once_or_periodic_warm_up_submitted_segment_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_index_size( + "file_cache_once_or_periodic_warm_up_submitted_index_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_index_num( + "file_cache_once_or_periodic_warm_up_submitted_index_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_segment_size( + "file_cache_once_or_periodic_warm_up_finished_segment_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_segment_num( + "file_cache_once_or_periodic_warm_up_finished_segment_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_index_size( + "file_cache_once_or_periodic_warm_up_finished_index_size"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_index_num( + "file_cache_once_or_periodic_warm_up_finished_index_num"); +bvar::Adder g_file_cache_recycle_cache_requested_segment_num( + "file_cache_recycle_cache_requested_segment_num"); +bvar::Adder g_file_cache_recycle_cache_requested_index_num( + "file_cache_recycle_cache_requested_index_num"); +bvar::Status g_file_cache_warm_up_rowset_last_call_unix_ts( + "file_cache_warm_up_rowset_last_call_unix_ts", 0); bvar::Adder file_cache_warm_up_failed_task_num("file_cache_warm_up", "failed_task_num"); CloudWarmUpManager::CloudWarmUpManager(CloudStorageEngine& engine) : _engine(engine) { @@ -65,7 +100,8 @@ std::unordered_map snapshot_rs_metas(BaseTable void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, int64_t expiration_time, - std::shared_ptr wait) { + std::shared_ptr wait, + bool is_index) { if (file_size < 0) { auto st = file_system->file_size(path, &file_size); if (!st.ok()) [[unlikely]] { @@ -75,6 +111,14 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, } } + if (is_index) { + g_file_cache_once_or_periodic_warm_up_submitted_index_num << 1; + g_file_cache_once_or_periodic_warm_up_submitted_index_size << file_size; + } else { + g_file_cache_once_or_periodic_warm_up_submitted_segment_num << 1; + g_file_cache_once_or_periodic_warm_up_submitted_segment_size << file_size; + } + const int64_t chunk_size = 10 * 1024 * 1024; // 10MB int64_t offset = 0; int64_t remaining_size = file_size; @@ -119,7 +163,10 @@ void CloudWarmUpManager::handle_jobs() { _cond.wait(lock); } if (_closed) break; - cur_job = _pending_job_metas.front(); + if (!_pending_job_metas.empty()) { + cur_job = _pending_job_metas.front(); + _pending_job_metas.pop_front(); + } } if (!cur_job) { @@ -176,6 +223,16 @@ void CloudWarmUpManager::handle_jobs() { auto idx_version = schema_ptr->get_inverted_index_storage_format(); const auto& idx_file_info = rs->inverted_index_file_info(seg_id); if (idx_version == InvertedIndexStorageFormatPB::V1) { + auto&& inverted_index_info = rs->inverted_index_file_info(seg_id); + std::unordered_map index_size_map; + for (const auto& info : inverted_index_info.index_info()) { + if (info.index_file_size() != -1) { + index_size_map[info.index_id()] = info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << seg_id + << ", index_id " << info.index_id(); + } + } for (const auto& index : schema_ptr->inverted_indexes()) { auto idx_path = storage_resource.value()->remote_idx_v1_path( *rs, seg_id, index->index_id(), index->get_index_suffix()); @@ -189,7 +246,7 @@ void CloudWarmUpManager::handle_jobs() { } } submit_download_tasks(idx_path, file_size, storage_resource.value()->fs, - expiration_time, wait); + expiration_time, wait, true); } } else { if (schema_ptr->has_inverted_index()) { @@ -198,7 +255,7 @@ void CloudWarmUpManager::handle_jobs() { file_size = idx_file_info.has_index_size() ? idx_file_info.index_size() : -1; submit_download_tasks(idx_path, file_size, storage_resource.value()->fs, - expiration_time, wait); + expiration_time, wait, true); } } } @@ -213,7 +270,6 @@ void CloudWarmUpManager::handle_jobs() { { std::unique_lock lock(_mtx); _finish_job.push_back(cur_job); - _pending_job_metas.pop_front(); } } #endif @@ -308,4 +364,255 @@ Status CloudWarmUpManager::clear_job(int64_t job_id) { return st; } +Status CloudWarmUpManager::set_event(int64_t job_id, TWarmUpEventType::type event, bool clear) { + DBUG_EXECUTE_IF("CloudWarmUpManager.set_event.ignore_all", { + LOG(INFO) << "Ignore set_event request, job_id=" << job_id << ", event=" << event + << ", clear=" << clear; + return Status::OK(); + }); + std::lock_guard lock(_mtx); + Status st = Status::OK(); + if (event == TWarmUpEventType::type::LOAD) { + if (clear) { + _tablet_replica_cache.erase(job_id); + LOG(INFO) << "Clear event driven sync, job_id=" << job_id << ", event=" << event; + } else if (!_tablet_replica_cache.contains(job_id)) { + static_cast(_tablet_replica_cache[job_id]); + LOG(INFO) << "Set event driven sync, job_id=" << job_id << ", event=" << event; + } + } else { + st = Status::InternalError("The event {} is not supported yet", event); + } + return st; +} + +std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_id) { + std::vector replicas; + std::vector cancelled_jobs; + std::lock_guard lock(_mtx); + for (auto& [job_id, cache] : _tablet_replica_cache) { + auto it = cache.find(tablet_id); + if (it != cache.end()) { + // check ttl expire + auto now = std::chrono::steady_clock::now(); + auto sec = std::chrono::duration_cast(now - it->second.first); + if (sec.count() < config::warmup_tablet_replica_info_cache_ttl_sec) { + replicas.push_back(it->second.second); + LOG(INFO) << "get_replica_info: cache hit, tablet_id=" << tablet_id + << ", job_id=" << job_id; + continue; + } else { + LOG(INFO) << "get_replica_info: cache expired, tablet_id=" << tablet_id + << ", job_id=" << job_id; + cache.erase(it); + } + } + LOG(INFO) << "get_replica_info: cache miss, tablet_id=" << tablet_id + << ", job_id=" << job_id; + + ClusterInfo* cluster_info = ExecEnv::GetInstance()->cluster_info(); + if (cluster_info == nullptr) { + LOG(WARNING) << "get_replica_info: have not get FE Master heartbeat yet, job_id=" + << job_id; + continue; + } + TNetworkAddress master_addr = cluster_info->master_fe_addr; + if (master_addr.hostname == "" || master_addr.port == 0) { + LOG(WARNING) << "get_replica_info: have not get FE Master heartbeat yet, job_id=" + << job_id; + continue; + } + + TGetTabletReplicaInfosRequest request; + TGetTabletReplicaInfosResult result; + request.warm_up_job_id = job_id; + request.__isset.warm_up_job_id = true; + request.tablet_ids.emplace_back(tablet_id); + Status rpc_st = ThriftRpcHelper::rpc( + master_addr.hostname, master_addr.port, + [&request, &result](FrontendServiceConnection& client) { + client->getTabletReplicaInfos(result, request); + }); + + if (!rpc_st.ok()) { + LOG(WARNING) << "get_replica_info: rpc failed error=" << rpc_st + << ", tablet id=" << tablet_id << ", job_id=" << job_id; + continue; + } + + auto st = Status::create(result.status); + if (!st.ok()) { + if (st.is()) { + LOG(INFO) << "get_replica_info: warm up job cancelled, tablet_id=" << tablet_id + << ", job_id=" << job_id; + cancelled_jobs.push_back(job_id); + } else { + LOG(WARNING) << "get_replica_info: failed status=" << st + << ", tablet id=" << tablet_id << ", job_id=" << job_id; + } + continue; + } + VLOG_DEBUG << "get_replica_info: got " << result.tablet_replica_infos.size() + << " tablets, tablet id=" << tablet_id << ", job_id=" << job_id; + + for (const auto& it : result.tablet_replica_infos) { + auto tablet_id = it.first; + VLOG_DEBUG << "get_replica_info: got " << it.second.size() + << " replica_infos, tablet id=" << tablet_id << ", job_id=" << job_id; + for (const auto& replica : it.second) { + cache[tablet_id] = std::make_pair(std::chrono::steady_clock::now(), replica); + replicas.push_back(replica); + LOG(INFO) << "get_replica_info: cache add, tablet_id=" << tablet_id + << ", job_id=" << job_id; + } + } + } + for (auto job_id : cancelled_jobs) { + LOG(INFO) << "get_replica_info: erasing cancelled job, job_id=" << job_id; + _tablet_replica_cache.erase(job_id); + } + VLOG_DEBUG << "get_replica_info: return " << replicas.size() + << " replicas, tablet id=" << tablet_id; + return replicas; +} + +void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta) { + auto replicas = get_replica_info(rs_meta.tablet_id()); + if (replicas.empty()) { + LOG(INFO) << "There is no need to warmup tablet=" << rs_meta.tablet_id() + << ", skipping rowset=" << rs_meta.rowset_id().to_string(); + return; + } + int64_t now_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + g_file_cache_warm_up_rowset_last_call_unix_ts.set_value(now_ts); + + PWarmUpRowsetRequest request; + request.add_rowset_metas()->CopyFrom(rs_meta.get_rowset_pb()); + request.set_unix_ts_us(now_ts); + for (auto& replica : replicas) { + // send sync request + std::string host = replica.host; + auto dns_cache = ExecEnv::GetInstance()->dns_cache(); + if (dns_cache == nullptr) { + LOG(WARNING) << "DNS cache is not initialized, skipping hostname resolve"; + } else if (!is_valid_ip(replica.host)) { + Status status = dns_cache->get(replica.host, &host); + if (!status.ok()) { + LOG(WARNING) << "failed to get ip from host " << replica.host << ": " + << status.to_string(); + return; + } + } + std::string brpc_addr = get_host_port(host, replica.brpc_port); + Status st = Status::OK(); + std::shared_ptr brpc_stub = + ExecEnv::GetInstance()->brpc_internal_client_cache()->get_new_client_no_cache( + brpc_addr); + if (!brpc_stub) { + st = Status::RpcError("Address {} is wrong", brpc_addr); + continue; + } + + // update metrics + auto schema_ptr = rs_meta.tablet_schema(); + bool has_inverted_index = schema_ptr->has_inverted_index(); + auto idx_version = schema_ptr->get_inverted_index_storage_format(); + for (int64_t segment_id = 0; segment_id < rs_meta.num_segments(); segment_id++) { + g_file_cache_event_driven_warm_up_requested_segment_num << 1; + g_file_cache_event_driven_warm_up_requested_segment_size + << rs_meta.segment_file_size(segment_id); + + if (has_inverted_index) { + if (idx_version == InvertedIndexStorageFormatPB::V1) { + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + if (inverted_index_info.index_info().empty()) { + VLOG_DEBUG << "No index info available for segment " << segment_id; + continue; + } + for (const auto& info : inverted_index_info.index_info()) { + g_file_cache_event_driven_warm_up_requested_index_num << 1; + if (info.index_file_size() != -1) { + g_file_cache_event_driven_warm_up_requested_index_size + << info.index_file_size(); + } else { + VLOG_DEBUG << "Invalid index_file_size for segment_id " << segment_id + << ", index_id " << info.index_id(); + } + } + } else { // InvertedIndexStorageFormatPB::V2 + auto&& inverted_index_info = rs_meta.inverted_index_file_info(segment_id); + g_file_cache_event_driven_warm_up_requested_index_num << 1; + if (inverted_index_info.has_index_size()) { + g_file_cache_event_driven_warm_up_requested_index_size + << inverted_index_info.index_size(); + } else { + VLOG_DEBUG << "index_size is not set for segment " << segment_id; + } + } + } + } + + brpc::Controller cntl; + PWarmUpRowsetResponse response; + brpc_stub->warm_up_rowset(&cntl, &request, &response, nullptr); + } +} + +void CloudWarmUpManager::recycle_cache( + int64_t tablet_id, const std::vector& rowset_ids, + const std::vector& num_segments, + const std::vector>& index_file_names) { + LOG(INFO) << "recycle_cache: tablet_id=" << tablet_id << ", num_rowsets=" << rowset_ids.size(); + auto replicas = get_replica_info(tablet_id); + if (replicas.empty()) { + return; + } + if (rowset_ids.size() != num_segments.size()) { + LOG(WARNING) << "recycle_cache: rowset_ids size mismatch with num_segments"; + return; + } + + PRecycleCacheRequest request; + for (int i = 0; i < rowset_ids.size(); i++) { + RecycleCacheMeta* meta = request.add_cache_metas(); + meta->set_tablet_id(tablet_id); + meta->set_rowset_id(rowset_ids[i].to_string()); + meta->set_num_segments(num_segments[i]); + for (const auto& name : index_file_names[i]) { + meta->add_index_file_names(name); + } + g_file_cache_recycle_cache_requested_segment_num << num_segments[i]; + g_file_cache_recycle_cache_requested_index_num << index_file_names[i].size(); + } + for (auto& replica : replicas) { + // send sync request + std::string host = replica.host; + auto dns_cache = ExecEnv::GetInstance()->dns_cache(); + if (dns_cache == nullptr) { + LOG(WARNING) << "DNS cache is not initialized, skipping hostname resolve"; + } else if (!is_valid_ip(replica.host)) { + Status status = dns_cache->get(replica.host, &host); + if (!status.ok()) { + LOG(WARNING) << "failed to get ip from host " << replica.host << ": " + << status.to_string(); + return; + } + } + std::string brpc_addr = get_host_port(host, replica.brpc_port); + Status st = Status::OK(); + std::shared_ptr brpc_stub = + ExecEnv::GetInstance()->brpc_internal_client_cache()->get_new_client_no_cache( + brpc_addr); + if (!brpc_stub) { + st = Status::RpcError("Address {} is wrong", brpc_addr); + continue; + } + brpc::Controller cntl; + PRecycleCacheResponse response; + brpc_stub->recycle_cache(&cntl, &request, &response, nullptr); + } +} + } // namespace doris diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 356d7284f6f3ee..13ba906a4e5844 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -69,11 +69,22 @@ class CloudWarmUpManager { // Cancel the job Status clear_job(int64_t job_id); + Status set_event(int64_t job_id, TWarmUpEventType::type event, bool clear = false); + + void warm_up_rowset(RowsetMeta& rs_meta); + + void recycle_cache(int64_t tablet_id, const std::vector& rowset_ids, + const std::vector& num_segments, + const std::vector>& index_file_names); + private: void handle_jobs(); + std::vector get_replica_info(int64_t tablet_id); + void submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, int64_t expiration_time, - std::shared_ptr wait); + std::shared_ptr wait, + bool is_index = false); std::mutex _mtx; std::condition_variable _cond; int64_t _cur_job_id {0}; @@ -84,6 +95,13 @@ class CloudWarmUpManager { bool _closed {false}; // the attribute for compile in ut [[maybe_unused]] CloudStorageEngine& _engine; + + // timestamp, info + using CacheEntry = std::pair; + // tablet_id -> entry + using Cache = std::unordered_map; + // job_id -> cache + std::unordered_map _tablet_replica_cache; }; } // namespace doris diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 234993ff0ab928..9c9eaa4d0dd2ab 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -88,5 +88,9 @@ DEFINE_mInt32(meta_service_conflict_error_retry_times, "10"); DEFINE_Bool(enable_check_storage_vault, "true"); +DEFINE_mInt64(warmup_tablet_replica_info_cache_ttl_sec, "600"); + +DEFINE_mInt64(warm_up_rowset_slow_log_ms, "1000"); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index 6f550ff25f86eb..cc477aa375fed2 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -124,5 +124,9 @@ DECLARE_mInt32(meta_service_conflict_error_retry_times); DECLARE_Bool(enable_check_storage_vault); +DECLARE_mInt64(warmup_tablet_replica_info_cache_ttl_sec); + +DECLARE_mInt64(warm_up_rowset_slow_log_ms); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/common/config.h b/be/src/common/config.h index 34fee6cabed955..caaf10c33cd8e8 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1170,6 +1170,8 @@ DECLARE_mInt64(file_cache_background_ttl_gc_batch); DECLARE_Int32(file_cache_downloader_thread_num_min); DECLARE_Int32(file_cache_downloader_thread_num_max); +DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); + // inverted index searcher cache // cache entry stay time after lookup DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s); diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index 05c18e0b945ce3..1732197e5b407c 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -40,6 +40,12 @@ namespace doris::io { +bvar::Adder g_file_cache_download_submitted_size("file_cache_download_submitted_size"); +bvar::Adder g_file_cache_download_finished_size("file_cache_download_finished_size"); +bvar::Adder g_file_cache_download_submitted_num("file_cache_download_submitted_num"); +bvar::Adder g_file_cache_download_finished_num("file_cache_download_finished_num"); +bvar::Adder g_file_cache_download_failed_num("file_cache_download_failed_num"); + FileCacheBlockDownloader::FileCacheBlockDownloader(CloudStorageEngine& engine) : _engine(engine) { _poller = std::thread(&FileCacheBlockDownloader::polling_download_task, this); auto st = ThreadPoolBuilder("FileCacheBlockDownloader") @@ -75,6 +81,14 @@ void FileCacheBlockDownloader::submit_download_task(DownloadTask task) { std::lock_guard lock(_inflight_mtx); for (auto& meta : std::get<0>(task.task_message)) { ++_inflight_tablets[meta.tablet_id()]; + if (meta.size() > 0) { + g_file_cache_download_submitted_size << meta.size(); + } + } + } else { + int64_t download_size = std::get<1>(task.task_message).download_size; + if (download_size > 0) { + g_file_cache_download_submitted_size << download_size; } } @@ -87,12 +101,14 @@ void FileCacheBlockDownloader::submit_download_task(DownloadTask task) { download_file_meta.download_done( Status::InternalError("The downloader queue is full")); } + g_file_cache_download_failed_num << 1; } _task_queue.pop_front(); // Eliminate the earliest task in the queue } _task_queue.push_back(std::move(task)); _empty.notify_all(); } + g_file_cache_download_submitted_num << 1; } void FileCacheBlockDownloader::polling_download_task() { @@ -212,6 +228,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met if (meta.download_done) { meta.download_done(std::move(st)); } + g_file_cache_download_failed_num << 1; return; } @@ -237,13 +254,16 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met if (meta.download_done) { meta.download_done(std::move(st)); } + g_file_cache_download_failed_num << 1; return; } + g_file_cache_download_finished_size << size; } if (meta.download_done) { meta.download_done(Status::OK()); } + g_file_cache_download_finished_num << 1; } void FileCacheBlockDownloader::download_blocks(DownloadTask& task) { diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp index 0da212ea8812d4..137f5c745cee43 100644 --- a/be/src/olap/rowset/rowset_meta.cpp +++ b/be/src/olap/rowset/rowset_meta.cpp @@ -210,7 +210,7 @@ void RowsetMeta::add_segments_file_size(const std::vector& seg_file_size } } -int64_t RowsetMeta::segment_file_size(int seg_id) { +int64_t RowsetMeta::segment_file_size(int seg_id) const { DCHECK(_rowset_meta_pb.segments_file_size().empty() || _rowset_meta_pb.segments_file_size_size() > seg_id) << _rowset_meta_pb.segments_file_size_size() << ' ' << seg_id; diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h index 4421b6dda1fb4e..2a4650611354c3 100644 --- a/be/src/olap/rowset/rowset_meta.h +++ b/be/src/olap/rowset/rowset_meta.h @@ -356,7 +356,7 @@ class RowsetMeta : public MetadataAdder { void add_segments_file_size(const std::vector& seg_file_size); // Return -1 if segment file size is unknown - int64_t segment_file_size(int seg_id); + int64_t segment_file_size(int seg_id) const; const auto& segments_file_size() const { return _rowset_meta_pb.segments_file_size(); } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 35d04db31fe469..8159018acc5085 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3213,6 +3213,9 @@ public static int metaServiceRpcRetryTimes() { @ConfField(mutable = true, masterOnly = true) public static long cloud_warm_up_job_max_bytes_per_batch = 21474836480L; // 20GB + @ConfField(mutable = true, masterOnly = true) + public static boolean cloud_warm_up_force_all_partitions = false; + @ConfField(mutable = true, masterOnly = true) public static boolean enable_fetch_cluster_cache_hotspot = true; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index bc205f6686f817..38acd9f6e9fa57 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -205,7 +205,8 @@ unsupportedOtherStatement | UNLOCK TABLES #unlockTables | WARM UP (CLUSTER | COMPUTE GROUP) destination=identifier WITH ((CLUSTER | COMPUTE GROUP) source=identifier | - (warmUpItem (AND warmUpItem)*)) FORCE? #warmUpCluster + (warmUpItem (AND warmUpItem)*)) FORCE? + properties=propertyClause? #warmUpCluster | BACKUP SNAPSHOT label=multipartIdentifier TO repo=identifier ((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)? properties=propertyClause? #backup diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index c1c5b0f4fb5687..0a0afa736348ae 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -1480,17 +1480,17 @@ alter_stmt ::= ; warm_up_stmt ::= - KW_WARM KW_UP KW_CLUSTER ident:dstClusterName KW_WITH KW_CLUSTER ident:srcClusterName opt_force:force + KW_WARM KW_UP KW_CLUSTER ident:dstClusterName KW_WITH KW_CLUSTER ident:srcClusterName opt_force:force opt_properties:properties {: - RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force); + RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force, properties); :} | KW_WARM KW_UP KW_CLUSTER ident:dstClusterName KW_WITH warm_up_list:list opt_force:force {: RESULT = new WarmUpClusterStmt(dstClusterName, list, force); :} - | KW_WARM KW_UP KW_COMPUTE KW_GROUP ident:dstClusterName KW_WITH KW_COMPUTE KW_GROUP ident:srcClusterName opt_force:force + | KW_WARM KW_UP KW_COMPUTE KW_GROUP ident:dstClusterName KW_WITH KW_COMPUTE KW_GROUP ident:srcClusterName opt_force:force opt_properties:properties {: - RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force); + RESULT = new WarmUpClusterStmt(dstClusterName, srcClusterName, force, properties); :} | KW_WARM KW_UP KW_COMPUTE KW_GROUP ident:dstClusterName KW_WITH warm_up_list:list opt_force:force {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java index 9ec063d3f76164..725025e77b8789 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowCloudWarmUpStmt.java @@ -36,10 +36,13 @@ public class ShowCloudWarmUpStmt extends ShowStmt implements NotFallbackInParser private static final ImmutableList WARM_UP_JOB_TITLE_NAMES = new ImmutableList.Builder() .add("JobId") - .add("ComputeGroup") + .add("SrcComputeGroup") + .add("DstComputeGroup") .add("Status") .add("Type") + .add("SyncMode") .add("CreateTime") + .add("StartTime") .add("FinishBatch") .add("AllBatch") .add("FinishTime") diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java index cca21d5c259c06..04d66eb8309281 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java @@ -34,6 +34,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -46,12 +47,15 @@ public class WarmUpClusterStmt extends StatementBase implements NotFallbackInPar private String srcClusterName; private boolean isWarmUpWithTable; private boolean isForce; + private Map properties; - public WarmUpClusterStmt(String dstClusterName, String srcClusterName, boolean isForce) { + public WarmUpClusterStmt(String dstClusterName, String srcClusterName, boolean isForce, + Map properties) { this.dstClusterName = dstClusterName; this.srcClusterName = srcClusterName; this.isForce = isForce; this.isWarmUpWithTable = false; + this.properties = properties; } public WarmUpClusterStmt(String dstClusterName, List> tableList, boolean isForce) { @@ -59,6 +63,7 @@ public WarmUpClusterStmt(String dstClusterName, List> tab this.tableList = tableList; this.isForce = isForce; this.isWarmUpWithTable = true; + this.properties = new HashMap<>(); } @Override @@ -161,4 +166,8 @@ public boolean isWarmUpWithTable() { public boolean isForce() { return isForce; } + + public Map getProperties() { + return properties; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index 4e073b21473566..167e413a02e996 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.cloud.CloudWarmUpJob.JobState; import org.apache.doris.cloud.CloudWarmUpJob.JobType; +import org.apache.doris.cloud.CloudWarmUpJob.SyncMode; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; @@ -70,6 +71,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -107,11 +109,137 @@ public class CacheHotspotManager extends MasterDaemon { private ConcurrentMap runnableCloudWarmUpJobs = Maps.newConcurrentMap(); - private Set runnableClusterSet = ConcurrentHashMap.newKeySet(); - private final ThreadPoolExecutor cloudWarmUpThreadPool = ThreadPoolManager.newDaemonCacheThreadPool( Config.max_active_cloud_warm_up_job, "cloud-warm-up-pool", true); + private static class JobKey { + private final String srcName; + private final String dstName; + private final CloudWarmUpJob.SyncMode syncMode; + + public JobKey(String srcName, String dstName, CloudWarmUpJob.SyncMode syncMode) { + this.srcName = srcName; + this.dstName = dstName; + this.syncMode = syncMode; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof JobKey)) { + return false; + } + JobKey jobKey = (JobKey) o; + return Objects.equals(srcName, jobKey.srcName) + && Objects.equals(dstName, jobKey.dstName) + && syncMode == jobKey.syncMode; + } + + @Override + public int hashCode() { + return Objects.hash(srcName, dstName, syncMode); + } + + @Override + public String toString() { + return "WarmUpJob src='" + srcName + "', dst='" + dstName + "', syncMode=" + String.valueOf(syncMode); + } + } + + // Tracks long-running jobs (event-driven and periodic). + // Ensures only one active job exists per tuple. + private Set repeatJobDetectionSet = ConcurrentHashMap.newKeySet(); + + private void registerJobForRepeatDetection(CloudWarmUpJob job, boolean replay) throws AnalysisException { + if (job.isDone()) { + return; + } + if (job.isEventDriven() || job.isPeriodic()) { + // For long lasting jobs, i.e. event-driven and periodic. + // It is meaningless to create more than one job for a given src, dst, and syncMode. + JobKey key = new JobKey(job.getSrcClusterName(), job.getDstClusterName(), job.getSyncMode()); + boolean added = this.repeatJobDetectionSet.add(key); + if (!added && !replay) { + throw new AnalysisException(key + " already has a runnable job"); + } + } + } + + // Tracks warm-up jobs scheduled by CacheHotSpotManager. + // Ensures that at most one job runs concurrently per destination cluster. + private Map clusterToRunningJobId = new ConcurrentHashMap<>(); + + /** + * Attempts to register a job as running for the given destination cluster. + *

+ * For one-time or periodic jobs, returns {@code false} if there is already a running job + * for the specified destination cluster. Returns {@code true} if this job is successfully + * registered as the only running job for that cluster. + *

+ * For event-driven jobs, this method does not perform any registration and always returns {@code true}. + * + * @param job the CloudWarmUpJob to register + * @return {@code true} if the job was registered successfully or is event-driven; {@code false} otherwise + */ + public boolean tryRegisterRunningJob(CloudWarmUpJob job) { + if (job.isEventDriven()) { + // Event-driven jobs do not require registration, always allow + return true; + } + + String clusterName = job.getDstClusterName(); + long jobId = job.getJobId(); + + // Try to register the job atomically if absent + Long existingJobId = clusterToRunningJobId.putIfAbsent(clusterName, jobId); + boolean success = (existingJobId == null) || (existingJobId == jobId); + if (!success) { + LOG.info("Job {} skipped: waiting for job {} to finish on destination cluster {}", + jobId, existingJobId, clusterName); + } + return success; + } + + /** + * Deregisters the given job from the running jobs map, allowing another job + * to run on the same destination cluster. + *

+ * For event-driven jobs, this method does nothing and always returns {@code true} + * since they are not registered. + *

+ * This method only removes the job if the currently registered job ID matches + * the job's ID, ensuring no accidental deregistration of other jobs. + * + * @param job the CloudWarmUpJob to deregister + * @return {@code true} if the job was successfully deregistered or is event-driven; {@code false} otherwise + */ + private boolean deregisterRunningJob(CloudWarmUpJob job) { + if (job.isEventDriven()) { + // Event-driven jobs are not registered, so nothing to deregister + return true; + } + + String clusterName = job.getDstClusterName(); + long jobId = job.getJobId(); + + return clusterToRunningJobId.remove(clusterName, jobId); + } + + public void notifyJobStop(CloudWarmUpJob job) { + if (job.isOnce() || job.isPeriodic()) { + this.deregisterRunningJob(job); + } + if (!job.isDone()) { + return; + } + if (job.isEventDriven() || job.isPeriodic()) { + this.repeatJobDetectionSet.remove(new JobKey( + job.getSrcClusterName(), job.getDstClusterName(), job.getSyncMode())); + } + } + public CacheHotspotManager(CloudSystemInfoService nodeMgr) { super("CacheHotspotManager", Config.fetch_cluster_cache_hotspot_interval_ms); this.nodeMgr = nodeMgr; @@ -364,13 +492,13 @@ Long getFileCacheCapacity(String clusterName) throws RuntimeException { return totalFileCache; } - private Map>> splitBatch(Map> beToWarmUpTablets) { + public Map>> splitBatch(Map> beToWarmUpTablets) { final Long maxSizePerBatch = Config.cloud_warm_up_job_max_bytes_per_batch; Map>> beToTabletIdBatches = new HashMap<>(); for (Map.Entry> entry : beToWarmUpTablets.entrySet()) { List> batches = new ArrayList<>(); List batch = new ArrayList<>(); - Long curBatchSize = 0L; + long curBatchSize = 0L; for (Tablet tablet : entry.getValue()) { if (curBatchSize + tablet.getDataSize(true) > maxSizePerBatch) { batches.add(batch); @@ -388,7 +516,7 @@ private Map>> splitBatch(Map> beToWarmU return beToTabletIdBatches; } - private Map> warmUpNewClusterByCluster(String dstClusterName, String srcClusterName) { + private List getHotTablets(String srcClusterName, String dstClusterName) { Long dstTotalFileCache = getFileCacheCapacity(dstClusterName); List> result = getClusterTopNHotPartitions(srcClusterName); Long warmUpTabletsSize = 0L; @@ -427,6 +555,39 @@ private Map> warmUpNewClusterByCluster(String dstClusterName, } } Collections.reverse(tablets); + return tablets; + } + + private List getAllTablets(String srcClusterName, String dstClusterName) { + List tablets = new ArrayList<>(); + List dbs = Env.getCurrentInternalCatalog().getDbs(); + for (Database db : dbs) { + List tables = db.getTables(); + for (Table table : tables) { + if (!(table instanceof OlapTable)) { + continue; + } + OlapTable olapTable = (OlapTable) table; + for (Partition partition : olapTable.getPartitions()) { + // Maybe IndexExtState.ALL + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (Tablet tablet : index.getTablets()) { + tablets.add(tablet); + } + } + } + } + } + return tablets; + } + + public Map> warmUpNewClusterByCluster(String dstClusterName, String srcClusterName) { + List tablets; + if (Config.cloud_warm_up_force_all_partitions) { + tablets = getAllTablets(srcClusterName, dstClusterName); + } else { + tablets = getHotTablets(srcClusterName, dstClusterName); + } List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) .getBackendsByClusterName(dstClusterName); Map> beToWarmUpTablets = new HashMap<>(); @@ -497,8 +658,8 @@ public Map getCloudWarmUpJobs() { return this.cloudWarmUpJobs; } - public Set getRunnableClusterSet() { - return this.runnableClusterSet; + public CloudWarmUpJob getCloudWarmUpJob(long jobId) { + return this.cloudWarmUpJobs.get(jobId); } public List> getAllJobInfos(int limit) { @@ -511,13 +672,11 @@ public List> getAllJobInfos(int limit) { return infos; } - public void addCloudWarmUpJob(CloudWarmUpJob job) { + public void addCloudWarmUpJob(CloudWarmUpJob job) throws AnalysisException { + registerJobForRepeatDetection(job, false); cloudWarmUpJobs.put(job.getJobId(), job); LOG.info("add cloud warm up job {}", job.getJobId()); runnableCloudWarmUpJobs.put(job.getJobId(), job); - if (!job.isDone()) { - runnableClusterSet.add(job.getCloudClusterName()); - } } public List getPartitionsFromTriple(Triple tableTriple) { @@ -616,51 +775,84 @@ public Map> warmUpNewClusterByTable(long jobId, String dstClu } public long createJob(WarmUpClusterStmt stmt) throws AnalysisException { - if (runnableClusterSet.contains(stmt.getDstClusterName())) { - throw new AnalysisException("cluster: " + stmt.getDstClusterName() + " already has a runnable job"); - } - Map> beToWarmUpTablets = new HashMap<>(); long jobId = Env.getCurrentEnv().getNextId(); - if (!FeConstants.runningUnitTest) { - if (stmt.isWarmUpWithTable()) { + CloudWarmUpJob warmUpJob; + if (stmt.isWarmUpWithTable()) { + Map> beToWarmUpTablets = new HashMap<>(); + if (!FeConstants.runningUnitTest) { beToWarmUpTablets = warmUpNewClusterByTable(jobId, stmt.getDstClusterName(), stmt.getTables(), stmt.isForce()); + } + Map>> beToTabletIdBatches = splitBatch(beToWarmUpTablets); + warmUpJob = new CloudWarmUpJob(jobId, null, stmt.getDstClusterName(), + beToTabletIdBatches, JobType.TABLE); + } else { + CloudWarmUpJob.Builder builder = new CloudWarmUpJob.Builder() + .setJobId(jobId) + .setSrcClusterName(stmt.getSrcClusterName()) + .setDstClusterName(stmt.getDstClusterName()) + .setJobType(JobType.CLUSTER); + + Map properties = stmt.getProperties(); + if ("periodic".equals(properties.get("sync_mode"))) { + String syncIntervalSecStr = properties.get("sync_interval_sec"); + if (syncIntervalSecStr == null) { + throw new AnalysisException("No sync_interval_sec is provided"); + } + long syncIntervalSec; + try { + syncIntervalSec = Long.parseLong(syncIntervalSecStr); + } catch (NumberFormatException e) { + throw new AnalysisException("Illegal sync_interval_sec: " + syncIntervalSecStr); + } + builder.setSyncMode(SyncMode.PERIODIC) + .setSyncInterval(syncIntervalSec); + } else if ("event_driven".equals(properties.get("sync_mode"))) { + String syncEventStr = properties.get("sync_event"); + if (syncEventStr == null) { + throw new AnalysisException("No sync_event is provided"); + } + CloudWarmUpJob.SyncEvent syncEvent; + try { + syncEvent = CloudWarmUpJob.SyncEvent.valueOf(syncEventStr.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new AnalysisException("Illegal sync_event: " + syncEventStr, e); + } + builder.setSyncMode(SyncMode.EVENT_DRIVEN) + .setSyncEvent(syncEvent); } else { - beToWarmUpTablets = warmUpNewClusterByCluster(stmt.getDstClusterName(), stmt.getSrcClusterName()); + builder.setSyncMode(SyncMode.ONCE); } + warmUpJob = builder.build(); } - Map>> beToTabletIdBatches = splitBatch(beToWarmUpTablets); - - CloudWarmUpJob.JobType jobType = stmt.isWarmUpWithTable() ? JobType.TABLE : JobType.CLUSTER; - CloudWarmUpJob warmUpJob; - if (jobType == JobType.TABLE) { - warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType, - stmt.getTables(), stmt.isForce()); - } else { - warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), beToTabletIdBatches, jobType); - } addCloudWarmUpJob(warmUpJob); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(warmUpJob); LOG.info("finished to create cloud warm up job: {}", warmUpJob.getJobId()); return jobId; - } public void cancel(CancelCloudWarmUpStmt stmt) throws DdlException { - CloudWarmUpJob job = cloudWarmUpJobs.get(stmt.getJobId()); + cancel(stmt.getJobId()); + } + + public void cancel(long jobId) throws DdlException { + CloudWarmUpJob job = cloudWarmUpJobs.get(jobId); if (job == null) { - throw new DdlException("job id: " + stmt.getJobId() + " does not exist."); + throw new DdlException("job id: " + jobId + " does not exist."); } - if (!job.cancel("user cancel")) { + if (!job.cancel("user cancel", true)) { throw new DdlException("job can not be cancelled. State: " + job.getJobState()); } } private void runCloudWarmUpJob() { runnableCloudWarmUpJobs.values().forEach(cloudWarmUpJob -> { + if (cloudWarmUpJob.shouldWait()) { + return; + } if (!cloudWarmUpJob.isDone() && !activeCloudWarmUpJobs.containsKey(cloudWarmUpJob.getJobId()) && activeCloudWarmUpJobs.size() < Config.max_active_cloud_warm_up_job) { if (FeConstants.runningUnitTest) { @@ -686,9 +878,9 @@ public void replayCloudWarmUpJob(CloudWarmUpJob cloudWarmUpJob) throws Exception cloudWarmUpJobs.put(cloudWarmUpJob.getJobId(), cloudWarmUpJob); LOG.info("replay cloud warm up job {}, state {}", cloudWarmUpJob.getJobId(), cloudWarmUpJob.getJobState()); if (cloudWarmUpJob.isDone()) { - runnableClusterSet.remove(cloudWarmUpJob.getCloudClusterName()); + notifyJobStop(cloudWarmUpJob); } else { - runnableClusterSet.add(cloudWarmUpJob.getCloudClusterName()); + registerJobForRepeatDetection(cloudWarmUpJob, true); } if (cloudWarmUpJob.jobState == JobState.DELETED) { if (cloudWarmUpJobs.remove(cloudWarmUpJob.getJobId()) != null diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java index 463a37c4635dc5..0bd655893a7a7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java @@ -18,6 +18,7 @@ package org.apache.doris.cloud; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.Tablet; import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.ClientPool; @@ -27,7 +28,9 @@ import org.apache.doris.common.Triple; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.metric.MetricRepo; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.Backend; @@ -36,6 +39,7 @@ import org.apache.doris.thrift.TJobMeta; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TStatusCode; +import org.apache.doris.thrift.TWarmUpEventType; import org.apache.doris.thrift.TWarmUpTabletsRequest; import org.apache.doris.thrift.TWarmUpTabletsRequestType; import org.apache.doris.thrift.TWarmUpTabletsResponse; @@ -75,26 +79,43 @@ public enum JobType { TABLE; } + public enum SyncMode { + ONCE, + PERIODIC, + EVENT_DRIVEN; + } + + public enum SyncEvent { + LOAD, + QUERY + } + @SerializedName(value = "jobId") protected long jobId; @SerializedName(value = "jobState") protected JobState jobState; @SerializedName(value = "createTimeMs") protected long createTimeMs = -1; + @SerializedName(value = "startTimeMs") + protected long startTimeMs = -1; @SerializedName(value = "errMsg") protected String errMsg = ""; @SerializedName(value = "finishedTimeMs") protected long finishedTimeMs = -1; + @SerializedName(value = "srcClusterName") + protected String srcClusterName = ""; + + // the serialized name is kept for compatibility reasons @SerializedName(value = "cloudClusterName") - protected String cloudClusterName = ""; + protected String dstClusterName = ""; @SerializedName(value = "lastBatchId") protected long lastBatchId = -1; @SerializedName(value = "beToTabletIdBatches") - protected Map>> beToTabletIdBatches; + protected Map>> beToTabletIdBatches = new HashMap<>(); @SerializedName(value = "beToThriftAddress") protected Map beToThriftAddress = new HashMap<>(); @@ -108,6 +129,15 @@ public enum JobType { @SerializedName(value = "force") protected boolean force = false; + @SerializedName(value = "syncMode") + protected SyncMode syncMode = SyncMode.ONCE; + + @SerializedName(value = "syncInterval") + protected long syncInterval; + + @SerializedName(value = "syncEvent") + protected SyncEvent syncEvent; + private Map beToClient; private Map beToAddr; @@ -120,17 +150,93 @@ public enum JobType { private boolean setJobDone = false; - public CloudWarmUpJob(long jobId, String cloudClusterName, - Map>> beToTabletIdBatches, JobType jobType) { + public static class Builder { + private long jobId; + private String srcClusterName; + private String dstClusterName; + private JobType jobType = JobType.CLUSTER; + private SyncMode syncMode = SyncMode.ONCE; + private SyncEvent syncEvent; + private long syncInterval; + + public Builder() {} + + public Builder setJobId(long jobId) { + this.jobId = jobId; + return this; + } + + public Builder setSrcClusterName(String srcClusterName) { + this.srcClusterName = srcClusterName; + return this; + } + + public Builder setDstClusterName(String dstClusterName) { + this.dstClusterName = dstClusterName; + return this; + } + + public Builder setJobType(JobType jobType) { + this.jobType = jobType; + return this; + } + + public Builder setSyncMode(SyncMode syncMode) { + this.syncMode = syncMode; + return this; + } + + public Builder setSyncEvent(SyncEvent syncEvent) { + this.syncEvent = syncEvent; + return this; + } + + public Builder setSyncInterval(long syncInterval) { + this.syncInterval = syncInterval; + return this; + } + + public CloudWarmUpJob build() { + if (jobId == 0 || srcClusterName == null || dstClusterName == null || jobType == null || syncMode == null) { + throw new IllegalStateException("Missing required fields for CloudWarmUpJob"); + } + return new CloudWarmUpJob(this); + } + } + + private CloudWarmUpJob(Builder builder) { + this.jobId = builder.jobId; + this.jobState = JobState.PENDING; + this.srcClusterName = builder.srcClusterName; + this.dstClusterName = builder.dstClusterName; + this.jobType = builder.jobType; + this.syncMode = builder.syncMode; + this.syncEvent = builder.syncEvent; + this.syncInterval = builder.syncInterval; + this.createTimeMs = System.currentTimeMillis(); + } + + private void fetchBeToThriftAddress() { + String clusterName = isEventDriven() ? srcClusterName : dstClusterName; + List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getBackendsByClusterName(clusterName); + for (Backend backend : backends) { + beToThriftAddress.put(backend.getId(), backend.getHost() + ":" + backend.getBePort()); + } + } + + public CloudWarmUpJob(long jobId, String srcClusterName, String dstClusterName, + Map>> beToTabletIdBatches, JobType jobType) { this.jobId = jobId; this.jobState = JobState.PENDING; - this.cloudClusterName = cloudClusterName; + this.srcClusterName = srcClusterName; + this.dstClusterName = dstClusterName; this.beToTabletIdBatches = beToTabletIdBatches; this.createTimeMs = System.currentTimeMillis(); this.jobType = jobType; if (!FeConstants.runningUnitTest) { List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getBackendsByClusterName(cloudClusterName); + .getBackendsByClusterName(dstClusterName); for (Backend backend : backends) { beToThriftAddress.put(backend.getId(), backend.getHost() + ":" + backend.getBePort()); } @@ -140,11 +246,68 @@ public CloudWarmUpJob(long jobId, String cloudClusterName, public CloudWarmUpJob(long jobId, String cloudClusterName, Map>> beToTabletIdBatches, JobType jobType, List> tables, boolean force) { - this(jobId, cloudClusterName, beToTabletIdBatches, jobType); + this(jobId, null, cloudClusterName, beToTabletIdBatches, jobType); this.tables = tables; this.force = force; } + public void fetchBeToTabletIdBatches() { + if (FeConstants.runningUnitTest) { + return; + } + if (jobType == JobType.TABLE) { + // warm up with table will have to set tablets on creation + return; + } + if (syncMode == null) { + // This job was created by an old FE version. + // It doesn't have the source cluster name, but tablets were already set. + // Return for backward compatibility. + return; + } + if (this.isEventDriven()) { + // Event-driven jobs do not need to calculate tablets + return; + } + CacheHotspotManager manager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); + Map> beToWarmUpTablets = + manager.warmUpNewClusterByCluster(dstClusterName, srcClusterName); + long totalTablets = beToWarmUpTablets.values().stream() + .mapToLong(List::size) + .sum(); + beToTabletIdBatches = manager.splitBatch(beToWarmUpTablets); + long totalBatches = beToTabletIdBatches.values().stream() + .mapToLong(List::size) + .sum(); + LOG.info("warm up job {} tablet num {}, batch num {}", jobId, totalTablets, totalBatches); + } + + public boolean shouldWait() { + if (!this.isPeriodic()) { + return false; + } + if (this.jobState != JobState.PENDING) { + return false; + } + long timeSinceLastStart = System.currentTimeMillis() - this.startTimeMs; + if (timeSinceLastStart < this.syncInterval * 1000L) { + return true; + } + return false; + } + + public boolean isOnce() { + return this.syncMode == SyncMode.ONCE || this.syncMode == null; + } + + public boolean isPeriodic() { + return this.syncMode == SyncMode.PERIODIC; + } + + public boolean isEventDriven() { + return this.syncMode == SyncMode.EVENT_DRIVEN; + } + public long getJobId() { return jobId; } @@ -181,20 +344,51 @@ public JobType getJobType() { return jobType; } + public SyncMode getSyncMode() { + return syncMode; + } + + public String getSyncModeString() { + if (syncMode == null) { + // For backward compatibility: older FE versions did not set syncMode for jobs, + // so default to ONCE when syncMode is missing. + return String.valueOf(SyncMode.ONCE); + } + StringBuilder sb = new StringBuilder().append(syncMode); + switch (syncMode) { + case PERIODIC: + sb.append(" ("); + sb.append(syncInterval); + sb.append("s)"); + break; + case EVENT_DRIVEN: + sb.append(" ("); + sb.append(syncEvent); + sb.append(")"); + break; + default: + break; + } + return sb.toString(); + } + public List getJobInfo() { List info = Lists.newArrayList(); info.add(String.valueOf(jobId)); - info.add(cloudClusterName); - info.add(jobState.name()); - info.add(jobType.name()); + info.add(srcClusterName); + info.add(dstClusterName); + info.add(String.valueOf(jobState)); + info.add(String.valueOf(jobType)); + info.add(this.getSyncModeString()); info.add(TimeUtils.longToTimeStringWithms(createTimeMs)); + info.add(TimeUtils.longToTimeStringWithms(startTimeMs)); info.add(Long.toString(lastBatchId + 1)); long maxBatchSize = 0; - for (List> list : beToTabletIdBatches.values()) { - long size = list.size(); - if (size > maxBatchSize) { - maxBatchSize = size; - } + if (beToTabletIdBatches != null) { + maxBatchSize = beToTabletIdBatches.values().stream() + .mapToLong(List::size) + .max() + .orElse(0); } info.add(Long.toString(maxBatchSize)); info.add(TimeUtils.longToTimeStringWithms(finishedTimeMs)); @@ -224,7 +418,7 @@ public void setFinishedTimeMs(long timeMs) { } public void setCloudClusterName(String name) { - this.cloudClusterName = name; + this.dstClusterName = name; } public void setLastBatchId(long id) { @@ -248,7 +442,8 @@ public boolean isDone() { } public boolean isTimeout() { - return (System.currentTimeMillis() - createTimeMs) / 1000 > Config.cloud_warm_up_timeout_second; + return jobState == JobState.RUNNING + && (System.currentTimeMillis() - startTimeMs) / 1000 > Config.cloud_warm_up_timeout_second; } public boolean isExpire() { @@ -256,20 +451,24 @@ public boolean isExpire() { > Config.history_cloud_warm_up_job_keep_max_second; } - public String getCloudClusterName() { - return cloudClusterName; + public String getDstClusterName() { + return dstClusterName; + } + + public String getSrcClusterName() { + return srcClusterName; } public synchronized void run() { if (isTimeout()) { - cancel("Timeout"); + cancel("Timeout", false); return; } if (Config.isCloudMode()) { LOG.debug("set context to job"); ConnectContext ctx = new ConnectContext(); ctx.setThreadLocalInfo(); - ctx.setCloudCluster(cloudClusterName); + ctx.setCloudCluster(dstClusterName); } try { switch (jobState) { @@ -294,6 +493,9 @@ public synchronized void run() { } public void initClients() throws Exception { + if (beToThriftAddress == null || beToThriftAddress.isEmpty()) { + fetchBeToThriftAddress(); + } if (beToClient == null) { beToClient = new HashMap<>(); beToAddr = new HashMap<>(); @@ -333,39 +535,77 @@ public void releaseClients() { beToAddr = null; } - public final synchronized boolean cancel(String errMsg) { - if (this.jobState.isFinalState()) { - return false; - } + private final void clearJobOnBEs() { try { initClients(); for (Map.Entry entry : beToClient.entrySet()) { TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); request.setType(TWarmUpTabletsRequestType.CLEAR_JOB); request.setJobId(jobId); - LOG.info("send warm up request. request_type=CLEAR_JOB"); + if (this.isEventDriven()) { + TWarmUpEventType event = getTWarmUpEventType(); + if (event == null) { + throw new IllegalArgumentException("Unknown SyncEvent " + syncEvent); + } + request.setEvent(event); + } + LOG.info("send warm up request to BE {}. job_id={}, request_type=CLEAR_JOB", + entry.getKey(), jobId); entry.getValue().warmUpTablets(request); } } catch (Exception e) { - LOG.warn("warm up job {} cancel exception: {}", jobId, e.getMessage()); + LOG.warn("send warm up request failed. job_id={}, request_type=CLEAR_JOB, exception={}", + jobId, e.getMessage()); } finally { releaseClients(); } - this.jobState = JobState.CANCELLED; + } + + public final synchronized boolean cancel(String errMsg, boolean force) { + if (this.jobState.isFinalState()) { + return false; + } + if (this.jobState == JobState.PENDING) { + // BE haven't started this job yet, skip RPC + } else { + clearJobOnBEs(); + } + if (this.isOnce() || force) { + this.jobState = JobState.CANCELLED; + } else { + this.jobState = JobState.PENDING; + } this.errMsg = errMsg; this.finishedTimeMs = System.currentTimeMillis(); + MetricRepo.updateClusterWarmUpJobLastFinishTime(String.valueOf(jobId), srcClusterName, + dstClusterName, finishedTimeMs); LOG.info("cancel cloud warm up job {}, err {}", jobId, errMsg); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().getRunnableClusterSet().remove(this.cloudClusterName); + ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().notifyJobStop(this); return true; - } private void runPendingJob() throws DdlException { Preconditions.checkState(jobState == JobState.PENDING, jobState); - // Todo: nothing to prepare yet + // make sure only one job runs concurrently for one destination cluster + if (!((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().tryRegisterRunningJob(this)) { + return; + } + // Todo: nothing to prepare yet + this.setJobDone = false; + this.lastBatchId = -1; + this.startTimeMs = System.currentTimeMillis(); + MetricRepo.updateClusterWarmUpJobLatestStartTime(String.valueOf(jobId), srcClusterName, + dstClusterName, startTimeMs); + this.fetchBeToTabletIdBatches(); + long totalTablets = beToTabletIdBatches.values().stream() + .flatMap(List::stream) + .mapToLong(List::size) + .sum(); + MetricRepo.increaseClusterWarmUpJobRequestedTablets(dstClusterName, totalTablets); + MetricRepo.increaseClusterWarmUpJobExecCount(dstClusterName); this.jobState = JobState.RUNNING; Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); LOG.info("transfer cloud warm up job {} state to {}", jobId, this.jobState); @@ -380,20 +620,67 @@ private List buildJobMetas(long beId, long batchId) { jobMeta.setDownloadType(TDownloadType.S3); jobMeta.setTabletIds(tabletIds); jobMetas.add(jobMeta); + MetricRepo.increaseClusterWarmUpJobFinishedTablets(dstClusterName, tabletIds.size()); } return jobMetas; } + private TWarmUpEventType getTWarmUpEventType() { + switch (syncEvent) { + case LOAD: + return TWarmUpEventType.LOAD; + case QUERY: + return TWarmUpEventType.QUERY; + default: + return null; + } + } + + private void runEventDrivenJob() throws Exception { + try { + initClients(); + for (Map.Entry entry : beToClient.entrySet()) { + TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); + request.setType(TWarmUpTabletsRequestType.SET_JOB); + request.setJobId(jobId); + TWarmUpEventType event = getTWarmUpEventType(); + if (event == null) { + throw new IllegalArgumentException("Unknown SyncEvent " + syncEvent); + } + request.setEvent(event); + LOG.debug("send warm up request to BE {}. job_id={}, event={}, request_type=SET_JOB(EVENT)", + entry.getKey(), jobId, syncEvent); + TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); + if (response.getStatus().getStatusCode() != TStatusCode.OK) { + if (!response.getStatus().getErrorMsgs().isEmpty()) { + errMsg = response.getStatus().getErrorMsgs().get(0); + } + LOG.warn("send warm up request failed. job_id={}, event={}, err={}", + jobId, syncEvent, errMsg); + } + } + } catch (Exception e) { + LOG.warn("send warm up request job_id={} failed with exception {}", + jobId, e); + } finally { + releaseClients(); + } + } + private void runRunningJob() throws Exception { Preconditions.checkState(jobState == JobState.RUNNING, jobState); if (FeConstants.runningUnitTest) { Thread.sleep(1000); this.jobState = JobState.FINISHED; this.finishedTimeMs = System.currentTimeMillis(); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().getRunnableClusterSet().remove(this.cloudClusterName); + ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().notifyJobStop(this); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); return; } + if (this.isEventDriven()) { + runEventDrivenJob(); + return; + } boolean changeToCancelState = false; try { initClients(); @@ -406,8 +693,9 @@ private void runRunningJob() throws Exception { request.setJobId(jobId); request.setBatchId(lastBatchId + 1); request.setJobMetas(buildJobMetas(entry.getKey(), request.batch_id)); - LOG.info("send warm up request. job_id={}, batch_id={}, job_sizes={}, request_type=SET_JOB", - jobId, request.batch_id, request.job_metas.size()); + LOG.info("send warm up request to BE {}. job_id={}, batch_id={}" + + ", job_size={}, request_type=SET_JOB", + entry.getKey(), jobId, request.batch_id, request.job_metas.size()); TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); if (response.getStatus().getStatusCode() != TStatusCode.OK) { if (!response.getStatus().getErrorMsgs().isEmpty()) { @@ -422,7 +710,8 @@ private void runRunningJob() throws Exception { for (Map.Entry entry : beToClient.entrySet()) { TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); request.setType(TWarmUpTabletsRequestType.GET_CURRENT_JOB_STATE_AND_LEASE); - LOG.info("send warm up request. request_type=GET_CURRENT_JOB_STATE_AND_LEASE"); + LOG.info("send warm up request to BE {}. job_id={}, request_type=GET_CURRENT_JOB_STATE_AND_LEASE", + entry.getKey(), jobId); TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); if (response.getStatus().getStatusCode() != TStatusCode.OK) { if (!response.getStatus().getErrorMsgs().isEmpty()) { @@ -434,6 +723,12 @@ private void runRunningJob() throws Exception { allLastBatchDone = false; break; } + // /api/debug_point/add/CloudWarmUpJob.FakeLastBatchNotDone + if (DebugPointUtil.isEnable("CloudWarmUpJob.FakeLastBatchNotDone")) { + allLastBatchDone = false; + LOG.info("DebugPoint:CloudWarmUpJob.FakeLastBatchNotDone, jobID={}", jobId); + break; + } } if (!changeToCancelState && allLastBatchDone) { if (retry) { @@ -454,9 +749,9 @@ private void runRunningJob() throws Exception { if (!request.job_metas.isEmpty()) { // check all batches is done or not allBatchesDone = false; - LOG.info("send warm up request. job_id={}, batch_id={}" - + "job_sizes={}, request_type=SET_BATCH", - jobId, request.batch_id, request.job_metas.size()); + LOG.info("send warm up request to BE {}. job_id={}, batch_id={}" + + ", job_size={}, request_type=SET_BATCH", + entry.getKey(), jobId, request.batch_id, request.job_metas.size()); TWarmUpTabletsResponse response = entry.getValue().warmUpTablets(request); if (response.getStatus().getStatusCode() != TStatusCode.OK) { if (!response.getStatus().getErrorMsgs().isEmpty()) { @@ -467,38 +762,23 @@ private void runRunningJob() throws Exception { } } if (allBatchesDone) { - // release job - this.jobState = JobState.FINISHED; - for (Map.Entry entry : beToClient.entrySet()) { - TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); - request.setType(TWarmUpTabletsRequestType.CLEAR_JOB); - request.setJobId(jobId); - LOG.info("send warm up request. request_type=CLEAR_JOB"); - entry.getValue().warmUpTablets(request); - } + clearJobOnBEs(); this.finishedTimeMs = System.currentTimeMillis(); - releaseClients(); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr() - .getRunnableClusterSet().remove(this.cloudClusterName); + if (this.isPeriodic()) { + // wait for next schedule + this.jobState = JobState.PENDING; + } else { + // release job + this.jobState = JobState.FINISHED; + } + ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().notifyJobStop(this); Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); } } } if (changeToCancelState) { // release job - this.jobState = JobState.CANCELLED; - for (Map.Entry entry : beToClient.entrySet()) { - TWarmUpTabletsRequest request = new TWarmUpTabletsRequest(); - request.setType(TWarmUpTabletsRequestType.CLEAR_JOB); - request.setJobId(jobId); - LOG.info("send warm up request. request_type=CLEAR_JOB"); - entry.getValue().warmUpTablets(request); - } - this.finishedTimeMs = System.currentTimeMillis(); - releaseClients(); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr() - .getRunnableClusterSet().remove(this.cloudClusterName); - Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); + cancel("job fail", false); } } catch (Exception e) { retryTime++; @@ -507,12 +787,7 @@ private void runRunningJob() throws Exception { LOG.warn("warm up job {} exception: {}", jobId, e.getMessage()); } else { // retry three times and release job - this.jobState = JobState.CANCELLED; - this.finishedTimeMs = System.currentTimeMillis(); - this.errMsg = "retry the warm up job until max retry time " + String.valueOf(maxRetryTime); - ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr() - .getRunnableClusterSet().remove(this.cloudClusterName); - Env.getCurrentEnv().getEditLog().logModifyCloudWarmUpJob(this); + cancel("retry the warm up job until max retry time " + String.valueOf(maxRetryTime), false); } releaseClients(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java index 9e57f087dbf7f8..20d4f145ec106a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/CloudMetrics.java @@ -38,6 +38,13 @@ public class CloudMetrics { protected static AutoMappedMetric> CLUSTER_BACKEND_ALIVE; protected static AutoMappedMetric> CLUSTER_BACKEND_ALIVE_TOTAL; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_EXEC_COUNT; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_FINISHED_TABLETS; + + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_LATEST_START_TIME; + protected static AutoMappedMetric CLUSTER_WARM_UP_JOB_LAST_FINISH_TIME; + protected static void init() { if (Config.isNotCloudMode()) { return; @@ -74,5 +81,22 @@ protected static void init() { + clusterId, "cluster_name=" + clusterName); return MetricRepo.METRIC_REGISTER.histogram(metricName); }); + + CLUSTER_WARM_UP_JOB_EXEC_COUNT = new AutoMappedMetric<>(name -> new LongCounterMetric( + "file_cache_warm_up_job_exec_count", MetricUnit.NOUNIT, "warm up job execution count")); + CLUSTER_WARM_UP_JOB_LATEST_START_TIME = new AutoMappedMetric<>(name -> new LongCounterMetric( + "file_cache_warm_up_job_latest_start_time", MetricUnit.MILLISECONDS, + "the latest start time (ms, epoch time) of the warm up job")); + CLUSTER_WARM_UP_JOB_LAST_FINISH_TIME = new AutoMappedMetric<>(name -> new LongCounterMetric( + "file_cache_warm_up_job_last_finish_time", MetricUnit.MILLISECONDS, + "the last finish time (ms, epoch time) of the warm up job")); + + CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS = new AutoMappedMetric<>( + name -> new LongCounterMetric("file_cache_warm_up_job_requested_tablets", + MetricUnit.NOUNIT, "warm up job requested tablets")); + + CLUSTER_WARM_UP_JOB_FINISHED_TABLETS = new AutoMappedMetric<>( + name -> new LongCounterMetric("file_cache_warm_up_job_finished_tablets", + MetricUnit.NOUNIT, "warm up job finished tablets")); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java index 64e973f78342a5..21e5e366ebccef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -105,7 +106,10 @@ public void removeMetricsByNameAndLabels(String name, List labels) private static String computeLabelId(List labels) { TreeMap labelMap = new TreeMap<>(); for (MetricLabel label : labels) { - labelMap.put(label.getKey(), label.getValue().replace("\\", "\\\\").replace("\"", "\\\"")); + labelMap.put(label.getKey(), + Optional.ofNullable(label.getValue()) + .map(v -> v.replace("\\", "\\\\").replace("\"", "\\\"")) + .orElse("")); } return labelMap.entrySet() .stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 3e21673972066b..45e738e8350f2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -975,6 +975,20 @@ public static void registerCloudMetrics(String clusterId, String clusterName) { queryErrCounter.setLabels(labels); MetricRepo.DORIS_METRIC_REGISTER.addMetrics(queryErrCounter); + LongCounterMetric warmUpJobExecCounter = CloudMetrics.CLUSTER_WARM_UP_JOB_EXEC_COUNT.getOrAdd(clusterId); + warmUpJobExecCounter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(warmUpJobExecCounter); + + LongCounterMetric warmUpJobRequestedTablets = + CloudMetrics.CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS.getOrAdd(clusterId); + warmUpJobRequestedTablets.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(warmUpJobRequestedTablets); + + LongCounterMetric warmUpJobFinishedTablets = + CloudMetrics.CLUSTER_WARM_UP_JOB_FINISHED_TABLETS.getOrAdd(clusterId); + warmUpJobFinishedTablets.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(warmUpJobFinishedTablets); + GaugeMetricImpl requestPerSecondGauge = CloudMetrics.CLUSTER_REQUEST_PER_SECOND_GAUGE .getOrAdd(clusterId); requestPerSecondGauge.setLabels(labels); @@ -1046,6 +1060,90 @@ public static void increaseClusterQueryErr(String clusterName) { MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); } + public static void increaseClusterWarmUpJobExecCount(String clusterName) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterName)) { + return; + } + String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterNameToId().get(clusterName); + if (Strings.isNullOrEmpty(clusterId)) { + return; + } + LongCounterMetric counter = CloudMetrics.CLUSTER_WARM_UP_JOB_EXEC_COUNT.getOrAdd(clusterId); + List labels = new ArrayList<>(); + counter.increase(1L); + labels.add(new MetricLabel("cluster_id", clusterId)); + labels.add(new MetricLabel("cluster_name", clusterName)); + counter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); + } + + public static void increaseClusterWarmUpJobRequestedTablets(String clusterName, long tablets) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterName)) { + return; + } + String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterNameToId().get(clusterName); + if (Strings.isNullOrEmpty(clusterId)) { + return; + } + LongCounterMetric counter = CloudMetrics.CLUSTER_WARM_UP_JOB_REQUESTED_TABLETS.getOrAdd(clusterId); + List labels = new ArrayList<>(); + counter.increase(tablets); + labels.add(new MetricLabel("cluster_id", clusterId)); + labels.add(new MetricLabel("cluster_name", clusterName)); + counter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); + } + + public static void increaseClusterWarmUpJobFinishedTablets(String clusterName, long bytes) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterName)) { + return; + } + String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterNameToId().get(clusterName); + if (Strings.isNullOrEmpty(clusterId)) { + return; + } + LongCounterMetric counter = CloudMetrics.CLUSTER_WARM_UP_JOB_FINISHED_TABLETS.getOrAdd(clusterId); + List labels = new ArrayList<>(); + counter.increase(bytes); + labels.add(new MetricLabel("cluster_id", clusterId)); + labels.add(new MetricLabel("cluster_name", clusterName)); + counter.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(counter); + } + + public static void updateClusterWarmUpJobLatestStartTime( + String jobId, String srcClusterName, String dstClusterName, long timeMs) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(jobId)) { + return; + } + LongCounterMetric time = CloudMetrics.CLUSTER_WARM_UP_JOB_LATEST_START_TIME.getOrAdd(jobId); + List labels = new ArrayList<>(); + time.update(timeMs); + labels.add(new MetricLabel("job_id", jobId)); + labels.add(new MetricLabel("src_cluster_name", srcClusterName)); + labels.add(new MetricLabel("dst_cluster_name", dstClusterName)); + time.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(time); + } + + public static void updateClusterWarmUpJobLastFinishTime( + String jobId, String srcClusterName, String dstClusterName, long timeMs) { + if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(jobId)) { + return; + } + LongCounterMetric time = CloudMetrics.CLUSTER_WARM_UP_JOB_LAST_FINISH_TIME.getOrAdd(jobId); + List labels = new ArrayList<>(); + time.update(timeMs); + labels.add(new MetricLabel("job_id", jobId)); + labels.add(new MetricLabel("src_cluster_name", srcClusterName)); + labels.add(new MetricLabel("dst_cluster_name", dstClusterName)); + time.setLabels(labels); + MetricRepo.DORIS_METRIC_REGISTER.addMetrics(time); + } + public static void updateClusterRequestPerSecond(String clusterId, double value, List labels) { if (!MetricRepo.isInit || Config.isNotCloudMode() || Strings.isNullOrEmpty(clusterId)) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index ddb49a0ee0ab7e..b95968604e3d14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -47,10 +47,13 @@ import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.TabletMeta; import org.apache.doris.catalog.View; +import org.apache.doris.cloud.CloudWarmUpJob; +import org.apache.doris.cloud.catalog.CloudEnv; import org.apache.doris.cloud.catalog.CloudPartition; import org.apache.doris.cloud.catalog.CloudReplica; import org.apache.doris.cloud.catalog.CloudTablet; import org.apache.doris.cloud.proto.Cloud.CommitTxnResponse; +import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.AuthenticationException; @@ -2744,6 +2747,21 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos TGetTabletReplicaInfosResult result = new TGetTabletReplicaInfosResult(); List tabletIds = request.getTabletIds(); Map> tabletReplicaInfos = Maps.newHashMap(); + String clusterId = ""; + if (Config.isCloudMode() && request.isSetWarmUpJobId()) { + CloudWarmUpJob job = ((CloudEnv) Env.getCurrentEnv()) + .getCacheHotspotMgr() + .getCloudWarmUpJob(request.getWarmUpJobId()); + if (job == null) { + LOG.info("warmup job {} is not running, notify caller BE {} to cancel job", + job.getJobId(), clientAddr); + // notify client to cancel this job + result.setStatus(new TStatus(TStatusCode.CANCELLED)); + return result; + } + clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getCloudClusterIdByName(job.getDstClusterName()); + } for (Long tabletId : tabletIds) { if (DebugPointUtil.isEnable("getTabletReplicaInfos.returnEmpty")) { LOG.info("enable getTabletReplicaInfos.returnEmpty"); @@ -2753,28 +2771,25 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos List replicas = Env.getCurrentEnv().getCurrentInvertedIndex() .getReplicasByTabletId(tabletId); for (Replica replica : replicas) { - if (!replica.isNormal()) { + if (!replica.isNormal() && !request.isSetWarmUpJobId()) { LOG.warn("replica {} not normal", replica.getId()); continue; } - List backends; + Backend backend; if (Config.isCloudMode()) { CloudReplica cloudReplica = (CloudReplica) replica; - backends = cloudReplica.getAllPrimaryBes(); + backend = cloudReplica.getPrimaryBackend(clusterId); } else { - Backend backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); - backends = Lists.newArrayList(backend); + backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); } - for (Backend backend : backends) { - if (backend != null) { - TReplicaInfo replicaInfo = new TReplicaInfo(); - replicaInfo.setHost(backend.getHost()); - replicaInfo.setBePort(backend.getBePort()); - replicaInfo.setHttpPort(backend.getHttpPort()); - replicaInfo.setBrpcPort(backend.getBrpcPort()); - replicaInfo.setReplicaId(replica.getId()); - replicaInfos.add(replicaInfo); - } + if (backend != null) { + TReplicaInfo replicaInfo = new TReplicaInfo(); + replicaInfo.setHost(backend.getHost()); + replicaInfo.setBePort(backend.getBePort()); + replicaInfo.setHttpPort(backend.getHttpPort()); + replicaInfo.setBrpcPort(backend.getBrpcPort()); + replicaInfo.setReplicaId(replica.getId()); + replicaInfos.add(replicaInfo); } } tabletReplicaInfos.put(tabletId, replicaInfos); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java index 0803fee02b6766..bca8c1eb0273e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/ModifyCloudWarmUpJobTest.java @@ -56,6 +56,7 @@ public void testSerialization() throws IOException { long createTimeMs = 11111; String errMsg = "testMsg"; long finishedTimesMs = 22222; + String srcClusterName = "cloudSrc"; String clusterName = "cloudTest"; long lastBatchId = 33333; Map>> beToTabletIdBatches = new HashMap<>(); @@ -68,7 +69,7 @@ public void testSerialization() throws IOException { beToThriftAddress.put(998L, "address"); CloudWarmUpJob.JobType jobType = JobType.TABLE; - CloudWarmUpJob warmUpJob = new CloudWarmUpJob(jobId, clusterName, beToTabletIdBatches, jobType); + CloudWarmUpJob warmUpJob = new CloudWarmUpJob(jobId, srcClusterName, clusterName, beToTabletIdBatches, jobType); warmUpJob.setJobState(jobState); warmUpJob.setCreateTimeMs(createTimeMs); warmUpJob.setErrMsg(errMsg); @@ -93,7 +94,7 @@ public void testSerialization() throws IOException { Assert.assertEquals(createTimeMs, warmUpJob2.getCreateTimeMs()); Assert.assertEquals(errMsg, warmUpJob2.getErrMsg()); Assert.assertEquals(finishedTimesMs, warmUpJob2.getFinishedTimeMs()); - Assert.assertEquals(clusterName, warmUpJob2.getCloudClusterName()); + Assert.assertEquals(clusterName, warmUpJob2.getDstClusterName()); Assert.assertEquals(lastBatchId, warmUpJob2.getLastBatchId()); Map>> beToTabletIdBatches2 = warmUpJob2.getBeToTabletIdBatches(); Assert.assertEquals(1, beToTabletIdBatches2.size()); diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 20de386b3eba7b..108f7d7613812d 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -837,6 +837,28 @@ message PGetFileCacheMetaResponse { repeated FileCacheBlockMeta file_cache_block_metas = 1; } +message PWarmUpRowsetRequest { + repeated RowsetMetaPB rowset_metas = 1; + optional int64 unix_ts_us = 2; +} + +message PWarmUpRowsetResponse { +} + +message RecycleCacheMeta { + optional int64 tablet_id = 1; + optional string rowset_id = 2; + optional int64 num_segments = 3; + repeated string index_file_names = 4; +} + +message PRecycleCacheRequest { + repeated RecycleCacheMeta cache_metas = 1; +} + +message PRecycleCacheResponse { +} + message PReportStreamLoadStatusRequest { optional PUniqueId load_id = 1; optional PStatus status = 2; @@ -1044,6 +1066,8 @@ service PBackendService { rpc fetch_table_schema(PFetchTableSchemaRequest) returns (PFetchTableSchemaResult); rpc multiget_data(PMultiGetRequest) returns (PMultiGetResponse); rpc get_file_cache_meta_by_tablet_id(PGetFileCacheMetaRequest) returns (PGetFileCacheMetaResponse); + rpc warm_up_rowset(PWarmUpRowsetRequest) returns (PWarmUpRowsetResponse); + rpc recycle_cache(PRecycleCacheRequest) returns (PRecycleCacheResponse); rpc tablet_fetch_data(PTabletKeyLookupRequest) returns (PTabletKeyLookupResponse); rpc get_column_ids_by_tablet_ids(PFetchColIdsRequest) returns (PFetchColIdsResponse); rpc get_tablet_rowset_versions(PGetTabletVersionsRequest) returns (PGetTabletVersionsResponse); diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 7895d8b83fa0ab..7cd09dc5cf7534 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -186,11 +186,16 @@ enum TDownloadType { S3 = 1, } +enum TWarmUpEventType { + LOAD = 0, + QUERY = 1, +} + enum TWarmUpTabletsRequestType { SET_JOB = 0, SET_BATCH = 1, GET_CURRENT_JOB_STATE_AND_LEASE = 2, - CLEAR_JOB = 3, + CLEAR_JOB = 3 } struct TJobMeta { @@ -205,6 +210,7 @@ struct TWarmUpTabletsRequest { 2: required i64 batch_id 3: optional list job_metas 4: required TWarmUpTabletsRequestType type + 5: optional TWarmUpEventType event } struct TWarmUpTabletsResponse { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 0723de2cf2f8d3..174ac9989d1c9e 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1360,6 +1360,7 @@ struct TGetBinlogResult { struct TGetTabletReplicaInfosRequest { 1: required list tablet_ids + 2: optional i64 warm_up_job_id } struct TGetTabletReplicaInfosResult { diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy index 3d22b75e98dfcf..b6ae1be881e2fa 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy index f9a5004a84e370..9a2aff3393333f 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy @@ -20,7 +20,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_cluster_batch") { def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy index e9be62cf9821ee..4f97116e120e7e 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster_bigsize") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy index bf3121b269f6e3..d86238c5be9ad4 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster_empty") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy new file mode 100644 index 00000000000000..60ae1819395ffe --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event.groovy @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy new file mode 100644 index 00000000000000..3bb78723b23663 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_add_new_be.groovy @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_add_new_be', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + // Add new backends to cluster 2 + cluster.addBackend(2, clusterName2) + + clearFileCacheOnAllBackends() + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy new file mode 100644 index 00000000000000..65ef46be4c84d1 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_active.groovy @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_cancel_active', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def srcSum = getClusterTTLCacheSizeSum(cluster1) + def dstSum = getClusterTTLCacheSizeSum(cluster2) + + logger.info("ttl_cache_size: src=${srcSum} dst=${dstSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, dstSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + // First make some entries in tablet location info ttl cache + sleep(15000) + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + def cacheSize0 = getClusterTTLCacheSizeSum(clusterName2) + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + sleep(5000) + + // At this point, cache should be expired, so we expect no more syncs + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + def cacheSize1 = getClusterTTLCacheSizeSum(clusterName1); + def cacheSize2 = getClusterTTLCacheSizeSum(clusterName2); + assertTrue(cacheSize1 > cacheSize0, "cache size in src cluster should increase") + assertEquals(cacheSize0, cacheSize2, "no more syncs after job cancel is expected") + + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy new file mode 100644 index 00000000000000..5c1cdb8ce0594f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy @@ -0,0 +1,248 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_cancel', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warmup_tablet_replica_info_cache_ttl_sec=10', + ] + options.cloudMode = true + + def setDebugPoint = {ip, port, op, name -> + def urlStr = "http://${ip}:${port}/api/debug_point/${op}/${name}" + def url = new URL(urlStr) + def conn = url.openConnection() + conn.requestMethod = 'POST' + conn.doOutput = true + + // Send empty body (required to trigger POST) + conn.outputStream.withWriter { it << "" } + + // Read response + def responseText = conn.inputStream.text + def json = new JsonSlurper().parseText(responseText) + + return json?.msg == "OK" && json?.code == 0 + } + + def setDebugPointsForCluster = { cluster, debug_point, enable -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[4] + if (enable) { + assertTrue(setDebugPoint(ip, port, 'add', debug_point)) + } else { + assertTrue(setDebugPoint(ip, port, 'remove', debug_point)) + } + } + } + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def srcSum = getClusterTTLCacheSizeSum(cluster1) + def dstSum = getClusterTTLCacheSizeSum(cluster2) + + logger.info("ttl_cache_size: src=${srcSum} dst=${dstSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, dstSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + // First make some entries in tablet location info ttl cache + sleep(15000) + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + def cacheSize0 = getClusterTTLCacheSizeSum(clusterName2) + + // Make BE ignore the cancel request + setDebugPointsForCluster(clusterName1, 'CloudWarmUpManager.set_event.ignore_all', true) + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Initially, the cache has not expired, so we expect some more syncs + for (int i = 0; i < 10; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + def cacheSize1 = getClusterTTLCacheSizeSum(clusterName2); + assertTrue(cacheSize1 > cacheSize0, "some more syncs before cache expire is expected") + + // At this point, cache should be expired, so we expect no more syncs + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + def cacheSize2 = getClusterTTLCacheSizeSum(clusterName2); + assertEquals(cacheSize1, cacheSize2, "no more syncs after cache expire is expected") + + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + + setDebugPointsForCluster(clusterName1, 'CloudWarmUpManager.set_event.ignore_all', false) + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy new file mode 100644 index 00000000000000..697bffeabfa931 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_compaction.groovy @@ -0,0 +1,241 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_compaction', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + def delta_time = 1000 + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(tableName + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def run_compaction = { + def timeout = 60000 + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test values (2, '{"a" : 111.1111}')""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + sql """insert into test values (6, '{"a" : "11111"}')""" + sql """insert into test values (7, '{"a" : 11111.11111}')""" + sql "alter table test modify column col1 variant;" + wait_for_latest_op_on_table_finish("test", timeout) + sql """insert into test values (1, '{"a" : 1.0}')""" + sql """insert into test values (2, '{"a" : 111.1111}')""" + sql """insert into test values (3, '{"a" : "11111"}')""" + sql """insert into test values (4, '{"a" : 1111111111}')""" + sql """insert into test values (5, '{"a" : 1111.11111}')""" + sql """insert into test values (6, '{"a" : "11111"}')""" + sql """insert into test values (7, '{"a" : 11111.11111}')""" + trigger_and_wait_compaction("test", "cumulative") + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ + create table test ( + col0 int not null, + col1 variant NOT NULL + ) UNIQUE KEY(`col0`) + DISTRIBUTED BY HASH(col0) BUCKETS 1 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "false"); + """ + + clearFileCacheOnAllBackends() + sleep(15000) + + run_compaction(); + sleep(15000) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy new file mode 100644 index 00000000000000..fc3ac52ae91d29 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy @@ -0,0 +1,251 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_rename', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + 'warmup_tablet_replica_info_cache_ttl_sec=1' + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(10000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + def clusterName3 = "warmup_other" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + srcSumOld = getClusterTTLCacheSizeSum(clusterName1) + dstSumOld = getClusterTTLCacheSizeSum(clusterName2) + + // rename + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName2} ${clusterName3}""" + sleep(5000) + + assertEquals(0, getClusterTTLCacheSizeSum(clusterName2)) + assertEquals(dstSumOld, getClusterTTLCacheSizeSum(clusterName3)) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + + assertEquals(0, getClusterTTLCacheSizeSum(clusterName2)) + assertEquals(dstSumOld, getClusterTTLCacheSizeSum(clusterName3)) + + // rename back + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName3} ${clusterName2}""" + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy new file mode 100644 index 00000000000000..1e2178af1fb69f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_all_be.groovy @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_restart_all_be', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + def restartAllBE = { + cluster.restartBackends() + sleep(5000) + + // wait for be restart + boolean ok = false + int cnt = 0 + for (; !ok && cnt < 30; cnt++) { + ok = true; + sql_return_maparray("show backends").each { be -> + if (!be.Alive.toBoolean()) { + ok = false + } + } + //def be = sql_return_maparray("show backends").get(0) + logger.info("wait for BE restart...") + Thread.sleep(1000) + } + if (!ok) { + logger.info("BE failed to restart") + assertTrue(false) + } + sleep(3000) + } + + restartAllBE() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy new file mode 100644 index 00000000000000..f96c0f0b4135bf --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_restart_master_fe.groovy @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_restart_master_fe', 'docker') { + def options = new ClusterOptions() + options.feNum = 3 + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def restartMasterFe = { + def oldMasterFe = cluster.getMasterFe() + cluster.restartFrontends(oldMasterFe.index) + boolean hasRestart = false + for (int i = 0; i < 30; i++) { + if (cluster.getFeByIndex(oldMasterFe.index).alive) { + hasRestart = true + break + } + sleep 1000 + } + assertTrue(hasRestart) + context.reconnectFe() + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + restartMasterFe() + sql """use @${clusterName1}""" + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + sleep(15000) + logWarmUpRowsetMetrics(clusterName2) + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy new file mode 100644 index 00000000000000..bc426a16d52f9a --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_schema_change.groovy @@ -0,0 +1,266 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_event_schema_change', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + def delta_time = 1000 + def useTime = 0 + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + def alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(tableName + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def run_schema_change = { + def table_name = "test" + def timeout = 60000 + def delta_time = 1000 + def useTime = 0 + + // sql "set experimental_enable_nereids_planner = true" + // add, drop columns + sql """INSERT INTO ${table_name} SELECT *, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}' FROM numbers("number" = "4096")""" + sql "alter table ${table_name} add column v2 variant default null" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name}""" + sql "alter table ${table_name} drop column v2" + sql """INSERT INTO ${table_name} SELECT k, v from ${table_name}""" + sql """select v['k1'] from ${table_name} order by k limit 10""" + sql "alter table ${table_name} add column vs string default null" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name}""" + sql """select v['k1'] from ${table_name} order by k desc limit 10""" + sql """select v['k1'], cast(v['k2'] as string) from ${table_name} order by k desc limit 10""" + + // sql "set experimental_enable_nereids_planner = true" + // add, drop index + sql "alter table ${table_name} add index btm_idxk (k) using bitmap ;" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name}""" + wait_for_latest_op_on_table_finish(table_name, timeout) + + // drop column is linked schema change + sql "drop index btm_idxk on ${table_name};" + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name} limit 1024""" + wait_for_latest_op_on_table_finish(table_name, timeout) + sql """select v['k1'] from ${table_name} order by k desc limit 10""" + sql """select v['k1'], cast(v['k2'] as string) from ${table_name} order by k desc limit 10""" + + // add, drop materialized view + createMV("""create materialized view var_order as select vs, k, v from ${table_name} order by vs""") + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name} limit 4096""" + createMV("""create materialized view var_cnt as select k, count(k) from ${table_name} group by k""") + sql """INSERT INTO ${table_name} SELECT k, v, v from ${table_name} limit 8101""" + sql """DROP MATERIALIZED VIEW var_cnt ON ${table_name}""" + sql """INSERT INTO ${table_name} SELECT k, v,v from ${table_name} limit 1111""" + // select from mv + sql """select v['k1'], cast(v['k2'] as string) from ${table_name} order by k desc limit 10""" + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + def table_name = "test" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 4 + properties("file_cache_ttl_seconds" = "3600"); + """ + + clearFileCacheOnAllBackends() + sleep(20000) + + run_schema_change(); + sleep(15000) + + logFileCacheDownloadMetrics(clusterName2) + logWarmUpRowsetMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy new file mode 100644 index 00000000000000..87938bd60af998 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + + clearFileCacheOnAllBackends() + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(5000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy new file mode 100644 index 00000000000000..5c58da8a564410 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_add_new_be', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(1, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + // Add new backends to cluster 2 + cluster.addBackend(2, clusterName2); + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + clearFileCacheOnAllBackends() + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(15000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy new file mode 100644 index 00000000000000..156c3e1697ae8b --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_and_event.groovy @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_and_event', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + + // Start warm up job 1 + def jobId1_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId1 = jobId1_[0][0] + logger.info("Periodic warm-up job ID: ${jobId1}") + + // Start warm up job 2 + def jobId2_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + def jobId2 = jobId2_[0][0] + logger.info("Event driven warm-up job ID: ${jobId2}") + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + clearFileCacheOnAllBackends() + + sleep(15000) + + for (int i = 0; i < 100; i++) { + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + } + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(15000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId1}""" + assertEquals(jobInfo[0][0], jobId1) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + def jobInfo2 = sql """SHOW WARM UP JOB WHERE ID = ${jobId2}""" + assertEquals(jobInfo2[0][0], jobId2) + assertEquals(jobInfo2[0][1], clusterName1) + assertEquals(jobInfo2[0][2], clusterName2) + assertEquals(jobInfo2[0][4], "CLUSTER") + assertTrue(jobInfo2[0][3] in ["RUNNING", "PENDING"]) + assertEquals(jobInfo2[0][5], "EVENT_DRIVEN (LOAD)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId1}""" + def cancelInfo1 = sql """SHOW WARM UP JOB WHERE ID = ${jobId1}""" + assertEquals(cancelInfo1[0][3], "CANCELLED") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId2}""" + def cancelInfo2 = sql """SHOW WARM UP JOB WHERE ID = ${jobId2}""" + assertEquals(cancelInfo2[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy new file mode 100644 index 00000000000000..271fec8bbfc9b0 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_rename', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + def clusterName3 = "warmup_other" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + + clearFileCacheOnAllBackends() + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName2} ${clusterName3}""" + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(5000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + assertTrue(getClusterTTLCacheSizeSum(clusterName1) > 0) + assertEquals(0, getClusterTTLCacheSizeSum(clusterName2)) + assertEquals(0, getClusterTTLCacheSizeSum(clusterName3)) + + sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName3} ${clusterName2}""" + sleep(5000) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy new file mode 100644 index 00000000000000..d7f49d3fc82a4f --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_restart_master_fe.groovy @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_periodic_restart_master_fe', 'docker') { + def options = new ClusterOptions() + options.feNum = 3 + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def logWarmUpRowsetMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") + def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") + def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") + def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_index=${submitted_index}" + + ", finished_index=${finished_index}" + + ", failed_index=${failed_index}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + srcSum += getTTLCacheSize(ip, port) + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + tgtSum += getTTLCacheSize(ip, port) + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + def restartMasterFe = { + def oldMasterFe = cluster.getMasterFe() + cluster.restartFrontends(oldMasterFe.index) + boolean hasRestart = false + for (int i = 0; i < 30; i++) { + if (cluster.getFeByIndex(oldMasterFe.index).alive) { + hasRestart = true + break + } + sleep 1000 + } + assertTrue(hasRestart) + context.reconnectFe() + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + restartMasterFe(); + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + clearFileCacheOnAllBackends() + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + sleep(5000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy new file mode 100644 index 00000000000000..5e64dddbbe0f31 --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_slow_job.groovy @@ -0,0 +1,288 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +import java.time.Duration +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.Base64 + +suite('test_warm_up_cluster_periodic_slow_job', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'fetch_cluster_cache_hotspot_interval_ms=1000', + 'enable_debug_points=true', + ] + options.beConfigs += [ + 'file_cache_enter_disk_resource_limit_mode_percent=99', + 'enable_evict_file_cache_in_advance=false', + 'file_cache_background_monitor_interval_ms=1000', + ] + options.cloudMode = true + + def setDebugPoint = {ip, port, op, name -> + def urlStr = "http://${ip}:${port}/api/debug_point/${op}/${name}" + def url = new URL(urlStr) + def conn = url.openConnection() + conn.requestMethod = 'POST' + conn.doOutput = true + + // Add Basic Auth header + def authString = "root:" + def encodedAuth = Base64.encoder.encodeToString(authString.getBytes("UTF-8")) + conn.setRequestProperty("Authorization", "Basic ${encodedAuth}") + + // Send empty body (required to trigger POST) + conn.outputStream.withWriter { it << "" } + + // Read response + def responseText = conn.inputStream.text + def json = new JsonSlurper().parseText(responseText) + + return json?.msg == "OK" && json?.code == 0 + } + + def addDebugPoint = { ip, http_port, name -> + return setDebugPoint(ip, http_port, 'add', name) + } + + def removeDebugPoint = { ip, http_port, name -> + return setDebugPoint(ip, http_port, 'remove', name) + } + + def addFEDebugPoint = { name -> + def fe = cluster.getMasterFe() + return addDebugPoint(fe.host, fe.httpPort, name) + } + + def removeFEDebugPoint = { name -> + def fe = cluster.getMasterFe() + return removeDebugPoint(fe.host, fe.httpPort, name) + } + + def getWarmUpJobInfo = { jobId -> + def jobInfo = sql """SHOW WARM UP JOB WHERE id = ${jobId}""" + + if (jobInfo == null || jobInfo.isEmpty()) { + fail("No warm up job found with ID: ${jobId}") + } + + def job = jobInfo[0] + def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS") + def sTime = LocalDateTime.parse(job[7], formatter) + def cTime = LocalDateTime.parse(job[6], formatter) + + return [ + status : job[3], + createTime: cTime, + startTime : sTime, + ] + } + + def clearFileCache = {ip, port -> + def url = "http://${ip}:${port}/api/file_cache?op=clear&sync=true" + def response = new URL(url).text + def json = new JsonSlurper().parseText(response) + + // Check the status + if (json.status != "OK") { + throw new RuntimeException("Clear cache on ${ip}:${port} failed: ${json.status}") + } + } + + def clearFileCacheOnAllBackends = { + def backends = sql """SHOW BACKENDS""" + + for (be in backends) { + def ip = be[1] + def port = be[4] + clearFileCache(ip, port) + } + + // clear file cache is async, wait it done + sleep(5000) + } + + def getBrpcMetrics = {ip, port, name -> + def url = "http://${ip}:${port}/brpc_metrics" + def metrics = new URL(url).text + def matcher = metrics =~ ~"${name}\\s+(\\d+)" + if (matcher.find()) { + return matcher[0][1] as long + } else { + throw new RuntimeException("${name} not found for ${ip}:${port}") + } + } + + def logFileCacheDownloadMetrics = { cluster -> + def backends = sql """SHOW BACKENDS""" + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def submitted = getBrpcMetrics(ip, port, "file_cache_download_submitted_num") + def finished = getBrpcMetrics(ip, port, "file_cache_download_finished_num") + def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") + logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + + ", finished=${finished}, failed=${failed}") + } + } + + def getTTLCacheSize = { ip, port -> + return getBrpcMetrics(ip, port, "ttl_cache_size") + } + + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> + def backends = sql """SHOW BACKENDS""" + + def srcBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster1}\"""") } + def tgtBes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster2}\"""") } + + long srcSum = 0 + for (src in srcBes) { + def ip = src[1] + def port = src[5] + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") + } + + long tgtSum = 0 + for (tgt in tgtBes) { + def ip = tgt[1] + def port = tgt[5] + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") + } + + logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") + assertTrue(srcSum > 0, "ttl_cache_size should > 0") + assertEquals(srcSum, tgtSum) + } + + docker(options) { + def clusterName1 = "warmup_source" + def clusterName2 = "warmup_target" + + // Add two clusters + cluster.addBackend(3, clusterName1) + cluster.addBackend(3, clusterName2) + + def tag1 = getCloudBeTagByName(clusterName1) + def tag2 = getCloudBeTagByName(clusterName2) + + logger.info("Cluster tag1: {}", tag1) + logger.info("Cluster tag2: {}", tag2) + + def jsonSlurper = new JsonSlurper() + def clusterId1 = jsonSlurper.parseText(tag1).compute_group_id + def clusterId2 = jsonSlurper.parseText(tag2).compute_group_id + + def getJobState = { jobId -> + def jobStateResult = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + return jobStateResult[0][3] + } + + // Ensure we are in source cluster + sql """use @${clusterName1}""" + + // Clear hotspot statistics + sql """truncate table __internal_schema.cloud_cache_hotspot;""" + + clearFileCacheOnAllBackends() + + // Simple setup to simulate data load and access + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """INSERT INTO customer VALUES (1, 'A'), (2, 'B'), (3, 'C')""" + + addFEDebugPoint("CloudWarmUpJob.FakeLastBatchNotDone") + + def sync_sec = 1 + + // Start warm up job + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "${sync_sec}" + ) + """ + + def jobId = jobId_[0][0] + logger.info("Warm-up job ID: ${jobId}") + + for (int i = 0; i < 1000; i++) { + sql """SELECT * FROM customer""" + } + + def wait_sec = sync_sec * 5 + sleep(wait_sec * 2 * 1000) + + def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" + logger.info("hotspot: {}", hotspot) + + def info1 = getWarmUpJobInfo(jobId) + logger.info("info1 {}", info1) + assertEquals("RUNNING", info1.status) + + sleep(wait_sec * 2 * 1000) + + def info2 = getWarmUpJobInfo(jobId) + logger.info("info2 {}", info2) + assertEquals("RUNNING", info2.status) + assertEquals(info1.createTime, info2.createTime) + assertEquals(info1.startTime, info2.startTime) + + removeFEDebugPoint("CloudWarmUpJob.FakeLastBatchNotDone") + sleep(wait_sec * 1000) + + def info3 = getWarmUpJobInfo(jobId) + logger.info("info3 {}", info3) + assertEquals(info1.createTime, info3.createTime) + assertTrue(Duration.between(info1.startTime, info3.startTime).seconds.abs() > wait_sec * 4) + + sleep(wait_sec * 1000) + def info4 = getWarmUpJobInfo(jobId) + logger.info("info4 {}", info4) + assertEquals(info1.createTime, info4.createTime) + assertTrue(Duration.between(info3.startTime, info4.startTime).seconds.abs() > sync_sec) + + logFileCacheDownloadMetrics(clusterName2) + checkTTLCacheSizeSumEqual(clusterName1, clusterName2) + + def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(jobInfo[0][0], jobId) + assertEquals(jobInfo[0][1], clusterName1) + assertEquals(jobInfo[0][2], clusterName2) + assertEquals(jobInfo[0][4], "CLUSTER") + assertTrue(jobInfo[0][3] in ["RUNNING", "PENDING"], + "JobState is ${jobInfo[0][3]}, expected RUNNING or PENDING") + assertEquals(jobInfo[0][5], "PERIODIC (1s)") + + // Cancel job and confirm + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + def cancelInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" + assertEquals(cancelInfo[0][3], "CANCELLED") + + // Clean up + sql """DROP TABLE IF EXISTS customer""" + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy new file mode 100644 index 00000000000000..8ad210def6adbc --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_repeat_jobs.groovy @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper + +suite('test_warm_up_cluster_repeat_jobs', 'docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + ] + options.cloudMode = true + + docker(options) { + def clusterName1 = "cluster1" + def clusterName2 = "cluster2" + def clusterName3 = "cluster3" + + // Add two clusters + cluster.addBackend(1, clusterName1) + cluster.addBackend(1, clusterName2) + cluster.addBackend(1, clusterName3) + + def jobId_ = sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + def jobId = jobId_[0][0] + + logger.info("JobID = {}", jobId) + + // For periodic jobs, it's not allowed to start a job + // with same src cluster, dst cluster, and sync_mode + try { + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "10" + ) + """ + assertTrue(false, "expected exception") + } catch (java.sql.SQLException e) { + assertTrue(e.getMessage().contains("already has a runnable job"), e.getMessage()); + } + + // It's allowed to start a job with same dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName3} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + // It's allowed to start a ONCE job with same src dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "once" + ) + """ + + // It's allowed to start a ONCE job with same src dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + """ + + // It's allowed to create a job in the opposite direction + sql """ + WARM UP CLUSTER ${clusterName1} WITH CLUSTER ${clusterName2} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + // after cancelling the old job, we can create another job with same attributes + sql """CANCEL WARM UP JOB WHERE ID = ${jobId}""" + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "periodic", + "sync_interval_sec" = "1" + ) + """ + + // It's allowed to start a event_driven job alongside with a periodic job + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + + // For event driven jobs, it's not allowed to start a job + // with same src cluster, dst cluster, and sync_mode + try { + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName1} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + assertTrue(false, "expected exception") + } catch (java.sql.SQLException e) { + assertTrue(e.getMessage().contains("already has a runnable job"), e.getMessage()); + } + + // For event driven jobs, it's allowed to start a job with same dst cluster + sql """ + WARM UP CLUSTER ${clusterName2} WITH CLUSTER ${clusterName3} + PROPERTIES ( + "sync_mode" = "event_driven", + "sync_event" = "load" + ) + """ + } +} diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy index a086731efffce4..5a16e92b36bb5d 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_compute_group") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy index 0eb93f2896c39d..c6819ad58ec20b 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_partition") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } List ipList = new ArrayList<>(); diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy index b7eb8761951049..258e9e87ef6121 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_table") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy index 77286717117578..03d45f1cce8e1e 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_tables") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0][3] } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ From a7c827ddfd62662201ee47c08152c6eb8518b8f0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 10:17:34 +0800 Subject: [PATCH 198/572] branch-3.0: [feat](recycler) Add http api for statistics recycler metrics #52523 (#53117) Cherry-picked from #52523 Co-authored-by: Uniqueyou --- cloud/src/common/bvars.cpp | 6 +- cloud/src/common/bvars.h | 6 +- cloud/src/common/config.h | 4 +- cloud/src/recycler/recycler.cpp | 1532 +++++++++++++---------- cloud/src/recycler/recycler.h | 55 +- cloud/src/recycler/recycler_service.cpp | 261 ++++ cloud/src/recycler/recycler_service.h | 2 + gensrc/proto/cloud.proto | 5 + 8 files changed, 1172 insertions(+), 699 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 4485fb9f5d6022..5999320af8311c 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -110,11 +110,11 @@ bvar::Adder g_bvar_recycler_instance_recycle_task_concurrency; // recycler's mbvars bvar::Adder g_bvar_recycler_instance_running_counter("recycler_instance_running_counter"); // cost time of the last whole recycle process -mBvarStatus g_bvar_recycler_instance_last_recycle_duration("recycler_instance_last_recycle_duration",{"instance_id"}); +mBvarStatus g_bvar_recycler_instance_last_round_recycle_duration("recycler_instance_last_round_recycle_duration",{"instance_id"}); mBvarStatus g_bvar_recycler_instance_next_ts("recycler_instance_next_ts",{"instance_id"}); // start and end timestamps of the recycle process -mBvarStatus g_bvar_recycler_instance_recycle_st_ts("recycler_instance_recycle_st_ts",{"instance_id"}); -mBvarStatus g_bvar_recycler_instance_recycle_ed_ts("recycler_instance_recycle_ed_ts",{"instance_id"}); +mBvarStatus g_bvar_recycler_instance_recycle_start_ts("recycler_instance_recycle_start_ts",{"instance_id"}); +mBvarStatus g_bvar_recycler_instance_recycle_end_ts("recycler_instance_recycle_end_ts",{"instance_id"}); mBvarStatus g_bvar_recycler_instance_recycle_last_success_ts("recycler_instance_recycle_last_success_ts",{"instance_id"}); // recycler's mbvars diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 83ab481764f926..3ca3e1b2cab1e0 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -262,10 +262,10 @@ extern BvarStatusWithTag g_bvar_recycler_recycle_expired_txn_label_earl extern bvar::Status g_bvar_recycler_task_max_concurrency; extern bvar::Adder g_bvar_recycler_instance_recycle_task_concurrency; extern bvar::Adder g_bvar_recycler_instance_running_counter; -extern mBvarStatus g_bvar_recycler_instance_last_recycle_duration; +extern mBvarStatus g_bvar_recycler_instance_last_round_recycle_duration; extern mBvarStatus g_bvar_recycler_instance_next_ts; -extern mBvarStatus g_bvar_recycler_instance_recycle_st_ts; -extern mBvarStatus g_bvar_recycler_instance_recycle_ed_ts; +extern mBvarStatus g_bvar_recycler_instance_recycle_start_ts; +extern mBvarStatus g_bvar_recycler_instance_recycle_end_ts; extern mBvarStatus g_bvar_recycler_instance_recycle_last_success_ts; extern mBvarIntAdder g_bvar_recycler_vault_recycle_status; diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 62cd5c9343b5b7..6fd933d3a152a6 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -89,6 +89,8 @@ CONF_Strings(recycle_whitelist, ""); // Comma seprated list CONF_Strings(recycle_blacklist, ""); // Comma seprated list // IO worker thread pool concurrency: object list, delete CONF_mInt32(instance_recycler_worker_pool_size, "32"); +// The worker pool size for http api `statistics_recycle` worker pool +CONF_mInt32(instance_recycler_statistics_recycle_worker_pool_size, "5"); CONF_Bool(enable_checker, "false"); // The parallelism for parallel recycle operation // s3_producer_pool recycle_tablet_pool, delete single object in this pool @@ -105,7 +107,7 @@ CONF_mInt32(scan_instances_interval_seconds, "60"); // 1min // interval for check object CONF_mInt32(check_object_interval_seconds, "43200"); // 12hours // enable recycler metrics statistics -CONF_Bool(enable_recycler_metrics, "false"); +CONF_Bool(enable_recycler_stats_metrics, "false"); CONF_mInt64(check_recycle_task_interval_seconds, "600"); // 10min CONF_mInt64(recycler_sleep_before_scheduling_seconds, "60"); diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index e285886acfb66f..e07dda0669a632 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -68,8 +68,8 @@ namespace doris::cloud { using namespace std::chrono; -static RecyclerMetricsContext tablet_metrics_context_("global_recycler", "recycle_tablet"); -static RecyclerMetricsContext segment_metrics_context_("global_recycler", "recycle_segment"); +RecyclerMetricsContext tablet_metrics_context_("global_recycler", "recycle_tablet"); +RecyclerMetricsContext segment_metrics_context_("global_recycler", "recycle_segment"); // return 0 for success get a key, 1 for key not found, negative for error [[maybe_unused]] static int txn_get(TxnKv* txn_kv, std::string_view key, std::string& val) { @@ -288,7 +288,7 @@ void Recycler::recycle_callback() { auto ctime_ms = duration_cast(system_clock::now().time_since_epoch()).count(); g_bvar_recycler_instance_recycle_task_concurrency << 1; g_bvar_recycler_instance_running_counter << 1; - g_bvar_recycler_instance_recycle_st_ts.put({instance_id}, ctime_ms); + g_bvar_recycler_instance_recycle_start_ts.put({instance_id}, ctime_ms); tablet_metrics_context_.reset(); segment_metrics_context_.reset(); ret = instance_recycler->do_recycle(); @@ -308,8 +308,8 @@ void Recycler::recycle_callback() { auto now = duration_cast(system_clock::now().time_since_epoch()).count(); auto elpased_ms = now - ctime_ms; - g_bvar_recycler_instance_recycle_ed_ts.put({instance_id}, now); - g_bvar_recycler_instance_last_recycle_duration.put({instance_id}, elpased_ms); + g_bvar_recycler_instance_recycle_end_ts.put({instance_id}, now); + g_bvar_recycler_instance_last_round_recycle_duration.put({instance_id}, elpased_ms); g_bvar_recycler_instance_next_ts.put({instance_id}, now + config::recycle_interval_seconds * 1000); LOG(INFO) << "recycle instance done, " @@ -812,6 +812,178 @@ int InstanceRecycler::recycle_deleted_instance() { return ret; } +bool is_txn_finished(std::shared_ptr txn_kv, const std::string& instance_id, + int64_t txn_id) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to create txn, txn_id=" << txn_id << " instance_id=" << instance_id; + return false; + } + + std::string index_val; + const std::string index_key = txn_index_key({instance_id, txn_id}); + err = txn->get(index_key, &index_val); + if (err != TxnErrorCode::TXN_OK) { + if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) { + TEST_SYNC_POINT_CALLBACK("is_txn_finished::txn_has_been_recycled"); + // txn has been recycled; + LOG(INFO) << "txn index key has been recycled, txn_id=" << txn_id + << " instance_id=" << instance_id; + return true; + } + LOG(WARNING) << "failed to get txn index key, txn_id=" << txn_id + << " instance_id=" << instance_id << " key=" << hex(index_key) + << " err=" << err; + return false; + } + + TxnIndexPB index_pb; + if (!index_pb.ParseFromString(index_val)) { + LOG(WARNING) << "failed to parse txn_index_pb, txn_id=" << txn_id + << " instance_id=" << instance_id; + return false; + } + + DCHECK(index_pb.has_tablet_index() == true); + if (!index_pb.tablet_index().has_db_id()) { + // In the previous version, the db_id was not set in the index_pb. + // If updating to the version which enable txn lazy commit, the db_id will be set. + LOG(INFO) << "txn index has no db_id, txn_id=" << txn_id << " instance_id=" << instance_id + << " index=" << index_pb.ShortDebugString(); + return true; + } + + int64_t db_id = index_pb.tablet_index().db_id(); + DCHECK_GT(db_id, 0) << "db_id=" << db_id << " txn_id=" << txn_id + << " instance_id=" << instance_id; + + std::string info_val; + const std::string info_key = txn_info_key({instance_id, db_id, txn_id}); + err = txn->get(info_key, &info_val); + if (err != TxnErrorCode::TXN_OK) { + if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) { + // txn info has been recycled; + LOG(INFO) << "txn info key has been recycled, db_id=" << db_id << " txn_id=" << txn_id + << " instance_id=" << instance_id; + return true; + } + + DCHECK(err != TxnErrorCode::TXN_KEY_NOT_FOUND); + LOG(WARNING) << "failed to get txn info key, txn_id=" << txn_id + << " instance_id=" << instance_id << " key=" << hex(info_key) + << " err=" << err; + return false; + } + + TxnInfoPB txn_info; + if (!txn_info.ParseFromString(info_val)) { + LOG(WARNING) << "failed to parse txn_info, txn_id=" << txn_id + << " instance_id=" << instance_id; + return false; + } + + DCHECK(txn_info.txn_id() == txn_id) << "txn_id=" << txn_id << " instance_id=" << instance_id + << " txn_info=" << txn_info.ShortDebugString(); + + if (TxnStatusPB::TXN_STATUS_ABORTED == txn_info.status() || + TxnStatusPB::TXN_STATUS_VISIBLE == txn_info.status()) { + TEST_SYNC_POINT_CALLBACK("is_txn_finished::txn_has_been_aborted", &txn_info); + return true; + } + + TEST_SYNC_POINT_CALLBACK("is_txn_finished::txn_not_finished", &txn_info); + return false; +} + +int64_t calculate_rowset_expired_time(const std::string& instance_id_, const RecycleRowsetPB& rs, + int64_t* earlest_ts /* rowset earliest expiration ts */) { + if (config::force_immediate_recycle) { + return 0L; + } + // RecycleRowsetPB created by compacted or dropped rowset has no expiration time, and will be recycled when exceed retention time + int64_t expiration = rs.expiration() > 0 ? rs.expiration() : rs.creation_time(); + int64_t retention_seconds = config::retention_seconds; + if (rs.type() == RecycleRowsetPB::COMPACT || rs.type() == RecycleRowsetPB::DROP) { + retention_seconds = std::min(config::compacted_rowset_retention_seconds, retention_seconds); + } + int64_t final_expiration = expiration + retention_seconds; + if (*earlest_ts > final_expiration) { + *earlest_ts = final_expiration; + g_bvar_recycler_recycle_rowset_earlest_ts.put(instance_id_, *earlest_ts); + } + return final_expiration; +} + +int64_t calculate_partition_expired_time( + const std::string& instance_id_, const RecyclePartitionPB& partition_meta_pb, + int64_t* earlest_ts /* partition earliest expiration ts */) { + if (config::force_immediate_recycle) { + return 0L; + } + int64_t expiration = partition_meta_pb.expiration() > 0 ? partition_meta_pb.expiration() + : partition_meta_pb.creation_time(); + int64_t retention_seconds = config::retention_seconds; + if (partition_meta_pb.state() == RecyclePartitionPB::DROPPED) { + retention_seconds = + std::min(config::dropped_partition_retention_seconds, retention_seconds); + } + int64_t final_expiration = expiration + retention_seconds; + if (*earlest_ts > final_expiration) { + *earlest_ts = final_expiration; + g_bvar_recycler_recycle_partition_earlest_ts.put(instance_id_, *earlest_ts); + } + return final_expiration; +} + +int64_t calculate_index_expired_time(const std::string& instance_id_, + const RecycleIndexPB& index_meta_pb, + int64_t* earlest_ts /* index earliest expiration ts */) { + if (config::force_immediate_recycle) { + return 0L; + } + int64_t expiration = index_meta_pb.expiration() > 0 ? index_meta_pb.expiration() + : index_meta_pb.creation_time(); + int64_t retention_seconds = config::retention_seconds; + if (index_meta_pb.state() == RecycleIndexPB::DROPPED) { + retention_seconds = std::min(config::dropped_index_retention_seconds, retention_seconds); + } + int64_t final_expiration = expiration + retention_seconds; + if (*earlest_ts > final_expiration) { + *earlest_ts = final_expiration; + g_bvar_recycler_recycle_index_earlest_ts.put(instance_id_, *earlest_ts); + } + return final_expiration; +} + +int64_t calculate_tmp_rowset_expired_time( + const std::string& instance_id_, const doris::RowsetMetaCloudPB& tmp_rowset_meta_pb, + int64_t* earlest_ts /* tmp_rowset earliest expiration ts */) { + // ATTN: `txn_expiration` should > 0, however we use `creation_time` + a large `retention_time` (> 1 day in production environment) + // when `txn_expiration` <= 0 in some unexpected situation (usually when there are bugs). This is usually safe, coz loading + // duration or timeout always < `retention_time` in practice. + int64_t expiration = tmp_rowset_meta_pb.txn_expiration() > 0 + ? tmp_rowset_meta_pb.txn_expiration() + : tmp_rowset_meta_pb.creation_time(); + expiration = config::force_immediate_recycle ? 0 : expiration; + int64_t final_expiration = expiration + config::retention_seconds; + if (*earlest_ts > final_expiration) { + *earlest_ts = final_expiration; + g_bvar_recycler_recycle_tmp_rowset_earlest_ts.put(instance_id_, *earlest_ts); + } + return final_expiration; +} + +int64_t calculate_txn_expired_time(const std::string& instance_id_, const RecycleTxnPB& txn_meta_pb, + int64_t* earlest_ts /* txn earliest expiration ts */) { + int64_t final_expiration = txn_meta_pb.creation_time() + config::label_keep_max_second * 1000L; + if (*earlest_ts > final_expiration / 1000) { + *earlest_ts = final_expiration / 1000; + g_bvar_recycler_recycle_expired_txn_label_earlest_ts.put(instance_id_, *earlest_ts); + } + return final_expiration; +} + int InstanceRecycler::recycle_indexes() { const std::string task_name = "recycle_indexes"; int64_t num_scanned = 0; @@ -845,24 +1017,6 @@ int InstanceRecycler::recycle_indexes() { int64_t earlest_ts = std::numeric_limits::max(); - auto calc_expiration = [&earlest_ts, this](const RecycleIndexPB& index) { - if (config::force_immediate_recycle) { - return 0L; - } - int64_t expiration = index.expiration() > 0 ? index.expiration() : index.creation_time(); - int64_t retention_seconds = config::retention_seconds; - if (index.state() == RecycleIndexPB::DROPPED) { - retention_seconds = - std::min(config::dropped_index_retention_seconds, retention_seconds); - } - int64_t final_expiration = expiration + retention_seconds; - if (earlest_ts > final_expiration) { - earlest_ts = final_expiration; - g_bvar_recycler_recycle_index_earlest_ts.put(instance_id_, earlest_ts); - } - return final_expiration; - }; - // Elements in `index_keys` has the same lifetime as `it` in `scan_and_recycle` std::vector index_keys; auto recycle_func = [&, this](std::string_view k, std::string_view v) -> int { @@ -873,7 +1027,8 @@ int InstanceRecycler::recycle_indexes() { return -1; } int64_t current_time = ::time(nullptr); - if (current_time < calc_expiration(index_pb)) { // not expired + if (current_time < + calculate_index_expired_time(instance_id_, index_pb, &earlest_ts)) { // not expired return 0; } ++num_expired; @@ -933,47 +1088,6 @@ int InstanceRecycler::recycle_indexes() { return 0; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { - RecycleIndexPB index_pb; - if (!index_pb.ParseFromArray(v.data(), v.size())) { - return 0; - } - int64_t current_time = ::time(nullptr); - if (current_time < calc_expiration(index_pb)) { - return 0; - } - // decode index_id - auto k1 = k; - k1.remove_prefix(1); - std::vector, int, int>> out; - decode_key(&k1, &out); - // 0x01 "recycle" ${instance_id} "index" ${index_id} -> RecycleIndexPB - auto index_id = std::get(std::get<0>(out[3])); - std::unique_ptr txn; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - std::string val; - err = txn->get(k, &val); - if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { - return 0; - } - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - index_pb.Clear(); - if (!index_pb.ParseFromString(val)) { - return 0; - } - if (scan_tablets_and_statistics(index_pb.table_id(), index_id, metrics_context) != 0) { - return 0; - } - metrics_context.total_need_recycle_num++; - return 0; - }; - auto loop_done = [&index_keys, this]() -> int { if (index_keys.empty()) return 0; DORIS_CLOUD_DEFER { @@ -986,9 +1100,11 @@ int InstanceRecycler::recycle_indexes() { return 0; }; - return scan_for_recycle_and_statistics(index_key0, index_key1, "indexes", metrics_context, - std::move(scan_and_statistics), std::move(recycle_func), - std::move(loop_done)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_indexes(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(index_key0, index_key1, std::move(recycle_func), std::move(loop_done)); } bool check_lazy_txn_finished(std::shared_ptr txn_kv, const std::string instance_id, @@ -1109,25 +1225,6 @@ int InstanceRecycler::recycle_partitions() { int64_t earlest_ts = std::numeric_limits::max(); - auto calc_expiration = [&earlest_ts, this](const RecyclePartitionPB& partition) { - if (config::force_immediate_recycle) { - return 0L; - } - int64_t expiration = - partition.expiration() > 0 ? partition.expiration() : partition.creation_time(); - int64_t retention_seconds = config::retention_seconds; - if (partition.state() == RecyclePartitionPB::DROPPED) { - retention_seconds = - std::min(config::dropped_partition_retention_seconds, retention_seconds); - } - int64_t final_expiration = expiration + retention_seconds; - if (earlest_ts > final_expiration) { - earlest_ts = final_expiration; - g_bvar_recycler_recycle_partition_earlest_ts.put(instance_id_, earlest_ts); - } - return final_expiration; - }; - // Elements in `partition_keys` has the same lifetime as `it` in `scan_and_recycle` std::vector partition_keys; std::vector partition_version_keys; @@ -1139,7 +1236,8 @@ int InstanceRecycler::recycle_partitions() { return -1; } int64_t current_time = ::time(nullptr); - if (current_time < calc_expiration(part_pb)) { // not expired + if (current_time < + calculate_partition_expired_time(instance_id_, part_pb, &earlest_ts)) { // not expired return 0; } ++num_expired; @@ -1213,54 +1311,6 @@ int InstanceRecycler::recycle_partitions() { return ret; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&, this](std::string_view k, std::string_view v) -> int { - RecyclePartitionPB part_pb; - if (!part_pb.ParseFromArray(v.data(), v.size())) { - return 0; - } - int64_t current_time = ::time(nullptr); - if (current_time < calc_expiration(part_pb)) { - return 0; - } - // decode partition_id - auto k1 = k; - k1.remove_prefix(1); - std::vector, int, int>> out; - decode_key(&k1, &out); - // 0x01 "recycle" ${instance_id} "partition" ${partition_id} -> RecyclePartitionPB - auto partition_id = std::get(std::get<0>(out[3])); - // Change state to RECYCLING - std::unique_ptr txn; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - std::string val; - err = txn->get(k, &val); - if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { - return 0; - } - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - part_pb.Clear(); - if (!part_pb.ParseFromString(val)) { - return 0; - } - // Partitions with PREPARED state MUST have no data - bool is_empty_tablet = part_pb.state() == RecyclePartitionPB::PREPARED; - int ret = 0; - for (int64_t index_id : part_pb.index_id()) { - if (scan_tablets_and_statistics(part_pb.table_id(), index_id, metrics_context, - partition_id, is_empty_tablet) != 0) { - ret = 0; - } - } - metrics_context.total_need_recycle_num++; - return ret; - }; - auto loop_done = [&partition_keys, &partition_version_keys, this]() -> int { if (partition_keys.empty()) return 0; DORIS_CLOUD_DEFER { @@ -1288,9 +1338,11 @@ int InstanceRecycler::recycle_partitions() { return 0; }; - return scan_for_recycle_and_statistics(part_key0, part_key1, "partitions", metrics_context, - std::move(scan_and_statistics), std::move(recycle_func), - std::move(loop_done)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_partitions(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(part_key0, part_key1, std::move(recycle_func), std::move(loop_done)); } int InstanceRecycler::recycle_versions() { @@ -1370,48 +1422,11 @@ int InstanceRecycler::recycle_versions() { return 0; }; - int64_t last_scanned_table_id_t = 0; - bool is_recycled_t = false; // Is last scanned kv recycled - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&metrics_context, &last_scanned_table_id_t, &is_recycled_t, this]( - std::string_view k, std::string_view) { - auto k1 = k; - k1.remove_prefix(1); - // 0x01 "version" ${instance_id} "partition" ${db_id} ${tbl_id} ${partition_id} - std::vector, int, int>> out; - decode_key(&k1, &out); - DCHECK_EQ(out.size(), 6) << k; - auto table_id = std::get(std::get<0>(out[4])); - if (table_id == last_scanned_table_id_t) { // Already handle kvs of this table - metrics_context.total_need_recycle_num += - is_recycled_t; // Version kv of this table has been recycled - return 0; - } - last_scanned_table_id_t = table_id; - is_recycled_t = false; - auto tablet_key_begin = stats_tablet_key({instance_id_, table_id, 0, 0, 0}); - auto tablet_key_end = stats_tablet_key({instance_id_, table_id, INT64_MAX, 0, 0}); - std::unique_ptr txn; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - std::unique_ptr iter; - err = txn->get(tablet_key_begin, tablet_key_end, &iter, false, 1); - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - if (iter->has_next()) { // Table is useful, should not recycle table and partition versions - return 0; - } - metrics_context.total_need_recycle_num++; - is_recycled_t = true; - return 0; - }; - - return scan_for_recycle_and_statistics(version_key_begin, version_key_end, "versions", - metrics_context, std::move(scan_and_statistics), - std::move(recycle_func)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_versions(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(version_key_begin, version_key_end, std::move(recycle_func)); } int InstanceRecycler::recycle_tablets(int64_t table_id, int64_t index_id, @@ -1935,7 +1950,12 @@ int InstanceRecycler::scan_tablets_and_statistics(int64_t table_id, int64_t inde } return 0; }; - return scan_and_recycle(tablet_key_begin, tablet_key_end, std::move(scan_and_statistics)); + return scan_and_recycle(tablet_key_begin, tablet_key_end, std::move(scan_and_statistics), + [&metrics_context]() -> int { + metrics_context.report(); + tablet_metrics_context_.report(); + return 0; + }); } int InstanceRecycler::scan_tablet_and_statistics(int64_t tablet_id, @@ -2033,7 +2053,7 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& std::string recyc_rs_key0 = recycle_rowset_key({instance_id_, tablet_id, ""}); std::string recyc_rs_key1 = recycle_rowset_key({instance_id_, tablet_id + 1, ""}); - std::vector rowset_meta; + std::set resource_ids; int64_t recycle_rowsets_number = 0; int64_t recycle_segments_number = 0; int64_t recycle_rowsets_data_size = 0; @@ -2139,17 +2159,16 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& max_rowset_creation_time = std::max(max_rowset_creation_time, rs_meta.creation_time()); min_rowset_expiration_time = std::min(min_rowset_expiration_time, rs_meta.txn_expiration()); max_rowset_expiration_time = std::max(max_rowset_expiration_time, rs_meta.txn_expiration()); - rowset_meta.emplace_back(rs_meta.resource_id()); - LOG(INFO) << "rs_meta.resource_id()=" << rs_meta.resource_id(); + resource_ids.emplace(rs_meta.resource_id()); } LOG_INFO("recycle tablet start to delete object") .tag("instance id", instance_id_) .tag("tablet id", tablet_id) .tag("recycle tablet resource ids are", - std::accumulate(rowset_meta.begin(), rowset_meta.begin(), std::string(), - [](std::string acc, const auto& it) { - return acc.empty() ? it : acc + ", " + it; + std::accumulate(resource_ids.begin(), resource_ids.begin(), std::string(), + [](std::string rs_id, const auto& it) { + return rs_id.empty() ? it : rs_id + ", " + it; })); SyncExecutor concurrent_delete_executor( @@ -2161,7 +2180,7 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& // ATTN: there may be data leak if not all accessor initilized successfully // partial data deleted if the tablet is stored cross-storage vault // vault id is not attached to TabletMeta... - for (const auto& resource_id : rowset_meta) { + for (const auto& resource_id : resource_ids) { concurrent_delete_executor.add([&, rs_id = resource_id, accessor_ptr = accessor_map_[resource_id]]() { std::unique_ptr> defer((int*)0x01, [&](int*) { @@ -2208,8 +2227,11 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext& segment_metrics_context_.total_recycled_num += recycle_segments_number; segment_metrics_context_.total_recycled_data_size += recycle_rowsets_data_size + recycle_rowsets_index_size; + metrics_context.total_recycled_data_size += + recycle_rowsets_data_size + recycle_rowsets_index_size; tablet_metrics_context_.report(); segment_metrics_context_.report(); + metrics_context.report(); txn.reset(); if (txn_kv_->create_txn(&txn) != TxnErrorCode::TXN_OK) { @@ -2339,77 +2361,23 @@ int InstanceRecycler::recycle_rowsets() { int64_t earlest_ts = std::numeric_limits::max(); - auto calc_expiration = [&earlest_ts, this](const RecycleRowsetPB& rs) { - if (config::force_immediate_recycle) { - return 0L; - } - // RecycleRowsetPB created by compacted or dropped rowset has no expiration time, and will be recycled when exceed retention time - int64_t expiration = rs.expiration() > 0 ? rs.expiration() : rs.creation_time(); - int64_t retention_seconds = config::retention_seconds; - if (rs.type() == RecycleRowsetPB::COMPACT || rs.type() == RecycleRowsetPB::DROP) { - retention_seconds = - std::min(config::compacted_rowset_retention_seconds, retention_seconds); - } - int64_t final_expiration = expiration + retention_seconds; - if (earlest_ts > final_expiration) { - earlest_ts = final_expiration; - g_bvar_recycler_recycle_rowset_earlest_ts.put(instance_id_, earlest_ts); - } - return final_expiration; - }; - - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { + auto handle_rowset_kv = [&, this](std::string_view k, std::string_view v) -> int { + ++num_scanned; + total_rowset_key_size += k.size(); + total_rowset_value_size += v.size(); RecycleRowsetPB rowset; if (!rowset.ParseFromArray(v.data(), v.size())) { - return 0; + LOG_WARNING("malformed recycle rowset").tag("key", hex(k)); + return -1; } + + int final_expiration = calculate_rowset_expired_time(instance_id_, rowset, &earlest_ts); + + VLOG_DEBUG << "recycle rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned + << " num_expired=" << num_expired << " expiration=" << final_expiration + << " RecycleRowsetPB=" << rowset.ShortDebugString(); int64_t current_time = ::time(nullptr); - if (current_time < calc_expiration(rowset)) { // not expired - return 0; - } - if (!rowset.has_type()) { - if (!rowset.has_resource_id()) [[unlikely]] { - return 0; - } - if (rowset.resource_id().empty()) [[unlikely]] { - return 0; - } - return 0; - } - auto* rowset_meta = rowset.mutable_rowset_meta(); - if (!rowset_meta->has_resource_id()) [[unlikely]] { - if (rowset.type() == RecycleRowsetPB::PREPARE || rowset_meta->num_segments() != 0) { - return 0; - } - } - if (rowset.type() != RecycleRowsetPB::PREPARE) { - if (rowset_meta->num_segments() > 0) { - metrics_context.total_need_recycle_num++; - segment_metrics_context_.total_need_recycle_num += rowset_meta->num_segments(); - segment_metrics_context_.total_need_recycle_data_size += - rowset_meta->total_disk_size(); - metrics_context.total_need_recycle_data_size += rowset_meta->total_disk_size(); - } - } - return 0; - }; - - auto handle_rowset_kv = [&](std::string_view k, std::string_view v) -> int { - ++num_scanned; - total_rowset_key_size += k.size(); - total_rowset_value_size += v.size(); - RecycleRowsetPB rowset; - if (!rowset.ParseFromArray(v.data(), v.size())) { - LOG_WARNING("malformed recycle rowset").tag("key", hex(k)); - return -1; - } - - VLOG_DEBUG << "recycle rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned - << " num_expired=" << num_expired << " expiration=" << calc_expiration(rowset) - << " RecycleRowsetPB=" << rowset.ShortDebugString(); - int64_t current_time = ::time(nullptr); - if (current_time < calc_expiration(rowset)) { // not expired + if (current_time < final_expiration) { // not expired return 0; } ++num_expired; @@ -2504,9 +2472,13 @@ int InstanceRecycler::recycle_rowsets() { return 0; }; - int ret = scan_for_recycle_and_statistics(recyc_rs_key0, recyc_rs_key1, "rowsets", - metrics_context, std::move(scan_and_statistics), - std::move(handle_rowset_kv), std::move(loop_done)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_rowsets(); + } + // recycle_func and loop_done for scan and recycle + int ret = scan_and_recycle(recyc_rs_key0, recyc_rs_key1, std::move(handle_rowset_kv), + std::move(loop_done)); + worker_pool->stop(); if (!async_recycled_rowset_keys.empty()) { @@ -2520,90 +2492,6 @@ int InstanceRecycler::recycle_rowsets() { return ret; } -bool is_txn_finished(std::shared_ptr txn_kv, const std::string& instance_id, - int64_t txn_id) { - std::unique_ptr txn; - TxnErrorCode err = txn_kv->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - LOG(WARNING) << "failed to create txn, txn_id=" << txn_id << " instance_id=" << instance_id; - return false; - } - - std::string index_val; - const std::string index_key = txn_index_key({instance_id, txn_id}); - err = txn->get(index_key, &index_val); - if (err != TxnErrorCode::TXN_OK) { - if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) { - TEST_SYNC_POINT_CALLBACK("is_txn_finished::txn_has_been_recycled"); - // txn has been recycled; - LOG(INFO) << "txn index key has been recycled, txn_id=" << txn_id - << " instance_id=" << instance_id; - return true; - } - LOG(WARNING) << "failed to get txn index key, txn_id=" << txn_id - << " instance_id=" << instance_id << " key=" << hex(index_key) - << " err=" << err; - return false; - } - - TxnIndexPB index_pb; - if (!index_pb.ParseFromString(index_val)) { - LOG(WARNING) << "failed to parse txn_index_pb, txn_id=" << txn_id - << " instance_id=" << instance_id; - return false; - } - - DCHECK(index_pb.has_tablet_index() == true); - if (!index_pb.tablet_index().has_db_id()) { - // In the previous version, the db_id was not set in the index_pb. - // If updating to the version which enable txn lazy commit, the db_id will be set. - LOG(INFO) << "txn index has no db_id, txn_id=" << txn_id << " instance_id=" << instance_id - << " index=" << index_pb.ShortDebugString(); - return true; - } - - int64_t db_id = index_pb.tablet_index().db_id(); - DCHECK_GT(db_id, 0) << "db_id=" << db_id << " txn_id=" << txn_id - << " instance_id=" << instance_id; - - std::string info_val; - const std::string info_key = txn_info_key({instance_id, db_id, txn_id}); - err = txn->get(info_key, &info_val); - if (err != TxnErrorCode::TXN_OK) { - if (TxnErrorCode::TXN_KEY_NOT_FOUND == err) { - // txn info has been recycled; - LOG(INFO) << "txn info key has been recycled, db_id=" << db_id << " txn_id=" << txn_id - << " instance_id=" << instance_id; - return true; - } - - DCHECK(err != TxnErrorCode::TXN_KEY_NOT_FOUND); - LOG(WARNING) << "failed to get txn info key, txn_id=" << txn_id - << " instance_id=" << instance_id << " key=" << hex(info_key) - << " err=" << err; - return false; - } - - TxnInfoPB txn_info; - if (!txn_info.ParseFromString(info_val)) { - LOG(WARNING) << "failed to parse txn_info, txn_id=" << txn_id - << " instance_id=" << instance_id; - return false; - } - - DCHECK(txn_info.txn_id() == txn_id) << "txn_id=" << txn_id << " instance_id=" << instance_id - << " txn_info=" << txn_info.ShortDebugString(); - - if (TxnStatusPB::TXN_STATUS_ABORTED == txn_info.status() || - TxnStatusPB::TXN_STATUS_VISIBLE == txn_info.status()) { - TEST_SYNC_POINT_CALLBACK("is_txn_finished::txn_has_been_aborted", &txn_info); - return true; - } - - TEST_SYNC_POINT_CALLBACK("is_txn_finished::txn_not_finished", &txn_info); - return false; -} - int InstanceRecycler::recycle_tmp_rowsets() { const std::string task_name = "recycle_tmp_rowsets"; int64_t num_scanned = 0; @@ -2648,25 +2536,10 @@ int InstanceRecycler::recycle_tmp_rowsets() { std::map tmp_rowsets; int64_t earlest_ts = std::numeric_limits::max(); - auto calc_expiration = [&earlest_ts, this](const doris::RowsetMetaCloudPB& rowset) { - // ATTN: `txn_expiration` should > 0, however we use `creation_time` + a large `retention_time` (> 1 day in production environment) - // when `txn_expiration` <= 0 in some unexpected situation (usually when there are bugs). This is usually safe, coz loading - // duration or timeout always < `retention_time` in practice. - int64_t expiration = - rowset.txn_expiration() > 0 ? rowset.txn_expiration() : rowset.creation_time(); - expiration = config::force_immediate_recycle ? 0 : expiration; - int64_t final_expiration = expiration + config::retention_seconds; - if (earlest_ts > final_expiration) { - earlest_ts = final_expiration; - g_bvar_recycler_recycle_tmp_rowset_earlest_ts.put(instance_id_, earlest_ts); - } - return final_expiration; - }; auto handle_rowset_kv = [&num_scanned, &num_expired, &tmp_rowset_keys, &tmp_rowsets, &expired_rowset_size, &total_rowset_key_size, &total_rowset_value_size, - &calc_expiration, - this](std::string_view k, std::string_view v) -> int { + &earlest_ts, this](std::string_view k, std::string_view v) -> int { ++num_scanned; total_rowset_key_size += k.size(); total_rowset_value_size += v.size(); @@ -2675,7 +2548,7 @@ int InstanceRecycler::recycle_tmp_rowsets() { LOG_WARNING("malformed rowset meta").tag("key", hex(k)); return -1; } - int64_t expiration = calc_expiration(rowset); + int64_t expiration = calculate_tmp_rowset_expired_time(instance_id_, rowset, &earlest_ts); VLOG_DEBUG << "recycle tmp rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned << " num_expired=" << num_expired << " expiration=" << expiration << " txn_expiration=" << rowset.txn_expiration() @@ -2724,41 +2597,6 @@ int InstanceRecycler::recycle_tmp_rowsets() { return 0; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { - doris::RowsetMetaCloudPB rowset; - if (!rowset.ParseFromArray(v.data(), v.size())) { - return 0; - } - int64_t expiration = calc_expiration(rowset); - int64_t current_time = ::time(nullptr); - if (current_time < expiration) { - return 0; - } - - DCHECK_GT(rowset.txn_id(), 0) - << "txn_id=" << rowset.txn_id() << " rowset=" << rowset.ShortDebugString(); - if (!is_txn_finished(txn_kv_, instance_id_, rowset.txn_id())) { - return 0; - } - - if (!rowset.has_resource_id()) { - if (rowset.num_segments() > 0) [[unlikely]] { // impossible - return 0; - } - metrics_context.total_need_recycle_num++; - return 0; - } - - metrics_context.total_need_recycle_num++; - if (rowset.num_segments() > 0) { - metrics_context.total_need_recycle_data_size += rowset.total_disk_size(); - segment_metrics_context_.total_need_recycle_data_size += rowset.total_disk_size(); - segment_metrics_context_.total_need_recycle_num += rowset.num_segments(); - } - return 0; - }; - auto loop_done = [&tmp_rowset_keys, &tmp_rowsets, &num_recycled, &metrics_context, this]() -> int { DORIS_CLOUD_DEFER { @@ -2778,30 +2616,12 @@ int InstanceRecycler::recycle_tmp_rowsets() { return 0; }; - return scan_for_recycle_and_statistics(tmp_rs_key0, tmp_rs_key1, "tmp_rowsets", metrics_context, - std::move(scan_and_statistics), - std::move(handle_rowset_kv), std::move(loop_done)); -} - -int InstanceRecycler::scan_for_recycle_and_statistics( - std::string begin, std::string_view end, std::string task_name, - RecyclerMetricsContext& metrics_context, - std::function statistics_func, - std::function recycle_func, - std::function loop_done) { - if (config::enable_recycler_metrics) { - scan_and_recycle(begin, end, std::move(statistics_func)); - - // report to bvar - metrics_context.report(true); - tablet_metrics_context_.report(true); - segment_metrics_context_.report(true); - - int ret = scan_and_recycle(begin, end, std::move(recycle_func), std::move(loop_done)); - return ret; - } else { - return scan_and_recycle(begin, end, std::move(recycle_func), std::move(loop_done)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_tmp_rowsets(); } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(tmp_rs_key0, tmp_rs_key1, std::move(handle_rowset_kv), + std::move(loop_done)); } int InstanceRecycler::scan_and_recycle( @@ -3002,50 +2822,12 @@ int InstanceRecycler::abort_timeout_txn() { return 0; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&metrics_context, ¤t_time, this](std::string_view k, - std::string_view v) -> int { - std::unique_ptr txn; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - std::string_view k1 = k; - k1.remove_prefix(1); - std::vector, int, int>> out; - if (decode_key(&k1, &out) != 0) { - return 0; - } - int64_t db_id = std::get(std::get<0>(out[3])); - int64_t txn_id = std::get(std::get<0>(out[4])); - // Update txn_info - std::string txn_inf_key, txn_inf_val; - txn_info_key({instance_id_, db_id, txn_id}, &txn_inf_key); - err = txn->get(txn_inf_key, &txn_inf_val); - if (err != TxnErrorCode::TXN_OK) { - return 0; - } - TxnInfoPB txn_info; - if (!txn_info.ParseFromString(txn_inf_val)) { - return 0; - } - - if (TxnStatusPB::TXN_STATUS_COMMITTED != txn_info.status()) { - TxnRunningPB txn_running_pb; - if (!txn_running_pb.ParseFromArray(v.data(), v.size())) { - return 0; - } - if (!config::force_immediate_recycle && txn_running_pb.timeout_time() > current_time) { - return 0; - } - metrics_context.total_need_recycle_num++; - } - return 0; - }; - - return scan_for_recycle_and_statistics( - begin_txn_running_key, end_txn_running_key, "abort_timeout_txns", metrics_context, - std::move(scan_and_statistics), std::move(handle_txn_running_kv)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_abort_timeout_txn(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(begin_txn_running_key, end_txn_running_key, + std::move(handle_txn_running_kv)); } int InstanceRecycler::recycle_expired_txn_label() { @@ -3081,15 +2863,6 @@ int InstanceRecycler::recycle_expired_txn_label() { }; int64_t earlest_ts = std::numeric_limits::max(); - auto calc_expiration = [&earlest_ts, this](const RecycleTxnPB& recycle_txn_pb) { - int64_t final_expiration = - recycle_txn_pb.creation_time() + config::label_keep_max_second * 1000L; - if (earlest_ts > final_expiration / 1000) { - earlest_ts = final_expiration / 1000; - g_bvar_recycler_recycle_expired_txn_label_earlest_ts.put(instance_id_, earlest_ts); - } - return final_expiration; - }; SyncExecutor concurrent_delete_executor( _thread_pool_group.s3_producer_pool, @@ -3099,7 +2872,7 @@ int InstanceRecycler::recycle_expired_txn_label() { int64_t current_time_ms = duration_cast(system_clock::now().time_since_epoch()).count(); - auto handle_recycle_txn_kv = [&](std::string_view k, std::string_view v) -> int { + auto handle_recycle_txn_kv = [&, this](std::string_view k, std::string_view v) -> int { ++num_scanned; RecycleTxnPB recycle_txn_pb; if (!recycle_txn_pb.ParseFromArray(v.data(), v.size())) { @@ -3108,7 +2881,8 @@ int InstanceRecycler::recycle_expired_txn_label() { } if ((config::force_immediate_recycle) || (recycle_txn_pb.has_immediate() && recycle_txn_pb.immediate()) || - (calc_expiration(recycle_txn_pb) <= current_time_ms)) { + (calculate_txn_expired_time(instance_id_, recycle_txn_pb, &earlest_ts) <= + current_time_ms)) { VLOG_DEBUG << "found recycle txn, key=" << hex(k); num_expired++; recycle_txn_info_keys.emplace_back(k); @@ -3116,20 +2890,6 @@ int InstanceRecycler::recycle_expired_txn_label() { return 0; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&](std::string_view k, std::string_view v) -> int { - RecycleTxnPB recycle_txn_pb; - if (!recycle_txn_pb.ParseFromArray(v.data(), v.size())) { - return 0; - } - if ((config::force_immediate_recycle) || - (recycle_txn_pb.has_immediate() && recycle_txn_pb.immediate()) || - (calc_expiration(recycle_txn_pb) <= current_time_ms)) { - metrics_context.total_need_recycle_num++; - } - return 0; - }; - auto delete_recycle_txn_kv = [&](const std::string& k) -> int { std::string_view k1 = k; //RecycleTxnKeyInfo 0:instance_id 1:db_id 2:txn_id @@ -3256,9 +3016,12 @@ int InstanceRecycler::recycle_expired_txn_label() { return ret; }; - return scan_for_recycle_and_statistics( - begin_recycle_txn_key, end_recycle_txn_key, "expired_txn_labels", metrics_context, - std::move(scan_and_statistics), std::move(handle_recycle_txn_kv), std::move(loop_done)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_expired_txn_label(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(begin_recycle_txn_key, end_recycle_txn_key, + std::move(handle_recycle_txn_kv), std::move(loop_done)); } struct CopyJobIdTuple { @@ -3523,46 +3286,12 @@ int InstanceRecycler::recycle_copy_jobs() { return 0; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&metrics_context](std::string_view k, std::string_view v) -> int { - CopyJobPB copy_job; - if (!copy_job.ParseFromArray(v.data(), v.size())) { - LOG_WARNING("malformed copy job").tag("key", hex(k)); - return 0; - } - - if (copy_job.job_status() == CopyJobPB::FINISH) { - if (copy_job.stage_type() == StagePB::EXTERNAL) { - int64_t current_time = - duration_cast(system_clock::now().time_since_epoch()).count(); - if (copy_job.finish_time_ms() > 0) { - if (!config::force_immediate_recycle && - current_time < copy_job.finish_time_ms() + - config::copy_job_max_retention_second * 1000) { - return 0; - } - } else { - if (!config::force_immediate_recycle && - current_time < copy_job.start_time_ms() + - config::copy_job_max_retention_second * 1000) { - return 0; - } - } - } - } else if (copy_job.job_status() == CopyJobPB::LOADING) { - int64_t current_time = - duration_cast(system_clock::now().time_since_epoch()).count(); - if (!config::force_immediate_recycle && current_time <= copy_job.timeout_time_ms()) { - return 0; - } - } - metrics_context.total_need_recycle_num++; - return 0; - }; - - return scan_for_recycle_and_statistics(key0, key1, "copy_jobs", metrics_context, - std::move(scan_and_statistics), std::move(recycle_func)); -} + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_copy_jobs(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(key0, key1, std::move(recycle_func)); +} int InstanceRecycler::init_copy_job_accessor(const std::string& stage_id, const StagePB::StageType& stage_type, @@ -3663,8 +3392,8 @@ int InstanceRecycler::init_copy_job_accessor(const std::string& stage_id, int InstanceRecycler::recycle_stage() { int64_t num_scanned = 0; int64_t num_recycled = 0; - RecyclerMetricsContext metrics_context(instance_id_, "recycle_stage"); const std::string task_name = "recycle_stage"; + RecyclerMetricsContext metrics_context(instance_id_, task_name); LOG_WARNING("begin to recycle stage").tag("instance_id", instance_id_); @@ -3751,51 +3480,6 @@ int InstanceRecycler::recycle_stage() { return 0; }; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&metrics_context, this](std::string_view k, - std::string_view v) -> int { - RecycleStagePB recycle_stage; - if (!recycle_stage.ParseFromArray(v.data(), v.size())) { - LOG_WARNING("malformed recycle stage").tag("key", hex(k)); - return 0; - } - - int idx = stoi(recycle_stage.stage().obj_info().id()); - if (idx > instance_info_.obj_info().size() || idx < 1) { - LOG(WARNING) << "invalid idx: " << idx; - return 0; - } - - std::shared_ptr accessor; - int ret = SYNC_POINT_HOOK_RETURN_VALUE( - [&] { - auto& old_obj = instance_info_.obj_info()[idx - 1]; - auto s3_conf = S3Conf::from_obj_store_info(old_obj); - if (!s3_conf) { - return 0; - } - - s3_conf->prefix = recycle_stage.stage().obj_info().prefix(); - std::shared_ptr s3_accessor; - int ret = S3Accessor::create(std::move(s3_conf.value()), &s3_accessor); - if (ret != 0) { - return 0; - } - - accessor = std::move(s3_accessor); - return 0; - }(), - "recycle_stage:get_accessor", &accessor); - - if (ret != 0) { - LOG(WARNING) << "failed to init accessor ret=" << ret; - return 0; - } - - metrics_context.total_need_recycle_num++; - return 0; - }; - auto loop_done = [&stage_keys, this]() -> int { if (stage_keys.empty()) return 0; DORIS_CLOUD_DEFER { @@ -3807,9 +3491,11 @@ int InstanceRecycler::recycle_stage() { } return 0; }; - return scan_for_recycle_and_statistics(key0, key1, "stages", metrics_context, - std::move(scan_and_statistics), std::move(recycle_func), - std::move(loop_done)); + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_stage(); + } + // recycle_func and loop_done for scan and recycle + return scan_and_recycle(key0, key1, std::move(recycle_func), std::move(loop_done)); } int InstanceRecycler::recycle_expired_stage_objects() { @@ -3827,108 +3513,69 @@ int InstanceRecycler::recycle_expired_stage_objects() { }; int ret = 0; - // for calculate the total num or bytes of recyled objects - auto scan_and_statistics = [&metrics_context, this]() { - for (const auto& stage : instance_info_.stages()) { - if (stopped()) { - break; - } - if (stage.type() == StagePB::EXTERNAL) { - continue; - } - int idx = stoi(stage.obj_info().id()); - if (idx > instance_info_.obj_info().size() || idx < 1) { - continue; - } - const auto& old_obj = instance_info_.obj_info()[idx - 1]; - auto s3_conf = S3Conf::from_obj_store_info(old_obj); - if (!s3_conf) { - continue; - } - s3_conf->prefix = stage.obj_info().prefix(); - std::shared_ptr accessor; - int ret1 = S3Accessor::create(*s3_conf, &accessor); - if (ret1 != 0) { - continue; - } - if (s3_conf->prefix.find("/stage/") == std::string::npos) { - continue; - } - metrics_context.total_need_recycle_num++; - } - }; - - auto handle_recycle_func = [&, this]() { - for (const auto& stage : instance_info_.stages()) { - std::stringstream ss; - ss << "instance_id=" << instance_id_ << ", stage_id=" << stage.stage_id() - << ", user_name=" - << (stage.mysql_user_name().empty() ? "null" : stage.mysql_user_name().at(0)) - << ", user_id=" - << (stage.mysql_user_id().empty() ? "null" : stage.mysql_user_id().at(0)) - << ", prefix=" << stage.obj_info().prefix(); - - if (stopped()) { - break; - } - if (stage.type() == StagePB::EXTERNAL) { - continue; - } - int idx = stoi(stage.obj_info().id()); - if (idx > instance_info_.obj_info().size() || idx < 1) { - LOG(WARNING) << "invalid idx: " << idx << ", id: " << stage.obj_info().id(); - continue; - } - const auto& old_obj = instance_info_.obj_info()[idx - 1]; - auto s3_conf = S3Conf::from_obj_store_info(old_obj); - if (!s3_conf) { - LOG(WARNING) << "failed to init s3_conf with obj_info=" - << old_obj.ShortDebugString(); - continue; - } + if (config::enable_recycler_stats_metrics) { + scan_and_statistics_expired_stage_objects(); + } - s3_conf->prefix = stage.obj_info().prefix(); - std::shared_ptr accessor; - int ret1 = S3Accessor::create(*s3_conf, &accessor); - if (ret1 != 0) { - LOG(WARNING) << "failed to init s3 accessor ret=" << ret1 << " " << ss.str(); - ret = -1; - continue; - } + for (const auto& stage : instance_info_.stages()) { + std::stringstream ss; + ss << "instance_id=" << instance_id_ << ", stage_id=" << stage.stage_id() << ", user_name=" + << (stage.mysql_user_name().empty() ? "null" : stage.mysql_user_name().at(0)) + << ", user_id=" << (stage.mysql_user_id().empty() ? "null" : stage.mysql_user_id().at(0)) + << ", prefix=" << stage.obj_info().prefix(); - if (s3_conf->prefix.find("/stage/") == std::string::npos) { - LOG(WARNING) << "try to delete illegal prefix, which is catastrophic, " << ss.str(); - ret = -1; - continue; - } + if (stopped()) { + break; + } + if (stage.type() == StagePB::EXTERNAL) { + continue; + } + int idx = stoi(stage.obj_info().id()); + if (idx > instance_info_.obj_info().size() || idx < 1) { + LOG(WARNING) << "invalid idx: " << idx << ", id: " << stage.obj_info().id(); + continue; + } - LOG(INFO) << "recycle expired stage objects, " << ss.str(); - int64_t expiration_time = - duration_cast(system_clock::now().time_since_epoch()).count() - - config::internal_stage_objects_expire_time_second; - if (config::force_immediate_recycle) { - expiration_time = INT64_MAX; - } - ret1 = accessor->delete_all(expiration_time); - if (ret1 != 0) { - LOG(WARNING) << "failed to recycle expired stage objects, ret=" << ret1 << " " - << ss.str(); - ret = -1; - continue; - } - metrics_context.total_recycled_num++; - metrics_context.report(); + const auto& old_obj = instance_info_.obj_info()[idx - 1]; + auto s3_conf = S3Conf::from_obj_store_info(old_obj); + if (!s3_conf) { + LOG(WARNING) << "failed to init s3_conf with obj_info=" << old_obj.ShortDebugString(); + continue; } - }; - // for calculate the total num or bytes of recyled objects - scan_and_statistics(); + s3_conf->prefix = stage.obj_info().prefix(); + std::shared_ptr accessor; + int ret1 = S3Accessor::create(*s3_conf, &accessor); + if (ret1 != 0) { + LOG(WARNING) << "failed to init s3 accessor ret=" << ret1 << " " << ss.str(); + ret = -1; + continue; + } - // report to bvar - metrics_context.report(true); + if (s3_conf->prefix.find("/stage/") == std::string::npos) { + LOG(WARNING) << "try to delete illegal prefix, which is catastrophic, " << ss.str(); + ret = -1; + continue; + } - handle_recycle_func(); + LOG(INFO) << "recycle expired stage objects, " << ss.str(); + int64_t expiration_time = + duration_cast(system_clock::now().time_since_epoch()).count() - + config::internal_stage_objects_expire_time_second; + if (config::force_immediate_recycle) { + expiration_time = INT64_MAX; + } + ret1 = accessor->delete_all(expiration_time); + if (ret1 != 0) { + LOG(WARNING) << "failed to recycle expired stage objects, ret=" << ret1 << " " + << ss.str(); + ret = -1; + continue; + } + metrics_context.total_recycled_num++; + metrics_context.report(); + } return ret; } @@ -3965,4 +3612,543 @@ bool InstanceRecycler::check_recycle_tasks() { return found; } +// Scan and statistics indexes that need to be recycled +int InstanceRecycler::scan_and_statistics_indexes() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_indexes"); + + RecycleIndexKeyInfo index_key_info0 {instance_id_, 0}; + RecycleIndexKeyInfo index_key_info1 {instance_id_, INT64_MAX}; + std::string index_key0; + std::string index_key1; + recycle_index_key(index_key_info0, &index_key0); + recycle_index_key(index_key_info1, &index_key1); + int64_t earlest_ts = std::numeric_limits::max(); + + auto handle_index_kv = [&, this](std::string_view k, std::string_view v) -> int { + RecycleIndexPB index_pb; + if (!index_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t current_time = ::time(nullptr); + if (current_time < + calculate_index_expired_time(instance_id_, index_pb, &earlest_ts)) { // not expired + return 0; + } + // decode index_id + auto k1 = k; + k1.remove_prefix(1); + std::vector, int, int>> out; + decode_key(&k1, &out); + // 0x01 "recycle" ${instance_id} "index" ${index_id} -> RecycleIndexPB + auto index_id = std::get(std::get<0>(out[3])); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::string val; + err = txn->get(k, &val); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return 0; + } + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + index_pb.Clear(); + if (!index_pb.ParseFromString(val)) { + return 0; + } + if (scan_tablets_and_statistics(index_pb.table_id(), index_id, metrics_context) != 0) { + return 0; + } + metrics_context.total_need_recycle_num++; + return 0; + }; + + return scan_and_recycle(index_key0, index_key1, std::move(handle_index_kv), + [&metrics_context]() -> int { + metrics_context.report(true); + segment_metrics_context_.report(true); + tablet_metrics_context_.report(true); + return 0; + }); +} + +// Scan and statistics partitions that need to be recycled +int InstanceRecycler::scan_and_statistics_partitions() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_partitions"); + + RecyclePartKeyInfo part_key_info0 {instance_id_, 0}; + RecyclePartKeyInfo part_key_info1 {instance_id_, INT64_MAX}; + std::string part_key0; + std::string part_key1; + int64_t earlest_ts = std::numeric_limits::max(); + + recycle_partition_key(part_key_info0, &part_key0); + recycle_partition_key(part_key_info1, &part_key1); + auto handle_partition_kv = [&, this](std::string_view k, std::string_view v) -> int { + RecyclePartitionPB part_pb; + if (!part_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t current_time = ::time(nullptr); + if (current_time < + calculate_partition_expired_time(instance_id_, part_pb, &earlest_ts)) { // not expired + return 0; + } + // decode partition_id + auto k1 = k; + k1.remove_prefix(1); + std::vector, int, int>> out; + decode_key(&k1, &out); + // 0x01 "recycle" ${instance_id} "partition" ${partition_id} -> RecyclePartitionPB + auto partition_id = std::get(std::get<0>(out[3])); + // Change state to RECYCLING + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::string val; + err = txn->get(k, &val); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return 0; + } + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + part_pb.Clear(); + if (!part_pb.ParseFromString(val)) { + return 0; + } + // Partitions with PREPARED state MUST have no data + bool is_empty_tablet = part_pb.state() == RecyclePartitionPB::PREPARED; + int ret = 0; + for (int64_t index_id : part_pb.index_id()) { + if (scan_tablets_and_statistics(part_pb.table_id(), index_id, metrics_context, + partition_id, is_empty_tablet) != 0) { + ret = 0; + } + } + metrics_context.total_need_recycle_num++; + return ret; + }; + return scan_and_recycle(part_key0, part_key1, std::move(handle_partition_kv), + [&metrics_context]() -> int { + metrics_context.report(true); + segment_metrics_context_.report(true); + tablet_metrics_context_.report(true); + return 0; + }); +} + +// Scan and statistics rowsets that need to be recycled +int InstanceRecycler::scan_and_statistics_rowsets() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_rowsets"); + RecycleRowsetKeyInfo recyc_rs_key_info0 {instance_id_, 0, ""}; + RecycleRowsetKeyInfo recyc_rs_key_info1 {instance_id_, INT64_MAX, ""}; + std::string recyc_rs_key0; + std::string recyc_rs_key1; + recycle_rowset_key(recyc_rs_key_info0, &recyc_rs_key0); + recycle_rowset_key(recyc_rs_key_info1, &recyc_rs_key1); + int64_t earlest_ts = std::numeric_limits::max(); + + auto handle_rowset_kv = [&, this](std::string_view k, std::string_view v) -> int { + RecycleRowsetPB rowset; + if (!rowset.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t current_time = ::time(nullptr); + if (current_time < + calculate_rowset_expired_time(instance_id_, rowset, &earlest_ts)) { // not expired + return 0; + } + if (!rowset.has_type()) { + if (!rowset.has_resource_id()) [[unlikely]] { + return 0; + } + if (rowset.resource_id().empty()) [[unlikely]] { + return 0; + } + return 0; + } + auto* rowset_meta = rowset.mutable_rowset_meta(); + if (!rowset_meta->has_resource_id()) [[unlikely]] { + if (rowset.type() == RecycleRowsetPB::PREPARE || rowset_meta->num_segments() != 0) { + return 0; + } + } + if (rowset.type() != RecycleRowsetPB::PREPARE) { + if (rowset_meta->num_segments() > 0) { + metrics_context.total_need_recycle_num++; + segment_metrics_context_.total_need_recycle_num += rowset_meta->num_segments(); + segment_metrics_context_.total_need_recycle_data_size += + rowset_meta->total_disk_size(); + metrics_context.total_need_recycle_data_size += rowset_meta->total_disk_size(); + } + } + return 0; + }; + return scan_and_recycle(recyc_rs_key0, recyc_rs_key1, std::move(handle_rowset_kv), + [&metrics_context]() -> int { + metrics_context.report(true); + segment_metrics_context_.report(true); + return 0; + }); +} + +// Scan and statistics tmp_rowsets that need to be recycled +int InstanceRecycler::scan_and_statistics_tmp_rowsets() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_tmp_rowsets"); + MetaRowsetTmpKeyInfo tmp_rs_key_info0 {instance_id_, 0, 0}; + MetaRowsetTmpKeyInfo tmp_rs_key_info1 {instance_id_, INT64_MAX, 0}; + std::string tmp_rs_key0; + std::string tmp_rs_key1; + meta_rowset_tmp_key(tmp_rs_key_info0, &tmp_rs_key0); + meta_rowset_tmp_key(tmp_rs_key_info1, &tmp_rs_key1); + + int64_t earlest_ts = std::numeric_limits::max(); + + auto handle_tmp_rowsets_kv = [&, this](std::string_view k, std::string_view v) -> int { + doris::RowsetMetaCloudPB rowset; + if (!rowset.ParseFromArray(v.data(), v.size())) { + return 0; + } + int64_t expiration = calculate_tmp_rowset_expired_time(instance_id_, rowset, &earlest_ts); + int64_t current_time = ::time(nullptr); + if (current_time < expiration) { + return 0; + } + + DCHECK_GT(rowset.txn_id(), 0) + << "txn_id=" << rowset.txn_id() << " rowset=" << rowset.ShortDebugString(); + if (!is_txn_finished(txn_kv_, instance_id_, rowset.txn_id())) { + return 0; + } + + if (!rowset.has_resource_id()) { + if (rowset.num_segments() > 0) [[unlikely]] { // impossible + return 0; + } + metrics_context.total_need_recycle_num++; + return 0; + } + + metrics_context.total_need_recycle_num++; + if (rowset.num_segments() > 0) { + metrics_context.total_need_recycle_data_size += rowset.total_disk_size(); + segment_metrics_context_.total_need_recycle_data_size += rowset.total_disk_size(); + segment_metrics_context_.total_need_recycle_num += rowset.num_segments(); + } + return 0; + }; + return scan_and_recycle(tmp_rs_key0, tmp_rs_key1, std::move(handle_tmp_rowsets_kv), + [&metrics_context]() -> int { + metrics_context.report(true); + segment_metrics_context_.report(true); + return 0; + }); +} + +// Scan and statistics abort_timeout_txn that need to be recycled +int InstanceRecycler::scan_and_statistics_abort_timeout_txn() { + RecyclerMetricsContext metrics_context(instance_id_, "abort_timeout_txn"); + + TxnRunningKeyInfo txn_running_key_info0 {instance_id_, 0, 0}; + TxnRunningKeyInfo txn_running_key_info1 {instance_id_, INT64_MAX, INT64_MAX}; + std::string begin_txn_running_key; + std::string end_txn_running_key; + txn_running_key(txn_running_key_info0, &begin_txn_running_key); + txn_running_key(txn_running_key_info1, &end_txn_running_key); + + int64_t current_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + + auto handle_abort_timeout_txn_kv = [&metrics_context, ¤t_time, this]( + std::string_view k, std::string_view v) -> int { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::string_view k1 = k; + k1.remove_prefix(1); + std::vector, int, int>> out; + if (decode_key(&k1, &out) != 0) { + return 0; + } + int64_t db_id = std::get(std::get<0>(out[3])); + int64_t txn_id = std::get(std::get<0>(out[4])); + // Update txn_info + std::string txn_inf_key, txn_inf_val; + txn_info_key({instance_id_, db_id, txn_id}, &txn_inf_key); + err = txn->get(txn_inf_key, &txn_inf_val); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + TxnInfoPB txn_info; + if (!txn_info.ParseFromString(txn_inf_val)) { + return 0; + } + + if (TxnStatusPB::TXN_STATUS_COMMITTED != txn_info.status()) { + TxnRunningPB txn_running_pb; + if (!txn_running_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + if (!config::force_immediate_recycle && txn_running_pb.timeout_time() > current_time) { + return 0; + } + metrics_context.total_need_recycle_num++; + } + return 0; + }; + return scan_and_recycle(begin_txn_running_key, end_txn_running_key, + std::move(handle_abort_timeout_txn_kv), [&metrics_context]() -> int { + metrics_context.report(true); + return 0; + }); +} + +// Scan and statistics expired_txn_label that need to be recycled +int InstanceRecycler::scan_and_statistics_expired_txn_label() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_expired_txn_label"); + + RecycleTxnKeyInfo recycle_txn_key_info0 {instance_id_, 0, 0}; + RecycleTxnKeyInfo recycle_txn_key_info1 {instance_id_, INT64_MAX, INT64_MAX}; + std::string begin_recycle_txn_key; + std::string end_recycle_txn_key; + recycle_txn_key(recycle_txn_key_info0, &begin_recycle_txn_key); + recycle_txn_key(recycle_txn_key_info1, &end_recycle_txn_key); + int64_t earlest_ts = std::numeric_limits::max(); + int64_t current_time_ms = + duration_cast(system_clock::now().time_since_epoch()).count(); + + // for calculate the total num or bytes of recyled objects + auto handle_expired_txn_label_kv = [&, this](std::string_view k, std::string_view v) -> int { + RecycleTxnPB recycle_txn_pb; + if (!recycle_txn_pb.ParseFromArray(v.data(), v.size())) { + return 0; + } + if ((config::force_immediate_recycle) || + (recycle_txn_pb.has_immediate() && recycle_txn_pb.immediate()) || + (calculate_txn_expired_time(instance_id_, recycle_txn_pb, &earlest_ts) <= + current_time_ms)) { + metrics_context.total_need_recycle_num++; + } + return 0; + }; + return scan_and_recycle(begin_recycle_txn_key, end_recycle_txn_key, + std::move(handle_expired_txn_label_kv), [&metrics_context]() -> int { + metrics_context.report(true); + return 0; + }); +} + +// Scan and statistics copy_jobs that need to be recycled +int InstanceRecycler::scan_and_statistics_copy_jobs() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_copy_jobs"); + CopyJobKeyInfo key_info0 {instance_id_, "", 0, "", 0}; + CopyJobKeyInfo key_info1 {instance_id_, "\xff", 0, "", 0}; + std::string key0; + std::string key1; + copy_job_key(key_info0, &key0); + copy_job_key(key_info1, &key1); + + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context](std::string_view k, std::string_view v) -> int { + CopyJobPB copy_job; + if (!copy_job.ParseFromArray(v.data(), v.size())) { + LOG_WARNING("malformed copy job").tag("key", hex(k)); + return 0; + } + + if (copy_job.job_status() == CopyJobPB::FINISH) { + if (copy_job.stage_type() == StagePB::EXTERNAL) { + int64_t current_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + if (copy_job.finish_time_ms() > 0) { + if (!config::force_immediate_recycle && + current_time < copy_job.finish_time_ms() + + config::copy_job_max_retention_second * 1000) { + return 0; + } + } else { + if (!config::force_immediate_recycle && + current_time < copy_job.start_time_ms() + + config::copy_job_max_retention_second * 1000) { + return 0; + } + } + } + } else if (copy_job.job_status() == CopyJobPB::LOADING) { + int64_t current_time = + duration_cast(system_clock::now().time_since_epoch()).count(); + if (!config::force_immediate_recycle && current_time <= copy_job.timeout_time_ms()) { + return 0; + } + } + metrics_context.total_need_recycle_num++; + return 0; + }; + + return scan_and_recycle(key0, key1, std::move(scan_and_statistics), + [&metrics_context]() -> int { + metrics_context.report(true); + return 0; + }); +} + +// Scan and statistics stage that need to be recycled +int InstanceRecycler::scan_and_statistics_stage() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_stage"); + RecycleStageKeyInfo key_info0 {instance_id_, ""}; + RecycleStageKeyInfo key_info1 {instance_id_, "\xff"}; + std::string key0 = recycle_stage_key(key_info0); + std::string key1 = recycle_stage_key(key_info1); + + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, this](std::string_view k, + std::string_view v) -> int { + RecycleStagePB recycle_stage; + if (!recycle_stage.ParseFromArray(v.data(), v.size())) { + LOG_WARNING("malformed recycle stage").tag("key", hex(k)); + return 0; + } + + int idx = stoi(recycle_stage.stage().obj_info().id()); + if (idx > instance_info_.obj_info().size() || idx < 1) { + LOG(WARNING) << "invalid idx: " << idx; + return 0; + } + + std::shared_ptr accessor; + int ret = SYNC_POINT_HOOK_RETURN_VALUE( + [&] { + auto& old_obj = instance_info_.obj_info()[idx - 1]; + auto s3_conf = S3Conf::from_obj_store_info(old_obj); + if (!s3_conf) { + return 0; + } + + s3_conf->prefix = recycle_stage.stage().obj_info().prefix(); + std::shared_ptr s3_accessor; + int ret = S3Accessor::create(std::move(s3_conf.value()), &s3_accessor); + if (ret != 0) { + return 0; + } + + accessor = std::move(s3_accessor); + return 0; + }(), + "recycle_stage:get_accessor", &accessor); + + if (ret != 0) { + LOG(WARNING) << "failed to init accessor ret=" << ret; + return 0; + } + + metrics_context.total_need_recycle_num++; + return 0; + }; + + return scan_and_recycle(key0, key1, std::move(scan_and_statistics), + [&metrics_context]() -> int { + metrics_context.report(true); + return 0; + }); +} + +// Scan and statistics expired_stage_objects that need to be recycled +int InstanceRecycler::scan_and_statistics_expired_stage_objects() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_expired_stage_objects"); + + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, this]() { + for (const auto& stage : instance_info_.stages()) { + if (stopped()) { + break; + } + if (stage.type() == StagePB::EXTERNAL) { + continue; + } + int idx = stoi(stage.obj_info().id()); + if (idx > instance_info_.obj_info().size() || idx < 1) { + continue; + } + const auto& old_obj = instance_info_.obj_info()[idx - 1]; + auto s3_conf = S3Conf::from_obj_store_info(old_obj); + if (!s3_conf) { + continue; + } + s3_conf->prefix = stage.obj_info().prefix(); + std::shared_ptr accessor; + int ret1 = S3Accessor::create(*s3_conf, &accessor); + if (ret1 != 0) { + continue; + } + if (s3_conf->prefix.find("/stage/") == std::string::npos) { + continue; + } + metrics_context.total_need_recycle_num++; + } + }; + + scan_and_statistics(); + metrics_context.report(true); + return 0; +} + +// Scan and statistics versions that need to be recycled +int InstanceRecycler::scan_and_statistics_versions() { + RecyclerMetricsContext metrics_context(instance_id_, "recycle_versions"); + auto version_key_begin = partition_version_key({instance_id_, 0, 0, 0}); + auto version_key_end = partition_version_key({instance_id_, INT64_MAX, 0, 0}); + + int64_t last_scanned_table_id = 0; + bool is_recycled = false; // Is last scanned kv recycled + // for calculate the total num or bytes of recyled objects + auto scan_and_statistics = [&metrics_context, &last_scanned_table_id, &is_recycled, this]( + std::string_view k, std::string_view) { + auto k1 = k; + k1.remove_prefix(1); + // 0x01 "version" ${instance_id} "partition" ${db_id} ${tbl_id} ${partition_id} + std::vector, int, int>> out; + decode_key(&k1, &out); + DCHECK_EQ(out.size(), 6) << k; + auto table_id = std::get(std::get<0>(out[4])); + if (table_id == last_scanned_table_id) { // Already handle kvs of this table + metrics_context.total_need_recycle_num += + is_recycled; // Version kv of this table has been recycled + return 0; + } + last_scanned_table_id = table_id; + is_recycled = false; + auto tablet_key_begin = stats_tablet_key({instance_id_, table_id, 0, 0, 0}); + auto tablet_key_end = stats_tablet_key({instance_id_, table_id, INT64_MAX, 0, 0}); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + std::unique_ptr iter; + err = txn->get(tablet_key_begin, tablet_key_end, &iter, false, 1); + if (err != TxnErrorCode::TXN_OK) { + return 0; + } + if (iter->has_next()) { // Table is useful, should not recycle table and partition versions + return 0; + } + metrics_context.total_need_recycle_num++; + is_recycled = true; + return 0; + }; + + return scan_and_recycle(version_key_begin, version_key_end, std::move(scan_and_statistics), + [&metrics_context]() -> int { + metrics_context.report(true); + return 0; + }); +} + } // namespace doris::cloud diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h index d1ae8a056c857f..97f9a8d57c5b73 100644 --- a/cloud/src/recycler/recycler.h +++ b/cloud/src/recycler/recycler.h @@ -204,6 +204,26 @@ class InstanceRecycler { bool check_recycle_tasks(); + int scan_and_statistics_indexes(); + + int scan_and_statistics_partitions(); + + int scan_and_statistics_rowsets(); + + int scan_and_statistics_tmp_rowsets(); + + int scan_and_statistics_abort_timeout_txn(); + + int scan_and_statistics_expired_txn_label(); + + int scan_and_statistics_copy_jobs(); + + int scan_and_statistics_stage(); + + int scan_and_statistics_expired_stage_objects(); + + int scan_and_statistics_versions(); + private: // returns 0 for success otherwise error int init_obj_store_accessors(); @@ -222,13 +242,6 @@ class InstanceRecycler { std::function recycle_func, std::function loop_done = nullptr); - int scan_for_recycle_and_statistics( - std::string begin, std::string_view end, std::string task_name, - RecyclerMetricsContext& metrics_context, - std::function statistics_func, - std::function recycle_func, - std::function loop_done = nullptr); - // return 0 for success otherwise error int delete_rowset_data(const doris::RowsetMetaCloudPB& rs_meta_pb); @@ -363,30 +376,34 @@ class RecyclerMetricsContext { // `is_begin` is used to initialize total num of items need to be recycled void report(bool is_begin = false) { if (!operation_type.empty()) { - if (total_need_recycle_data_size > 0) { - // is init - if (is_begin) { + // is init + if (is_begin) { + if (total_need_recycle_data_size) { g_bvar_recycler_instance_last_round_to_recycle_bytes.put( {instance_id, operation_type}, total_need_recycle_data_size); - } else { + } + } else { + if (total_recycled_data_size.load()) { g_bvar_recycler_instance_last_round_recycled_bytes.put( {instance_id, operation_type}, total_recycled_data_size.load()); - g_bvar_recycler_instance_recycle_total_bytes_since_started.put( - {instance_id, operation_type}, total_recycled_data_size.load()); } + g_bvar_recycler_instance_recycle_total_bytes_since_started.put( + {instance_id, operation_type}, total_recycled_data_size.load()); } - if (total_need_recycle_num > 0) { - // is init - if (is_begin) { + // is init + if (is_begin) { + if (total_need_recycle_num) { g_bvar_recycler_instance_last_round_to_recycle_num.put( {instance_id, operation_type}, total_need_recycle_num); - } else { + } + } else { + if (total_recycled_num.load()) { g_bvar_recycler_instance_last_round_recycled_num.put( {instance_id, operation_type}, total_recycled_num.load()); - g_bvar_recycler_instance_recycle_total_num_since_started.put( - {instance_id, operation_type}, total_recycled_num.load()); } + g_bvar_recycler_instance_recycle_total_num_since_started.put( + {instance_id, operation_type}, total_recycled_num.load()); } } } diff --git a/cloud/src/recycler/recycler_service.cpp b/cloud/src/recycler/recycler_service.cpp index b812fbd27eec62..48b2fe370ef61e 100644 --- a/cloud/src/recycler/recycler_service.cpp +++ b/cloud/src/recycler/recycler_service.cpp @@ -23,6 +23,13 @@ #include #include +#include +#include +#include +#include +#include +#include + #include "common/config.h" #include "common/defer.h" #include "common/logging.h" @@ -34,6 +41,7 @@ #include "recycler/meta_checker.h" #include "recycler/recycler.h" #include "recycler/s3_accessor.h" +#include "recycler/util.h" namespace doris::cloud { @@ -52,6 +60,245 @@ RecyclerServiceImpl::RecyclerServiceImpl(std::shared_ptr txn_kv, Recycler RecyclerServiceImpl::~RecyclerServiceImpl() = default; +void RecyclerServiceImpl::statistics_recycle(StatisticsRecycleRequest& req, MetaServiceCode& code, + std::string& msg) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = MetaServiceCode::KV_TXN_CREATE_ERR; + msg = "failed to create txn"; + return; + } + + static std::map> resource_handlers = { + {"recycle_indexes", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_indexes(); + }}, + {"recycle_partitions", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_partitions(); + }}, + {"recycle_tmp_rowsets", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_tmp_rowsets(); + }}, + {"recycle_rowsets", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_rowsets(); + }}, + {"abort_timeout_txn", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_abort_timeout_txn(); + }}, + {"recycle_expired_txn_label", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_expired_txn_label(); + }}, + {"recycle_versions", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_versions(); + }}, + {"recycle_copy_jobs", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_copy_jobs(); + }}, + {"recycle_stage", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_stage(); + }}, + {"recycle_expired_stage_objects", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_expired_stage_objects(); + }}, + {"recycle_tablet", + [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_partitions(); + instance_recycler.scan_and_statistics_indexes(); + }}, + {"recycle_segment", [](InstanceRecycler& instance_recycler) { + instance_recycler.scan_and_statistics_partitions(); + instance_recycler.scan_and_statistics_indexes(); + instance_recycler.scan_and_statistics_rowsets(); + instance_recycler.scan_and_statistics_tmp_rowsets(); + }}}; + + std::set resource_types; + for (const auto& resource_type : req.resource_type()) { + if (resource_type == "*") { + std::for_each(resource_handlers.begin(), resource_handlers.end(), + [&](const auto& it) { resource_types.emplace(it.first); }); + break; + } else { + if (!resource_handlers.contains(resource_type)) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = fmt::format( + "invalid resource type: {}, valid resource_type have [{}]", resource_type, + std::accumulate(resource_handlers.begin(), resource_handlers.end(), + std::string(), [](const std::string& acc, const auto& it) { + return acc.empty() ? it.first : acc + ", " + it.first; + })); + LOG_WARNING(msg); + return; + } else { + resource_types.emplace(resource_type); + } + } + } + + std::set instance_ids; + std::vector instances; + get_all_instances(txn_kv_.get(), instances); + + for (const auto& instance_id : req.instance_ids()) { + if (instance_id == "*") { + std::for_each(instances.begin(), instances.end(), [&](const InstanceInfoPB& instance) { + instance_ids.emplace(instance.instance_id()); + }); + break; + } else { + if (std::find_if(instances.begin(), instances.end(), + [&](const InstanceInfoPB& instance) { + return instance.instance_id() == instance_id; + }) == instances.end()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = fmt::format("invalid instance id: {}", instance_id); + LOG_WARNING(msg); + return; + } else { + instance_ids.emplace(instance_id); + } + } + } + + LOG(INFO) << "begin to statistics recycle for " + << std::accumulate(instance_ids.begin(), instance_ids.end(), std::string(), + [](const std::string& acc, const std::string& id) { + return acc.empty() ? id : acc + ", " + id; + }); + + auto worker_pool = std::make_unique( + config::instance_recycler_statistics_recycle_worker_pool_size, "statistics_recycle"); + worker_pool->start(); + + for (const auto& id : instance_ids) { + InstanceKeyInfo key_info {id}; + std::string key; + instance_key(key_info, &key); + std::string val; + err = txn->get(key, &val); + if (err != TxnErrorCode::TXN_OK) { + code = MetaServiceCode::KV_TXN_GET_ERR; + msg = fmt::format("failed to get instance, instance_id={}, err={}", id, err); + LOG_WARNING(msg); + continue; + } + InstanceInfoPB instance; + if (!instance.ParseFromString(val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = fmt::format("malformed instance info, key={}, val={}", hex(key), hex(val)); + LOG_WARNING(msg); + continue; + } + auto instance_recycler = std::make_shared( + txn_kv_, instance, recycler_->_thread_pool_group, txn_lazy_committer_); + + if (int r = instance_recycler->init(); r != 0) { + LOG(WARNING) << "failed to init instance recycler, instance_id=" << id << " ret=" << r; + continue; + } + // if empty, statistics all resources + if (resource_types.empty()) { + for (const auto& [_, func] : resource_handlers) { + worker_pool->submit([&instance_recycler, &func]() { func(*instance_recycler); }); + } + } else { + for (const auto& resource_type : resource_types) { + if (auto it = resource_handlers.find(resource_type); + it != resource_handlers.end()) { + worker_pool->submit( + [&it, &instance_recycler]() { it->second(*instance_recycler); }); + } + } + } + } + + worker_pool->stop(); + std::stringstream ss; + for_each(instance_ids.begin(), instance_ids.end(), [&](const std::string& id) { + ss << "Instance ID: " << id << "\n"; + ss << "----------------------------------------\n"; + + for_each(resource_types.begin(), resource_types.end(), [&](const auto& resource_type) { + int64_t to_recycle_num = 0; + int64_t to_recycle_bytes = 0; + if (resource_type == "recycle_segment" || resource_type == "recycle_tablet") { + to_recycle_num = g_bvar_recycler_instance_last_round_to_recycle_num.get( + {"global_recycler", resource_type}); + to_recycle_bytes = g_bvar_recycler_instance_last_round_to_recycle_bytes.get( + {"global_recycler", resource_type}); + } else { + to_recycle_num = + g_bvar_recycler_instance_last_round_to_recycle_num.get({id, resource_type}); + to_recycle_bytes = g_bvar_recycler_instance_last_round_to_recycle_bytes.get( + {id, resource_type}); + } + + ss << "Task Type: " << resource_type << "\n"; + + // Add specific counts for different resource types + if (resource_type == "recycle_partitions") { + ss << " • Need to recycle partition count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle partition size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_rowsets") { + ss << " • Need to recycle rowset count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle rowset size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_tmp_rowsets") { + ss << " • Need to recycle tmp rowset count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle tmp rowset size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_indexes") { + ss << " • Need to recycle index count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle index size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_segment") { + ss << " • Need to recycle segment count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle segment size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_tablet") { + ss << " • Need to recycle tablet count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle tablet size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_versions") { + ss << " • Need to recycle version count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle version size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "abort_timeout_txn") { + ss << " • Need to abort timeout txn count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle timeout txn size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_expired_txn_label") { + ss << " • Need to recycle expired txn label count: " << to_recycle_num + << " items\n"; + ss << " • Need to recycle expired txn label size: " << to_recycle_bytes + << " bytes\n"; + } else if (resource_type == "recycle_copy_jobs") { + ss << " • Need to recycle copy job count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle copy job size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_stage") { + ss << " • Need to recycle stage count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle stage size: " << to_recycle_bytes << " bytes\n"; + } else if (resource_type == "recycle_expired_stage_objects") { + ss << " • Need to recycle expired stage object count: " << to_recycle_num + << " items\n"; + ss << " • Need to recycle expired stage object size: " << to_recycle_bytes + << " bytes\n"; + } else { + ss << " • Need to recycle count: " << to_recycle_num << " items\n"; + ss << " • Need to recycle size: " << to_recycle_bytes << " bytes\n"; + } + + ss << "----------------------------------------\n"; + }); + ss << "\n"; + }); + msg = ss.str(); +} + void RecyclerServiceImpl::recycle_instance(::google::protobuf::RpcController* controller, const ::doris::cloud::RecycleInstanceRequest* request, ::doris::cloud::RecycleInstanceResponse* response, @@ -330,6 +577,20 @@ void RecyclerServiceImpl::http(::google::protobuf::RpcController* controller, return; } + if (unresolved_path == "statistics_recycle") { + StatisticsRecycleRequest req; + auto st = google::protobuf::util::JsonStringToMessage(request_body, &req); + if (!st.ok()) { + msg = "failed to StatisticsRecycleRequest, error: " + st.message().ToString(); + response_body = msg; + LOG(WARNING) << msg; + return; + } + statistics_recycle(req, code, msg); + response_body = msg; + return; + } + if (unresolved_path == "recycle_copy_jobs") { auto instance_id = uri.GetQuery("instance_id"); if (instance_id == nullptr || instance_id->empty()) { diff --git a/cloud/src/recycler/recycler_service.h b/cloud/src/recycler/recycler_service.h index 67eb9d21333075..2ddf311310bf6a 100644 --- a/cloud/src/recycler/recycler_service.h +++ b/cloud/src/recycler/recycler_service.h @@ -44,6 +44,8 @@ class RecyclerServiceImpl : public cloud::RecyclerService { ::google::protobuf::Closure* done) override; private: + void statistics_recycle(StatisticsRecycleRequest& req, MetaServiceCode& code, std::string& msg); + void check_instance(const std::string& instance_id, MetaServiceCode& code, std::string& msg); private: diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 4e5c27f6a4382b..571bd0b26e2429 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -1342,6 +1342,11 @@ message RecycleInstanceRequest { repeated string instance_ids = 1; } +message StatisticsRecycleRequest { + repeated string instance_ids = 1; + repeated string resource_type = 2; +} + message RecycleInstanceResponse { optional MetaServiceResponseStatus status = 1; } From 358c2aa3aa155df0017b8b3673c5b0794c88d537 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 10:18:21 +0800 Subject: [PATCH 199/572] branch-3.0: [enhance](meta-service)collect fdb kv meta range info as metrics #52430 (#53116) Cherry-picked from #52430 Co-authored-by: koarz --- cloud/src/common/bvars.cpp | 3 + cloud/src/common/bvars.h | 11 ++- cloud/src/common/metric.cpp | 89 ++++++++++++++++++++ cloud/src/common/metric.h | 2 + cloud/src/meta-service/meta_service_http.cpp | 37 ++------ cloud/src/meta-store/keys.cpp | 19 ++++- 6 files changed, 126 insertions(+), 35 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 5999320af8311c..7e4e36ce14059b 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -383,4 +383,7 @@ mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv_clean_txn_labe // get_txn_id mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv_get_txn_id_get_counter",{"instance_id"}); +// meta ranges +mBvarStatus g_bvar_fdb_kv_ranges_count("fdb_kv_ranges_count", {"category","instance_id", "sub_category"}); + // clang-format on diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index 3ca3e1b2cab1e0..d57e14e2697278 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -139,8 +140,7 @@ class mBvarWrapper { void put(const std::initializer_list& dim_values, ValType value) { BvarType* stats = counter_.get_stats(std::list(dim_values)); if (stats) { - if constexpr (std::is_same_v> || - std::is_same_v>) { + if constexpr (is_bvar_status::value) { stats->set_value(value); } else { *stats << value; @@ -170,6 +170,10 @@ class mBvarWrapper { struct is_valid_bvar_type> : std::true_type {}; template <> struct is_valid_bvar_type : std::true_type {}; + template + struct is_bvar_status : std::false_type {}; + template + struct is_bvar_status> : std::true_type {}; bvar::MultiDimension counter_; }; @@ -464,3 +468,6 @@ extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter; + +// meta ranges +extern mBvarStatus g_bvar_fdb_kv_ranges_count; diff --git a/cloud/src/common/metric.cpp b/cloud/src/common/metric.cpp index 124a5f26a063af..ac0193a9921ec3 100644 --- a/cloud/src/common/metric.cpp +++ b/cloud/src/common/metric.cpp @@ -25,16 +25,20 @@ #include #include #include +#include #include #include +#include #include #include "common/bvars.h" #include "common/logging.h" +#include "meta-store/keys.h" #include "meta-store/txn_kv.h" #include "meta-store/txn_kv_error.h" namespace doris::cloud { +extern std::set get_key_prefix_contants(); // The format of the output is shown in "test/fdb_metric_example.json" static const std::string FDB_STATUS_KEY = "\xff\xff/status/json"; @@ -298,10 +302,95 @@ static void export_fdb_status_details(const std::string& status_str) { get_process_metric("memory"); } +// boundaries include the key category{meta, txn, recycle...}, instance_id and sub_category{rowset, txn_label...} +// encode look like +// 0x01 "txn" ${instance_id} "txn_label" ${db_id} ${label} +// 0x01 "meta" ${instance_id} "rowset" ${tablet_id} ${version} +// the func count same key to hashmap kv_range_count +// exmaple: +// kv_range_boundaries: meta|instance1|rowset|..., meta|instance1|rowset|..., meta|instance2|rowset|..., txn|instance1|txn_label|... +// kv_range_count output: , , +void get_kv_range_boundaries_count(std::vector& kv_range_boundaries, + std::unordered_map& kv_range_count) { + size_t prefix_size = FdbTxnKv::fdb_partition_key_prefix().size(); + for (auto&& boundary : kv_range_boundaries) { + if (boundary.size() < prefix_size + 1 || boundary[prefix_size] != CLOUD_USER_KEY_SPACE01) { + continue; + } + + std::string_view user_key(boundary); + user_key.remove_prefix(prefix_size + 1); // Skip the KEY_SPACE prefix. + std::vector, int, int>> out; + decode_key(&user_key, &out); // ignore any error, since the boundary key might be truncated. + + auto visitor = [](auto&& arg) -> std::string { + using T = std::decay_t; + if constexpr (std::is_same_v) { + return arg; + } else { + return std::to_string(arg); + } + }; + + if (!out.empty()) { + std::string key; + // whatever the boundary's category have similar encode part: + // category, instance_id, sub_category + // we can distinguish boundary using the three parts + // some boundaries do not contain all three parts, so restrictions based on size are also necessary + for (size_t i = 0; i < 3 && i < out.size(); ++i) { + key += std::visit(visitor, std::get<0>(out[i])) + '|'; + } + key.pop_back(); + kv_range_count[key]++; + } + } +} + +static void export_fdb_kv_ranges_details(TxnKv* kv) { + auto* txn_kv = dynamic_cast(kv); + if (!txn_kv) { + LOG(WARNING) << "this method only support fdb txn kv"; + return; + } + + std::vector partition_boundaries; + TxnErrorCode code = txn_kv->get_partition_boundaries(&partition_boundaries); + if (code != TxnErrorCode::TXN_OK) { + auto msg = fmt::format("failed to get boundaries, code={}", code); + return; + } + + std::unordered_map partition_count; + get_kv_range_boundaries_count(partition_boundaries, partition_count); + + auto key_prefix_set = get_key_prefix_contants(); + std::unordered_map category_count; + for (auto&& [key, count] : partition_count) { + std::vector keys; + size_t pos {}; + // split key with '|' + do { + size_t p = std::min(key.size(), key.find('|', pos)); + keys.emplace_back(key.substr(pos, p - pos)); + pos = p + 1; + } while (pos < key.size()); + keys.resize(3); + if (key_prefix_set.contains(keys[0])) { + category_count[keys[0]] += count; + g_bvar_fdb_kv_ranges_count.put({keys[0], keys[1], keys[2]}, count); + } else { + LOG(WARNING) << fmt::format("Unknow meta range type: {}", keys[0]); + continue; + } + } +} + void FdbMetricExporter::export_fdb_metrics(TxnKv* txn_kv) { int64_t busyness = 0; std::string fdb_status = get_fdb_status(txn_kv); export_fdb_status_details(fdb_status); + export_fdb_kv_ranges_details(txn_kv); if (auto* kv = dynamic_cast(txn_kv); kv != nullptr) { busyness = static_cast(kv->get_client_thread_busyness() * 100); g_bvar_fdb_client_thread_busyness_percent.set_value(busyness); diff --git a/cloud/src/common/metric.h b/cloud/src/common/metric.h index ad918482daa460..72f508c84e7e74 100644 --- a/cloud/src/common/metric.h +++ b/cloud/src/common/metric.h @@ -28,6 +28,8 @@ #include "meta-store/txn_kv.h" namespace doris::cloud { +extern void get_kv_range_boundaries_count(std::vector& partition_boundaries, + std::unordered_map& partition_count); class FdbMetricExporter { public: diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp index 21db2ec0357b64..7a78b97eaa6f45 100644 --- a/cloud/src/meta-service/meta_service_http.cpp +++ b/cloud/src/meta-service/meta_service_http.cpp @@ -75,6 +75,9 @@ extern int decrypt_instance_info(InstanceInfoPB& instance, const std::string& in MetaServiceCode& code, std::string& msg, std::shared_ptr& txn); +extern void get_kv_range_boundaries_count(std::vector& partition_boundaries, + std::unordered_map& partition_count); + template static google::protobuf::util::Status parse_json_message(const std::string& unresolved_path, const std::string& body, Message* req) { @@ -532,38 +535,8 @@ static HttpResponse process_show_meta_ranges(MetaServiceImpl* service, brpc::Con auto msg = fmt::format("failed to get boundaries, code={}", code); return http_json_reply(MetaServiceCode::UNDEFINED_ERR, msg); } - std::unordered_map partition_count; - size_t prefix_size = FdbTxnKv::fdb_partition_key_prefix().size(); - for (auto&& boundary : partition_boundaries) { - if (boundary.size() < prefix_size + 1 || boundary[prefix_size] != CLOUD_USER_KEY_SPACE01) { - continue; - } - - std::string_view user_key(boundary); - user_key.remove_prefix(prefix_size + 1); // Skip the KEY_SPACE prefix. - std::vector, int, int>> out; - decode_key(&user_key, &out); // ignore any error, since the boundary key might be truncated. - - auto visitor = [](auto&& arg) -> std::string { - using T = std::decay_t; - if constexpr (std::is_same_v) { - return arg; - } else { - return std::to_string(arg); - } - }; - - if (!out.empty()) { - std::string key; - for (size_t i = 0; i < 3 && i < out.size(); ++i) { - key += std::visit(visitor, std::get<0>(out[i])); - key += '|'; - } - key.pop_back(); // omit the last '|' - partition_count[key]++; - } - } + get_kv_range_boundaries_count(partition_boundaries, partition_count); // sort ranges by count std::vector> meta_ranges; @@ -575,7 +548,7 @@ static HttpResponse process_show_meta_ranges(MetaServiceImpl* service, brpc::Con std::sort(meta_ranges.begin(), meta_ranges.end(), [](const auto& lhs, const auto& rhs) { return lhs.second > rhs.second; }); - std::string body = fmt::format("total partitions: {}\n", partition_boundaries.size()); + std::string body = fmt::format("total meta ranges: {}\n", partition_boundaries.size()); for (auto&& [key, count] : meta_ranges) { body += fmt::format("{}: {}\n", key, count); } diff --git a/cloud/src/meta-store/keys.cpp b/cloud/src/meta-store/keys.cpp index dff688c22d75d5..83ba12deba7614 100644 --- a/cloud/src/meta-store/keys.cpp +++ b/cloud/src/meta-store/keys.cpp @@ -17,6 +17,8 @@ #include "meta-store/keys.h" +#include + #include "meta-store/codec.h" namespace doris::cloud { @@ -526,5 +528,20 @@ int decode_key(std::string_view* in, } return 0; } - +//================================================================================== +// Key Prefix Map +//================================================================================== +std::set get_key_prefix_contants() { + std::set key_prefix_set; + key_prefix_set.insert(INSTANCE_KEY_PREFIX); + key_prefix_set.insert(TXN_KEY_PREFIX); + key_prefix_set.insert(VERSION_KEY_PREFIX); + key_prefix_set.insert(META_KEY_PREFIX); + key_prefix_set.insert(RECYCLE_KEY_PREFIX); + key_prefix_set.insert(STATS_KEY_PREFIX); + key_prefix_set.insert(JOB_KEY_PREFIX); + key_prefix_set.insert(COPY_KEY_PREFIX); + key_prefix_set.insert(VAULT_KEY_PREFIX); + return key_prefix_set; +} } // namespace doris::cloud From 7ebf1cfacc61f4be6c65704512b9f4f157aba1b0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 10:21:36 +0800 Subject: [PATCH 200/572] branch-3.0: [improvement](cloud-mow) change update_delete_bitmap rpc timeout to 30s #51671 (#53109) Cherry-picked from #51671 Co-authored-by: zhannngchen --- be/src/cloud/cloud_meta_mgr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 21a91d7a64e6c7..622a38c3159c16 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -400,7 +400,7 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res, std::shared_ptr stub; RETURN_IF_ERROR(proxy->get(&stub)); brpc::Controller cntl; - if (op_name == "get delete bitmap") { + if (op_name == "get delete bitmap" || op_name == "update delete bitmap") { cntl.set_timeout_ms(3 * config::meta_service_brpc_timeout_ms); } else { cntl.set_timeout_ms(config::meta_service_brpc_timeout_ms); From 883fb539a22b40b7f343a85d828be34d1863c022 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 11:02:28 +0800 Subject: [PATCH 201/572] branch-3.0: [fix](job) fix show routine load job result incorrect after alter job property #53038 (#53098) Cherry-picked from #53038 Co-authored-by: hui lai --- .../org/apache/doris/load/routineload/RoutineLoadJob.java | 1 + .../suites/load_p0/routine_load/test_routine_load.groovy | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index d7e3c6fc49450f..5ea984ebfac23f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -2060,6 +2060,7 @@ protected void modifyCommonJobProperties(Map jobProperties) { if (jobProperties.containsKey(CreateRoutineLoadStmt.MAX_FILTER_RATIO_PROPERTY)) { this.maxFilterRatio = Double.parseDouble( jobProperties.remove(CreateRoutineLoadStmt.MAX_FILTER_RATIO_PROPERTY)); + this.jobProperties.put(CreateRoutineLoadStmt.MAX_FILTER_RATIO_PROPERTY, String.valueOf(maxFilterRatio)); } if (jobProperties.containsKey(CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_SEC_PROPERTY)) { diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy index 75b8ddce80d3b8..e5a24c2a44c16a 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy @@ -1834,6 +1834,7 @@ suite("test_routine_load","p0") { sql "ALTER ROUTINE LOAD FOR ${jobs[i]} PROPERTIES(\"num_as_string\" = \"true\");" sql "ALTER ROUTINE LOAD FOR ${jobs[i]} PROPERTIES(\"fuzzy_parse\" = \"true\");" sql "ALTER ROUTINE LOAD FOR ${jobs[i]} PROPERTIES(\"workload_group\" = \"alter_routine_load_group\");" + sql "ALTER ROUTINE LOAD FOR ${jobs[i]} PROPERTIES(\"max_filter_ratio\" = \"0.5\");" res = sql "show routine load for ${jobs[i]}" log.info("routine load job properties: ${res[0][11].toString()}".toString()) @@ -1844,8 +1845,7 @@ suite("test_routine_load","p0") { assertEquals("300001", json.max_batch_rows.toString()) assertEquals("209715201", json.max_batch_size.toString()) assertEquals("6", json.max_batch_interval.toString()) - //TODO(bug): Can not update - //assertEquals("0.5", json.max_filter_ratio.toString()) + assertEquals("0.5", json.max_filter_ratio.toString()) assertEquals("jsonpaths", json.jsonpaths.toString()) assertEquals("json_root", json.json_root.toString()) assertEquals("true", json.strict_mode.toString()) From 13607471e0930115340193d75f0d7d640ffc815b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 12 Jul 2025 11:03:32 +0800 Subject: [PATCH 202/572] branch-3.0: [fix](httpserver) Fix lsan check error when stop libevent server #53081 (#53111) Cherry-picked from #53081 Co-authored-by: Xin Liao --- be/src/http/ev_http_server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp index 1bbd2c0e178fd2..457cf0e0322ba5 100644 --- a/be/src/http/ev_http_server.cpp +++ b/be/src/http/ev_http_server.cpp @@ -148,9 +148,9 @@ void EvHttpServer::stop() { for (int i = 0; i < _num_workers; ++i) { event_base_loopbreak(_event_bases[i].get()); } - _event_bases.clear(); } _workers->shutdown(); + _event_bases.clear(); close(_server_fd); _started = false; } From 02b25b9e8e395f3a9ed05c15790b5666a95cc588 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sat, 12 Jul 2025 18:28:06 -0700 Subject: [PATCH 203/572] branch-3.0: [chore](ci)Replace local setup-maven with setup-java for more stable and simplified Maven setup (#52229) (#53154) bp #52229 Co-authored-by: Calvin Kirs --- .github/actions/setup-maven | 1 - .github/workflows/checkstyle.yaml | 8 +++++--- .gitmodules | 3 --- 3 files changed, 5 insertions(+), 7 deletions(-) delete mode 160000 .github/actions/setup-maven diff --git a/.github/actions/setup-maven b/.github/actions/setup-maven deleted file mode 160000 index 417e1a9899611c..00000000000000 --- a/.github/actions/setup-maven +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 417e1a9899611c0350621d1fb0c2770f35105c69 diff --git a/.github/workflows/checkstyle.yaml b/.github/workflows/checkstyle.yaml index 13ab46b2cd50b2..9d4e2385c151bc 100644 --- a/.github/workflows/checkstyle.yaml +++ b/.github/workflows/checkstyle.yaml @@ -42,11 +42,13 @@ jobs: - 'gensrc/proto/**' - 'gensrc/thrift/**' - - name: Setup Maven Action + - name: Set up JDK with Maven if: steps.filter.outputs.fe_changes == 'true' - uses: ./.github/actions/setup-maven + uses: actions/setup-java@v4 with: - maven-version: 3.8.4 + distribution: 'temurin' + java-version: '17' + cache: 'maven' - name: Run java checkstyle if: steps.filter.outputs.fe_changes == 'true' diff --git a/.gitmodules b/.gitmodules index 80afde96cd7985..2ea37f04b332f5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,9 +4,6 @@ [submodule ".github/actions/get-workflow-origin"] path = .github/actions/get-workflow-origin url = https://github.com/potiuk/get-workflow-origin.git -[submodule ".github/actions/setup-maven"] - path = .github/actions/setup-maven - url = https://github.com/stCarolas/setup-maven.git [submodule ".github/actions/paths-filter"] path = .github/actions/paths-filter url = https://github.com/dorny/paths-filter From 9ceae903f6d767f679146bd28c9e09c19446200f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 09:27:11 +0800 Subject: [PATCH 204/572] branch-3.0: [feat](fe) Upgrade azure.sdk.version from 1.2.30 to 1.2.36 #52983 (#53096) Cherry-picked from #52983 Co-authored-by: Lei Zhang --- fe/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/pom.xml b/fe/pom.xml index 215d887370cb79..e88c6b383ff5a2 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -376,7 +376,7 @@ under the License. 2.1.1 9.4 202 - 1.2.30 + 1.2.36 12.22.0 5.3.0 3.15.0 From 837125a666681670b80f90ec04b416121a63c2b9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 09:27:55 +0800 Subject: [PATCH 205/572] branch-3.0: [fix](test) fix test_routine_load_timeout_value unstable #53115 (#53142) Cherry-picked from #53115 Co-authored-by: hui lai --- .../load_p0/routine_load/test_routine_load_timeout_value.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy index de813084c09f7e..873b945857212c 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy @@ -183,7 +183,7 @@ suite("test_routine_load_timeout_value","nonConcurrent") { } } finally { sql "stop routine load for ${jobName}" - sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP TABLE IF EXISTS ${tableName} FORCE" } } } \ No newline at end of file From 7838ed94fb4c0c62447897127d91ea90f2245708 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 09:29:06 +0800 Subject: [PATCH 206/572] branch-3.0: [enhance](auth)ranger support storage vault and compute group #47925 (#51288) Cherry-picked from #47925 Co-authored-by: zhangdong --- .../ranger/doris/DorisAccessType.java | 2 ++ .../ranger/doris/DorisObjectType.java | 2 +- .../doris/RangerDorisAccessController.java | 30 +++++++++++++++-- .../ranger/doris/RangerDorisResource.java | 8 +++++ .../doris/mysql/privilege/RangerTest.java | 33 +++++++++++++++++-- 5 files changed, 70 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisAccessType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisAccessType.java index 77d7bfefc239d5..68a926f39cbe45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisAccessType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisAccessType.java @@ -51,6 +51,8 @@ public static DorisAccessType toAccessType(Privilege privilege) { case DROP_PRIV: return DROP; case USAGE_PRIV: + case STAGE_USAGE_PRIV: + case CLUSTER_USAGE_PRIV: return USAGE; case SHOW_VIEW_PRIV: return SHOW_VIEW; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisObjectType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisObjectType.java index cafff6bd6e1747..334098aeb49f7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisObjectType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/DorisObjectType.java @@ -18,5 +18,5 @@ package org.apache.doris.catalog.authorizer.ranger.doris; public enum DorisObjectType { - NONE, CATALOG, DATABASE, TABLE, COLUMN, RESOURCE, WORKLOAD_GROUP, GLOBAL + NONE, CATALOG, DATABASE, TABLE, COLUMN, RESOURCE, WORKLOAD_GROUP, GLOBAL, COMPUTE_GROUP, STORAGE_VAULT } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisAccessController.java index 509afc63c36f2d..359910aba49b9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisAccessController.java @@ -260,12 +260,38 @@ private boolean checkColPrivInternal(UserIdentity currentUser, String ctl, Strin @Override public boolean checkCloudPriv(UserIdentity currentUser, String cloudName, PrivPredicate wanted, ResourceTypeEnum type) { - return false; + // only support CLUSTER, + // STORAGE_VAULT should call `checkStorageVaultPriv` + // GENERAL should call `checkResourcePriv` + // STAGE is used to support `copy into`, but this feature will soon expire, + // so it is no longer supported through Ranger + if (!ResourceTypeEnum.CLUSTER.equals(type)) { + return false; + } + PrivBitSet checkedPrivs = PrivBitSet.of(); + return checkGlobalPrivInternal(currentUser, wanted, checkedPrivs) + || checkComputeGroupPrivInternal(currentUser, cloudName, wanted, checkedPrivs); + } + + private boolean checkComputeGroupPrivInternal(UserIdentity currentUser, String computeGroupName, + PrivPredicate wanted, + PrivBitSet checkedPrivs) { + RangerDorisResource resource = new RangerDorisResource(DorisObjectType.COMPUTE_GROUP, computeGroupName); + return checkPrivilege(currentUser, wanted, resource, checkedPrivs); } @Override public boolean checkStorageVaultPriv(UserIdentity currentUser, String storageVaultName, PrivPredicate wanted) { - return false; + PrivBitSet checkedPrivs = PrivBitSet.of(); + return checkGlobalPrivInternal(currentUser, wanted, checkedPrivs) + || checkStorageVaultPrivInternal(currentUser, storageVaultName, wanted, checkedPrivs); + } + + private boolean checkStorageVaultPrivInternal(UserIdentity currentUser, String storageVaultName, + PrivPredicate wanted, + PrivBitSet checkedPrivs) { + RangerDorisResource resource = new RangerDorisResource(DorisObjectType.STORAGE_VAULT, storageVaultName); + return checkPrivilege(currentUser, wanted, resource, checkedPrivs); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisResource.java index db173a25354d4f..672c06ec2813dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/doris/RangerDorisResource.java @@ -27,6 +27,8 @@ public class RangerDorisResource extends RangerAccessResourceImpl { public static final String KEY_COLUMN = "column"; public static final String KEY_RESOURCE = "resource"; public static final String KEY_WORKLOAD_GROUP = "workload_group"; + public static final String KEY_COMPUTE_GROUP = "compute_group"; + public static final String KEY_STORAGE_VAULT = "storage_vault"; // FirstLevelResource => Catalog / Resource / WorkloadGroup / GLOBAL // SecondLevelResource => Database @@ -76,6 +78,12 @@ public RangerDorisResource(DorisObjectType objectType, String firstLevelResource case WORKLOAD_GROUP: setValue(KEY_WORKLOAD_GROUP, firstLevelResource); break; + case STORAGE_VAULT: + setValue(KEY_STORAGE_VAULT, firstLevelResource); + break; + case COMPUTE_GROUP: + setValue(KEY_COMPUTE_GROUP, firstLevelResource); + break; case NONE: default: break; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerTest.java index a0f0ef0f2d13b1..e0e1dd36b2a4d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/RangerTest.java @@ -17,6 +17,7 @@ package org.apache.doris.mysql.privilege; +import org.apache.doris.analysis.ResourceTypeEnum; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.authorizer.ranger.doris.RangerDorisAccessController; import org.apache.doris.catalog.authorizer.ranger.doris.RangerDorisResource; @@ -67,8 +68,10 @@ public RangerAccessResult isAccessAllowed(RangerAccessRequest request) { String col = (String) resource.getValue(RangerDorisResource.KEY_COLUMN); String rs = (String) resource.getValue(RangerDorisResource.KEY_RESOURCE); String wg = (String) resource.getValue(RangerDorisResource.KEY_WORKLOAD_GROUP); + String cg = (String) resource.getValue(RangerDorisResource.KEY_COMPUTE_GROUP); + String sv = (String) resource.getValue(RangerDorisResource.KEY_STORAGE_VAULT); String user = request.getUser(); - return returnAccessResult(request, ctl, db, tbl, col, rs, wg, user); + return returnAccessResult(request, ctl, db, tbl, col, rs, wg, cg, sv, user); } @Override @@ -98,7 +101,7 @@ public RangerAccessResult evalDataMaskPolicies(RangerAccessRequest request, private RangerAccessResult returnAccessResult( RangerAccessRequest request, String ctl, String db, String tbl, - String col, String rs, String wg, String user) { + String col, String rs, String wg, String cg, String sv, String user) { RangerAccessResult result = new RangerAccessResult(1, "test", null, request); if (!Strings.isNullOrEmpty(wg)) { result.setIsAllowed(wg.equals("wg1")); @@ -114,6 +117,10 @@ private RangerAccessResult returnAccessResult( result.setIsAllowed("ctl3".equals(ctl) && "db3".equals(db)); } else if (!Strings.isNullOrEmpty(ctl)) { result.setIsAllowed("ctl4".equals(ctl)); + } else if (!Strings.isNullOrEmpty(cg)) { + result.setIsAllowed("cg1".equals(cg)); + } else if (!Strings.isNullOrEmpty(sv)) { + result.setIsAllowed("sv1".equals(sv)); } else { result.setIsAllowed(false); } @@ -227,4 +234,26 @@ public void testDataMask() { policy = ac.evalDataMaskPolicy(ui, "ctl1", "db1", "tbl1", "col4"); Assertions.assertTrue(!policy.isPresent()); } + + @Test + public void testComputeGroupAuth() { + DorisTestPlugin plugin = new DorisTestPlugin("test"); + RangerDorisAccessController ac = new RangerDorisAccessController(plugin); + UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%"); + boolean cg1 = ac.checkCloudPriv(ui, "cg1", PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER); + Assertions.assertTrue(cg1); + boolean cg2 = ac.checkCloudPriv(ui, "cg2", PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER); + Assertions.assertFalse(cg2); + } + + @Test + public void testStorageVaultAuth() { + DorisTestPlugin plugin = new DorisTestPlugin("test"); + RangerDorisAccessController ac = new RangerDorisAccessController(plugin); + UserIdentity ui = UserIdentity.createAnalyzedUserIdentWithIp("user1", "%"); + boolean cg1 = ac.checkStorageVaultPriv(ui, "sv1", PrivPredicate.USAGE); + Assertions.assertTrue(cg1); + boolean cg2 = ac.checkStorageVaultPriv(ui, "sv2", PrivPredicate.USAGE); + Assertions.assertFalse(cg2); + } } From 6413ebcd753dc6c0e6cc541ff22715d4e5c14db6 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sun, 13 Jul 2025 20:20:43 -0700 Subject: [PATCH 207/572] branch-3.0: [fix](iceberg)Fix the thread pool issue used for commit. (#51508) (#53155) bp #51508 --------- Co-authored-by: wuwenchi --- .../doris/common/ThreadPoolManager.java | 68 +++++++++++++++++++ .../doris/datasource/ExternalCatalog.java | 12 ++++ .../datasource/hive/HMSExternalCatalog.java | 6 ++ .../iceberg/IcebergExternalCatalog.java | 7 ++ .../iceberg/IcebergMetadataOps.java | 5 ++ .../iceberg/IcebergTransaction.java | 4 +- 6 files changed, 100 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java index dcb1f704271eab..24f6a9de2bcaf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java @@ -18,6 +18,7 @@ package org.apache.doris.common; +import org.apache.doris.common.security.authentication.PreExecutionAuthenticator; import org.apache.doris.metric.Metric; import org.apache.doris.metric.Metric.MetricUnit; import org.apache.doris.metric.MetricLabel; @@ -33,6 +34,7 @@ import java.util.Map; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; import java.util.concurrent.FutureTask; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.PriorityBlockingQueue; @@ -68,6 +70,7 @@ */ public class ThreadPoolManager { + private static final Logger LOG = LogManager.getLogger(ThreadPoolManager.class); private static Map nameToThreadPoolMap = Maps.newConcurrentMap(); @@ -119,6 +122,50 @@ TimeUnit.SECONDS, new SynchronousQueue(), new LogDiscardPolicy(poolName), poolName, needRegisterMetric); } + public static ThreadPoolExecutor newDaemonFixedThreadPoolWithPreAuth( + int numThread, + int queueSize, + String poolName, + boolean needRegisterMetric, + PreExecutionAuthenticator preAuth) { + return newDaemonThreadPoolWithPreAuth(numThread, numThread, KEEP_ALIVE_TIME, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(queueSize), new BlockedPolicy(poolName, 60), + poolName, needRegisterMetric, preAuth); + } + + public static ThreadPoolExecutor newDaemonThreadPoolWithPreAuth( + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + RejectedExecutionHandler handler, + String poolName, + boolean needRegisterMetric, + PreExecutionAuthenticator preAuth) { + ThreadFactory threadFactory = namedThreadFactoryWithPreAuth(poolName, preAuth); + ThreadPoolExecutor threadPool = new ThreadPoolExecutor(corePoolSize, maximumPoolSize, + keepAliveTime, unit, workQueue, threadFactory, handler); + if (needRegisterMetric) { + nameToThreadPoolMap.put(poolName, threadPool); + } + return threadPool; + } + + private static ThreadFactory namedThreadFactoryWithPreAuth(String poolName, PreExecutionAuthenticator preAuth) { + return new ThreadFactoryBuilder() + .setDaemon(true) + .setNameFormat(poolName + "-%d") + .setThreadFactory(runnable -> new Thread(() -> { + try { + preAuth.execute(runnable); + } catch (Exception e) { + throw new RuntimeException(e); + } + })) + .build(); + } + public static ThreadPoolExecutor newDaemonCacheThreadPoolUseBlockedPolicy(int maxNumThread, String poolName, boolean needRegisterMetric) { return newDaemonThreadPool(0, maxNumThread, KEEP_ALIVE_TIME, @@ -384,4 +431,25 @@ public void rejectedExecution(Runnable task, ThreadPoolExecutor executor) { } } } + + public static void shutdownExecutorService(ExecutorService executorService) { + // Disable new tasks from being submitted + executorService.shutdown(); + try { + // Wait a while for existing tasks to terminate + if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) { + // Cancel currently executing tasks + executorService.shutdownNow(); + // Wait a while for tasks to respond to being cancelled + if (!executorService.awaitTermination(60, TimeUnit.SECONDS)) { + LOG.warn("ExecutorService did not terminate"); + } + } + } catch (InterruptedException e) { + // (Re-)Cancel if current thread also interrupted + executorService.shutdownNow(); + // Preserve interrupt status + Thread.currentThread().interrupt(); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 23d6cb958d414d..86c5a3ae3e676e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -35,6 +35,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; +import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; import org.apache.doris.common.Version; import org.apache.doris.common.io.Text; @@ -94,6 +95,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; /** @@ -128,6 +130,8 @@ public abstract class ExternalCatalog CREATE_TIME, USE_META_CACHE); + protected static final int ICEBERG_CATALOG_EXECUTOR_THREAD_NUM = Runtime.getRuntime().availableProcessors(); + // Unique id of this catalog, will be assigned after catalog is loaded. @SerializedName(value = "id") protected long id; @@ -168,6 +172,7 @@ public abstract class ExternalCatalog protected Optional useMetaCache = Optional.empty(); protected MetaCache> metaCache; protected PreExecutionAuthenticator preExecutionAuthenticator; + protected ThreadPoolExecutor threadPoolWithPreAuth; private volatile Configuration cachedConf = null; private byte[] confLock = new byte[0]; @@ -776,6 +781,9 @@ public void onClose() { if (null != transactionManager) { transactionManager = null; } + if (threadPoolWithPreAuth != null) { + ThreadPoolManager.shutdownExecutorService(threadPoolWithPreAuth); + } CatalogIf.super.onClose(); } @@ -1197,4 +1205,8 @@ public void notifyPropertiesUpdated(Map updatedProps) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidSchemaCache(id); } } + + public ThreadPoolExecutor getThreadPoolExecutor() { + return threadPoolWithPreAuth; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 2442eed0b9c36a..5d9128bcc04bd7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -188,6 +188,12 @@ protected void initLocalObjectsImpl() { String.valueOf(Config.hive_metastore_client_timeout_second)); } HiveMetadataOps hiveOps = ExternalMetadataOperations.newHiveMetadataOps(hiveConf, jdbcClientConfig, this); + threadPoolWithPreAuth = ThreadPoolManager.newDaemonFixedThreadPoolWithPreAuth( + ICEBERG_CATALOG_EXECUTOR_THREAD_NUM, + Integer.MAX_VALUE, + String.format("hms_iceberg_catalog_%s_executor_pool", name), + true, + preExecutionAuthenticator); FileSystemProvider fileSystemProvider = new FileSystemProviderImpl(Env.getCurrentEnv().getExtMetaCacheMgr(), this.bindBrokerName(), this.catalogProperty.getHadoopProperties()); this.fileSystemExecutor = ThreadPoolManager.newDaemonFixedThreadPool(FILE_SYSTEM_EXECUTOR_THREAD_NUM, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 30590f5af26070..e25199adfbeaf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.security.authentication.PreExecutionAuthenticator; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InitCatalogLog; @@ -65,6 +66,12 @@ protected void initLocalObjectsImpl() { initCatalog(); IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog); transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops); + threadPoolWithPreAuth = ThreadPoolManager.newDaemonFixedThreadPoolWithPreAuth( + ICEBERG_CATALOG_EXECUTOR_THREAD_NUM, + Integer.MAX_VALUE, + String.format("iceberg_catalog_%s_executor_pool", name), + true, + preExecutionAuthenticator); metadataOps = ops; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 787d706132e3cb..14a64163265384 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ThreadPoolExecutor; import java.util.stream.Collectors; public class IcebergMetadataOps implements ExternalMetadataOps { @@ -292,4 +293,8 @@ private Namespace getNamespace(String dbName) { private Namespace getNamespace() { return externalCatalogName.map(Namespace::of).orElseGet(() -> Namespace.empty()); } + + public ThreadPoolExecutor getThreadPoolWithPreAuth() { + return dorisCatalog.getThreadPoolExecutor(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index e36db86022e360..797caea0deaf1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -155,7 +155,7 @@ private synchronized Table getNativeTable(SimpleTableInfo tableInfo) { private void commitAppendTxn(Table table, List pendingResults) { // commit append files. - AppendFiles appendFiles = table.newAppend(); + AppendFiles appendFiles = table.newAppend().scanManifestsWith(ops.getThreadPoolWithPreAuth()); for (WriteResult result : pendingResults) { Preconditions.checkState(result.referencedDataFiles().length == 0, "Should have no referenced data files for append."); @@ -171,7 +171,7 @@ private void commitReplaceTxn(Table table, List pendingResults) { // 1. if dst_tb is a partitioned table, it will return directly. // 2. if dst_tb is an unpartitioned table, the `dst_tb` table will be emptied. if (!table.spec().isPartitioned()) { - OverwriteFiles overwriteFiles = table.newOverwrite(); + OverwriteFiles overwriteFiles = table.newOverwrite().scanManifestsWith(ops.getThreadPoolWithPreAuth()); try (CloseableIterable fileScanTasks = table.newScan().planFiles()) { fileScanTasks.forEach(f -> overwriteFiles.deleteFile(f.file())); } catch (IOException e) { From d18faa3db4cb9ec62d0c46367ab0d6b9eba739be Mon Sep 17 00:00:00 2001 From: morrySnow Date: Mon, 14 Jul 2025 14:21:53 +0800 Subject: [PATCH 208/572] branch-3.0: [fix](test) fix lazy_materialize_topn unstable #52545 (#53181) picked from #52545 --- .../defer_materialize_topn/lazy_materialize_topn.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy index cd683b08f280f2..14bf68b4fa5dd1 100644 --- a/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy +++ b/regression-test/suites/nereids_rules_p0/defer_materialize_topn/lazy_materialize_topn.groovy @@ -16,7 +16,8 @@ // under the License. suite("lazy_materialize_topn") { sql """ - set enable_two_phase_read_opt = true + set enable_two_phase_read_opt = true; + set topn_opt_limit_threshold = 1000; """ sql """ From 119f8d4ace6ffa066a68e4a15d504d6901fdbf6d Mon Sep 17 00:00:00 2001 From: Socrates Date: Mon, 14 Jul 2025 14:22:29 +0800 Subject: [PATCH 209/572] branch-3.0: [fix](tvf) support compressed json file for tvf and refactor code (#51983) (#53165) bp: #51983 --- be/src/vec/exec/format/json/new_json_reader.cpp | 4 +++- .../json_format_test/simple_object_json.json.gz | Bin 0 -> 211 bytes .../data/external_table_p0/tvf/test_hdfs_tvf.out | 14 ++++++++++++++ .../external_table_p0/tvf/test_hdfs_tvf.groovy | 10 ++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json_format_test/simple_object_json.json.gz diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index c8969c6d4c314e..fb1b04e867c44b 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -258,7 +258,9 @@ Status NewJsonReader::get_columns(std::unordered_map* col_names, std::vector* col_types) { RETURN_IF_ERROR(_get_range_params()); - + // create decompressor. + // _decompressor may be nullptr if this is not a compressed file + RETURN_IF_ERROR(Decompressor::create_decompressor(_file_compress_type, &_decompressor)); RETURN_IF_ERROR(_open_file_reader(true)); if (_read_json_by_line) { RETURN_IF_ERROR(_open_line_reader()); diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json_format_test/simple_object_json.json.gz b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/json_format_test/simple_object_json.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..8a6db90241ffc25e12d351e459c1f4ce374289af GIT binary patch literal 211 zcmV;^04)C>iwFo!J2H?9- zVL0y~h^TuTrVLmN9ya2p@!f}6QoGdcKth_2KP_FD@7X!$r<`^E@k&>AG5+R*Wm{?N zO)7~}R;uP}wtTce`tEeFPJ6-y@Dgzw6QA7}i7&y+uXq#YbO@KgE5zNe^a{8_Jm_#Z zssddjJ;fAHYz?+Swo5(Rp7zmzZjpZagWwh@FYt)wYUpqCV%KPyAr1qf`y#pc6TH1$ Nvp1P-u Date: Mon, 14 Jul 2025 18:01:37 +0800 Subject: [PATCH 210/572] [feature](cloud)Implement vcg related functions #52515 (#52433) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Luwei <814383175@qq.com> pick from https://github.com/apache/doris/pull/52515 … conflict --- cloud/src/meta-service/meta_service_http.cpp | 3 + .../meta-service/meta_service_resource.cpp | 733 +++++++++------ .../src/resource-manager/resource_manager.cpp | 275 +++++- cloud/src/resource-manager/resource_manager.h | 48 +- cloud/test/meta_service_http_test.cpp | 881 +++++++++++++++++- cloud/test/mock_resource_manager.h | 3 +- .../doris/analysis/ShowClusterStmt.java | 4 +- .../doris/analysis/WarmUpClusterStmt.java | 37 +- .../doris/cloud/CacheHotspotManager.java | 3 +- .../cloud/catalog/CloudClusterChecker.java | 17 +- .../catalog/CloudInstanceStatusChecker.java | 445 ++++++++- .../doris/cloud/catalog/CloudReplica.java | 13 +- .../doris/cloud/catalog/ComputeGroup.java | 177 ++++ .../cloud/system/CloudSystemInfoService.java | 317 ++++++- .../httpv2/rest/manager/ClusterAction.java | 2 +- .../org/apache/doris/metric/MetricRepo.java | 14 +- .../apache/doris/mysql/privilege/Role.java | 15 +- .../org/apache/doris/qe/AuditLogHelper.java | 30 +- .../org/apache/doris/qe/ConnectProcessor.java | 12 +- .../org/apache/doris/qe/ShowExecutor.java | 42 +- .../system/CloudSystemInfoServiceTest.java | 318 +++++++ .../doris/mysql/privilege/CloudAuthTest.java | 623 +++++++++++++ gensrc/proto/cloud.proto | 19 + .../multi_cluster/vcluster/all_types.csv | 20 + .../cloud/multi_cluster/vcluster/vcluster.out | 7 + .../virtual_compute_group/all_types.csv | 20 + .../use_vcg_read_write.out | 37 + .../virtual_compute_group/vcluster.out | 7 + .../doris/regression/suite/Suite.groovy | 125 ++- .../multi_cluster/vcluster/vcluster.groovy | 173 ++++ .../default_vcg_auto_failover.groovy | 391 ++++++++ .../virtual_compute_group/test_vcg.groovy | 430 +++++++++ .../test_vcg_metrics.groovy | 169 ++++ .../use_default_vcg_read_write.groovy | 451 +++++++++ .../use_vcg_read_write.groovy | 453 +++++++++ .../vcg_auto_failover.groovy | 393 ++++++++ 36 files changed, 6327 insertions(+), 380 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java create mode 100644 regression-test/data/cloud/multi_cluster/vcluster/all_types.csv create mode 100644 regression-test/data/cloud/multi_cluster/vcluster/vcluster.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/all_types.csv create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcluster.out create mode 100644 regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp index 7a78b97eaa6f45..8d7f6bba6e4a31 100644 --- a/cloud/src/meta-service/meta_service_http.cpp +++ b/cloud/src/meta-service/meta_service_http.cpp @@ -201,6 +201,7 @@ static HttpResponse process_alter_cluster(MetaServiceImpl* service, brpc::Contro {"decommission_node", AlterClusterRequest::DECOMMISSION_NODE}, {"set_cluster_status", AlterClusterRequest::SET_CLUSTER_STATUS}, {"notify_decommissioned", AlterClusterRequest::NOTIFY_DECOMMISSIONED}, + {"alter_vcluster_info", AlterClusterRequest::ALTER_VCLUSTER_INFO}, }; auto& path = ctrl->http_request().unresolved_path(); @@ -680,6 +681,7 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller, {"decommission_node", process_alter_cluster}, {"set_cluster_status", process_alter_cluster}, {"notify_decommissioned", process_alter_cluster}, + {"alter_vcluster_info", process_alter_cluster}, {"v1/add_cluster", process_alter_cluster}, {"v1/drop_cluster", process_alter_cluster}, {"v1/rename_cluster", process_alter_cluster}, @@ -689,6 +691,7 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller, {"v1/drop_node", process_alter_cluster}, {"v1/decommission_node", process_alter_cluster}, {"v1/set_cluster_status", process_alter_cluster}, + {"v1/alter_vcluster_info", process_alter_cluster}, // for alter instance {"create_instance", process_create_instance}, {"drop_instance", process_alter_instance}, diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index db8d76eab8d1ac..f69ed65786db6e 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -2084,6 +2084,408 @@ std::pair MetaServiceImpl::alter_instance( return std::make_pair(code, msg); } +void handle_add_cluster(const std::string& instance_id, const ClusterInfo& cluster, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + auto r = resource_mgr->add_cluster(instance_id, cluster); + code = r.first; + msg = r.second; +} + +void handle_drop_cluster(const std::string& instance_id, const ClusterInfo& cluster, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + auto r = resource_mgr->drop_cluster(instance_id, cluster); + code = r.first; + msg = r.second; +} + +void handle_update_cluster_mySQL_username(const std::string& instance_id, + const ClusterInfo& cluster, + std::shared_ptr resource_mgr, + std::string& msg, MetaServiceCode& code) { + msg = resource_mgr->update_cluster( + instance_id, cluster, + [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, + [&](ClusterPB& c, std::vector&) { + auto& mysql_user_names = cluster.cluster.mysql_user_name(); + c.mutable_mysql_user_name()->CopyFrom(mysql_user_names); + return ""; + }); +} + +void handle_add_node(const std::string& instance_id, const AlterClusterRequest* request, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, false); + if (!msg.empty()) { + LOG(WARNING) << msg; + return; + } + std::vector to_add; + std::vector to_del; + for (auto& n : request->cluster().nodes()) { + NodeInfo node; + node.instance_id = request->instance_id(); + node.node_info = n; + node.cluster_id = request->cluster().cluster_id(); + node.cluster_name = request->cluster().cluster_name(); + node.role = (request->cluster().type() == ClusterPB::SQL + ? Role::SQL_SERVER + : (request->cluster().type() == ClusterPB::COMPUTE ? Role::COMPUTE_NODE + : Role::UNDEFINED)); + node.node_info.set_status(NodeStatusPB::NODE_STATUS_RUNNING); + to_add.emplace_back(std::move(node)); + } + msg = resource_mgr->modify_nodes(instance_id, to_add, to_del); +} + +void handle_drop_node(const std::string& instance_id, const AlterClusterRequest* request, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, false); + if (!msg.empty()) { + LOG(WARNING) << msg; + return; + } + std::vector to_add; + std::vector to_del; + for (auto& n : request->cluster().nodes()) { + NodeInfo node; + node.instance_id = request->instance_id(); + node.node_info = n; + node.cluster_id = request->cluster().cluster_id(); + node.cluster_name = request->cluster().cluster_name(); + node.role = (request->cluster().type() == ClusterPB::SQL + ? Role::SQL_SERVER + : (request->cluster().type() == ClusterPB::COMPUTE ? Role::COMPUTE_NODE + : Role::UNDEFINED)); + to_del.emplace_back(std::move(node)); + } + msg = resource_mgr->modify_nodes(instance_id, to_add, to_del); +} + +void handle_decommission_node(const std::string& instance_id, const AlterClusterRequest* request, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, false); + if (msg != "") { + LOG(WARNING) << msg; + return; + } + + std::string be_unique_id = (request->cluster().nodes())[0].cloud_unique_id(); + std::vector nodes; + std::string err = resource_mgr->get_node(be_unique_id, &nodes); + if (!err.empty()) { + LOG(INFO) << "failed to check instance info, err=" << err; + msg = err; + return; + } + + std::vector decomission_nodes; + for (auto& node : nodes) { + for (auto req_node : request->cluster().nodes()) { + bool ip_processed = false; + if (node.node_info.has_ip() && req_node.has_ip()) { + std::string endpoint = + node.node_info.ip() + ":" + std::to_string(node.node_info.heartbeat_port()); + std::string req_endpoint = + req_node.ip() + ":" + std::to_string(req_node.heartbeat_port()); + if (endpoint == req_endpoint) { + decomission_nodes.push_back(node); + node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING); + } + ip_processed = true; + } + + if (!ip_processed && node.node_info.has_host() && req_node.has_host()) { + std::string endpoint = node.node_info.host() + ":" + + std::to_string(node.node_info.heartbeat_port()); + std::string req_endpoint = + req_node.host() + ":" + std::to_string(req_node.heartbeat_port()); + if (endpoint == req_endpoint) { + decomission_nodes.push_back(node); + node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING); + } + } + } + } + + { + std::vector to_add; + std::vector& to_del = decomission_nodes; + msg = resource_mgr->modify_nodes(instance_id, to_add, to_del); + } + { + std::vector& to_add = decomission_nodes; + std::vector to_del; + for (auto& node : to_add) { + node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING); + LOG(INFO) << "decomission node, " + << "size: " << to_add.size() << " " << node.node_info.DebugString() << " " + << node.cluster_id << " " << node.cluster_name; + } + msg = resource_mgr->modify_nodes(instance_id, to_add, to_del); + } +} + +void handle_notify_decommissioned(const std::string& instance_id, + const AlterClusterRequest* request, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + resource_mgr->check_cluster_params_valid(request->cluster(), &msg, false, false); + if (msg != "") { + LOG(WARNING) << msg; + return; + } + + std::string be_unique_id = (request->cluster().nodes())[0].cloud_unique_id(); + std::vector nodes; + std::string err = resource_mgr->get_node(be_unique_id, &nodes); + if (!err.empty()) { + LOG(INFO) << "failed to check instance info, err=" << err; + msg = err; + return; + } + + std::vector decomission_nodes; + for (auto& node : nodes) { + for (auto req_node : request->cluster().nodes()) { + bool ip_processed = false; + if (node.node_info.has_ip() && req_node.has_ip()) { + std::string endpoint = + node.node_info.ip() + ":" + std::to_string(node.node_info.heartbeat_port()); + std::string req_endpoint = + req_node.ip() + ":" + std::to_string(req_node.heartbeat_port()); + if (endpoint == req_endpoint) { + decomission_nodes.push_back(node); + } + ip_processed = true; + } + + if (!ip_processed && node.node_info.has_host() && req_node.has_host()) { + std::string endpoint = node.node_info.host() + ":" + + std::to_string(node.node_info.heartbeat_port()); + std::string req_endpoint = + req_node.host() + ":" + std::to_string(req_node.heartbeat_port()); + if (endpoint == req_endpoint) { + decomission_nodes.push_back(node); + } + } + } + } + + { + std::vector to_add; + std::vector& to_del = decomission_nodes; + msg = resource_mgr->modify_nodes(instance_id, to_add, to_del); + } + { + std::vector& to_add = decomission_nodes; + std::vector to_del; + for (auto& node : to_add) { + node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONED); + LOG(INFO) << "notify node decomissioned, " + << " size: " << to_add.size() << " " << node.node_info.DebugString() << " " + << node.cluster_id << " " << node.cluster_name; + } + msg = resource_mgr->modify_nodes(instance_id, to_add, to_del); + } +} + +void handle_rename_cluster(const std::string& instance_id, const ClusterInfo& cluster, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + msg = resource_mgr->update_cluster( + instance_id, cluster, + [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, + [&](ClusterPB& c, std::vector& clusters_in_instance) { + std::string msg; + std::stringstream ss; + std::set cluster_names; + for (auto cluster_in_instance : clusters_in_instance) { + cluster_names.emplace(cluster_in_instance.cluster_name()); + } + auto it = cluster_names.find(cluster.cluster.cluster_name()); + if (it != cluster_names.end()) { + code = MetaServiceCode::INVALID_ARGUMENT; + ss << "failed to rename cluster, a cluster with the same name already exists " + "in this instance " + << proto_to_json(c); + msg = ss.str(); + return msg; + } + if (c.cluster_name() == cluster.cluster.cluster_name()) { + code = MetaServiceCode::INVALID_ARGUMENT; + ss << "failed to rename cluster, name eq original name, original cluster is " + << proto_to_json(c); + msg = ss.str(); + return msg; + } + c.set_cluster_name(cluster.cluster.cluster_name()); + return msg; + }); +} + +void handle_update_cluster_endpoint(const std::string& instance_id, const ClusterInfo& cluster, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + msg = resource_mgr->update_cluster( + instance_id, cluster, + [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, + [&](ClusterPB& c, std::vector&) { + std::string msg; + std::stringstream ss; + if (!cluster.cluster.has_private_endpoint() || + cluster.cluster.private_endpoint().empty()) { + code = MetaServiceCode::CLUSTER_ENDPOINT_MISSING; + ss << "missing private endpoint"; + msg = ss.str(); + return msg; + } + c.set_public_endpoint(cluster.cluster.public_endpoint()); + c.set_private_endpoint(cluster.cluster.private_endpoint()); + return msg; + }); +} + +void handle_set_cluster_status(const std::string& instance_id, const ClusterInfo& cluster, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + msg = resource_mgr->update_cluster( + instance_id, cluster, + [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, + [&](ClusterPB& c, std::vector&) { + std::string msg; + std::stringstream ss; + if (ClusterPB::COMPUTE != c.type()) { + code = MetaServiceCode::INVALID_ARGUMENT; + ss << "just support set COMPUTE cluster status"; + msg = ss.str(); + return msg; + } + if (c.cluster_status() == cluster.cluster.cluster_status()) { + code = MetaServiceCode::INVALID_ARGUMENT; + ss << "failed to set cluster status, status eq original status, original " + "cluster is " + << print_cluster_status(c.cluster_status()); + msg = ss.str(); + return msg; + } + // status from -> to + std::set> + can_work_directed_edges { + {ClusterStatus::UNKNOWN, ClusterStatus::NORMAL}, + {ClusterStatus::NORMAL, ClusterStatus::SUSPENDED}, + {ClusterStatus::SUSPENDED, ClusterStatus::TO_RESUME}, + {ClusterStatus::TO_RESUME, ClusterStatus::NORMAL}, + {ClusterStatus::SUSPENDED, ClusterStatus::NORMAL}, + {ClusterStatus::NORMAL, ClusterStatus::MANUAL_SHUTDOWN}, + {ClusterStatus::MANUAL_SHUTDOWN, ClusterStatus::NORMAL}, + }; + auto from = c.cluster_status(); + auto to = cluster.cluster.cluster_status(); + if (can_work_directed_edges.count({from, to}) == 0) { + code = MetaServiceCode::INVALID_ARGUMENT; + ss << "failed to set cluster status, original cluster is " + << print_cluster_status(from) << " and want set " + << print_cluster_status(to); + msg = ss.str(); + return msg; + } + c.set_cluster_status(cluster.cluster.cluster_status()); + return msg; + }); +} + +void handle_alter_vcluster_Info(const std::string& instance_id, const ClusterInfo& cluster, + std::shared_ptr resource_mgr, std::string& msg, + MetaServiceCode& code) { + msg = resource_mgr->update_cluster( + instance_id, cluster, + [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, + [&](ClusterPB& c, std::vector& clusters_in_instance) { + std::string msg; + // Clear existing cluster names and set new ones if provided + for (auto it = clusters_in_instance.begin(); it != clusters_in_instance.end();) { + if (c.cluster_name() == it->cluster_name()) { + it = clusters_in_instance.erase(it); + } else { + ++it; + } + } + if (cluster.cluster.cluster_names_size() > 0) { + c.clear_cluster_names(); // Clear existing cluster names + for (const auto& name : cluster.cluster.cluster_names()) { + auto [ret_code, msg] = + resource_mgr->validate_sub_clusters({name}, clusters_in_instance); + if (ret_code != MetaServiceCode::OK) { + LOG(WARNING) << msg; + return msg; + } + c.add_cluster_names(name); // Add each new name + } + } + + // Check and set cluster policy if provided + if (cluster.cluster.has_cluster_policy()) { + const auto& policy = cluster.cluster.cluster_policy(); + if (policy.has_active_cluster_name()) { + auto [ret_code, msg] = resource_mgr->validate_sub_clusters( + {policy.active_cluster_name()}, clusters_in_instance); + if (ret_code != MetaServiceCode::OK) { + LOG(WARNING) << msg; + return msg; + } + c.mutable_cluster_policy()->set_active_cluster_name( + policy.active_cluster_name()); + } + + for (const auto& standby_name : policy.standby_cluster_names()) { + auto [ret_code, msg] = resource_mgr->validate_sub_clusters( + {standby_name}, clusters_in_instance); + if (ret_code != MetaServiceCode::OK) { + LOG(WARNING) << msg; + return msg; + } + c.mutable_cluster_policy()->clear_standby_cluster_names(); + c.mutable_cluster_policy()->add_standby_cluster_names(standby_name); + // current just support one stadby; + break; + } + + if (policy.has_type()) { + c.mutable_cluster_policy()->set_type(policy.type()); + } + + if (policy.has_failover_failure_threshold()) { + c.mutable_cluster_policy()->set_failover_failure_threshold( + policy.failover_failure_threshold()); + } + + if (policy.has_unhealthy_node_threshold_percent()) { + c.mutable_cluster_policy()->set_unhealthy_node_threshold_percent( + policy.unhealthy_node_threshold_percent()); + } + + if (!policy.cache_warmup_jobids().empty()) { + c.mutable_cluster_policy()->clear_cache_warmup_jobids(); + } + + for (const auto& warmup_jobid : policy.cache_warmup_jobids()) { + c.mutable_cluster_policy()->add_cache_warmup_jobids(warmup_jobid); + } + } + + // Validate the virtual cluster after alterations + if (!resource_mgr->validate_virtual_cluster(c, &msg)) { + return msg; // Return validation error + } + return msg; // Return success or empty message + }); +} + void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, const AlterClusterRequest* request, AlterClusterResponse* response, @@ -2091,6 +2493,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, RPC_PREPROCESS(alter_cluster, get); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; instance_id = request->has_instance_id() ? request->instance_id() : ""; + if (!cloud_unique_id.empty() && instance_id.empty()) { auto [is_degraded_format, id] = ResourceManager::get_instance_id_by_cloud_unique_id(cloud_unique_id); @@ -2128,304 +2531,48 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, cluster.cluster.CopyFrom(request->cluster()); switch (request->op()) { - case AlterClusterRequest::ADD_CLUSTER: { - auto r = resource_mgr_->add_cluster(instance_id, cluster); - code = r.first; - msg = r.second; - } break; - case AlterClusterRequest::DROP_CLUSTER: { - auto r = resource_mgr_->drop_cluster(instance_id, cluster); - code = r.first; - msg = r.second; - } break; - case AlterClusterRequest::UPDATE_CLUSTER_MYSQL_USER_NAME: { - msg = resource_mgr_->update_cluster( - instance_id, cluster, - [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, - [&](ClusterPB& c, std::set&) { - auto& mysql_user_names = cluster.cluster.mysql_user_name(); - c.mutable_mysql_user_name()->CopyFrom(mysql_user_names); - return ""; - }); - } break; - case AlterClusterRequest::ADD_NODE: { - resource_mgr_->check_cluster_params_valid(request->cluster(), &msg, false, false); - if (msg != "") { - LOG(WARNING) << msg; - break; - } - std::vector to_add; - std::vector to_del; - for (auto& n : request->cluster().nodes()) { - NodeInfo node; - node.instance_id = request->instance_id(); - node.node_info = n; - node.cluster_id = request->cluster().cluster_id(); - node.cluster_name = request->cluster().cluster_name(); - node.role = - (request->cluster().type() == ClusterPB::SQL - ? Role::SQL_SERVER - : (request->cluster().type() == ClusterPB::COMPUTE ? Role::COMPUTE_NODE - : Role::UNDEFINED)); - node.node_info.set_status(NodeStatusPB::NODE_STATUS_RUNNING); - to_add.emplace_back(std::move(node)); - } - msg = resource_mgr_->modify_nodes(instance_id, to_add, to_del); - } break; - case AlterClusterRequest::DROP_NODE: { - resource_mgr_->check_cluster_params_valid(request->cluster(), &msg, false, false); - if (msg != "") { - LOG(WARNING) << msg; - break; - } - std::vector to_add; - std::vector to_del; - for (auto& n : request->cluster().nodes()) { - NodeInfo node; - node.instance_id = request->instance_id(); - node.node_info = n; - node.cluster_id = request->cluster().cluster_id(); - node.cluster_name = request->cluster().cluster_name(); - node.role = - (request->cluster().type() == ClusterPB::SQL - ? Role::SQL_SERVER - : (request->cluster().type() == ClusterPB::COMPUTE ? Role::COMPUTE_NODE - : Role::UNDEFINED)); - to_del.emplace_back(std::move(node)); - } - msg = resource_mgr_->modify_nodes(instance_id, to_add, to_del); - } break; - case AlterClusterRequest::DECOMMISSION_NODE: { - resource_mgr_->check_cluster_params_valid(request->cluster(), &msg, false, false); - if (msg != "") { - LOG(WARNING) << msg; - break; - } - - std::string be_unique_id = (request->cluster().nodes())[0].cloud_unique_id(); - std::vector nodes; - std::string err = resource_mgr_->get_node(be_unique_id, &nodes); - if (!err.empty()) { - LOG(INFO) << "failed to check instance info, err=" << err; - msg = err; - break; - } - - std::vector decomission_nodes; - for (auto& node : nodes) { - for (auto req_node : request->cluster().nodes()) { - bool ip_processed = false; - if (node.node_info.has_ip() && req_node.has_ip()) { - std::string endpoint = node.node_info.ip() + ":" + - std::to_string(node.node_info.heartbeat_port()); - std::string req_endpoint = - req_node.ip() + ":" + std::to_string(req_node.heartbeat_port()); - if (endpoint == req_endpoint) { - decomission_nodes.push_back(node); - node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING); - } - ip_processed = true; - } - - if (!ip_processed && node.node_info.has_host() && req_node.has_host()) { - std::string endpoint = node.node_info.host() + ":" + - std::to_string(node.node_info.heartbeat_port()); - std::string req_endpoint = - req_node.host() + ":" + std::to_string(req_node.heartbeat_port()); - if (endpoint == req_endpoint) { - decomission_nodes.push_back(node); - node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING); - } - } - } - } - - { - std::vector to_add; - std::vector& to_del = decomission_nodes; - msg = resource_mgr_->modify_nodes(instance_id, to_add, to_del); - } - { - std::vector& to_add = decomission_nodes; - std::vector to_del; - for (auto& node : to_add) { - node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONING); - LOG(INFO) << "decomission node, " - << "size: " << to_add.size() << " " << node.node_info.DebugString() << " " - << node.cluster_id << " " << node.cluster_name; - } - msg = resource_mgr_->modify_nodes(instance_id, to_add, to_del); - } - } break; - case AlterClusterRequest::NOTIFY_DECOMMISSIONED: { - resource_mgr_->check_cluster_params_valid(request->cluster(), &msg, false, false); - if (msg != "") { - LOG(WARNING) << msg; - break; - } - - std::string be_unique_id = (request->cluster().nodes())[0].cloud_unique_id(); - std::vector nodes; - std::string err = resource_mgr_->get_node(be_unique_id, &nodes); - if (!err.empty()) { - LOG(INFO) << "failed to check instance info, err=" << err; - msg = err; - break; - } - - std::vector decomission_nodes; - for (auto& node : nodes) { - for (auto req_node : request->cluster().nodes()) { - bool ip_processed = false; - if (node.node_info.has_ip() && req_node.has_ip()) { - std::string endpoint = node.node_info.ip() + ":" + - std::to_string(node.node_info.heartbeat_port()); - std::string req_endpoint = - req_node.ip() + ":" + std::to_string(req_node.heartbeat_port()); - if (endpoint == req_endpoint) { - decomission_nodes.push_back(node); - } - ip_processed = true; - } - - if (!ip_processed && node.node_info.has_host() && req_node.has_host()) { - std::string endpoint = node.node_info.host() + ":" + - std::to_string(node.node_info.heartbeat_port()); - std::string req_endpoint = - req_node.host() + ":" + std::to_string(req_node.heartbeat_port()); - if (endpoint == req_endpoint) { - decomission_nodes.push_back(node); - } - } - } - } - - { - std::vector to_add; - std::vector& to_del = decomission_nodes; - msg = resource_mgr_->modify_nodes(instance_id, to_add, to_del); - } - { - std::vector& to_add = decomission_nodes; - std::vector to_del; - for (auto& node : to_add) { - node.node_info.set_status(NodeStatusPB::NODE_STATUS_DECOMMISSIONED); - LOG(INFO) << "notify node decomissioned, " - << " size: " << to_add.size() << " " << node.node_info.DebugString() - << " " << node.cluster_id << " " << node.cluster_name; - } - msg = resource_mgr_->modify_nodes(instance_id, to_add, to_del); - } - } break; - case AlterClusterRequest::RENAME_CLUSTER: { - // SQL mode, cluster cluster name eq empty cluster name, need drop empty cluster first. - // but in http api, cloud control will drop empty cluster - bool replace_if_existing_empty_target_cluster = - request->has_replace_if_existing_empty_target_cluster() - ? request->replace_if_existing_empty_target_cluster() - : false; - - msg = resource_mgr_->update_cluster( - instance_id, cluster, - [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, - [&](ClusterPB& c, std::set& cluster_names) { - std::string msg; - auto it = cluster_names.find(cluster.cluster.cluster_name()); - LOG(INFO) << "cluster.cluster.cluster_name(): " - << cluster.cluster.cluster_name(); - for (auto itt : cluster_names) { - LOG(INFO) << "instance's cluster name : " << itt; - } - if (it != cluster_names.end()) { - code = MetaServiceCode::INVALID_ARGUMENT; - ss << "failed to rename cluster, a cluster with the same name already " - "exists in this instance " - << proto_to_json(c); - msg = ss.str(); - return msg; - } - if (c.cluster_name() == cluster.cluster.cluster_name()) { - code = MetaServiceCode::INVALID_ARGUMENT; - ss << "failed to rename cluster, name eq original name, original cluster " - "is " - << proto_to_json(c); - msg = ss.str(); - return msg; - } - c.set_cluster_name(cluster.cluster.cluster_name()); - return msg; - }, - replace_if_existing_empty_target_cluster); - } break; - case AlterClusterRequest::UPDATE_CLUSTER_ENDPOINT: { - msg = resource_mgr_->update_cluster( - instance_id, cluster, - [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, - [&](ClusterPB& c, std::set&) { - std::string msg; - if (!cluster.cluster.has_private_endpoint() || - cluster.cluster.private_endpoint().empty()) { - code = MetaServiceCode::CLUSTER_ENDPOINT_MISSING; - ss << "missing private endpoint"; - msg = ss.str(); - return msg; - } - - c.set_public_endpoint(cluster.cluster.public_endpoint()); - c.set_private_endpoint(cluster.cluster.private_endpoint()); - - return msg; - }); - } break; - case AlterClusterRequest::SET_CLUSTER_STATUS: { - msg = resource_mgr_->update_cluster( - instance_id, cluster, - [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, - [&](ClusterPB& c, std::set&) { - std::string msg; - if (c.cluster_status() == request->cluster().cluster_status()) { - code = MetaServiceCode::INVALID_ARGUMENT; - ss << "failed to set cluster status, status eq original status, original " - "cluster is " - << print_cluster_status(c.cluster_status()); - msg = ss.str(); - return msg; - } - // status from -> to - std::set> - can_work_directed_edges { - {ClusterStatus::UNKNOWN, ClusterStatus::NORMAL}, - {ClusterStatus::NORMAL, ClusterStatus::SUSPENDED}, - {ClusterStatus::SUSPENDED, ClusterStatus::TO_RESUME}, - {ClusterStatus::TO_RESUME, ClusterStatus::NORMAL}, - {ClusterStatus::SUSPENDED, ClusterStatus::NORMAL}, - {ClusterStatus::NORMAL, ClusterStatus::MANUAL_SHUTDOWN}, - {ClusterStatus::MANUAL_SHUTDOWN, ClusterStatus::NORMAL}, - }; - auto from = c.cluster_status(); - auto to = request->cluster().cluster_status(); - if (can_work_directed_edges.count({from, to}) == 0) { - // can't find a directed edge in set, so refuse it - code = MetaServiceCode::INVALID_ARGUMENT; - ss << "failed to set cluster status, original cluster is " - << print_cluster_status(from) << " and want set " - << print_cluster_status(to); - msg = ss.str(); - return msg; - } - c.set_cluster_status(request->cluster().cluster_status()); - return msg; - }); - } break; - default: { + case AlterClusterRequest::ADD_CLUSTER: + handle_add_cluster(instance_id, cluster, resource_mgr(), msg, code); + break; + case AlterClusterRequest::DROP_CLUSTER: + handle_drop_cluster(instance_id, cluster, resource_mgr(), msg, code); + break; + case AlterClusterRequest::UPDATE_CLUSTER_MYSQL_USER_NAME: + handle_update_cluster_mySQL_username(instance_id, cluster, resource_mgr(), msg, code); + break; + case AlterClusterRequest::ADD_NODE: + handle_add_node(instance_id, request, resource_mgr(), msg, code); + break; + case AlterClusterRequest::DROP_NODE: + handle_drop_node(instance_id, request, resource_mgr(), msg, code); + break; + case AlterClusterRequest::DECOMMISSION_NODE: + handle_decommission_node(instance_id, request, resource_mgr(), msg, code); + break; + case AlterClusterRequest::NOTIFY_DECOMMISSIONED: + handle_notify_decommissioned(instance_id, request, resource_mgr(), msg, code); + break; + case AlterClusterRequest::RENAME_CLUSTER: + handle_rename_cluster(instance_id, cluster, resource_mgr(), msg, code); + break; + case AlterClusterRequest::UPDATE_CLUSTER_ENDPOINT: + handle_update_cluster_endpoint(instance_id, cluster, resource_mgr(), msg, code); + break; + case AlterClusterRequest::SET_CLUSTER_STATUS: + handle_set_cluster_status(instance_id, cluster, resource_mgr(), msg, code); + break; + case AlterClusterRequest::ALTER_VCLUSTER_INFO: + handle_alter_vcluster_Info(instance_id, cluster, resource_mgr(), msg, code); + break; + default: code = MetaServiceCode::INVALID_ARGUMENT; ss << "invalid request op, op=" << request->op(); msg = ss.str(); return; } - } + if (!msg.empty() && code == MetaServiceCode::OK) { - code = MetaServiceCode::UNDEFINED_ERR; + code = MetaServiceCode::INVALID_ARGUMENT; } // ugly but easy to repair @@ -2451,7 +2598,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, LOG(WARNING) << "notify refresh instance inplace, instance_id=" << request->instance_id(); run_bthread_work(f); } -} // alter cluster +} void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, const GetClusterRequest* request, GetClusterResponse* response, diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp index 548478a85395d5..0e4133d0198e45 100644 --- a/cloud/src/resource-manager/resource_manager.cpp +++ b/cloud/src/resource-manager/resource_manager.cpp @@ -140,29 +140,55 @@ std::string ResourceManager::get_node(const std::string& cloud_unique_id, bool ResourceManager::check_cluster_params_valid(const ClusterPB& cluster, std::string* err, bool check_master_num, bool check_cluster_name) { - // check + // Check if the cluster has a type if (!cluster.has_type()) { *err = "cluster must have type arg"; return false; } - const char* cluster_pattern_str = "^[a-zA-Z][a-zA-Z0-9_]*$"; - std::regex txt_regex(cluster_pattern_str); - if (config::enable_cluster_name_check && cluster.has_cluster_name() && - !std::regex_match(cluster.cluster_name(), txt_regex)) { - *err = "cluster name not regex with ^[a-zA-Z][a-zA-Z0-9_]*$, please check it"; + // Validate cluster name + if (!validate_cluster_name(cluster, err, check_cluster_name)) { + return false; + } + + // Validate nodes + if (!validate_nodes(cluster, err, check_master_num)) { return false; } + // Validate virtual cluster specifics + if (ClusterPB::VIRTUAL == cluster.type()) { + if (!validate_virtual_cluster(cluster, err)) { + return false; + } + } + + return true; +} + +bool ResourceManager::validate_cluster_name(const ClusterPB& cluster, std::string* err, + bool check_cluster_name) { if (check_cluster_name && (!cluster.has_cluster_name() || cluster.cluster_name() == "")) { *err = "not have cluster name"; return false; } + const char* cluster_pattern_str = "^[a-zA-Z][a-zA-Z0-9_]*$"; + std::regex txt_regex(cluster_pattern_str); + if (config::enable_cluster_name_check && cluster.has_cluster_name() && + !std::regex_match(cluster.cluster_name(), txt_regex)) { + *err = "cluster name not regex with ^[a-zA-Z][a-zA-Z0-9_]*$, please check it"; + return false; + } + return true; +} +bool ResourceManager::validate_nodes(const ClusterPB& cluster, std::string* err, + bool check_master_num) { std::stringstream ss; bool no_err = true; int master_num = 0; int follower_num = 0; + for (auto& n : cluster.nodes()) { // check here cloud_unique_id std::string cloud_unique_id = n.cloud_unique_id(); @@ -185,6 +211,8 @@ bool ResourceManager::check_cluster_params_valid(const ClusterPB& cluster, std:: } else if (ClusterPB::COMPUTE == cluster.type() && n.has_heartbeat_port() && n.heartbeat_port()) { continue; + } else if (ClusterPB::VIRTUAL) { + continue; } ss << "check cluster params failed, edit_log_port is required for frontends while " "heatbeat_port is required for banckends, node : " @@ -195,17 +223,136 @@ bool ResourceManager::check_cluster_params_valid(const ClusterPB& cluster, std:: } if (check_master_num && ClusterPB::SQL == cluster.type()) { - if (master_num == 0 && follower_num == 0) { - ss << "cluster is SQL type, but not set master and follower node, master count=" - << master_num << " follower count=" << follower_num - << " so sql cluster can't get a Master node"; - no_err = false; - } - *err = ss.str(); + no_err = validate_master_follower_count(master_num, follower_num, err); } + return no_err; } +bool ResourceManager::validate_master_follower_count(int master_num, int follower_num, + std::string* err) { + std::stringstream ss; + if (master_num > 0 && follower_num > 0) { + ss << "cluster is SQL type, and use multi follower mode, cant set master node, master " + "count: " + << master_num << " follower count: " << follower_num; + *err = ss.str(); + return false; + } else if (!follower_num && master_num != 1) { + ss << "cluster is SQL type, must have only one master node, now master count: " + << master_num; + *err = ss.str(); + return false; + } + return true; +} + +void set_default_failover_threshold(doris::cloud::ClusterPolicy& policy) { + // if not set failover_failure_threshold in api, set here default value + if (!policy.has_failover_failure_threshold()) { + policy.set_failover_failure_threshold(3); + } + // if not set unhealthy_node_threshold_percent in api, set here default value + if (!policy.has_unhealthy_node_threshold_percent()) { + policy.set_unhealthy_node_threshold_percent(100); + } +} + +bool ResourceManager::validate_virtual_cluster(const ClusterPB& cluster, std::string* err) { + if (cluster.nodes().size() || cluster.mysql_user_name().size() || + cluster.has_public_endpoint() || cluster.has_private_endpoint() || + cluster.has_cluster_status()) { + *err = "Inconsistent virtual cluster args"; + LOG(WARNING) << *err; + return false; + } + + if ((cluster.cluster_names().size() && !cluster.has_cluster_policy()) || + (cluster.cluster_names().size() == 0 && cluster.has_cluster_policy())) { + *err = "subcgs and policy must be Incoming at the same time or do not transmit at the same " + "time"; + LOG(WARNING) << *err; + return false; + } + // Validate cluster names + if (cluster.cluster_names().size()) { + // Currently, the number of sub clusters is limited to 2 + if (cluster.cluster_names().size() != 2) { + *err = "Currently, just support two sub clusters"; + LOG(WARNING) << *err; + return false; + } + + for (const auto& sub_cluster : cluster.cluster_names()) { + if (!std::regex_match(sub_cluster, std::regex("^[a-zA-Z][a-zA-Z0-9_]*$"))) { + *err = "cluster name " + sub_cluster + + " does not match regex ^[a-zA-Z][a-zA-Z0-9_]*$"; + LOG(WARNING) << *err; + return false; + } + } + } + + // Validate cluster policy + if (cluster.has_cluster_policy()) { + const auto& policy = cluster.cluster_policy(); + if (!policy.has_type()) { + *err = "plz set cluster policy type, use virtual cluster policy"; + LOG(WARNING) << *err; + return false; + } + if (policy.type() != ClusterPolicy::ActiveStandby) { + *err = "cluster policy type must be ActiveStandby"; + LOG(WARNING) << *err; + return false; + } + if ((policy.active_cluster_name().empty() && !policy.standby_cluster_names().empty()) || + (!policy.active_cluster_name().empty() && policy.standby_cluster_names().empty())) { + *err = "Inconsistent cluster policy: active_cluster_name must be set if " + "standby_cluster_names are present, and vice versa."; + LOG(WARNING) << *err; + return false; + } + if (policy.active_cluster_name().empty() || + std::find(cluster.cluster_names().begin(), cluster.cluster_names().end(), + policy.active_cluster_name()) == cluster.cluster_names().end()) { + *err = "active_cluster_name must not be empty and must be in cluster_names"; + LOG(WARNING) << *err; + return false; + } + for (const auto& standby_name : policy.standby_cluster_names()) { + if (std::find(cluster.cluster_names().begin(), cluster.cluster_names().end(), + standby_name) == cluster.cluster_names().end()) { + *err = "standby_cluster_name " + standby_name + " must be in cluster_names"; + LOG(WARNING) << *err; + return false; + } + if (standby_name == policy.active_cluster_name()) { + *err = "active_cluster_name is same of standby_cluster_name"; + LOG(WARNING) << *err; + return false; + } + } + + set_default_failover_threshold(const_cast&>(policy)); + + if (policy.failover_failure_threshold() <= 0 || + policy.unhealthy_node_threshold_percent() > INT64_MAX) { + *err = "failover_failure_threshold must be greater than 0 and less than max(int64)"; + LOG(WARNING) << *err; + return false; + } + if (policy.unhealthy_node_threshold_percent() <= 0 || + policy.unhealthy_node_threshold_percent() > 100) { + *err = "unhealthy_node_threshold_percent must be greater than 0 and less than or equal " + "to 100"; + LOG(WARNING) << *err; + return false; + } + } + return true; +} + std::pair ResourceManager::get_instance_id_by_cloud_unique_id( const std::string& cloud_unique_id) { auto v = split(cloud_unique_id, ':'); @@ -377,17 +524,21 @@ std::pair ResourceManager::add_cluster(const std:: // Check id and name, they need to be unique // One cluster id per name, name is alias of cluster id - for (auto& instance_cluster : instance.clusters()) { - if (instance_cluster.cluster_id() == req_cluster.cluster_id()) { + std::vector clusters_in_instance; + for (auto& i : instance.clusters()) { + clusters_in_instance.emplace_back(i); + } + for (auto& i : clusters_in_instance) { + if (i.cluster_id() == cluster.cluster.cluster_id()) { ss << "try to add a existing cluster id," - << " existing_cluster_id=" << instance_cluster.cluster_id(); + << " existing_cluster_id=" << i.cluster_id(); msg = ss.str(); return std::make_pair(MetaServiceCode::ALREADY_EXISTED, msg); } - if (instance_cluster.cluster_name() == req_cluster.cluster_name()) { + if (i.cluster_name() == req_cluster.cluster_name()) { ss << "try to add a existing cluster name," - << " existing_cluster_name=" << instance_cluster.cluster_name(); + << " existing_cluster_name=" << i.cluster_name(); msg = ss.str(); return std::make_pair(MetaServiceCode::ALREADY_EXISTED, msg); } @@ -415,6 +566,16 @@ std::pair ResourceManager::add_cluster(const std:: } } } + // vitrual cluster + if (cluster.cluster.type() == ClusterPB::VIRTUAL) { + // if has cluster_names, check it's cluster has been added + std::vector check_clusters_vec(cluster.cluster.cluster_names().begin(), + cluster.cluster.cluster_names().end()); + auto validation_result = validate_sub_clusters(check_clusters_vec, clusters_in_instance); + if (validation_result.first != MetaServiceCode::OK) { + return validation_result; + } + } auto to_add_cluster = instance.add_clusters(); to_add_cluster->CopyFrom(cluster.cluster); @@ -422,6 +583,8 @@ std::pair ResourceManager::add_cluster(const std:: if (cluster.cluster.type() == ClusterPB::COMPUTE) { to_add_cluster->set_cluster_status(ClusterStatus::NORMAL); } + to_add_cluster->set_ctime(time); + to_add_cluster->set_mtime(time); LOG(INFO) << "instance " << instance_id << " has " << instance.clusters().size() << " clusters"; InstanceKeyInfo key_info {instance_id}; @@ -548,9 +711,26 @@ std::pair ResourceManager::drop_cluster( << " cluster_name=" << cluster.cluster.cluster_name() << " help Msg=" << cache_err_help_msg; msg = ss.str(); + LOG(WARNING) << msg; return std::make_pair(MetaServiceCode::CLUSTER_NOT_FOUND, msg); } + // Checks whether the deleted cluster belongs to a virtual cluster. + // If so, drop failed, need drop virtual cluster first + for (auto& c : instance.clusters()) { + if (ClusterPB::VIRTUAL == c.type() && + std::find(c.cluster_names().begin(), c.cluster_names().end(), to_del.cluster_name()) != + c.cluster_names().end()) { + ss << "failed to drop cluster, this cluster owned by virtual cluster=" + << c.cluster_name() + << " if you want drop this cluster, please drop virtual cluster=" << c.cluster_name() + << " firstly"; + msg = ss.str(); + LOG(WARNING) << msg; + return std::make_pair(MetaServiceCode::ALREADY_EXISTED, msg); + } + } + InstanceInfoPB new_instance(instance); new_instance.mutable_clusters()->DeleteSubrange(idx, 1); // Remove it @@ -583,7 +763,7 @@ std::pair ResourceManager::drop_cluster( std::string ResourceManager::update_cluster( const std::string& instance_id, const ClusterInfo& cluster, std::function filter, - std::function& cluster_names)> action, + std::function& clusters_in_instance)> action, bool replace_if_existing_empty_target_cluster) { std::stringstream ss; std::string msg; @@ -619,10 +799,12 @@ std::string ResourceManager::update_cluster( return msg; } + std::vector clusters_in_instance; std::set cluster_names; - // collect cluster_names + // collect cluster in instance pb for check for (auto& i : instance.clusters()) { cluster_names.emplace(i.cluster_name()); + clusters_in_instance.emplace_back(i); } bool found = false; @@ -679,11 +861,15 @@ std::string ResourceManager::update_cluster( // do update ClusterPB original = clusters[idx]; - msg = action(clusters[idx], cluster_names); + msg = action(clusters[idx], clusters_in_instance); if (!msg.empty()) { return msg; } ClusterPB now = clusters[idx]; + auto now_time = std::chrono::system_clock::now(); + uint64_t time = + std::chrono::duration_cast(now_time.time_since_epoch()).count(); + now.set_mtime(time); LOG(INFO) << "before update cluster original: " << proto_to_json(original) << " after update now: " << proto_to_json(now); @@ -932,6 +1118,12 @@ std::string ResourceManager::modify_nodes(const std::string& instance_id, if (msg != "") { return msg; } + auto now_time = std::chrono::system_clock::now(); + uint64_t time = std::chrono::duration_cast( + now_time.time_since_epoch()) + .count(); + ClusterPB modify_cluster = const_cast&>(c); + modify_cluster.set_mtime(time); } } return ""; @@ -1261,4 +1453,45 @@ std::pair ResourceManager::refresh_instance( return ret0; } +std::pair ResourceManager::validate_sub_clusters( + const std::vector& check_clusters, + const std::vector& clusters_in_instance) { + std::string msg; + for (const auto& check_cluster : check_clusters) { + auto it = std::find_if(clusters_in_instance.begin(), clusters_in_instance.end(), + [&check_cluster](const auto& cluster) { + return check_cluster == cluster.cluster_name(); + }); + if (it == clusters_in_instance.end()) { + msg = "sub cluster " + check_cluster + + " not been added in instance, plz add it before create virtual cluster"; + LOG(WARNING) << msg; + return std::make_pair(MetaServiceCode::INVALID_ARGUMENT, msg); + } + // sub cluster type must be COMPUTE + if (ClusterPB::COMPUTE != it->type()) { + msg = "sub cluster " + check_cluster + " 's type must be eq COMPUTE"; + LOG(WARNING) << msg; + return std::make_pair(MetaServiceCode::INVALID_ARGUMENT, msg); + } + // check one cluster can only be mounted by one virtual cluster at most. + for (const auto& cluster_in_instance : clusters_in_instance) { + if (ClusterPB::VIRTUAL == cluster_in_instance.type()) { + auto it = std::find_if( + cluster_in_instance.cluster_names().begin(), + cluster_in_instance.cluster_names().end(), + [&check_cluster](const auto& cluster) { return check_cluster == cluster; }); + if (it != cluster_in_instance.cluster_names().end()) { + msg = "sub cluster " + check_cluster + + " has been add by other vcg=" + cluster_in_instance.cluster_name(); + LOG(WARNING) << msg; + return std::make_pair(MetaServiceCode::INVALID_ARGUMENT, msg); + } + } + } + } + + return std::make_pair(MetaServiceCode::OK, ""); +} + } // namespace doris::cloud diff --git a/cloud/src/resource-manager/resource_manager.h b/cloud/src/resource-manager/resource_manager.h index 0c0bc461c2d1f6..6b6db4091403ec 100644 --- a/cloud/src/resource-manager/resource_manager.h +++ b/cloud/src/resource-manager/resource_manager.h @@ -95,7 +95,8 @@ class ResourceManager { virtual std::string update_cluster( const std::string& instance_id, const ClusterInfo& cluster, std::function filter, - std::function& cluster_names)> action, + std::function& clusters_in_instance)> + action, bool replace_if_existing_empty_target_cluster = false); /** @@ -134,6 +135,47 @@ class ResourceManager { bool check_cluster_params_valid(const ClusterPB& cluster, std::string* err, bool check_master_num, bool check_cluster_name); + /** + * Validates the cluster name against a regex pattern. + * + * @param cluster The ClusterPB object containing the cluster information. + * @param err Output parameter to store error message if validation fails. + * @param need check cluster name + * @return true if the cluster name is valid, false otherwise. + */ + bool validate_cluster_name(const ClusterPB& cluster, std::string* err, bool check_cluster_name); + + /** + * Validates the nodes in the cluster, checking for cloud unique IDs + * and counting master and follower nodes. + * + * @param cluster The ClusterPB object containing the cluster information. + * @param err Output parameter to store error message if validation fails. + * @param check_master_num Flag indicating whether to check master and follower counts. + * @return true if the nodes are valid, false otherwise. + */ + bool validate_nodes(const ClusterPB& cluster, std::string* err, bool check_master_num); + + /** + * Validates the counts of master and follower nodes for SQL clusters. + * + * @param master_num The number of master nodes. + * @param follower_num The number of follower nodes. + * @param err Output parameter to store error message if validation fails. + * @return true if the counts are valid, false otherwise. + */ + bool validate_master_follower_count(int master_num, int follower_num, std::string* err); + + /** + * Validates the specifics of virtual clusters, including cluster names + * and policies. + * + * @param cluster The ClusterPB object containing the cluster information. + * @param err Output parameter to store error message if validation fails. + * @return true if the virtual cluster is valid, false otherwise. + */ + bool validate_virtual_cluster(const ClusterPB& cluster, std::string* err); + /** * Check cloud_unique_id is degraded format, and get instance_id from cloud_unique_id * degraded format : "${version}:${instance_id}:${unique_id}" @@ -163,6 +205,10 @@ class ResourceManager { virtual std::pair refresh_instance( const std::string& instance_id); + std::pair validate_sub_clusters( + const std::vector& check_clusters, + const std::vector& clusters_in_instance); + private: void add_cluster_to_index(const std::string& instance_id, const ClusterPB& cluster); diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp index 855ffbb33cb990..7ade292966354a 100644 --- a/cloud/test/meta_service_http_test.cpp +++ b/cloud/test/meta_service_http_test.cpp @@ -717,7 +717,7 @@ TEST(MetaServiceHttpTest, AlterClusterTest) { AlterClusterRequest req; req.set_instance_id(mock_instance); req.mutable_cluster()->set_type(ClusterPB::COMPUTE); - req.mutable_cluster()->set_cluster_name(""); + req.mutable_cluster()->set_cluster_name("%"); req.mutable_cluster()->set_cluster_id(mock_cluster_id + "1"); auto [status_code, resp] = ctx.forward("add_cluster", req); ASSERT_EQ(status_code, 400); @@ -1972,4 +1972,883 @@ TEST(MetaServiceHttpTest, CreateInstanceWithIamRoleTest) { } } +static std::tuple add_cluster( + HttpContext& ctx, std::string cluster_name, std::string cluster_id, + const ClusterPB::Type type, const std::vector& cluster_names, + const ClusterPolicy* policy) { + AlterClusterRequest req; + req.set_instance_id(mock_instance); + req.mutable_cluster()->set_type(type); + req.mutable_cluster()->set_cluster_id(cluster_id); + req.mutable_cluster()->set_cluster_name(cluster_name); + if (type == ClusterPB::VIRTUAL) { + for (auto cg_name : cluster_names) { + req.mutable_cluster()->add_cluster_names(cg_name); + } + if (policy != nullptr) { + req.mutable_cluster()->mutable_cluster_policy()->CopyFrom(*policy); + } + } + return ctx.forward("add_cluster", req); +} + +static std::tuple alter_virtual_cluster( + HttpContext& ctx, std::string instance_id, std::string cluster_name, std::string cluster_id, + const std::vector& cluster_names, const ClusterPolicy* policy) { + AlterClusterRequest alter_req; + alter_req.set_instance_id(instance_id); + alter_req.mutable_cluster()->set_cluster_id(cluster_id); + alter_req.mutable_cluster()->set_cluster_name(cluster_name); + alter_req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + for (auto subcg_name : cluster_names) { + alter_req.mutable_cluster()->add_cluster_names(subcg_name); + } + if (policy != nullptr) { + alter_req.mutable_cluster()->mutable_cluster_policy()->CopyFrom(*policy); + } + return ctx.forward("alter_vcluster_info", alter_req); +} + +static std::tuple> get_cluster_info(HttpContext& ctx, + std::string cluster_name, + std::string cluster_id) { + GetClusterRequest req; + req.set_cloud_unique_id("1:" + mock_instance + ":xxxx"); + req.set_cluster_id(cluster_id); + req.set_cluster_name(cluster_name); + return ctx.forward_with_result("get_cluster", req); +} + +static std::tuple drop_cluster(HttpContext& ctx, + std::string cluster_name, + std::string cluster_id) { + AlterClusterRequest req_drop; + req_drop.set_instance_id(mock_instance); + if ("" != cluster_id) { + req_drop.mutable_cluster()->set_cluster_id(cluster_id); + } + if ("" != cluster_name) { + req_drop.mutable_cluster()->set_cluster_name(cluster_name); + } + return ctx.forward("drop_cluster", req_drop); +} + +TEST(MetaServiceHttpTest, VirtualClusterTest) { + config::enable_cluster_name_check = true; + + HttpContext ctx; + { + CreateInstanceRequest req; + req.set_instance_id(mock_instance); + req.set_user_id("test_user_virtual"); + req.set_name("test_name_virtual"); + ObjectStoreInfoPB obj; + obj.set_ak("123_virtual"); + obj.set_sk("321_virtual"); + obj.set_bucket("456_virtual"); + obj.set_prefix("654_virtual"); + obj.set_endpoint("789_virtual"); + obj.set_region("987_virtual"); + obj.set_external_endpoint("888_virtual"); + obj.set_provider(ObjectStoreInfoPB::BOS); + req.mutable_obj_info()->CopyFrom(obj); + + auto [status_code, resp] = ctx.forward("create_instance", req); + ASSERT_EQ(status_code, 200); + ASSERT_EQ(resp.code(), MetaServiceCode::OK); + } + + //case: no type + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.set_instance_id(mock_instance); + req.mutable_cluster()->set_cluster_name(mock_vcg_name); + req.mutable_cluster()->set_cluster_id(mock_vcg_id); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "cluster must have type arg"); + } + + //case: regex not ok + { + std::string mock_vcg_name = "*virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.set_instance_id(mock_instance); + req.mutable_cluster()->set_cluster_name(mock_vcg_name); + req.mutable_cluster()->set_cluster_id(mock_vcg_id); + req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), + "cluster name not regex with ^[a-zA-Z][a-zA-Z0-9_]*$, please check it"); + } + + //case: no instance + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.mutable_cluster()->set_cluster_name(mock_vcg_name); + req.mutable_cluster()->set_cluster_id(mock_vcg_id); + req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "invalid request instance_id or cluster not given"); + } + + //case: no cluster name + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.mutable_cluster()->set_cluster_id(mock_vcg_id); + req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "invalid request instance_id or cluster not given"); + } + + //case: no cluster id + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.mutable_cluster()->set_cluster_name(mock_vcg_name); + req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "invalid request instance_id or cluster not given"); + } + + //case: use ordinary cluster's args, public_endpoint, mysql_user_name, private_endpoint, cluster_status + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.set_instance_id(mock_instance); + req.mutable_cluster()->set_cluster_name(mock_vcg_name); + req.mutable_cluster()->set_cluster_id(mock_vcg_id); + req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + req.mutable_cluster()->set_public_endpoint("public_endpoint"); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "Inconsistent virtual cluster args"); + } + + //case: use ordinary cluster's args, nodes + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + AlterClusterRequest req; + req.set_instance_id(mock_instance); + req.mutable_cluster()->set_cluster_name(mock_vcg_name); + req.mutable_cluster()->set_cluster_id(mock_vcg_id); + req.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + auto node = req.mutable_cluster()->add_nodes(); + node->set_ip("127.0.0.1"); + node->set_heartbeat_port(9999); + auto [status_code, resp] = ctx.forward("add_cluster", req); + std::cout << resp.DebugString() << std::endl; + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "Inconsistent virtual cluster args"); + } + + // case: add a empty cluster + { + std::string mock_vcg_name = "virtual_cluster_name"; + std::string mock_vcg_id = "virtual_cluster_id"; + auto [status_code, resp] = + add_cluster(ctx, mock_vcg_name, mock_vcg_id, ClusterPB::VIRTUAL, {}, nullptr); + ASSERT_EQ(status_code, 200); + ASSERT_EQ(resp.code(), MetaServiceCode::OK); + + // test alter, add two subcg to empty cluster + { + add_cluster(ctx, "subcg_name1", "subcg_id1", ClusterPB::COMPUTE, {}, nullptr); + add_cluster(ctx, "subcg_name2", "subcg_id2", ClusterPB::COMPUTE, {}, nullptr); + ClusterPolicy policy; + policy.set_active_cluster_name("subcg_name2"); + policy.add_standby_cluster_names("subcg_name1"); + const std::vector cluster_names = {"subcg_name2", "subcg_name1"}; + auto [status_code, resp] = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name, + mock_vcg_id, cluster_names, &policy); + ASSERT_EQ(status_code, 400); + ASSERT_EQ(resp.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp.msg(), "plz set cluster policy type, use virtual cluster policy"); + + // just has cluster_names + auto [status_code2, resp2] = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name, + mock_vcg_id, cluster_names, nullptr); + ASSERT_EQ(status_code2, 400); + ASSERT_EQ(resp2.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp2.msg(), + "subcgs and policy must be Incoming at the same time or do not transmit at " + "the same time"); + + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("subcg_name2"); + policy.add_standby_cluster_names("subcg_name1"); + auto [status_code1, resp1] = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name, + mock_vcg_id, cluster_names, &policy); + ASSERT_EQ(status_code1, 200); + ASSERT_EQ(resp1.code(), MetaServiceCode::OK); + auto ret = get_cluster_info(ctx, mock_vcg_name, mock_vcg_id); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).status.code(), MetaServiceCode::OK); + ASSERT_TRUE(std::get<1>(ret).result.has_value()); + for (auto showrt : std::get<1>(ret).result->cluster_names()) { + ASSERT_TRUE(std::find(cluster_names.begin(), cluster_names.end(), showrt) != + cluster_names.end()); + } + ASSERT_EQ("subcg_name2", + std::get<1>(ret).result->cluster_policy().active_cluster_name()); + ASSERT_EQ("subcg_name1", + std::get<1>(ret).result->cluster_policy().standby_cluster_names().at(0)); + // check default value + ASSERT_EQ(3, std::get<1>(ret).result->cluster_policy().failover_failure_threshold()); + ASSERT_EQ(100, + std::get<1>(ret).result->cluster_policy().unhealthy_node_threshold_percent()); + } + + // case: disable alter vcluster status + AlterClusterRequest req1; + req1.set_instance_id(mock_instance); + req1.mutable_cluster()->set_cluster_id(mock_vcg_id); + req1.mutable_cluster()->set_cluster_status(ClusterStatus::SUSPENDED); + req1.set_op(AlterClusterRequest::SET_CLUSTER_STATUS); + auto [status_code1, resp1] = + ctx.forward("set_cluster_status", req1); + ASSERT_EQ(status_code1, 400); + ASSERT_EQ(resp1.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(resp1.msg(), "just support set COMPUTE cluster status"); + + // rename vcluster + AlterClusterRequest req2; + req2.set_instance_id(mock_instance); + req2.mutable_cluster()->set_cluster_id(mock_vcg_id); + req2.mutable_cluster()->set_cluster_name("virtual_cluster_name"); + auto [status_code2, resp2] = ctx.forward("rename_cluster", req2); + ASSERT_EQ(status_code2, 400); + ASSERT_EQ(resp2.code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_TRUE( + resp2.msg().starts_with("failed to rename cluster, a cluster with the same name " + "already exists in this instance")); + + req2.mutable_cluster()->set_cluster_name("virtual_cluster_new_name"); + auto [status_code3, resp3] = ctx.forward("rename_cluster", req2); + ASSERT_EQ(status_code3, 200); + ASSERT_EQ(resp3.code(), MetaServiceCode::OK); + } + + // case: add non-empty vcg failed + { + // has subcgs, not has policy + std::string mock_vcg_name1 = "virtual_cluster_name_1"; + std::string mock_vcg_id1 = "virtual_cluster_id_1"; + auto ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1"}, nullptr); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "subcgs and policy must be Incoming at the same time or do not transmit at the " + "same time"); + + // 2 subcg, and not set active_cluster_name + ClusterPolicy policy; + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_failover_failure_threshold(1); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "active_cluster_name must not be empty and must be in cluster_names"); + + // 2 subcg, seted active_cluster_name but not set standby_cluster_names + policy.set_active_cluster_name("not_exist_cluster_1"); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "Inconsistent cluster policy: active_cluster_name must be set if " + "standby_cluster_names are present, and vice versa."); + + // 2 subcg, seted active_cluster_name, one standby_cluster_names, active_cluster_name not create before + policy.add_standby_cluster_names("not_exist_cluster_2"); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "sub cluster not_exist_cluster_1 not been added in instance, plz add it before " + "create virtual cluster"); + + // 2 subcg, seted active_cluster_name, but not_exist_cluster_3 not in subcgs + policy.set_active_cluster_name("not_exist_cluster_3"); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "active_cluster_name must not be empty and must be in cluster_names"); + + // 2 subcg, seted active_cluster_name, two standby_cluster_names, but subcgs not have not_exist_cluster_3 + policy.set_active_cluster_name("not_exist_cluster_1"); + policy.add_standby_cluster_names("not_exist_cluster_3"); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "standby_cluster_name not_exist_cluster_3 must be in cluster_names"); + + // just support 2 subcg + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2", "not_exist_cluster_3"}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "Currently, just support two sub clusters"); + + // just support 2 subcg + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"only_one_cluster"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "Currently, just support two sub clusters"); + + // subcg regex cluster name + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"*_regex_err_cluster", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "cluster name *_regex_err_cluster does not match regex ^[a-zA-Z][a-zA-Z0-9_]*$"); + + // subcgs and policy must be Incoming at the same time or do not transmit at the same time + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.add_standby_cluster_names("not_exist_cluster_2"); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, {}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "subcgs and policy must be Incoming at the same time or do not transmit at the " + "same time"); + + // policy has standy but not have active + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "Inconsistent cluster policy: active_cluster_name must be set if " + "standby_cluster_names are present, and vice versa."); + + policy.set_active_cluster_name("not_exist_cluster_2"); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "active_cluster_name is same of standby_cluster_name"); + + policy.set_active_cluster_name("not_exist_cluster_1"); + // failover_failure_threshold 0, failed + policy.set_failover_failure_threshold(0); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "failover_failure_threshold must be greater than 0 and less than max(int64)"); + + // failover_failure_threshold -1, failed + policy.set_failover_failure_threshold(-1); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "failover_failure_threshold must be greater than 0 and less than max(int64)"); + + policy.set_failover_failure_threshold(88); + policy.set_unhealthy_node_threshold_percent(-1); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "unhealthy_node_threshold_percent must be greater than 0 and less than or equal " + "to 100"); + + policy.set_unhealthy_node_threshold_percent(0); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "unhealthy_node_threshold_percent must be greater than 0 and less than or equal " + "to 100"); + + policy.set_failover_failure_threshold(1); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {"not_exist_cluster_1", "not_exist_cluster_2"}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "unhealthy_node_threshold_percent must be greater than 0 and less than or equal " + "to 100"); + } + + // case: add non-empty vcg succ + { + std::string mock_vcg_name1 = "virtual_cluster_name_1"; + std::string mock_vcg_id1 = "virtual_cluster_id_1"; + std::string mock_vcg_name2 = "virtual_cluster_name_2"; + std::string mock_vcg_id2 = "virtual_cluster_id_2"; + std::string mock_exist_cluster_name1 = "exist_cluster_name_1"; + std::string mock_exist_cluster_id1 = "exist_cluster_id_1"; + std::string mock_exist_cluster_name2 = "exist_cluster_name_2"; + std::string mock_exist_cluster_id2 = "exist_cluster_id_2"; + std::string mock_exist_cluster_name3 = "fe_exist_cluster_name_3"; + std::string mock_exist_cluster_id3 = "fe_exist_cluster_id_3"; + // add two exist cluster before + auto ret = add_cluster(ctx, mock_exist_cluster_name1, mock_exist_cluster_id1, + ClusterPB::COMPUTE, {}, nullptr); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + ret = add_cluster(ctx, mock_exist_cluster_name2, mock_exist_cluster_id2, ClusterPB::COMPUTE, + {}, nullptr); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + AlterClusterRequest req_before_fe; + req_before_fe.set_instance_id(mock_instance); + req_before_fe.mutable_cluster()->set_type(ClusterPB::SQL); + req_before_fe.mutable_cluster()->set_cluster_id(mock_exist_cluster_id3); + req_before_fe.mutable_cluster()->set_cluster_name(mock_exist_cluster_name3); + auto node = req_before_fe.mutable_cluster()->add_nodes(); + node->set_ip("127.0.0.1"); + node->set_edit_log_port(9990); + node->set_node_type(NodeInfoPB::FE_MASTER); + ret = ctx.forward("add_cluster", req_before_fe); + ASSERT_EQ(std::get<1>(ret).msg(), ""); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // test normal succ + ClusterPolicy policy; + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name2); + policy.add_standby_cluster_names(mock_exist_cluster_name1); + ret = add_cluster(ctx, mock_vcg_name1, mock_vcg_id1, ClusterPB::VIRTUAL, + {mock_exist_cluster_name1, mock_exist_cluster_name2}, &policy); + ASSERT_EQ(std::get<1>(ret).msg(), ""); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // one cluster just can be added by one vcg + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name1); + policy.add_standby_cluster_names(mock_exist_cluster_name2); + ret = add_cluster(ctx, mock_vcg_name2, mock_vcg_id2, ClusterPB::VIRTUAL, + {mock_exist_cluster_name1, mock_exist_cluster_name2}, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "sub cluster exist_cluster_name_1 has been add by other " + "vcg=virtual_cluster_name_1"); + + // case alter cluster + // no instance_id or cloud_unique_id + { + ClusterPolicy policy; + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name1); + policy.add_standby_cluster_names(mock_exist_cluster_name2); + auto ret = alter_virtual_cluster(ctx, "", mock_vcg_name2, mock_vcg_id2, + {mock_exist_cluster_name1, mock_exist_cluster_name2}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "invalid request instance_id or cluster not given"); + } + + // no cluster_id + { + ClusterPolicy policy; + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name1); + policy.add_standby_cluster_names(mock_exist_cluster_name2); + auto ret = alter_virtual_cluster(ctx, mock_instance, "", "", + {mock_exist_cluster_name1, mock_exist_cluster_name2}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "missing cluster_id="); + } + + { + // current vcg (active:exist_cluster_name2 Standby:exist_cluster_name1) + // alter to (active:exist_cluster_name1 Standby:exist_cluster_name_4) + add_cluster(ctx, "exist_cluster_name_4", "exist_cluster_id_4", ClusterPB::COMPUTE, {}, + nullptr); + std::vector cluster_names = {mock_exist_cluster_name1, + "exist_cluster_name_4"}; + auto ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, nullptr); + // not change policy, so alter failed + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "active_cluster_name must not be empty and must be in cluster_names"); + + // cluster_names and policy modify together, succ + ClusterPolicy policy; + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name1); + policy.add_standby_cluster_names("exist_cluster_name_4"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // change back to vcg (active:exist_cluster_name2 Standby:exist_cluster_name1) + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name2); + policy.add_standby_cluster_names(mock_exist_cluster_name1); + cluster_names = {mock_exist_cluster_name1, mock_exist_cluster_name2}; + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // current vcg (active:exist_cluster_name2 Standby:exist_cluster_name1) + // alter to (active:exist_cluster_name_5 Standby:exist_cluster_name_4) + policy.Clear(); + add_cluster(ctx, "exist_cluster_name_5", "exist_cluster_id_5", ClusterPB::COMPUTE, {}, + nullptr); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_5"); + policy.add_standby_cluster_names("exist_cluster_name_4"); + cluster_names = {"exist_cluster_name_4", "exist_cluster_name_5"}; + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // active A standby B -> active A standby B -> active B standby A + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_4"); + policy.add_standby_cluster_names("exist_cluster_name_5"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + auto show_rest = get_cluster_info(ctx, mock_vcg_name1, mock_vcg_id1); + ASSERT_EQ(std::get<0>(show_rest), 200); + ASSERT_EQ(std::get<1>(show_rest).status.code(), MetaServiceCode::OK); + ASSERT_TRUE(std::get<1>(show_rest).result.has_value()); + for (auto showrt : std::get<1>(show_rest).result->cluster_names()) { + ASSERT_TRUE(std::find(cluster_names.begin(), cluster_names.end(), showrt) != + cluster_names.end()); + } + ASSERT_EQ("exist_cluster_name_4", + std::get<1>(show_rest).result->cluster_policy().active_cluster_name()); + ASSERT_EQ( + "exist_cluster_name_5", + std::get<1>(show_rest).result->cluster_policy().standby_cluster_names().at(0)); + // check default value + ASSERT_EQ(3, + std::get<1>(show_rest).result->cluster_policy().failover_failure_threshold()); + ASSERT_EQ(100, std::get<1>(show_rest) + .result->cluster_policy() + .unhealthy_node_threshold_percent()); + ASSERT_EQ(ClusterPolicy::ActiveStandby, + std::get<1>(show_rest).result->cluster_policy().type()); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_5"); + policy.add_standby_cluster_names("exist_cluster_name_4"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // active A standby B -> active B standby C -> active C standby D -> active D standby A + cluster_names = {mock_exist_cluster_name2, "exist_cluster_name_4"}; + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name(mock_exist_cluster_name2); + policy.add_standby_cluster_names("exist_cluster_name_4"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + cluster_names = {"exist_cluster_name_4", "exist_cluster_name_5"}; + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_4"); + policy.add_standby_cluster_names("exist_cluster_name_5"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + cluster_names = {"exist_cluster_name_5", "exist_cluster_name_1"}; + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_5"); + policy.add_standby_cluster_names("exist_cluster_name_1"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // failed, not_exist_cluster_name_6 + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_5"); + policy.add_standby_cluster_names("not_exist_cluster_name_6"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "sub cluster not_exist_cluster_name_6 not been added in instance, plz " + "add it before create virtual cluster"); + + cluster_names = {"exist_cluster_name_5", "not_exist_cluster_name_6"}; + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_5"); + policy.add_standby_cluster_names("not_exist_cluster_name_6"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "sub cluster not_exist_cluster_name_6 not been added in instance, plz " + "add it before create virtual cluster"); + + show_rest = get_cluster_info(ctx, mock_vcg_name1, mock_vcg_id1); + ASSERT_EQ(std::get<0>(show_rest), 200); + ASSERT_EQ(std::get<1>(show_rest).status.code(), MetaServiceCode::OK); + ASSERT_TRUE(std::get<1>(show_rest).result.has_value()); + cluster_names = {"exist_cluster_name_5", "exist_cluster_name_1"}; + for (auto showrt : std::get<1>(show_rest).result->cluster_names()) { + ASSERT_TRUE(std::find(cluster_names.begin(), cluster_names.end(), showrt) != + cluster_names.end()); + } + ASSERT_EQ("exist_cluster_name_5", + std::get<1>(show_rest).result->cluster_policy().active_cluster_name()); + ASSERT_EQ( + "exist_cluster_name_1", + std::get<1>(show_rest).result->cluster_policy().standby_cluster_names().at(0)); + // check default value + ASSERT_EQ(3, + std::get<1>(show_rest).result->cluster_policy().failover_failure_threshold()); + ASSERT_EQ(100, std::get<1>(show_rest) + .result->cluster_policy() + .unhealthy_node_threshold_percent()); + ASSERT_EQ(ClusterPolicy::ActiveStandby, + std::get<1>(show_rest).result->cluster_policy().type()); + + // just change active,standby, so can not send cluster_names + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_1"); + policy.add_standby_cluster_names("exist_cluster_name_5"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // just change cluster_names, so can not send policy, nothing changed + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, nullptr); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // just alter active_cluster_name, failed + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_active_cluster_name("exist_cluster_name_5"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "active_cluster_name is same of standby_cluster_name"); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.add_standby_cluster_names("exist_cluster_name_1"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "active_cluster_name is same of standby_cluster_name"); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_failover_failure_threshold(55); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + show_rest = get_cluster_info(ctx, mock_vcg_name1, mock_vcg_id1); + ASSERT_EQ(std::get<0>(show_rest), 200); + ASSERT_EQ(std::get<1>(show_rest).status.code(), MetaServiceCode::OK); + // check default value + ASSERT_EQ(55, + std::get<1>(show_rest).result->cluster_policy().failover_failure_threshold()); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_unhealthy_node_threshold_percent(66); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + show_rest = get_cluster_info(ctx, mock_vcg_name1, mock_vcg_id1); + ASSERT_EQ(std::get<0>(show_rest), 200); + ASSERT_EQ(std::get<1>(show_rest).status.code(), MetaServiceCode::OK); + // check default value + ASSERT_EQ(66, std::get<1>(show_rest) + .result->cluster_policy() + .unhealthy_node_threshold_percent()); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_unhealthy_node_threshold_percent(200); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "unhealthy_node_threshold_percent must be greater than 0 and less than " + "or equal to 100"); + + policy.Clear(); + policy.set_type(ClusterPolicy::ActiveStandby); + policy.set_unhealthy_node_threshold_percent(88); + cluster_names = {"exist_cluster_name_5"}; + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "Currently, just support two sub clusters"); + + cluster_names = {"exist_cluster_name_5", "exist_cluster_name_4", + mock_exist_cluster_name1}; + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, + cluster_names, &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "Currently, just support two sub clusters"); + + policy.Clear(); + policy.set_active_cluster_name("exist_cluster_name_1"); + policy.add_standby_cluster_names("exist_cluster_name_1"); + ret = alter_virtual_cluster(ctx, mock_instance, mock_vcg_name1, mock_vcg_id1, {}, + &policy); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "active_cluster_name is same of standby_cluster_name"); + } + + { + // sub cg must be COMPUTE cluster + AlterClusterRequest req2; + req2.set_instance_id(mock_instance); + req2.mutable_cluster()->set_cluster_name("vcg_name_3"); + req2.mutable_cluster()->set_cluster_id("vcg_id_3"); + req2.mutable_cluster()->set_type(ClusterPB::VIRTUAL); + req2.mutable_cluster()->add_cluster_names("exist_cluster_name_4"); + req2.mutable_cluster()->add_cluster_names(mock_exist_cluster_name3); + auto* policy2 = req2.mutable_cluster()->mutable_cluster_policy(); + policy2->set_type(ClusterPolicy::ActiveStandby); + policy2->set_active_cluster_name(mock_exist_cluster_name3); + policy2->add_standby_cluster_names("exist_cluster_name_4"); + ret = ctx.forward("add_cluster", req2); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), + "sub cluster fe_exist_cluster_name_3 's type must be eq COMPUTE"); + } + + // drop subcg failed, if not drop vcg + ret = drop_cluster(ctx, "", mock_exist_cluster_id1); + // ASSERT_EQ(std::get<0>(ret), 409); + // ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::ALREADY_EXISTED); + ASSERT_EQ(std::get<1>(ret).msg(), + "failed to drop cluster, this cluster owned by virtual " + "cluster=virtual_cluster_name_1 if you want drop this cluster, please drop " + "virtual " + "cluster=virtual_cluster_name_1 firstly"); + + // case drop cluster no cluster_id + ret = drop_cluster(ctx, mock_vcg_name1, ""); + ASSERT_EQ(std::get<0>(ret), 400); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::INVALID_ARGUMENT); + ASSERT_EQ(std::get<1>(ret).msg(), "missing cluster_id="); + + // case drop cluster + ret = drop_cluster(ctx, mock_vcg_name1, mock_vcg_id1); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // ATTN: meta_service_http_test.cpp:203 check, parse json err, but I don't want change pb add errcode + // drop_cluster(ctx, mock_vcg_name1, mock_vcg_id1); + + // after drop vcg, cg can be drop + ret = drop_cluster(ctx, "", mock_exist_cluster_id1); + ASSERT_EQ(std::get<0>(ret), 200); + ASSERT_EQ(std::get<1>(ret).code(), MetaServiceCode::OK); + + // ret = drop_cluster(ctx, "", "not_exist_cg_id_1"); + // ATTN: meta_service_http_test.cpp:203 check, parse json err, but I don't want change pb add errcode + // std::tuple doris::cloud::HttpContext::forward(std::string_view, const Request &) [Response = doris::cloud::MetaServiceResponseStatus, Request = doris::cloud::AlterClusterRequest] Parse JSON: INVALID_ARGUMENT:(code): invalid value "NOT_FOUND" for type type.googleapis.com/doris.cloud.MetaServiceCode, body: { + // "code": "NOT_FOUND", + // "msg": "failed to find cluster to drop, instance_id=test_instance cluster_id=not_exist_cg_id_1 cluster_name=" + // } + + } // namespace doris::cloud +} } // namespace doris::cloud diff --git a/cloud/test/mock_resource_manager.h b/cloud/test/mock_resource_manager.h index eccb6ee93e90a1..59d0365c8e6150 100644 --- a/cloud/test/mock_resource_manager.h +++ b/cloud/test/mock_resource_manager.h @@ -59,7 +59,8 @@ class MockResourceManager : public ResourceManager { std::string update_cluster( const std::string& instance_id, const ClusterInfo& cluster, std::function filter, - std::function& cluster_names)> action, + std::function& clusters_in_instance)> + action, bool replace_if_existing_empty_target_cluster) override { return ""; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java index d1a0486d68675d..19f536316d1391 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowClusterStmt.java @@ -35,10 +35,10 @@ public class ShowClusterStmt extends ShowStmt implements NotFallbackInParser { public static final ImmutableList CLUSTER_TITLE_NAMES = new ImmutableList.Builder() - .add("cluster").add("is_current").add("users").add("backend_num").build(); + .add("cluster").add("is_current").add("users").add("backend_num").add("sub_clusters").add("policy").build(); public static final ImmutableList COMPUTE_GROUP_TITLE_NAMES = new ImmutableList.Builder() - .add("Name").add("IsCurrent").add("Users").add("BackendNum").build(); + .add("Name").add("IsCurrent").add("Users").add("BackendNum").add("SubClusters").add("Policy").build(); boolean isComputeGroup = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java index 04d66eb8309281..f2e5961c38f4a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WarmUpClusterStmt.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -72,11 +73,14 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { throw new UserException("The sql is illegal in disk mode "); } super.analyze(analyzer); - if (!((CloudSystemInfoService) Env.getCurrentSystemInfo()).containClusterName(dstClusterName)) { + CloudSystemInfoService cloudSys = ((CloudSystemInfoService) Env.getCurrentSystemInfo()); + if (!cloudSys.containClusterName(dstClusterName)) { throw new AnalysisException("The dstClusterName " + dstClusterName + " doesn't exist"); } - if (!isWarmUpWithTable - && !((CloudSystemInfoService) Env.getCurrentSystemInfo()).containClusterName(srcClusterName)) { + + checkWarmupCgs(cloudSys); + + if (!isWarmUpWithTable && !cloudSys.containClusterName(srcClusterName)) { boolean contains = false; try { contains = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr().containsCluster(srcClusterName); @@ -119,6 +123,33 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { } } + private void checkWarmupCgs(CloudSystemInfoService cloudSys) throws AnalysisException { + if (!Strings.isNullOrEmpty(srcClusterName)) { + ComputeGroup srcCg = cloudSys.getComputeGroupByName(srcClusterName); + if (srcCg != null && srcCg.isVirtual()) { + throw new AnalysisException("The srcClusterName " + srcClusterName + + " is a virtual compute group, not support"); + } + } + + if (!Strings.isNullOrEmpty(dstClusterName)) { + ComputeGroup dstCg = cloudSys.getComputeGroupByName(dstClusterName); + if (dstCg != null && dstCg.isVirtual()) { + throw new AnalysisException("The dstClusterName " + dstClusterName + + " is a virtual compute group, not support"); + } + } + + if (!Strings.isNullOrEmpty(srcClusterName) && !Strings.isNullOrEmpty(dstClusterName)) { + String srcMayOwnedVcg = cloudSys.ownedByVirtualComputeGroup(srcClusterName); + String dstMayOwnedVcg = cloudSys.ownedByVirtualComputeGroup(srcClusterName); + if (srcMayOwnedVcg != null && srcMayOwnedVcg.equals(dstMayOwnedVcg)) { + throw new AnalysisException("The srcClusterName " + srcClusterName + " dstClusterName " + dstClusterName + + " is owned by virtual compute group " + srcMayOwnedVcg + " not support"); + } + } + } + @Override public String toSql() { StringBuilder sb = new StringBuilder(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index 167e413a02e996..6b7b0e5282a3c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -267,7 +267,7 @@ public void runAfterCatalogReady() { } traverseAllDatabaseForTable(); // it's thread safe to iterate through this concurrent map's ref - nodeMgr.getCloudClusterIdToBackend().entrySet().forEach(clusterToBeList -> { + nodeMgr.getCloudClusterIdToBackend(false).entrySet().forEach(clusterToBeList -> { List, Backend>> futureList = new ArrayList<>(); clusterToBeList.getValue().forEach(backend -> { @@ -303,6 +303,7 @@ public boolean containsCluster(String clusterName) { // table_id table_name, index_id, partition_id public List> getClusterTopNHotPartitions(String clusterName) { + LOG.debug("getClusterTopNHotPartitions called with clusterName={}", clusterName); return CacheHotspotManagerUtils.getClusterTopNPartitions(nodeMgr.getCloudClusterIdByName(clusterName)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java index 731ab0675882bf..ca81b165cb96ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java @@ -370,7 +370,7 @@ private void checkFeNodesMapValid() { if (LOG.isDebugEnabled()) { LOG.debug("begin checkFeNodesMapValid"); } - Map> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(); + Map> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(false); Set clusterIds = new HashSet<>(); Set clusterNames = new HashSet<>(); clusterIdToBackend.forEach((clusterId, bes) -> { @@ -384,7 +384,7 @@ private void checkFeNodesMapValid() { }); }); - Map nameToId = cloudSystemInfoService.getCloudClusterNameToId(); + Map nameToId = cloudSystemInfoService.getCloudClusterNameToId(false); nameToId.forEach((clusterName, clusterId) -> { if (!clusterIdToBackend.containsKey(clusterId)) { LOG.warn("impossible, somewhere err, clusterId {}, clusterName {}, clusterNameToIdMap {}", @@ -517,7 +517,7 @@ private void checkCloudFes() { } private void checkCloudBackends() { - Map> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(); + Map> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(false); //rpc to ms, to get mysql user can use cluster_id // NOTE: rpc args all empty, use cluster_unique_id to get a instance's all cluster info. Cloud.GetClusterResponse response = cloudSystemInfoService.getCloudCluster("", "", ""); @@ -531,7 +531,7 @@ private void checkCloudBackends() { Set localClusterIds = clusterIdToBackend.keySet(); // clusterId -> clusterPB Map remoteClusterIdToPB = response.getClusterList().stream() - .filter(c -> c.getType() != Type.SQL) + .filter(c -> c.getType() == Type.COMPUTE) .collect(Collectors.toMap(ClusterPB::getClusterId, clusterPB -> clusterPB)); LOG.info("get cluster info clusterIds: {}", remoteClusterIdToPB); @@ -543,7 +543,7 @@ private void checkCloudBackends() { // local - remote > 0, drop bes from local checkToDelCluster(remoteClusterIdToPB, localClusterIds, clusterIdToBackend); - clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(); + clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(false); if (remoteClusterIdToPB.keySet().size() != clusterIdToBackend.keySet().size()) { LOG.warn("impossible cluster id size not match, check it local {}, remote {}", @@ -559,13 +559,14 @@ private void checkCloudBackends() { } LOG.info("daemon cluster get cluster info succ, current cloudClusterIdToBackendMap: {} clusterNameToId {}", - cloudSystemInfoService.getCloudClusterIdToBackend(), cloudSystemInfoService.getCloudClusterNameToId()); + cloudSystemInfoService.getCloudClusterIdToBackend(false), + cloudSystemInfoService.getCloudClusterNameToId(false)); } private void updateCloudMetrics() { // Metric - Map> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(); - Map clusterNameToId = cloudSystemInfoService.getCloudClusterNameToId(); + Map> clusterIdToBackend = cloudSystemInfoService.getCloudClusterIdToBackend(false); + Map clusterNameToId = cloudSystemInfoService.getCloudClusterNameToId(true); for (Map.Entry entry : clusterNameToId.entrySet()) { int aliveNum = 0; List bes = clusterIdToBackend.get(entry.getValue()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java index 45369f3a8bbba6..e40ee50ae58feb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java @@ -17,25 +17,41 @@ package org.apache.doris.cloud.catalog; +import org.apache.doris.analysis.WarmUpClusterStmt; import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.CacheHotspotManager; +import org.apache.doris.cloud.CloudWarmUpJob; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.apache.doris.common.util.MasterDaemon; +import org.apache.doris.metric.MetricRepo; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + + public class CloudInstanceStatusChecker extends MasterDaemon { private static final Logger LOG = LogManager.getLogger(CloudInstanceStatusChecker.class); private CloudSystemInfoService cloudSystemInfoService; + // if find vcg failed sync, record it timestamp, virtual compute group name <-> timestamp + private Map lastFailedSyncTimeMap = new ConcurrentHashMap<>(); public CloudInstanceStatusChecker(CloudSystemInfoService cloudSystemInfoService) { - super("cloud instance check"); + super("cloud instance check", Config.cloud_cluster_check_interval_second * 1000L); this.cloudSystemInfoService = cloudSystemInfoService; } @@ -46,25 +62,420 @@ protected void runAfterCatalogReady() { if (LOG.isDebugEnabled()) { LOG.debug("get from ms response {}", response); } - if (response == null || !response.hasStatus() || !response.getStatus().hasCode() - || response.getStatus().getCode() != Cloud.MetaServiceCode.OK) { - LOG.warn("failed to get cloud instance due to incomplete response, " - + "cloud_unique_id={}, response={}", Config.cloud_unique_id, response); - } else { - cloudSystemInfoService.setInstanceStatus(response.getInstance().getStatus()); - Map vaultMap = new HashMap<>(); - int cnt = response.getInstance().getResourceIdsCount(); - for (int i = 0; i < cnt; i++) { - String name = response.getInstance().getStorageVaultNames(i); - String id = response.getInstance().getResourceIds(i); - vaultMap.put(name, id); - } - Env.getCurrentEnv().getStorageVaultMgr().refreshVaultMap(vaultMap, - Pair.of(response.getInstance().getDefaultStorageVaultName(), - response.getInstance().getDefaultStorageVaultId())); + if (!isResponseValid(response)) { + return; } + + Cloud.InstanceInfoPB instance = response.getInstance(); + cloudSystemInfoService.setInstanceStatus(instance.getStatus()); + syncStorageVault(instance); + processVirtualClusters(instance.getClustersList()); + } catch (Exception e) { LOG.warn("get instance from ms exception", e); } } + + private boolean isResponseValid(Cloud.GetInstanceResponse response) { + if (response == null || !response.hasStatus() || !response.getStatus().hasCode() + || response.getStatus().getCode() != Cloud.MetaServiceCode.OK) { + LOG.warn("failed to get cloud instance due to incomplete response, " + + "cloud_unique_id={}, response={}", Config.cloud_unique_id, response); + return false; + } + return true; + } + + private void syncStorageVault(Cloud.InstanceInfoPB instance) { + Map vaultMap = new HashMap<>(); + int cnt = instance.getResourceIdsCount(); + for (int i = 0; i < cnt; i++) { + String name = instance.getStorageVaultNames(i); + String id = instance.getResourceIds(i); + vaultMap.put(name, id); + } + Env.getCurrentEnv().getStorageVaultMgr().refreshVaultMap(vaultMap, + Pair.of(instance.getDefaultStorageVaultName(), instance.getDefaultStorageVaultId())); + } + + private void processVirtualClusters(List clusters) { + List virtualClusters = new ArrayList<>(); + List computeClusters = new ArrayList<>(); + categorizeClusters(clusters, virtualClusters, computeClusters); + handleVirtualClusters(virtualClusters, computeClusters); + removeObsoleteVirtualGroups(virtualClusters); + } + + private void categorizeClusters(List clusters, + List virtualClusters, List computeClusters) { + for (Cloud.ClusterPB cluster : clusters) { + if (!cluster.hasType()) { + LOG.warn("found a cluster {} which has no type", cluster); + continue; + } + if (Cloud.ClusterPB.Type.COMPUTE == cluster.getType()) { + computeClusters.add(cluster); + } + if (Cloud.ClusterPB.Type.VIRTUAL == cluster.getType()) { + virtualClusters.add(cluster); + } + } + } + + private void handleVirtualClusters(List virtualGroups, List computeClusters) { + for (Cloud.ClusterPB virtualGroupInMs : virtualGroups) { + ComputeGroup virtualGroupInFe = cloudSystemInfoService + .getComputeGroupById(virtualGroupInMs.getClusterId()); + if (virtualGroupInFe != null) { + handleExistingVirtualComputeGroup(virtualGroupInMs, virtualGroupInFe); + } else { + handleNewVirtualComputeGroup(virtualGroupInMs, computeClusters); + } + // just fe master gen file cache sync task + if (Env.getCurrentEnv().isMaster()) { + // get again in fe mem + virtualGroupInFe = cloudSystemInfoService + .getComputeGroupById(virtualGroupInMs.getClusterId()); + if (virtualGroupInFe == null) { + LOG.info("virtual compute can not find virtual group {} after handle, may be a empty vcg", + virtualGroupInMs); + continue; + } + syncFileCacheTasksForVirtualGroup(virtualGroupInMs, virtualGroupInFe); + } + } + } + + private void cancelCacheJobs(ComputeGroup vcgInFe, List jobIds) { + CacheHotspotManager cacheHotspotManager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); + for (String jobId : jobIds) { + try { + if (Env.getCurrentEnv().isMaster()) { + // cancel old jobId, will write editlog, so just master can do + cacheHotspotManager.cancel(Long.parseLong(jobId)); + LOG.info("virtual compute group {}, cancel jobId {}", vcgInFe.getName(), jobId); + } + } catch (DdlException e) { + LOG.warn("virtual compute err, name {}, failed to cancel expired jobId failed {}", + vcgInFe.getName(), jobId, e); + } + } + } + + private void checkNeedRebuildFileCache(ComputeGroup virtualGroupInFe, List jobIdsInMs) { + CacheHotspotManager cacheHotspotManager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); + // check jobIds in Ms valid, if been cancelled, start new jobs + for (String jobId : jobIdsInMs) { + CloudWarmUpJob job = cacheHotspotManager.getCloudWarmUpJob(Long.parseLong(jobId)); + if (job == null) { + LOG.warn("virtual compute err, clusterName {} jobId {} not found, need rebuild file cache", + virtualGroupInFe.getName(), jobId); + virtualGroupInFe.setNeedRebuildFileCache(true); + return; + } + if (job.getSrcClusterName() == null || job.getDstClusterName() == null) { + LOG.info("may be Upgrade after downgrade, warm up job info lost," + + " so just rebuild job, clusterName {}, jobId {}", + virtualGroupInFe.getName(), jobId); + virtualGroupInFe.setNeedRebuildFileCache(true); + return; + } + // check src + String expectedSrc = virtualGroupInFe.getActiveComputeGroup(); + if (!job.getSrcClusterName().equals(expectedSrc)) { + LOG.debug("file cache job src mismatch: jobId {} jobSrc {} expectedSrc {}, need rebuild", + jobId, job.getSrcClusterName(), expectedSrc); + virtualGroupInFe.setNeedRebuildFileCache(true); + return; + } + // check dest + String expectedDst = virtualGroupInFe.getStandbyComputeGroup(); + if (!job.getDstClusterName().equals(expectedDst)) { + LOG.debug("file cache job dest mismatch: jobId {} jobDst {} expectedDst {}, need rebuild", + jobId, job.getDstClusterName(), expectedDst); + virtualGroupInFe.setNeedRebuildFileCache(true); + return; + } + // check job state + CloudWarmUpJob.JobState jobState = job.getJobState(); + if (jobState == CloudWarmUpJob.JobState.CANCELLED) { + LOG.warn("virtual compute err, clusterName {} jobId {} has been cancelled, need rebuild", + virtualGroupInFe.getName(), jobId); + virtualGroupInFe.setNeedRebuildFileCache(true); + return; + } + } + } + + /** + * Generates and synchronizes file cache related tasks for virtual computing groups on the FE master. + */ + private void syncFileCacheTasksForVirtualGroup(Cloud.ClusterPB virtualGroupInMs, ComputeGroup virtualGroupInFe) { + if (!virtualGroupInMs.hasClusterPolicy()) { + LOG.warn("virtual compute err, clusterName {}, no cluster policy {}", + virtualGroupInFe.getName(), virtualGroupInMs); + return; + } + CacheHotspotManager cacheHotspotManager = ((CloudEnv) Env.getCurrentEnv()).getCacheHotspotMgr(); + List jobIdsInMs = + new ArrayList<>(virtualGroupInMs.getClusterPolicy().getCacheWarmupJobidsList()); + + checkNeedRebuildFileCache(virtualGroupInFe, jobIdsInMs); + LOG.debug("virtual compute group {}, get from ms file cache sync task jobIds {}", + virtualGroupInFe, jobIdsInMs); + // virtual group has been changed in before step + if (virtualGroupInFe.isNeedRebuildFileCache()) { + String srcCg = virtualGroupInFe.getActiveComputeGroup(); + String dstCg = virtualGroupInFe.getStandbyComputeGroup(); + cancelCacheJobs(virtualGroupInFe, jobIdsInMs); + try { + // all + Map periodicProperties = new HashMap<>(); + // "sync_mode" = "periodic", "sync_interval_sec" = "fetch_cluster_cache_hotspot_interval_ms" + periodicProperties.put("sync_mode", "periodic"); + long syncInterValSec = Config.fetch_cluster_cache_hotspot_interval_ms / 1000; + if (syncInterValSec <= 0) { + LOG.warn("invalid Config fetch_cluster_cache_hotspot_interval_ms set it to 600s"); + syncInterValSec = 600; + } + periodicProperties.put("sync_interval_sec", String.valueOf(syncInterValSec)); + WarmUpClusterStmt periodicStmtPeriodic = + new WarmUpClusterStmt(dstCg, srcCg, true, periodicProperties); + long jobIdPeriodic = cacheHotspotManager.createJob(periodicStmtPeriodic); + + // load event + Map eventProperties = new HashMap<>(); + // "sync_mode" = "event_driven", "sync_event" = "load" + eventProperties.put("sync_mode", "event_driven"); + eventProperties.put("sync_event", "load"); + WarmUpClusterStmt eventStmtPeriodic = + new WarmUpClusterStmt(dstCg, srcCg, true, eventProperties); + long jobIdEvent = cacheHotspotManager.createJob(eventStmtPeriodic); + // send jobIds to ms + List newJobIds = Arrays.asList(Long.toString(jobIdPeriodic), Long.toString(jobIdEvent)); + CloudSystemInfoService.updateFileCacheJobIds(virtualGroupInFe, newJobIds); + LOG.info("virtual compute group {}, generate new jobIds periodic={}, event={}, and old jobIds {}", + virtualGroupInFe, jobIdPeriodic, jobIdEvent, jobIdsInMs); + } catch (AnalysisException e) { + LOG.warn("virtual compute err, name: {}, analysis error", virtualGroupInFe.getName(), e); + return; + } + virtualGroupInFe.setNeedRebuildFileCache(false); + } + } + + private void handleExistingVirtualComputeGroup(Cloud.ClusterPB clusterInMs, ComputeGroup virtualGroupInFe) { + if (!isClusterIdConsistent(clusterInMs, virtualGroupInFe)) { + return; + } + + if (!isClusterPolicyValid(clusterInMs)) { + return; + } + + if (!areSubComputeGroupsValid(clusterInMs, virtualGroupInFe)) { + return; + } + + diffAndUpdateComputeGroup(clusterInMs, virtualGroupInFe); + } + + private boolean isClusterIdConsistent(Cloud.ClusterPB cluster, ComputeGroup computeGroup) { + if (!cluster.getClusterId().equals(computeGroup.getId())) { + LOG.warn("virtual compute err, group id changed, in fe={} but in ms={}, " + + "verbose {}, please check it", + computeGroup.getId(), cluster.getClusterId(), computeGroup); + return false; + } + return true; + } + + private boolean isClusterPolicyValid(Cloud.ClusterPB cluster) { + if (!cluster.hasClusterPolicy()) { + LOG.warn("virtual compute err, no cluster policy {}", cluster); + } + if (!cluster.getClusterPolicy().hasType() + || cluster.getClusterPolicy().getType() != Cloud.ClusterPolicy.PolicyType.ActiveStandby) { + LOG.warn("virtual compute err, current just support Virtual compute group policy ActiveStandby"); + return false; + } + if (cluster.getClusterPolicy().getStandbyClusterNamesList().size() != 1) { + LOG.warn("virtual compute err, current just support one Standby compute group policy ActiveStandby," + + " verbose {}", cluster); + return false; + } + return true; + } + + private boolean areSubComputeGroupsValid(Cloud.ClusterPB clusterInMs, ComputeGroup virtualGroupInFe) { + List subComputeGroups = clusterInMs.getClusterNamesList(); + if (subComputeGroups.isEmpty() || virtualGroupInFe.getSubComputeGroups() == null) { + LOG.warn("virtual compute err, please check it, verbose {}", virtualGroupInFe); + return false; + } + if (subComputeGroups.size() != virtualGroupInFe.getSubComputeGroups().size() || subComputeGroups.size() != 2) { + LOG.warn("virtual compute err, sub compute group in fe {}, in ms {}", + virtualGroupInFe, subComputeGroups); + return false; + } + return true; + } + + private void diffAndUpdateComputeGroup(Cloud.ClusterPB cluster, ComputeGroup computeGroup) { + // vcg rename logic, here cluster_id same, but cluster_name changed, so vcg renamed + String clusterNameInMs = cluster.getClusterName(); + String computeGroupNameInFe = computeGroup.getName(); + if (!clusterNameInMs.equals(computeGroupNameInFe)) { + LOG.info("virtual compute group renamed from {} to {}", computeGroupNameInFe, clusterNameInMs); + computeGroup.setName(clusterNameInMs); + cloudSystemInfoService.renameVirtualComputeGroup(computeGroup.getId(), computeGroupNameInFe, computeGroup); + } + + List subCgsInFe = computeGroup.getSubComputeGroups(); + List subCgsInMs = new ArrayList<>(cluster.getClusterNamesList()); + Collections.sort(subCgsInFe); + Collections.sort(subCgsInMs); + + if (!subCgsInFe.equals(subCgsInMs)) { + LOG.info("virtual compute group change sub cgs from {} to {}", subCgsInFe, subCgsInMs); + computeGroup.setSubComputeGroups(subCgsInMs); + computeGroup.setNeedRebuildFileCache(true); + } + + if (!cluster.getClusterPolicy().getActiveClusterName() + .equals(computeGroup.getPolicy().activeComputeGroup)) { + LOG.info("virtual compute group change active group from {} to {}", + computeGroup.getPolicy().activeComputeGroup, + cluster.getClusterPolicy().getActiveClusterName()); + computeGroup.getPolicy().setActiveComputeGroup(cluster.getClusterPolicy().getActiveClusterName()); + computeGroup.setNeedRebuildFileCache(true); + } + + if (!cluster.getClusterPolicy().getStandbyClusterNames(0) + .equals(computeGroup.getPolicy().standbyComputeGroup)) { + LOG.info("virtual compute group change standby group from {} to {}", + computeGroup.getPolicy().standbyComputeGroup, + cluster.getClusterPolicy().getStandbyClusterNames(0)); + computeGroup.getPolicy().setStandbyComputeGroup(cluster.getClusterPolicy().getStandbyClusterNames(0)); + computeGroup.setNeedRebuildFileCache(true); + } + + if (cluster.getClusterPolicy().getFailoverFailureThreshold() + != computeGroup.getPolicy().failoverFailureThreshold) { + LOG.info("virtual compute group change failover failure threshold from {} to {}", + computeGroup.getPolicy().failoverFailureThreshold, + cluster.getClusterPolicy().getFailoverFailureThreshold()); + computeGroup.getPolicy() + .setFailoverFailureThreshold(cluster.getClusterPolicy().getFailoverFailureThreshold()); + } + + if (cluster.getClusterPolicy().getUnhealthyNodeThresholdPercent() + != computeGroup.getPolicy().unhealthyNodeThresholdPercent) { + LOG.info("virtual compute group change unhealthy node threshold from {} to {}", + computeGroup.getPolicy().unhealthyNodeThresholdPercent, + cluster.getClusterPolicy().getUnhealthyNodeThresholdPercent()); + computeGroup.getPolicy() + .setUnhealthyNodeThresholdPercent(cluster.getClusterPolicy().getUnhealthyNodeThresholdPercent()); + } + + List jobIdsInMs = + new ArrayList<>(cluster.getClusterPolicy().getCacheWarmupJobidsList()); + List jobIdsInFe = computeGroup.getPolicy().getCacheWarmupJobIds(); + if (!jobIdsInMs.equals(jobIdsInFe)) { + LOG.debug("set exist vcg {}, jobIds in FE {} in Ms ms {}", + cluster.getClusterName(), jobIdsInFe, jobIdsInMs); + computeGroup.getPolicy().setCacheWarmupJobIds(jobIdsInMs); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("after diff virtual cg info {}", computeGroup); + } + } + + private void handleNewVirtualComputeGroup(Cloud.ClusterPB cluster, List computeClusters) { + List subComputeGroups = cluster.getClusterNamesList(); + if (subComputeGroups.isEmpty()) { + LOG.info("found virtual cluster {} which has no sub clusters, skip empty virtual cluster", cluster); + return; + } + if (subComputeGroups.size() != 2) { + LOG.warn("virtual compute err, sub compute group size not eq 2, in ms {}", subComputeGroups); + return; + } + if (!cluster.hasClusterPolicy()) { + LOG.warn("virtual compute err, no cluster policy {}", cluster); + return; + } + if (!cluster.getClusterPolicy().hasActiveClusterName()) { + LOG.warn("virtual compute err, active cluster empty in ms {}", cluster); + return; + } + if (cluster.getClusterPolicy().getStandbyClusterNamesList().size() != 1) { + LOG.warn("virtual compute err, standby cluster size not eq 1 in ms {}", cluster); + return; + } + checkSubClusters(subComputeGroups, cluster, computeClusters); + ComputeGroup computeGroup = new ComputeGroup(cluster.getClusterId(), + cluster.getClusterName(), ComputeGroup.ComputeTypeEnum.VIRTUAL); + computeGroup.setSubComputeGroups(new ArrayList<>(subComputeGroups)); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(cluster.getClusterPolicy().getActiveClusterName()); + policy.setStandbyComputeGroup(cluster.getClusterPolicy().getStandbyClusterNames(0)); + policy.setFailoverFailureThreshold(cluster.getClusterPolicy().getFailoverFailureThreshold()); + policy.setUnhealthyNodeThresholdPercent(cluster.getClusterPolicy().getUnhealthyNodeThresholdPercent()); + computeGroup.setPolicy(policy); + computeGroup.setNeedRebuildFileCache(true); + cloudSystemInfoService.addComputeGroup(cluster.getClusterId(), computeGroup); + MetricRepo.registerCloudMetrics(cluster.getClusterId(), cluster.getClusterName()); + } + + private void checkSubClusters(List subClusterNames, Cloud.ClusterPB cluster, + List computeClustersInPB) { + for (String subClusterName : subClusterNames) { + if (cloudSystemInfoService.getCloudClusterIdByName(subClusterName) == null) { + handleFailedSync(cluster, subClusterName, computeClustersInPB); + continue; + } + // CloudClusterChecker find sub compute group + lastFailedSyncTimeMap.remove(cluster.getClusterName()); + } + } + + private void handleFailedSync(Cloud.ClusterPB cluster, String subClusterName, + List computeClustersInPB) { + if (!lastFailedSyncTimeMap.containsKey(cluster.getClusterName())) { + lastFailedSyncTimeMap.put(cluster.getClusterName(), System.currentTimeMillis()); + } else { + List computeGroupsInPb = computeClustersInPB.stream() + .map(Cloud.ClusterPB::getClusterName).collect(Collectors.toList()); + if (computeGroupsInPb.contains(subClusterName)) { + LOG.warn("fe mem cant find {}, it may be wait cluster check to sync", subClusterName); + } else { + LOG.warn("fe mem and ms cant find {}, it may be dropped or renamed", subClusterName); + } + // sub cluster may be dropped or rename, or fe may be slowly, + // need manual intervention + if (System.currentTimeMillis() - lastFailedSyncTimeMap.get(cluster.getClusterName()) + > 3 * Config.cloud_cluster_check_interval_second * 1000L) { + LOG.warn("virtual compute err, cant find cluster info by cluster checker, " + + "sub cluster: {}, virtual cluster: {}", subClusterName, cluster.getClusterName()); + } + } + } + + private void removeObsoleteVirtualGroups(List virtualClusters) { + List msVirtualClusters = virtualClusters.stream().map(Cloud.ClusterPB::getClusterId) + .collect(Collectors.toList()); + for (ComputeGroup computeGroup : cloudSystemInfoService.getComputeGroups(true)) { + // in fe mem, but not in meta server + if (!msVirtualClusters.contains(computeGroup.getId())) { + LOG.info("virtual compute group {} will be removed.", computeGroup.getName()); + cloudSystemInfoService.removeComputeGroup(computeGroup.getId(), computeGroup.getName()); + // cancel invalid job + if (!computeGroup.getPolicy().getCacheWarmupJobIds().isEmpty()) { + cancelCacheJobs(computeGroup, computeGroup.getPolicy().getCacheWarmupJobIds()); + } + } + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java index fce12193e3571f..6e75bd6ee09e93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudReplica.java @@ -164,12 +164,15 @@ public long getBackendId() throws ComputeGroupException { public long getBackendId(String beEndpoint) { String clusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getClusterNameByBeAddr(beEndpoint); + String physicalClusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getPhysicalCluster(clusterName); + try { - String clusterId = getCloudClusterIdByName(clusterName); + String clusterId = getCloudClusterIdByName(physicalClusterName); return getBackendIdImpl(clusterId); } catch (ComputeGroupException e) { if (LOG.isDebugEnabled()) { - LOG.debug("failed to get compute group name {}", clusterName, e); + LOG.debug("failed to get compute group name {}", physicalClusterName, e); } return -1; } @@ -213,7 +216,8 @@ private String getCurrentClusterId() throws ComputeGroupException { ConnectContext context = ConnectContext.get(); if (context != null) { if (!Strings.isNullOrEmpty(context.getSessionVariable().getCloudCluster())) { - cluster = context.getSessionVariable().getCloudCluster(); + cluster = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getPhysicalCluster(context.getSessionVariable().getCloudCluster()); try { ((CloudEnv) Env.getCurrentEnv()).checkCloudClusterPriv(cluster); } catch (Exception e) { @@ -226,7 +230,8 @@ private String getCurrentClusterId() throws ComputeGroupException { LOG.debug("get compute group by session context compute group: {}", cluster); } } else { - cluster = context.getCloudCluster(false); + cluster = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getPhysicalCluster(context.getCloudCluster(false)); if (LOG.isDebugEnabled()) { LOG.debug("get compute group by context {}", cluster); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java new file mode 100644 index 00000000000000..82c24afd7c5d75 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/ComputeGroup.java @@ -0,0 +1,177 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.catalog; + +import org.apache.doris.cloud.proto.Cloud; + +import com.google.gson.Gson; +import lombok.Getter; +import lombok.Setter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class ComputeGroup { + private static final Logger LOG = LogManager.getLogger(ComputeGroup.class); + + private enum PolicyTypeEnum { + ActiveStandby, + } + + public enum ComputeTypeEnum { + SQL, + COMPUTE, + VIRTUAL, + } + + @Getter + @Setter + public static class Policy { + PolicyTypeEnum policyType; + String activeComputeGroup; + String standbyComputeGroup; + long failoverFailureThreshold = 3; + long unhealthyNodeThresholdPercent = 100; + List cacheWarmupJobIds = new ArrayList<>(); + + public Policy() { + policyType = PolicyTypeEnum.ActiveStandby; + } + + @Override + public String toString() { + Map showMap = new LinkedHashMap<>(); + showMap.put("policyType", policyType.toString()); + showMap.put("activeComputeGroup", activeComputeGroup); + showMap.put("standbyComputeGroup", standbyComputeGroup); + showMap.put("failoverFailureThreshold", String.valueOf(failoverFailureThreshold)); + showMap.put("unhealthyNodeThresholdPercent", String.valueOf(unhealthyNodeThresholdPercent)); + showMap.put("cacheWarmupJobIds", String.valueOf(cacheWarmupJobIds)); + Gson gson = new Gson(); + return gson.toJson(showMap); + } + + public Cloud.ClusterPolicy toPb() { + return Cloud.ClusterPolicy.newBuilder() + .setType(Cloud.ClusterPolicy.PolicyType.ActiveStandby) + .setActiveClusterName(activeComputeGroup) + .addStandbyClusterNames(standbyComputeGroup) + .setFailoverFailureThreshold(failoverFailureThreshold) + .setUnhealthyNodeThresholdPercent(unhealthyNodeThresholdPercent) + .build(); + } + } + + @Getter + private String id; + @Getter + // cg can be renamed + @Setter + private String name; + @Getter + private ComputeTypeEnum type; + + // record sub cluster name + @Getter + @Setter + private List subComputeGroups; + @Getter + @Setter + private long unavailableSince = -1; + @Getter + @Setter + private long availableSince = -1; + @Getter + @Setter + private Policy policy; + @Getter + @Setter + private String currentClusterName; + @Getter + @Setter + private boolean needRebuildFileCache = false; + + public ComputeGroup(String id, String name, ComputeTypeEnum type) { + this.id = id; + this.name = name; + this.type = type; + } + + public boolean isVirtual() { + return type == ComputeTypeEnum.VIRTUAL; + } + + public String getActiveComputeGroup() { + if (policy == null) { + return "empty_policy"; + } + return policy.getActiveComputeGroup(); + } + + public String getStandbyComputeGroup() { + if (policy == null) { + return "empty_policy"; + } + return policy.getStandbyComputeGroup(); + } + + @Override + public String toString() { + Map showMap = new LinkedHashMap<>(); + showMap.put("id", id); + showMap.put("name", name); + showMap.put("type", type.toString()); + showMap.put("unavailableSince", String.valueOf(unavailableSince)); + showMap.put("availableSince", String.valueOf(availableSince)); + showMap.put("policy", policy == null ? "no_policy" : policy.toString()); + Gson gson = new Gson(); + return gson.toJson(showMap); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ComputeGroup)) { + return false; + } + ComputeGroup that = (ComputeGroup) o; + return unavailableSince == that.unavailableSince + && availableSince == that.availableSince + && id.equals(that.id) + && name.equals(that.name) + && type == that.type + && (policy != null ? policy.equals(that.policy) : that.policy == null); + } + + @Override + public int hashCode() { + int result = id.hashCode(); + result = 31 * result + name.hashCode(); + result = 31 * result + (type != null ? type.hashCode() : 0); + result = 31 * result + (int) (unavailableSince ^ (unavailableSince >>> 32)); + result = 31 * result + (int) (availableSince ^ (availableSince >>> 32)); + result = 31 * result + (policy != null ? policy.hashCode() : 0); + return result; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index f3583fa54877cb..a20972760772a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.ClusterPB; import org.apache.doris.cloud.proto.Cloud.InstanceInfoPB; @@ -88,8 +89,120 @@ public class CloudSystemInfoService extends SystemInfoService { // clusterName -> clusterId protected Map clusterNameToId = new ConcurrentHashMap<>(); + // clusterId -> ComputeGroup + protected Map computeGroupIdToComputeGroup = new ConcurrentHashMap<>(); + private InstanceInfoPB.Status instanceStatus; + public void addVirtualClusterInfoToMapsNoLock(String clusterId, String clusterName) { + LOG.info("add virtual cluster info to maps, clusterId={}, clusterName={}", clusterId, clusterName); + clusterNameToId.put(clusterName, clusterId); + clusterIdToBackend.computeIfAbsent(clusterId, k -> new ArrayList<>()); + } + + public void removeVirtualClusterInfoFromMapsNoLock(String clusterId, String clusterName) { + LOG.info("remove virtual cluster info from maps, clusterId={}, clusterName={}", clusterId, clusterName); + clusterIdToBackend.remove(clusterId); + clusterNameToId.remove(clusterName); + } + + public void renameVirtualClusterInfoFromMapsNoLock(String clusterId, String oldClusterName, String newClusterName) { + LOG.info("remove virtual cluster info from maps, clusterId={}, name from {} to {}", + clusterId, oldClusterName, newClusterName); + clusterNameToId.put(newClusterName, clusterId); + clusterNameToId.remove(oldClusterName); + } + + public ComputeGroup getComputeGroupByName(String computeGroupName) { + LOG.debug("get id {} computeGroupIdToComputeGroup : {} ", computeGroupName, computeGroupIdToComputeGroup); + try { + rlock.lock(); + if (!clusterNameToId.containsKey(computeGroupName)) { + return null; + } + return computeGroupIdToComputeGroup.get(clusterNameToId.get(computeGroupName)); + } finally { + rlock.unlock(); + } + } + + public ComputeGroup getComputeGroupById(String computeGroupId) { + try { + rlock.lock(); + return computeGroupIdToComputeGroup.get(computeGroupId); + } finally { + rlock.unlock(); + } + } + + public void addComputeGroup(String computeGroupId, ComputeGroup computeGroup) { + LOG.debug("add id {} computeGroupIdToComputeGroup : {} ", computeGroupId, computeGroupIdToComputeGroup); + try { + wlock.lock(); + computeGroupIdToComputeGroup.put(computeGroupId, computeGroup); + addVirtualClusterInfoToMapsNoLock(computeGroupId, computeGroup.getName()); + } finally { + wlock.unlock(); + } + } + + public List getComputeGroups(boolean virtual) { + LOG.debug("get virtual {} computeGroupIdToComputeGroup : {} ", virtual, computeGroupIdToComputeGroup); + try { + rlock.lock(); + return computeGroupIdToComputeGroup.values().stream().filter(computeGroup -> { + if (virtual) { + return computeGroup.isVirtual(); + } + return true; + }).collect(Collectors.toList()); + } finally { + rlock.unlock(); + } + } + + // get sub compute group Owned by which virtual compute group + public String ownedByVirtualComputeGroup(String computeGroupName) { + try { + rlock.lock(); + for (ComputeGroup vcg : getComputeGroups(true)) { + if (computeGroupName.equals(vcg.getPolicy().getActiveComputeGroup())) { + return vcg.getName(); + } + if (vcg.getPolicy().getStandbyComputeGroup().contains(computeGroupName)) { + return vcg.getName(); + } + if (vcg.getSubComputeGroups().contains(computeGroupName)) { + return vcg.getName(); + } + } + return null; + } finally { + rlock.unlock(); + } + } + + public void removeComputeGroup(String computeGroupId, String computeGroupName) { + try { + wlock.lock(); + computeGroupIdToComputeGroup.remove(computeGroupId); + removeVirtualClusterInfoFromMapsNoLock(computeGroupId, computeGroupName); + } finally { + wlock.unlock(); + } + } + + public void renameVirtualComputeGroup(String computeGroupId, String oldComputeGroupName, + ComputeGroup newComputeGroup) { + try { + wlock.lock(); + computeGroupIdToComputeGroup.put(computeGroupId, newComputeGroup); + renameVirtualClusterInfoFromMapsNoLock(computeGroupId, oldComputeGroupName, newComputeGroup.getName()); + } finally { + wlock.unlock(); + } + } + @Override public Pair>, TStorageMedium> selectBackendIdsForReplicaCreation( ReplicaAllocation replicaAlloc, Map nextIndexs, @@ -232,6 +345,10 @@ public void updateCloudClusterMapNoLock(List toAdd, List toDel } clusterNameToId.put(clusterName, clusterId); + // add to computeGroupIdToComputeGroup + ComputeGroup cg = new ComputeGroup(clusterId, clusterName, ComputeGroup.ComputeTypeEnum.COMPUTE); + addComputeGroup(clusterId, cg); + List be = clusterIdToBackend.get(clusterId); if (be == null) { be = new ArrayList<>(); @@ -278,6 +395,10 @@ public void updateCloudClusterMapNoLock(List toAdd, List toDel if (be.isEmpty()) { LOG.info("del clusterId {} and clusterName {} due to be nodes eq 0", clusterId, clusterName); boolean succ = clusterNameToId.remove(clusterName, clusterId); + + // remove from computeGroupIdToComputeGroup + removeComputeGroup(clusterId, clusterName); + if (!succ) { LOG.warn("impossible, somewhere err, clusterNameToId {}, " + "want remove cluster name {}, cluster id {}", @@ -322,6 +443,82 @@ public synchronized void updateFrontends(List toAdd, List to } } + public static void updateFileCacheJobIds(ComputeGroup cg, List jobIds) { + Cloud.ClusterPolicy policy = Cloud.ClusterPolicy.newBuilder() + .setType(Cloud.ClusterPolicy.PolicyType.ActiveStandby) + .addAllCacheWarmupJobids(jobIds).build(); + + Cloud.ClusterPB clusterPB = Cloud.ClusterPB.newBuilder() + .setClusterId(cg.getId()) + .setClusterPolicy(policy) + .build(); + Cloud.AlterClusterRequest request = Cloud.AlterClusterRequest.newBuilder() + .setCloudUniqueId(Config.cloud_unique_id) + .setOp(Cloud.AlterClusterRequest.Operation.ALTER_VCLUSTER_INFO) + .setCluster(clusterPB) + .build(); + + Cloud.AlterClusterResponse response; + try { + response = MetaServiceProxy.getInstance().alterCluster(request); + LOG.info("update file cache jobIds, request: {}, response: {}", request, response); + if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) { + LOG.warn("update file cache jobIds, response: {}", response); + } + } catch (RpcException e) { + LOG.warn("failed to update file cache jobIds {}", cg, e); + } + } + + /* + message ClusterPolicy { + enum PolicyType { + ActiveStandby = 0; + } + optional PolicyType type = 1; + optional string active_cluster_name = 2; + repeated string standby_cluster_names = 3; + optional int64 failover_failure_threshold = 4; + optional int64 unhealthy_node_threshold_percent = 5; + repeated string cacheWarmupJobIds = 6; + } + */ + + private void switchActiveStandby(ComputeGroup cg, String active, String standby) { + Cloud.ClusterPolicy policy = cg.getPolicy().toPb().toBuilder() + .clearStandbyClusterNames() + .addStandbyClusterNames(active) + .setActiveClusterName(standby) + .build(); + + Cloud.ClusterPB clusterPB = Cloud.ClusterPB.newBuilder() + .setClusterId(cg.getId()) + .setClusterPolicy(policy) + .build(); + + Cloud.AlterClusterRequest request = Cloud.AlterClusterRequest.newBuilder() + .setInstanceId(((CloudEnv) Env.getCurrentEnv()).getCloudInstanceId()) + .setOp(Cloud.AlterClusterRequest.Operation.ALTER_VCLUSTER_INFO) + .setCluster(clusterPB) + .build(); + + Cloud.AlterClusterResponse response; + try { + response = MetaServiceProxy.getInstance().alterCluster(request); + LOG.info("switch compute group, request: {}, response: {}", request, response); + if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) { + LOG.warn("failed to switch compute group, response: {}", response); + /* + throw new DdlException("failed to alter backends errorCode: " + response.getStatus().getCode() + + " msg: " + response.getStatus().getMsg()); + */ + } + } catch (RpcException e) { + //throw new DdlException("failed to alter backends", e); + LOG.warn("failed to switch compute group {}", e); + } + } + private void alterBackendCluster(List hostInfos, String computeGroupId, String cloudUniqueId, Cloud.AlterClusterRequest.Operation operation) throws DdlException { if (Strings.isNullOrEmpty(((CloudEnv) Env.getCurrentEnv()).getCloudInstanceId())) { @@ -546,7 +743,8 @@ public ImmutableMap getBackendsByCurrentCluster() throws Analysis } public List getBackendsByClusterName(final String clusterName) { - String clusterId = clusterNameToId.getOrDefault(clusterName, ""); + String physicalClusterName = getPhysicalCluster(clusterName); + String clusterId = clusterNameToId.getOrDefault(physicalClusterName, ""); if (clusterId.isEmpty()) { return new ArrayList<>(); } @@ -555,8 +753,86 @@ public List getBackendsByClusterName(final String clusterName) { } public List getBackendsByClusterId(final String clusterId) { + String clusterName = getClusterNameByClusterId(clusterId); + String physicalClusterName = getPhysicalCluster(clusterName); + String physicalClusterId = getCloudClusterIdByName(physicalClusterName); + // copy a new List - return new ArrayList<>(clusterIdToBackend.getOrDefault(clusterId, new ArrayList<>())); + return new ArrayList<>(clusterIdToBackend.getOrDefault(physicalClusterId, new ArrayList<>())); + } + + public String getPhysicalCluster(String clusterName) { + ComputeGroup cg = getComputeGroupByName(clusterName); + if (cg == null) { + return clusterName; + } + + if (!cg.isVirtual()) { + return clusterName; + } + + ComputeGroup.Policy policy = cg.getPolicy(); + // todo check policy + String acgName = policy.getActiveComputeGroup(); + if (acgName != null) { + ComputeGroup acg = getComputeGroupByName(acgName); + if (acg != null) { + if (isComputeGroupAvailable(acgName, policy.getUnhealthyNodeThresholdPercent())) { + acg.setUnavailableSince(-1); + return acgName; + } else { + if (acg.getUnavailableSince() <= 0) { + acg.setUnavailableSince(System.currentTimeMillis()); + } + } + } + } + + String scgName = policy.getStandbyComputeGroup(); + if (scgName != null) { + ComputeGroup scg = getComputeGroupByName(scgName); + if (scg != null) { + if (isComputeGroupAvailable(scgName, policy.getUnhealthyNodeThresholdPercent())) { + scg.setUnavailableSince(-1); + ComputeGroup acg = getComputeGroupByName(acgName); + if (acg == null || System.currentTimeMillis() - acg.getUnavailableSince() + > policy.getFailoverFailureThreshold() * Config.heartbeat_interval_second * 1000) { + switchActiveStandby(cg, acgName, scgName); + policy.setActiveComputeGroup(scgName); + policy.setStandbyComputeGroup(acgName); + cg.setNeedRebuildFileCache(true); + } + return scgName; + } else { + scg.setUnavailableSince(System.currentTimeMillis()); + } + } + } + + if (acgName != null) { + return acgName; + } else { + return clusterName; + } + } + + public boolean isComputeGroupAvailable(String cg, long unhealthyNodeThresholdPercent) { + List bes = getBackendsByClusterName(cg); + if (bes == null || bes.isEmpty()) { + return false; + } + + long deadBeNum = 0; + for (Backend be : bes) { + if (!be.isAlive()) { + deadBeNum++; + } + } + if (deadBeNum * 100 / bes.size() >= unhealthyNodeThresholdPercent) { + return false; + } + + return true; } public String getClusterNameByBeAddr(String beEndpoint) { @@ -623,6 +899,7 @@ public void updateClusterNameToId(final String newName, } public String getCloudClusterIdByName(String clusterName) { + LOG.debug("clusterNameToId {}", clusterNameToId); return clusterNameToId.get(clusterName); } @@ -754,10 +1031,24 @@ public String addCloudCluster(final String clusterName, final String userName) t } // Return the ref of concurrentMap clusterIdToBackend - public Map> getCloudClusterIdToBackend() { + public Map> getCloudClusterIdToBackend(boolean needVirtual) { + Map> map = new ConcurrentHashMap<>(); rlock.lock(); try { - return new ConcurrentHashMap<>(clusterIdToBackend); + for (Map.Entry> idBe : clusterIdToBackend.entrySet()) { + String clusterId = idBe.getKey(); + if (!computeGroupIdToComputeGroup.containsKey(clusterId)) { + LOG.warn("cant find clusterId {} in computeGroupIdToComputeGroup {}", + clusterId, computeGroupIdToComputeGroup); + continue; + } + ComputeGroup computeGroup = computeGroupIdToComputeGroup.get(clusterId); + if (!needVirtual && computeGroup.isVirtual()) { + continue; + } + map.put(clusterId, idBe.getValue()); + } + return map; } finally { rlock.unlock(); } @@ -787,10 +1078,24 @@ public ImmutableMap getCloudIdToBackendNoLock(String clusterName) } // Return the ref of concurrentMap clusterNameToId - public Map getCloudClusterNameToId() { + public Map getCloudClusterNameToId(boolean needVirtual) { + Map map = new ConcurrentHashMap<>(); rlock.lock(); try { - return new ConcurrentHashMap<>(clusterNameToId); + for (Map.Entry nameAndId : clusterNameToId.entrySet()) { + String clusterId = nameAndId.getValue(); + ComputeGroup computeGroup = computeGroupIdToComputeGroup.get(clusterId); + if (computeGroup == null) { + LOG.warn("cant find clusterId {} in computeGroupIdToComputeGroup {}", + clusterId, computeGroupIdToComputeGroup); + continue; + } + if (!needVirtual && computeGroup.isVirtual()) { + continue; + } + map.put(nameAndId.getKey(), clusterId); + } + return map; } finally { rlock.unlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java index 5d5b083664ead9..ef435ad0d41d62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/manager/ClusterAction.java @@ -116,7 +116,7 @@ public Object cloudClusterInfo(HttpServletRequest request, HttpServletResponse r // Key: cluster_name Value: be status Map> result = Maps.newHashMap(); - ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getCloudClusterIdToBackend() + ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getCloudClusterIdToBackend(false) .forEach((clusterId, backends) -> { List bis = backends.stream().map(backend -> { BeClusterInfo bi = new BeClusterInfo(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java index 45e738e8350f2b..18707eeb6faea8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java @@ -1011,7 +1011,7 @@ public static void increaseClusterRequestAll(String clusterName) { return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } @@ -1029,7 +1029,7 @@ public static void increaseClusterQueryAll(String clusterName) { return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } @@ -1047,7 +1047,7 @@ public static void increaseClusterQueryErr(String clusterName) { return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } @@ -1065,7 +1065,7 @@ public static void increaseClusterWarmUpJobExecCount(String clusterName) { return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } @@ -1083,7 +1083,7 @@ public static void increaseClusterWarmUpJobRequestedTablets(String clusterName, return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } @@ -1101,7 +1101,7 @@ public static void increaseClusterWarmUpJobFinishedTablets(String clusterName, l return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } @@ -1210,7 +1210,7 @@ public static void updateClusterQueryLatency(String clusterName, long elapseMs) return; } String clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterNameToId().get(clusterName); + .getCloudClusterNameToId(true).get(clusterName); if (Strings.isNullOrEmpty(clusterId)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java index 4f87574924b00e..3d6f496d54b200 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Role.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.UserIdentity; import org.apache.doris.analysis.WorkloadGroupPattern; import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; @@ -460,7 +461,8 @@ public boolean checkCloudPriv(String cloudName, } if (checkGlobalInternal(wanted, savedPrivs) - || checkCloudInternal(cloudName, wanted, savedPrivs, cloudPrivTable, type)) { + || checkCloudInternal(cloudName, wanted, savedPrivs, cloudPrivTable, type) + || checkCloudVirtualComputeGroup(cloudName, wanted, savedPrivs, cloudPrivTable)) { return true; } @@ -532,6 +534,17 @@ private boolean checkCloudInternal(String cloudName, PrivPredicate wanted, return Privilege.satisfy(savedPrivs, wanted); } + private boolean checkCloudVirtualComputeGroup(String cloudName, PrivPredicate wanted, + PrivBitSet savedPrivs, ResourcePrivTable table) { + String virtualComputeGroupName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .ownedByVirtualComputeGroup(cloudName); + if (virtualComputeGroupName == null) { + return false; + } + table.getPrivs(virtualComputeGroupName, savedPrivs); + return Privilege.satisfy(savedPrivs, wanted); + } + public boolean checkWorkloadGroupPriv(String workloadGroupName, PrivPredicate wanted, PrivBitSet savedPrivs) { // For compatibility with older versions, it is not needed to check the privileges of the default group. if (WorkloadGroupMgr.DEFAULT_GROUP_NAME.equals(workloadGroupName)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java index ecfd08aaa710b9..ebed76bf78bc6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java @@ -26,6 +26,7 @@ import org.apache.doris.analysis.ValueList; import org.apache.doris.catalog.Env; import org.apache.doris.cloud.qe.ComputeGroupException; +import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; import org.apache.doris.common.util.DebugUtil; @@ -226,14 +227,23 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme MetricRepo.COUNTER_QUERY_ALL.increase(1L); MetricRepo.USER_COUNTER_QUERY_ALL.getOrAdd(ctx.getQualifiedUser()).increase(1L); } + String physicalClusterName = ""; try { if (Config.isCloudMode()) { cloudCluster = ctx.getCloudCluster(false); + physicalClusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getPhysicalCluster(cloudCluster); + if (!cloudCluster.equals(physicalClusterName)) { + // vcg + MetricRepo.increaseClusterQueryAll(physicalClusterName); + } } } catch (ComputeGroupException e) { - LOG.warn("Failed to get cloud cluster", e); + LOG.warn("Failed to get cloud cluster, cloudCluster={}, physicalClusterName={} ", + cloudCluster, physicalClusterName, e); return; } + MetricRepo.increaseClusterQueryAll(cloudCluster); if (ctx.getState().getStateType() == MysqlStateType.ERR && ctx.getState().getErrType() != QueryState.ErrType.ANALYSIS_ERR) { @@ -241,7 +251,14 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme if (!ctx.getSessionVariable().internalSession && MetricRepo.isInit) { MetricRepo.COUNTER_QUERY_ERR.increase(1L); MetricRepo.USER_COUNTER_QUERY_ERR.getOrAdd(ctx.getQualifiedUser()).increase(1L); - MetricRepo.increaseClusterQueryErr(cloudCluster); + if (cloudCluster.equals(physicalClusterName)) { + // not vcg + MetricRepo.increaseClusterQueryErr(cloudCluster); + } else { + // vcg + MetricRepo.increaseClusterQueryErr(cloudCluster); + MetricRepo.increaseClusterQueryErr(physicalClusterName); + } } } else if (ctx.getState().getStateType() == MysqlStateType.OK || ctx.getState().getStateType() == MysqlStateType.EOF) { @@ -249,7 +266,14 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme if (!ctx.getSessionVariable().internalSession && MetricRepo.isInit) { MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs); MetricRepo.USER_HISTO_QUERY_LATENCY.getOrAdd(ctx.getQualifiedUser()).update(elapseMs); - MetricRepo.updateClusterQueryLatency(cloudCluster, elapseMs); + if (cloudCluster.equals(physicalClusterName)) { + // not vcg + MetricRepo.updateClusterQueryLatency(cloudCluster, elapseMs); + } else { + // vcg + MetricRepo.updateClusterQueryLatency(cloudCluster, elapseMs); + MetricRepo.updateClusterQueryLatency(physicalClusterName, elapseMs); + } } if (elapseMs > Config.qe_slow_log_ms) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 6b3640c1d887f0..cb3c3478fd01ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -265,7 +265,17 @@ public void executeQuery(String originStmt) throws Exception { MetricRepo.COUNTER_REQUEST_ALL.increase(1L); if (Config.isCloudMode()) { try { - MetricRepo.increaseClusterRequestAll(ctx.getCloudCluster(false)); + String clusterName = ctx.getCloudCluster(false); + String physicalClusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getPhysicalCluster(clusterName); + if (clusterName.equals(physicalClusterName)) { + // not vcg + MetricRepo.increaseClusterRequestAll(clusterName); + } else { + // vcg + MetricRepo.increaseClusterRequestAll(clusterName); + MetricRepo.increaseClusterRequestAll(physicalClusterName); + } } catch (ComputeGroupException e) { LOG.warn("metrics get cluster exception", e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index c2330cb84d7af2..cdf2d79995396e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -154,6 +154,7 @@ import org.apache.doris.catalog.View; import org.apache.doris.clone.DynamicPartitionScheduler; import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.ComputeGroup; import org.apache.doris.cloud.datasource.CloudInternalCatalog; import org.apache.doris.cloud.load.CloudLoadManager; import org.apache.doris.cloud.proto.Cloud; @@ -803,8 +804,14 @@ private void handleShowCluster() throws AnalysisException { final ShowClusterStmt showStmt = (ShowClusterStmt) stmt; final List> rows = Lists.newArrayList(); List clusterNames = null; - clusterNames = ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getCloudClusterNames(); - + CloudSystemInfoService cloudSys = ((CloudSystemInfoService) Env.getCurrentSystemInfo()); + clusterNames = cloudSys.getCloudClusterNames(); + // virtual cluster info + List virtualComputeGroup = cloudSys.getComputeGroups(true); + List virtualComputeGroupNames = virtualComputeGroup.stream() + .map(ComputeGroup::getName).collect(Collectors.toList()); + + clusterNames.addAll(virtualComputeGroupNames); final Set clusterNameSet = Sets.newTreeSet(); clusterNameSet.addAll(clusterNames); @@ -836,10 +843,35 @@ private void handleShowCluster() throws AnalysisException { String result = Joiner.on(", ").join(users); row.add(result); - int backendNum = ((CloudSystemInfoService) Env.getCurrentEnv().getCurrentSystemInfo()) - .getBackendsByClusterName(clusterName).size(); - row.add(String.valueOf(backendNum)); + + // subClusters + String subClusterNames = ""; + // Policy + String policy = ""; + if (!virtualComputeGroupNames.contains(clusterName)) { + int backendNum = cloudSys.getBackendsByClusterName(clusterName).size(); + row.add(String.valueOf(backendNum)); + rows.add(row); + row.add(subClusterNames); + row.add(policy); + continue; + } + // virtual compute group + // virtual cg backends eq 0 + row.add(String.valueOf(0)); rows.add(row); + ComputeGroup cg = cloudSys.getComputeGroupByName(clusterName); + if (cg == null) { + continue; + } + String activeCluster = cg.getPolicy().getActiveComputeGroup(); + String standbyCluster = cg.getPolicy().getStandbyComputeGroup(); + // first active, second standby + subClusterNames = Joiner.on(", ").join(activeCluster, standbyCluster); + row.add(subClusterNames); + + // Policy + row.add(cg.getPolicy().toString()); } resultSet = new ShowResultSet(showStmt.getMetaData(), rows); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java new file mode 100644 index 00000000000000..e638bca1b32225 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.system; + +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.resource.Tag; +import org.apache.doris.system.Backend; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CloudSystemInfoServiceTest { + private CloudSystemInfoService infoService; + + @Before + public void setUp() { + //infoService = new CloudSystemInfoService(); + } + + @Test + public void testGetPhysicalClusterNotExist() { + infoService = new CloudSystemInfoService(); + // not exist cluster + String c1 = "not_exist_cluster_1"; + String res = infoService.getPhysicalCluster(c1); + Assert.assertEquals(c1, res); + } + + @Test + public void testGetPhysicalClusterPhysicalCluster() { + infoService = new CloudSystemInfoService(); + String c1 = "physical_cluster_1"; + String res = infoService.getPhysicalCluster(c1); + Assert.assertEquals(c1, res); + } + + // virtual cluster does not contain physical cluster + //@Test + //public void testGetPhysicalClusterEmptyVirtualCluster() { + // infoService = new CloudSystemInfoService(); + // String vcgName = "v_cluster_1"; + // ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + // infoService.addComputeGroup(vcgName, vcg); + + // String res = infoService.getPhysicalCluster(vcgName); + // Assert.assertEquals(vcgName, res); + //} + + // active and standby are both empty cluster + @Test + public void testGetPhysicalClusterEmptyCluster() { + infoService = new CloudSystemInfoService(); + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + + String res = infoService.getPhysicalCluster(vcgName); + Assert.assertEquals(pcgName1, res); + } + + // active is empty cluster and standby has 3 alive be + @Test + public void testGetPhysicalClusterStandbyAvailable() { + infoService = new CloudSystemInfoService(); + + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + + List toAdd = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName2); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id3"); + b.setTagMap(newTagMap); + b.setAlive(true); + toAdd.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd, new ArrayList<>()); + + String res = infoService.getPhysicalCluster(vcgName); + Assert.assertEquals(pcgName2, res); + } + + // active has 3 alive be and standby is empty cluster + @Test + public void testGetPhysicalClusterActiveAvailable() { + infoService = new CloudSystemInfoService(); + + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + + List toAdd = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName1); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id2"); + b.setTagMap(newTagMap); + b.setAlive(true); + toAdd.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd, new ArrayList<>()); + + String res = infoService.getPhysicalCluster(vcgName); + Assert.assertEquals(pcgName1, res); + } + + // active has 3 alive be and standby has 3 dead be + @Test + public void testGetPhysicalClusterActive3AliveBe() { + infoService = new CloudSystemInfoService(); + + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + + List toAdd1 = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName1); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id2"); + b.setTagMap(newTagMap); + b.setAlive(true); + toAdd1.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd1, new ArrayList<>()); + + List toAdd2 = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName2); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id3"); + b.setTagMap(newTagMap); + b.setAlive(false); + toAdd2.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd2, new ArrayList<>()); + + String res = infoService.getPhysicalCluster(vcgName); + Assert.assertEquals(pcgName1, res); + } + + // active has 3 dead be and standby has 3 alive be + @Test + public void testGetPhysicalClusterStandby3AliveBe() { + infoService = new CloudSystemInfoService(); + + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + + List toAdd1 = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName1); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id2"); + b.setTagMap(newTagMap); + b.setAlive(false); + toAdd1.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd1, new ArrayList<>()); + + List toAdd2 = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName2); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id3"); + b.setTagMap(newTagMap); + b.setAlive(true); + toAdd2.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd2, new ArrayList<>()); + + String res = infoService.getPhysicalCluster(vcgName); + Assert.assertEquals(pcgName2, res); + } + + // active has 1 alive be and 2 dead be, standby has 3 alive be + @Test + public void testGetPhysicalClusterActive1AliveBe2DeadBe() { + infoService = new CloudSystemInfoService(); + + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + + List toAdd1 = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName1); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id2"); + b.setTagMap(newTagMap); + if (i == 2) { + b.setAlive(true); + } else { + b.setAlive(false); + } + toAdd1.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd1, new ArrayList<>()); + + List toAdd2 = new ArrayList<>(); + for (int i = 0; i < 3; ++i) { + Backend b = new Backend(Env.getCurrentEnv().getNextId(), "", i); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, pcgName2); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "id3"); + b.setTagMap(newTagMap); + b.setAlive(true); + toAdd2.add(b); + } + infoService.updateCloudClusterMapNoLock(toAdd2, new ArrayList<>()); + + String res = infoService.getPhysicalCluster(vcgName); + Assert.assertEquals(pcgName1, res); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java new file mode 100644 index 00000000000000..8334aa3a835884 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mysql/privilege/CloudAuthTest.java @@ -0,0 +1,623 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql.privilege; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateRoleStmt; +import org.apache.doris.analysis.CreateUserStmt; +import org.apache.doris.analysis.DropRoleStmt; +import org.apache.doris.analysis.DropUserStmt; +import org.apache.doris.analysis.GrantStmt; +import org.apache.doris.analysis.ResourcePattern; +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.RevokeStmt; +import org.apache.doris.analysis.ShowGrantsStmt; +import org.apache.doris.analysis.TablePattern; +import org.apache.doris.analysis.UserDesc; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.AccessPrivilege; +import org.apache.doris.catalog.AccessPrivilegeWithCols; +import org.apache.doris.catalog.Env; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.persist.EditLog; +import org.apache.doris.persist.PrivInfo; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; +import org.apache.doris.qe.ShowExecutor; +import org.apache.doris.qe.ShowResultSet; + +import com.google.common.collect.Lists; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +public class CloudAuthTest { + + private Auth auth; + private AccessControllerManager accessManager; + @Mocked + public CloudEnv env; + @Mocked + private Analyzer analyzer; + @Mocked + private EditLog editLog; + @Mocked + private ConnectContext ctx; + private CloudSystemInfoService systemInfoService = new CloudSystemInfoService(); + + @Before + public void setUp() throws NoSuchMethodException, SecurityException { + auth = new Auth(); + accessManager = new AccessControllerManager(auth); + new Expectations() { + { + + analyzer.getDefaultCatalog(); + minTimes = 0; + result = InternalCatalog.INTERNAL_CATALOG_NAME; + + Env.getCurrentEnv(); + minTimes = 0; + result = env; + + env.getAccessManager(); + minTimes = 0; + result = accessManager; + + env.getAuth(); + minTimes = 0; + result = auth; + + env.getEditLog(); + minTimes = 0; + result = editLog; + + editLog.logCreateUser((PrivInfo) any); + minTimes = 0; + + ConnectContext.get(); + minTimes = 0; + result = ctx; + + ctx.getQualifiedUser(); + minTimes = 0; + result = "root"; + + ctx.getRemoteIP(); + minTimes = 0; + result = "192.168.1.1"; + + ctx.getState(); + minTimes = 0; + result = new QueryState(); + + ctx.getCurrentUserIdentity(); + minTimes = 0; + result = UserIdentity.createAnalyzedUserIdentWithIp("root", "%"); + + Env.getCurrentSystemInfo(); + minTimes = 0; + result = systemInfoService; + + // systemInfoService.addComputeGroup(anyString, (ComputeGroup) any); + // minTimes = 0; + } + }; + + } + + private void dropUser(UserIdentity userIdentity) throws UserException { + DropUserStmt dropUserStmt = new DropUserStmt(userIdentity); + dropUserStmt.analyze(analyzer); + auth.dropUser(dropUserStmt); + } + + public ShowResultSet testShowGrants(UserIdentity userIdent) throws AnalysisException { + ShowGrantsStmt stmt = new ShowGrantsStmt(userIdent, false); + ShowExecutor executor = new ShowExecutor(ctx, stmt); + return executor.execute(); + } + + @Test + public void testComputeGroup() throws UserException { + UserIdentity userIdentity = new UserIdentity("testUser", "%"); + String role = "role0"; + String computeGroup1 = "cg1"; + ResourcePattern resourcePattern = new ResourcePattern(computeGroup1, ResourceTypeEnum.CLUSTER); + String anyResource = "%"; + ResourcePattern anyResourcePattern = new ResourcePattern(anyResource, ResourceTypeEnum.CLUSTER); + List usagePrivileges = Lists + .newArrayList(new AccessPrivilegeWithCols(AccessPrivilege.USAGE_PRIV)); + UserDesc userDesc = new UserDesc(userIdentity, "12345", true); + + // ------ grant|revoke cluster to|from user ------ + // 1. create user with no role + CreateUserStmt createUserStmt = new CreateUserStmt(false, userDesc, null); + try { + createUserStmt.analyze(analyzer); + auth.createUser(createUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // 2. grant usage_priv on cluster 'cg1' to 'testUser'@'%' + GrantStmt grantStmt = new GrantStmt(userIdentity, null, resourcePattern, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + + // 3. revoke usage_priv on cluster 'cg1' from 'testUser'@'%' + RevokeStmt revokeStmt = new RevokeStmt(userIdentity, null, resourcePattern, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + // 3.1 grant 'notBelongToResourcePrivileges' on cluster 'cg1' to 'testUser'@'%' + for (int i = 0; i < Privilege.notBelongToResourcePrivileges.length; i++) { + List notAllowedPrivileges = Lists + .newArrayList(new AccessPrivilegeWithCols( + AccessPrivilege.fromName(Privilege.notBelongToResourcePrivileges[i].getName()))); + grantStmt = new GrantStmt(userIdentity, null, resourcePattern, notAllowedPrivileges, + ResourceTypeEnum.GENERAL); + try { + grantStmt.analyze(analyzer); + Assert.fail(String.format("Can not grant/revoke %s to/from any other users or roles", + Privilege.notBelongToWorkloadGroupPrivileges[i])); + } catch (UserException e) { + e.printStackTrace(); + } + } + // 4. drop user + DropUserStmt dropUserStmt = new DropUserStmt(userIdentity); + try { + dropUserStmt.analyze(analyzer); + auth.dropUser(dropUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // ------ grant|revoke resource to|from role ------ + // 1. create role + CreateRoleStmt roleStmt = new CreateRoleStmt(role); + try { + roleStmt.analyze(analyzer); + auth.createRole(roleStmt); + } catch (UserException e1) { + e1.printStackTrace(); + Assert.fail(); + } + // grant usage_priv on cluster 'cg1' to role 'role0' + grantStmt = new GrantStmt(null, role, resourcePattern, usagePrivileges, ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // 2. create user with role + createUserStmt = new CreateUserStmt(false, userDesc, role); + try { + createUserStmt.analyze(analyzer); + auth.createUser(createUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + + // 3. revoke usage_priv on cluster 'cg1' from role 'role0' + revokeStmt = new RevokeStmt(null, role, resourcePattern, usagePrivileges, ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + // also revoke from user with this role + Assert.assertFalse(accessManager.checkResourcePriv(userIdentity, computeGroup1, PrivPredicate.USAGE)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + + // 4. drop user and role + dropUserStmt = new DropUserStmt(userIdentity); + try { + dropUserStmt.analyze(analyzer); + auth.dropUser(dropUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + DropRoleStmt dropRoleStmt = new DropRoleStmt(role); + try { + dropRoleStmt.analyze(analyzer); + auth.dropRole(dropRoleStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // ------ grant|revoke any compute group to|from user ------ + // 1. create user with no role + createUserStmt = new CreateUserStmt(false, userDesc, null); + try { + createUserStmt.analyze(analyzer); + auth.createUser(createUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // 2. grant usage_priv on cluster '*' to 'testUser'@'%' + grantStmt = new GrantStmt(userIdentity, null, anyResourcePattern, usagePrivileges, ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + // anyResource not belong to global auth + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.SHOW_RESOURCES)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.SHOW)); + + // 3. revoke usage_priv on cluster '*' from 'testUser'@'%' + revokeStmt = new RevokeStmt(userIdentity, null, anyResourcePattern, usagePrivileges, ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.SHOW_RESOURCES)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.SHOW)); + + // 4. drop user + dropUserStmt = new DropUserStmt(userIdentity); + try { + dropUserStmt.analyze(analyzer); + auth.dropUser(dropUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // ------ grant|revoke any cluster to|from role ------ + // 1. create role + roleStmt = new CreateRoleStmt(role); + try { + roleStmt.analyze(analyzer); + auth.createRole(roleStmt); + } catch (UserException e1) { + e1.printStackTrace(); + Assert.fail(); + } + // grant usage_priv on cluster '*' to role 'role0' + grantStmt = new GrantStmt(null, role, anyResourcePattern, usagePrivileges, ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e1) { + e1.printStackTrace(); + Assert.fail(); + } + + // 2. create user with role + createUserStmt = new CreateUserStmt(false, userDesc, role); + try { + createUserStmt.analyze(analyzer); + auth.createUser(createUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + + // 3. revoke usage_priv on cluster '*' from role 'role0' + revokeStmt = new RevokeStmt(null, role, anyResourcePattern, usagePrivileges, ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + // also revoke from user with this role + Assert.assertFalse(accessManager.checkResourcePriv(userIdentity, computeGroup1, PrivPredicate.USAGE)); + Assert.assertFalse(accessManager.checkGlobalPriv(userIdentity, PrivPredicate.USAGE)); + + // 4. drop user and role + dropUserStmt = new DropUserStmt(userIdentity); + try { + dropUserStmt.analyze(analyzer); + auth.dropUser(dropUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + dropRoleStmt = new DropRoleStmt(role); + try { + dropRoleStmt.analyze(analyzer); + auth.dropRole(dropRoleStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // ------ error case ------ + boolean hasException = false; + createUserStmt = new CreateUserStmt(false, userDesc, null); + try { + createUserStmt.analyze(analyzer); + auth.createUser(createUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // 1. grant db table priv to cluster + List privileges = Lists + .newArrayList(new AccessPrivilegeWithCols(AccessPrivilege.SELECT_PRIV)); + grantStmt = new GrantStmt(userIdentity, null, resourcePattern, privileges, ResourceTypeEnum.GENERAL); + hasException = false; + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e) { + e.printStackTrace(); + hasException = true; + } + Assert.assertTrue(hasException); + + // 2. grant cluster priv to db table + TablePattern tablePattern = new TablePattern("db1", "*"); + GrantStmt grantStmt2 = new GrantStmt(userIdentity, null, tablePattern, usagePrivileges); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, + "Can not grant/revoke Usage_priv to/from any other users or roles", + () -> grantStmt2.analyze(analyzer)); + + // 3. grant cluster prov to role on db.table + tablePattern = new TablePattern("db1", "*"); + GrantStmt grantStmt3 = new GrantStmt(userIdentity, "test_role", tablePattern, usagePrivileges); + ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, + "Can not grant/revoke Usage_priv to/from any other users or roles", + () -> grantStmt3.analyze(analyzer)); + + // 4.drop user + dropUser(userIdentity); + } + + @Test + public void testVirtualComputeGroup() throws UserException { + UserIdentity userIdentity = new UserIdentity("testUser", "%"); + // String role = "role0"; + String computeGroup1 = "cg1"; + ResourcePattern resourcePattern1 = new ResourcePattern(computeGroup1, ResourceTypeEnum.CLUSTER); + String computeGroup2 = "cg2"; + ResourcePattern resourcePattern2 = new ResourcePattern(computeGroup2, ResourceTypeEnum.CLUSTER); + String virtualComputeGroup = "vcg"; + ResourcePattern resourcePatternVcg = new ResourcePattern(virtualComputeGroup, ResourceTypeEnum.CLUSTER); + + List usagePrivileges = Lists + .newArrayList(new AccessPrivilegeWithCols(AccessPrivilege.USAGE_PRIV)); + + UserDesc userDesc = new UserDesc(userIdentity, "12345", true); + + // create user with no role + CreateUserStmt createUserStmt = new CreateUserStmt(false, userDesc, null); + try { + createUserStmt.analyze(analyzer); + auth.createUser(createUserStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // -------------------- case 1 ------------------------- + // grant usage_priv on cluster 'vcg' to 'testUser'@'%' + GrantStmt grantStmt = new GrantStmt(userIdentity, null, resourcePatternVcg, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + // create vcg, sub cg(cg1, cg2), add to systemInfoService + ComputeGroup vcg = new ComputeGroup("vcg_id", virtualComputeGroup, ComputeGroup.ComputeTypeEnum.VIRTUAL); + vcg.setSubComputeGroups(Lists.newArrayList(computeGroup2, computeGroup1)); + systemInfoService.addComputeGroup(virtualComputeGroup, vcg); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(computeGroup1); + policy.setStandbyComputeGroup(computeGroup2); + vcg.setPolicy(policy); + + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, virtualComputeGroup, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + + // testUser has vcg, but not have cg1,cg2, he can use cg1,cg2 + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup2, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + ShowResultSet showResultSet = testShowGrants(userIdentity); + // cluster field + Assert.assertEquals("vcg: Cluster_usage_priv", showResultSet.getResultRows().get(0).get(10)); + // compute group field + Assert.assertEquals("vcg: Cluster_usage_priv", showResultSet.getResultRows().get(0).get(14)); + + // -------------------- case 2 ------------------------- + // grant usage_priv on cluster 'cg1' to 'testUser'@'%' + grantStmt = new GrantStmt(userIdentity, null, resourcePattern1, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + // testUser can use cg1, because he has vcg,cg1 auth + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + showResultSet = testShowGrants(userIdentity); + // cluster field + Assert.assertEquals("cg1: Cluster_usage_priv; vcg: Cluster_usage_priv", + showResultSet.getResultRows().get(0).get(10)); + // compute group field + Assert.assertEquals("cg1: Cluster_usage_priv; vcg: Cluster_usage_priv", + showResultSet.getResultRows().get(0).get(14)); + + // revoke cg1 from test user + RevokeStmt revokeStmt = new RevokeStmt(userIdentity, null, resourcePattern1, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + // testUser can use cg1, because he has vcg auth + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + showResultSet = testShowGrants(userIdentity); + // cluster field + Assert.assertEquals("vcg: Cluster_usage_priv", + showResultSet.getResultRows().get(0).get(10)); + // compute group field + Assert.assertEquals("vcg: Cluster_usage_priv", + showResultSet.getResultRows().get(0).get(14)); + + // grant cg2 to user + grantStmt = new GrantStmt(userIdentity, null, resourcePattern2, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + grantStmt.analyze(analyzer); + auth.grant(grantStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // revoke vcg from test user + revokeStmt = new RevokeStmt(userIdentity, null, resourcePatternVcg, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // currently, user has cg2 auth, not have vcg auth + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, virtualComputeGroup, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + + // testUser has cg2, but not have vcg, he can use cg2, can't use cg1, vcg + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertTrue(accessManager.checkCloudPriv(userIdentity, computeGroup2, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + showResultSet = testShowGrants(userIdentity); + // cluster field + Assert.assertEquals("cg2: Cluster_usage_priv", + showResultSet.getResultRows().get(0).get(10)); + // compute group field + Assert.assertEquals("cg2: Cluster_usage_priv", + showResultSet.getResultRows().get(0).get(14)); + // revoke cg2 from user + revokeStmt = new RevokeStmt(userIdentity, null, resourcePattern2, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + + // revoke vcg from user, he can't use it + revokeStmt = new RevokeStmt(userIdentity, null, resourcePatternVcg, usagePrivileges, + ResourceTypeEnum.CLUSTER); + try { + revokeStmt.analyze(analyzer); + auth.revoke(revokeStmt); + } catch (UserException e) { + e.printStackTrace(); + Assert.fail(); + } + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, virtualComputeGroup, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + + // testUser after revoke vcg, not have cg1,cg2, it should can use, vcg,cg1,cg2 + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, computeGroup1, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + Assert.assertFalse(accessManager.checkCloudPriv(userIdentity, computeGroup2, + PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)); + showResultSet = testShowGrants(userIdentity); + // cluster field + Assert.assertEquals("\\N", showResultSet.getResultRows().get(0).get(10)); + // compute group field + Assert.assertEquals("\\N", showResultSet.getResultRows().get(0).get(14)); + + // drop user + dropUser(userIdentity); + } +} diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 571bd0b26e2429..56060be67bae15 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -121,10 +121,24 @@ enum ClusterStatus { MANUAL_SHUTDOWN = 4; } +// just for virtual cluster +message ClusterPolicy { + enum PolicyType { + ActiveStandby = 0; + } + optional PolicyType type = 1; + optional string active_cluster_name = 2; + repeated string standby_cluster_names = 3; + optional int64 failover_failure_threshold = 4; + optional int64 unhealthy_node_threshold_percent = 5; + repeated string cache_warmup_jobids = 6; +} + message ClusterPB { enum Type { SQL = 0; COMPUTE = 1; + VIRTUAL = 2; } optional string cluster_id = 1; optional string cluster_name = 2; @@ -135,6 +149,10 @@ message ClusterPB { optional string public_endpoint = 7; optional string private_endpoint = 8; optional ClusterStatus cluster_status = 9; + optional int64 ctime = 10; + optional int64 mtime = 11; + repeated string cluster_names = 12; // clusters in virtual cluster + optional ClusterPolicy cluster_policy = 13; // virtual cluster policy } message NodeInfoPB { @@ -1116,6 +1134,7 @@ message AlterClusterRequest { NOTIFY_DECOMMISSIONED = 8; UPDATE_CLUSTER_ENDPOINT = 9; SET_CLUSTER_STATUS = 10; + ALTER_VCLUSTER_INFO = 11; } optional string instance_id = 1; optional string cloud_unique_id = 2; // For auth diff --git a/regression-test/data/cloud/multi_cluster/vcluster/all_types.csv b/regression-test/data/cloud/multi_cluster/vcluster/all_types.csv new file mode 100644 index 00000000000000..64d38b1db8fd46 --- /dev/null +++ b/regression-test/data/cloud/multi_cluster/vcluster/all_types.csv @@ -0,0 +1,20 @@ +0,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +1,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +2,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +3,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +4,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +5,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +6,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +7,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +8,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +9,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +10,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +11,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +12,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +13,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +14,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +15,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +16,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +17,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +18,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +19,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 diff --git a/regression-test/data/cloud/multi_cluster/vcluster/vcluster.out b/regression-test/data/cloud/multi_cluster/vcluster/vcluster.out new file mode 100644 index 00000000000000..369f2e3af7889b --- /dev/null +++ b/regression-test/data/cloud/multi_cluster/vcluster/vcluster.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/all_types.csv b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/all_types.csv new file mode 100644 index 00000000000000..64d38b1db8fd46 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/all_types.csv @@ -0,0 +1,20 @@ +0,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +1,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +2,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +3,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +4,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +5,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +6,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +7,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +8,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +9,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +10,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +11,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +12,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +13,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +14,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +15,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +16,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +17,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +18,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 +19,2,3,4,5,6.6,7.7,8.8,abc,def,ghiaaaaaa,2020-10-10,2020-10-10 11:12:59 diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.out new file mode 100644 index 00000000000000..0abde7ca45bf71 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcluster.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcluster.out new file mode 100644 index 00000000000000..369f2e3af7889b --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcluster.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 8d3a7834fee3fc..465dc139166f31 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2572,7 +2572,46 @@ class Suite implements GroovyInterceptable { } } - def get_cluster = { be_unique_id , MetaService ms=null-> + def add_vcluster = { cluster_name, cluster_id, active, standby -> + def jsonOutput = new JsonOutput() + def ci = [ + type: "VIRTUAL", + cluster_name : cluster_name, + cluster_id : cluster_id, + cluster_names: [ + active, + standby + ], + cluster_policy: [ + type: "ActiveStandby", + active_cluster_name: active, + standby_cluster_names: [ + standby + ] + ] + ] + def map = [instance_id: "${instance_id}", cluster: ci] + def js = jsonOutput.toJson(map) + log.info("add cluster req: ${js} ".toString()) + + def add_cluster_api = { request_body, check_func -> + httpTest { + endpoint context.config.metaServiceHttpAddress + uri "/MetaService/http/add_cluster?token=${token}" + body request_body + check check_func + } + } + + add_cluster_api.call(js) { + respCode, body -> + log.info("add cluster resp: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK") || json.code.equalsIgnoreCase("ALREADY_EXISTED")) + } + } + + def get_cluster = { be_unique_id, MetaService ms=null -> def jsonOutput = new JsonOutput() def map = [instance_id: "${instance_id}", cloud_unique_id: "${be_unique_id}" ] def js = jsonOutput.toJson(map) @@ -2611,7 +2650,6 @@ class Suite implements GroovyInterceptable { def drop_cluster = { cluster_name, cluster_id, MetaService ms=null -> def jsonOutput = new JsonOutput() def reqBody = [ - type: "COMPUTE", cluster_name : cluster_name, cluster_id : cluster_id, nodes: [ @@ -2815,7 +2853,82 @@ class Suite implements GroovyInterceptable { } } - def rename_cloud_cluster = { cluster_name, cluster_id -> + def checkProfileNew = { addrSet -> + def query_profile_api = { check_func -> + httpTest { + op "get" + endpoint context.config.feHttpAddress + uri "/rest/v1/query_profile" + check check_func + basicAuthorization "${context.config.feCloudHttpUser}","${context.config.feCloudHttpPassword}" + } + } + + query_profile_api.call() { + respCode, body -> + log.info("query profile resp: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.msg.equalsIgnoreCase("success")) + log.info("lw query profile resp: ${json.data.rows[0]}".toString()) + log.info("lw query profile resp: ${json.data.rows[0]['Profile ID']}".toString()) + checkProfileNew1.call(addrSet, json.data.rows[0]['Profile ID']) + } + } + + def checkProfileNew1 = {addrSet, query_id -> + def query_profile_api = { check_func -> + httpTest { + op "get" + endpoint context.config.feHttpAddress + uri "/api/profile?query_id=${query_id}" + check check_func + basicAuthorization "${context.config.feCloudHttpUser}","${context.config.feCloudHttpPassword}" + } + } + + query_profile_api.call() { + respCode, body -> + //log.info("query profile resp: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.msg.equalsIgnoreCase("success")) + //log.info("lw query profile resp: ${json.data.rows[0]}".toString()) + + def instanceLineMatcher = json =~ /Instances\s+Num\s+Per\s+BE:\s*(.*)/ + if (instanceLineMatcher.find()) { + // 提取出IP等信息的部分 + def instancesStr = instanceLineMatcher.group(1).trim() + + // 拆分各个实例，实例格式类似 "10.16.10.11:9713:4" + def instanceEntries = instancesStr.split(/\s*,\s*/) + + // 定义存储解析结果的列表 + def result = [] + + // 每个实例使用正则表达式解析IP和端口（忽略最后一个数字） + instanceEntries.each { entry -> + def matcher = entry =~ /(\d{1,3}(?:\.\d{1,3}){3}):(\d+):\d+/ + if(matcher.matches()){ + def ip = matcher.group(1) + def port = matcher.group(2) + //result << [ip: ip, port: port] + //result << [ip:port] + result.add(ip+":"+port) + } + } + + // 输出解析结果 + println "提取的IP和端口：" + result.each { println it } + addrSet.each { println it } + //result.each { assertTrue(addrSet.contains(it)) } + assertTrue(addrSet.containsAll(result)) + } else { + println "未找到实例信息。" + } + } + } + + def rename_cloud_cluster = { cluster_name, cluster_id, MetaService ms=null -> def jsonOutput = new JsonOutput() def reqBody = [ cluster_name : cluster_name, @@ -2827,7 +2940,11 @@ class Suite implements GroovyInterceptable { def rename_cluster_api = { request_body, check_func -> httpTest { - endpoint context.config.metaServiceHttpAddress + if (ms) { + endpoint ms.host+':'+ms.httpPort + } else { + endpoint context.config.metaServiceHttpAddress + } uri "/MetaService/http/rename_cluster?token=${token}" body request_body check check_func diff --git a/regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy b/regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy new file mode 100644 index 00000000000000..390e65d4b39da4 --- /dev/null +++ b/regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy @@ -0,0 +1,173 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonOutput + +suite("vcluster") { + def token = context.config.metaServiceToken + def instance_id = context.config.multiClusterInstance + String tableName = "test_all_vcluster" + + List ipList = new ArrayList<>() + List hbPortList = new ArrayList<>() + List httpPortList = new ArrayList<>() + List beUniqueIdList = new ArrayList<>() + List bePortList = new ArrayList<>() + + String[] bes = context.config.multiClusterBes.split(','); + println("the value is " + context.config.multiClusterBes); + for(String values : bes) { + println("the value is " + values); + String[] beInfo = values.split(':'); + ipList.add(beInfo[0]); + hbPortList.add(beInfo[1]); + httpPortList.add(beInfo[2]); + beUniqueIdList.add(beInfo[3]); + } + + println("the ip is " + ipList); + println("the heartbeat port is " + hbPortList); + println("the http port is " + httpPortList); + println("the be unique id is " + beUniqueIdList); + + for (unique_id : beUniqueIdList) { + resp = get_cluster.call(unique_id); + for (cluster : resp) { + log.info("lw test drop : ${cluster.type} ".toString()) + if (cluster.type == "COMPUTE" || cluster.type == "VIRTUAL") { + drop_cluster.call(cluster.cluster_name, cluster.cluster_id); + } + } + } + wait_cluster_change() + + List> result = sql "show clusters" + assertTrue(result.size() == 0); + + add_cluster.call(beUniqueIdList[0], ipList[0], hbPortList[0], + "regression_cluster_name0", "regression_cluster_id0"); + add_cluster.call(beUniqueIdList[1], ipList[1], hbPortList[1], + "regression_cluster_name1", "regression_cluster_id1"); + add_vcluster.call("regression_vcluster_name0", "regression_vcluster_id0", + "regression_cluster_name1", "regression_cluster_name0"); + wait_cluster_change() + + result = sql "show clusters" + assertTrue(result.size() == 3); + + for (row : result) { + println row + } + + try { + sql """ use @regression_vcluster_name0 """ + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + ; + """ + + sql """ set enable_profile = true """ + + before_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); + log.info("before_cluster0_load_rows : ${before_cluster0_load_rows}".toString()) + before_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); + log.info("before_cluster0_flush : ${before_cluster0_flush}".toString()) + + before_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); + log.info("before_cluster1_load_rows : ${before_cluster1_load_rows}".toString()) + before_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); + log.info("before_cluster1_flush : ${before_cluster1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'regression_vcluster_name0' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); + log.info("after_cluster0_load_rows : ${after_cluster0_load_rows}".toString()) + after_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); + log.info("after_cluster0_flush : ${after_cluster0_flush}".toString()) + + after_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); + log.info("after_cluster1_load_rows : ${after_cluster1_load_rows}".toString()) + after_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); + log.info("after_cluster1_flush : ${after_cluster1_flush}".toString()) + + assertTrue(before_cluster0_load_rows == after_cluster0_load_rows) + assertTrue(before_cluster0_flush == after_cluster0_flush) + + assertTrue(before_cluster1_load_rows < after_cluster1_load_rows) + assertTrue(before_cluster1_flush < after_cluster1_flush) + + // fill bePortList + for (int i = 0; i < ipList.size(); ++i) { + result = sql """show backends""" + for (row : result) { + println row + println row[2] + if (ipList[i] == row[1] && hbPortList[i] == row[2]) { + bePortList.add(row[5]); + } + } + } + + set = [ipList[1] + ":" +bePortList[1]] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + //checkProfileNew1.call("e329bc41f42c49f5-9326cb8429dacc06") + } finally { + sql """ drop table if exists ${tableName} """ + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy new file mode 100644 index 00000000000000..22bdf8ce4fa505 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy @@ -0,0 +1,391 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 stop a backend of c1 +// 3 stop a backend of c1 +// 4 start 2 backends of c1 +// 5 long-term stop 2 backends of c1 + +suite('default_vcg_auto_failover', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 10] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + + showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """ SET PROPERTY 'default_cloud_cluster' = 'normalVirtualClusterName' """ + + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + cluster.stopBackends(4, 5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + log.info("policy {}", vcgInShow.Policy) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + + cluster.startBackends(4, 5) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + cluster.stopBackends(4, 5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + sleep(60000) + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + } + // connect to follower, run again + //options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy new file mode 100644 index 00000000000000..74aef9e628e3b6 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy @@ -0,0 +1,430 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +suite('test_vcg', 'multi_cluster,docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + 'fetch_cluster_cache_hotspot_interval_ms=600000', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + def checkWarmupJob = { job, warmUpIds, expectedSrc, expectedDst -> + assertTrue(job.SyncMode.contains("EVENT_DRIVEN (LOAD)") || job.SyncMode.contains("PERIODIC (600s)")) + if (warmUpIds.contains(job.JobId)) { + // old + assertTrue(job.Status.contains("CANCELLED") && (job.ErrMsg.contains("user cancel") || job.ErrMsg.contains("vcg cancel"))) + } else { + // new + assertTrue(job.SrcComputeGroup.contains(expectedSrc) && job.DstComputeGroup.contains(expectedDst)) + warmUpIds.add(job.JobId) + } + } + + options.connectToFollower = false + def tbl = "test_virtual_compute_group_tbl" + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // 添加一个新的cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // 添加一个新的cluster newcluster2 + cluster.addBackend(1, clusterName2) + + def clusterName3 = "newcluster3" + // 添加一个新的cluster newcluster3 + cluster.addBackend(3, clusterName3) + + // create test user + def user = "regression_test_cloud_user" + sql """create user ${user} identified by 'Cloud12345'""" + sql """grant select_priv,load_priv on *.*.* to ${user}""" + + // add empty vcg, test not show in fe + /* + curl '175.43.101.1:5000/MetaService/http/add_cluster?token=greedisgood9999' -d '{ + "instance_id":"default_instance_id", + "cluster":{ + "type":"VIRTUAL", + "cluster_name":"emptyClusterName", + "cluster_id":"emptyClusterId" + } + }' + */ + def vClusterNameEmpty = "emptyClusterName" + def vClusterIdEmpty = "emptyClusterId" + def instance_id = "default_instance_id" + def clusterMap = [cluster_name: "${vClusterNameEmpty}", cluster_id:"${vClusterIdEmpty}", type:"VIRTUAL"] + def instance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def addEmptyVComputeGroupBody = jsonOutput.toJson(instance) + add_cluster_api.call(msHttpPort, addEmptyVComputeGroupBody) { + respCode, body -> + log.info("add empty vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + // empty vcg, add succ, but not show in fe + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def find = showComputeGroup.any { it.Name == vClusterNameEmpty } + assertFalse(find) + + + // add one vcg, test normal vcg + // contain newCluster1, newCluster2 + + /* + curl '175.43.101.1:5000/MetaService/http/add_cluster?token=greedisgood9999' -d '{ + "instance_id": "default_instance_id", + "cluster": { + "type": "VIRTUAL", + "cluster_name": "normalVirtualClusterName", + "cluster_id": "normalVirtualClusterId", + "cluster_names": [ + "newcluster1", + "newcluster2" + ], + "cluster_policy": { + "type": "ActiveStandby", + "active_cluster_name": "newcluster1", + "standby_cluster_names": [ + "newcluster2" + ] + } + } + }' + */ + + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + // show cluster + sleep(5000) + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + /* + [{IsCurrent=TRUE, BackendNum=3, Policy=, SubClusters=, Users=, Name=compute_cluster}, {IsCurrent=FALSE, BackendNum=2, Policy=, SubClu + sters=, Users=, Name=newcluster1}, {IsCurrent=FALSE, BackendNum=1, Policy=, SubClusters=, Users=, Name=newcluster2}, {IsCurrent=FALSE, BackendNum=3, Policy=, SubClusters=, Users=, Name=newcluster3}, {IsCurrent=FALSE, BackendNum=0 + , Policy={policyType=ActiveStandby, activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2', failoverFailureThreshold=0, unhealthyNodeThresholdPercent=0}, SubClusters=newcluster1, newcluster2, Users=, Name=normalVirtualClusterName}] + */ + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + + assertNotNull(vcgInShow) + // {policyType=ActiveStandby, activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2', failoverFailureThreshold=0, unhealthyNodeThresholdPercent=0} + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + def showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + log.info("show warm up after create {}", showWarmup) + def warmUpIds = [] +/* + [{Status=RUNNING, AllBatch=0, Type=CLUSTER, FinishTime=null, ErrMsg= + │ , DstComputeGroup=newcluster2, CreateTime=2025-05-11 20:38:48.479, SrcComputeGroup=newcluster1, StartTime=2025-05-11 20:38:48.479, FinishBatch=0, JobId=1746966973833, Sync + │ Mode=EVENT_DRIVEN (LOAD)}, {Status=WAITING, AllBatch=0, Type=CLUSTER, FinishTime=2025-05-11 20:38:50.720, ErrMsg=, DstComputeGroup=newcluster2, CreateTime=2025-05-11 20:38 + │ :48.437, SrcComputeGroup=newcluster1, StartTime=2025-05-11 20:38:48.437, FinishBatch=1, JobId=1746966973832, SyncMode=PERIODIC (600s)}] + */ + assertEquals(showWarmup.size(), 2) + showWarmup.each { + assertTrue(it.SyncMode.contains("EVENT_DRIVEN (LOAD)") || it.SyncMode.contains("PERIODIC (600s)")) + if (it.SyncMode.contains("EVENT_DRIVEN (LOAD)")) { + assertTrue(it.SrcComputeGroup.contains("newcluster1") && it.DstComputeGroup.contains("newcluster2")) + warmUpIds.add(it.JobId) + } else if (it.SyncMode.contains("PERIODIC (600s)")) { + assertTrue(it.SrcComputeGroup.contains("newcluster1") && it.DstComputeGroup.contains("newcluster2")) + warmUpIds.add(it.JobId) + } else { + assertTrue(false) + } + } + assertEquals(warmUpIds.size(), 2) + + // test manual cancel warm up job, generate new jobs + sql """CANCEL WARM UP JOB WHERE ID=${showWarmup[0].JobId}""" + + dockerAwaitUntil(50, 3) { + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + // cancel 2, generate 2 + showWarmup.size() == 4 + } + + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + assertEquals(showWarmup.size(), 4) + showWarmup.each { job -> + checkWarmupJob(job, warmUpIds, "newcluster1", "newcluster2") + } + assertEquals(warmUpIds.size(), 4) + + // grant usage_priv on cluster vcg to user + sql """GRANT USAGE_PRIV ON CLUSTER '${normalVclusterName}' TO '${user}'""" + // show grant + def result = sql_return_maparray """show grants for '${user}'""" + log.info("show grant for ${user}, ret={}", result) + def ret = result.find {it.CloudClusterPrivs == "normalVirtualClusterName: Cluster_usage_priv"} + // more auth test, use fe ut, see org.apache.doris.mysql.privilege.CloudAuthTest + + // use vcg + + sql """use @${normalVclusterName}""" + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ); + """ + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + result = sql """select count(*) from ${tbl}""" + log.info("result = {}", result) + assertEquals(result.size(), 1) + + def db = context.dbName + connectInDocker(user, 'Cloud12345') { + sql """insert into ${db}.${tbl} (k1, k2) values (2, "20")""" + result = sql """select * from ${db}.${tbl}""" + assertEquals(result.size(), 2) + } + + try { + sql """WARM UP COMPUTE GROUP ${normalVclusterName} WITH TABLE ${tbl} """ + assertTrue(false) + } catch (Exception e) { + logger.info("exception: {}", e.getMessage()) + assertTrue(e.getMessage().contains("not support")) + } + + try { + sql """WARM UP COMPUTE GROUP ${clusterName1} WITH COMPUTE GROUP ${clusterName2} + PROPERTIES ( + "sync_mode" = "periodic", -- 周期性的 warm up job + "sync_interval_sec" = "600" -- 单位是秒 + )""" + assertTrue(false) + } catch (Exception e) { + logger.info("exception: {}", e.getMessage()) + assertTrue(e.getMessage().contains("not support")) + } + + try { + sql """WARM UP COMPUTE GROUP ${clusterName1} WITH COMPUTE GROUP ${clusterName2} + PROPERTIES ( + "sync_mode" = "event_driven", -- 事件触发的 warm up job + "sync_event" = "load" -- load 包含 compaction 和 sc，后续可能扩展 query + )""" + assertTrue(false) + } catch (Exception e) { + logger.info("exception: {}", e.getMessage()) + assertTrue(e.getMessage().contains("not support")) + } + + // alter cluster info, change standbyComputeGroup to newcluster1, activeComputeGroup to newcluster2 + clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName2}", standby_cluster_names: ["${clusterName1}"]] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_policy:clusterPolicy] + normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + normalVcgBody = jsonOutput.toJson(normalInstance) + alter_cluster_info_api(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("alter virtual cluster result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + dockerAwaitUntil(20) { + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group after alter {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + vcgInShow != null + } + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + // {Policy={policyType=ActiveStandby, activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1', failoverFailureThreshold=3, unhealthyNodeThresholdPercent=100}, SubClusters=newcluster2, newcluster1, Users=, Name=normalVirtualClusterName} + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + // alter active -> sync to fe -> cancel old jobs -> generate new jobs + // -> sync to ms -> sync to fe -> save new jobs + dockerAwaitUntil(50, 3) { + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + // cancel 2, generate 2 + showWarmup.size() == 6 + } + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + log.info("show warm up after alter {}", showWarmup) + + assertEquals(showWarmup.size(), 6) + showWarmup.each { job -> + checkWarmupJob(job, warmUpIds, "newcluster2", "newcluster1") + } + assertEquals(warmUpIds.size(), 6) + + // test switch active cluster, use fe switch + // stop 6, stop clusterName2 + cluster.stopBackends(6) + + // test warm up job destory and generate new jobs + dockerAwaitUntil(50, 3) { + sql """USE @${normalVclusterName}""" + sql """select count(*) from ${tbl}""" + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"') + } + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + assertEquals(showWarmup.size(), 6) + + cluster.startBackends(6) + dockerAwaitUntil(50, 3) { + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + showWarmup.size() == 8 + } + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + showWarmup.each { job -> + checkWarmupJob(job, warmUpIds, "newcluster1", "newcluster2") + } + assertEquals(warmUpIds.size(), 8) + + // test rename vcg + def newNormalVclusterName = "newNormalVirtualClusterName" + rename_cloud_cluster.call(newNormalVclusterName, normalVclusterId, ms) + dockerAwaitUntil(20) { + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group after rename {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + vcgInShow == null + } + // show cluster again + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNull(vcgInShow) + vcgInShow = showComputeGroup.find { it.Name == newNormalVclusterName } + assertNotNull(vcgInShow) + + // rename back to + rename_cloud_cluster.call(normalVclusterName, normalVclusterId, ms) + dockerAwaitUntil(20) { + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group after rename back {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + vcgInShow != null + } + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + vcgInShow = showComputeGroup.find { it.Name == newNormalVclusterName } + assertNull(vcgInShow) + + // drop vcg + drop_cluster(normalVclusterName, normalVclusterId, ms) + + // drop_cluster, drop vcg succ + def tag = getCloudBeTagByName(clusterName1) + logger.info("tag1 = {}", tag) + + def jsonSlurper = new JsonSlurper() + def jsonObject = jsonSlurper.parseText(tag) + def cloudClusterId = jsonObject.compute_group_id + drop_cluster(clusterName1, cloudClusterId, ms) + + tag = getCloudBeTagByName(clusterName2) + logger.info("tag2 = {}", tag) + jsonSlurper = new JsonSlurper() + jsonObject = jsonSlurper.parseText(tag) + cloudClusterId = jsonObject.compute_group_id + drop_cluster(clusterName2, cloudClusterId, ms) + dockerAwaitUntil(20) { + def showRet = sql """SHOW COMPUTE GROUPS""" + log.info("show cgs: {}", showRet) + showRet.size() == 2 + } + + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group after drop {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNull(vcgInShow) + def cg1InShow = showComputeGroup.find { it.Name == clusterName1 } + assertNull(cg1InShow) + def cg2InShow = showComputeGroup.find { it.Name == clusterName2 } + assertNull(cg1InShow) + + showWarmup = sql_return_maparray """SHOW WARM UP JOB""" + log.info("show warm up after drop vcg {}", showWarmup) + assertEquals(showWarmup.size(), 8) + showWarmup.each { + assertTrue(it.SyncMode.contains("EVENT_DRIVEN (LOAD)") || it.SyncMode.contains("PERIODIC (600s)")) + assertTrue(warmUpIds.contains(it.JobId)) + if (it.SyncMode.contains("EVENT_DRIVEN (LOAD)")) { + assertTrue(it.Status.contains("CANCELLED") && (it.ErrMsg.contains("user cancel") || it.ErrMsg.contains("vcg cancel"))) + } else if (it.SyncMode.contains("PERIODIC (600s)")) { + assertTrue(it.Status.contains("CANCELLED") && it.ErrMsg.contains("vcg cancel")) + } else { + assertTrue(false) + } + } + } + // connect to follower, run again + options.connectToFollower = true + logger.info("Successfully run {} times", j + 1) + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy new file mode 100644 index 00000000000000..9939539568c46e --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +suite('test_vcg_metrics', 'multi_cluster,docker') { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + def tbl = "test_virtual_compute_group_metrics_tbl" + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // 添加一个新的cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // 添加一个新的cluster newcluster2 + cluster.addBackend(3, clusterName2) + + // create test user + def user = "regression_test_cloud_user" + sql """create user ${user} identified by 'Cloud12345'""" + sql """grant select_priv,load_priv on *.*.* to ${user}""" + + // add one vcg, test normal vcg + // contain newCluster1, newCluster2 + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + // grant usage_priv on cluster vcg to user + sql """GRANT USAGE_PRIV ON CLUSTER '${normalVclusterName}' TO '${user}'""" + // show grant + def result = sql_return_maparray """show grants for '${user}'""" + log.info("show grant for ${user}, ret={}", result) + def ret = result.find {it.CloudClusterPrivs == "normalVirtualClusterName: Cluster_usage_priv"} + // use vcg + + sql """use @${normalVclusterName}""" + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ); + """ + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + result = sql """select count(*) from ${tbl}""" + log.info("result = {}", result) + assertEquals(result.size(), 1) + + def db = context.dbName + connectInDocker(user, 'Cloud12345') { + sql """insert into ${db}.${tbl} (k1, k2) values (2, "20")""" + result = sql """select * from ${db}.${tbl}""" + assertEquals(result.size(), 2) + } + + def fe = cluster.getFeByIndex(j + 1) + + httpTest { + endpoint fe.host + ":" + fe.httpPort + uri "/metrics" + op "get" + check { code, body -> + logger.debug("code:${code} body:${body}"); + assertEquals(200, code) + assertTrue(body.contains("""doris_fe_query_total{cluster_id="normalVirtualClusterId", cluster_name="normalVirtualClusterName"} 1""")) + assertTrue(body.contains("""doris_fe_query_total{cluster_id="newcluster2_id", cluster_name="newcluster2"} 0""")) + } + } + + // stop clusterName1, switch active + cluster.stopBackends(4, 5) + + // test warm up job destory and generate new jobs + dockerAwaitUntil(50, 3) { + sql """USE @${normalVclusterName}""" + sql """select count(*) from ${tbl}""" + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"') + } + + // check after switch metrics + httpTest { + endpoint fe.host + ":" + fe.httpPort + uri "/metrics" + op "get" + check { code, body -> + logger.debug("code:${code} body:${body}"); + assertEquals(200, code) + assertFalse(body.contains("""doris_fe_query_total{cluster_id="normalVirtualClusterId", cluster_name="normalVirtualClusterName"} 1""")) + assertFalse(body.contains("""doris_fe_query_total{cluster_id="newcluster2_id", cluster_name="newcluster2"} 0""")) + } + } + } + + + // connect to follower, run again + options.connectToFollower = true + logger.info("Successfully run {} times", j + 1) + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy new file mode 100644 index 00000000000000..4c8c03e5f61e46 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy @@ -0,0 +1,451 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 stop a backend of c1 +// 3 stop a backend of c1 +// 4 start 2 backends of c1 +// 5 long-term stop 2 backends of c1 + +suite('use_vcg_read_write', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + + + showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """ SET PROPERTY 'default_cloud_cluster' = 'normalVirtualClusterName' """ + + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + cluster.stopBackends(4) + sleep(6000) + + showResult = sql "show backends" + for (row : showResult) { + println row + } + cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + + def alive = entry[9] + log.info("alive : ${alive}".toString()) + info.compute_group_name == clusterName1 && alive == "true" + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + set = [cluster1Ips[0] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + cluster.stopBackends(5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + + //// use vcg + //sql """use @${normalVclusterName}""" + //sql """ + //CREATE TABLE ${tbl} ( + //`k1` int(11) NULL, + //`k2` char(5) NULL + //) + //DUPLICATE KEY(`k1`, `k2`) + //COMMENT 'OLAP' + //DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + //PROPERTIES ( + //"replication_num"="1" + //); + //""" + + //sql """ + // insert into ${tbl} (k1, k2) values (1, "10"); + //""" + + //result = sql """select count(*) from ${tbl}""" + //log.info("result = {}", result) + //assertEquals(result.size(), 1) + + // alter cluster info, change standbyComputeGroup to newcluster1, activeComputeGroup to newcluster2 + //clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName2}", standby_cluster_names: ["${clusterName1}"]] + //clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_policy:clusterPolicy] + //normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + //jsonOutput = new JsonOutput() + //normalVcgBody = jsonOutput.toJson(normalInstance) + //alter_cluster_info_api(msHttpPort, normalVcgBody) { + // respCode, body -> + // log.info("alter virtual cluster result: ${body} ${respCode}".toString()) + // def json = parseJson(body) + // assertTrue(json.code.equalsIgnoreCase("OK")) + //} + //sleep(5000) + //showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + //log.info("show compute group after alter {}", showComputeGroup) + //vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + //assertNotNull(vcgInShow) + //assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + } + // connect to follower, run again + //options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy new file mode 100644 index 00000000000000..8a87bf71c2c83f --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy @@ -0,0 +1,453 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 stop a backend of c1 +// 3 stop a backend of c1 +// 4 start 2 backends of c1 +// 5 long-term stop 2 backends of c1 + +suite('use_vcg_read_write', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + + + showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + cluster.stopBackends(4) + sleep(6000) + + showResult = sql "show backends" + for (row : showResult) { + println row + } + cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + + def alive = entry[9] + log.info("alive : ${alive}".toString()) + info.compute_group_name == clusterName1 && alive == "true" + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + set = [cluster1Ips[0] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + cluster.stopBackends(5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + + //// use vcg + //sql """use @${normalVclusterName}""" + //sql """ + //CREATE TABLE ${tbl} ( + //`k1` int(11) NULL, + //`k2` char(5) NULL + //) + //DUPLICATE KEY(`k1`, `k2`) + //COMMENT 'OLAP' + //DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + //PROPERTIES ( + //"replication_num"="1" + //); + //""" + + //sql """ + // insert into ${tbl} (k1, k2) values (1, "10"); + //""" + + //result = sql """select count(*) from ${tbl}""" + //log.info("result = {}", result) + //assertEquals(result.size(), 1) + + // alter cluster info, change standbyComputeGroup to newcluster1, activeComputeGroup to newcluster2 + //clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName2}", standby_cluster_names: ["${clusterName1}"]] + //clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_policy:clusterPolicy] + //normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + //jsonOutput = new JsonOutput() + //normalVcgBody = jsonOutput.toJson(normalInstance) + //alter_cluster_info_api(msHttpPort, normalVcgBody) { + // respCode, body -> + // log.info("alter virtual cluster result: ${body} ${respCode}".toString()) + // def json = parseJson(body) + // assertTrue(json.code.equalsIgnoreCase("OK")) + //} + //sleep(5000) + //showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + //log.info("show compute group after alter {}", showComputeGroup) + //vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + //assertNotNull(vcgInShow) + //assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + } + // connect to follower, run again + //options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy new file mode 100644 index 00000000000000..87c5ffa7aecd7f --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy @@ -0,0 +1,393 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 stop a backend of c1 +// 3 stop a backend of c1 +// 4 start 2 backends of c1 +// 5 long-term stop 2 backends of c1 + +suite('vcg_failover_1', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 10] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + + showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + cluster.stopBackends(4, 5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + log.info("policy {}", vcgInShow.Policy) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + + cluster.startBackends(4, 5) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + cluster.stopBackends(4, 5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set) + + sleep(60000) + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + } + // connect to follower, run again + //options.connectToFollower = true + } +} From fbe74004ad00cc046330e0cce6f2f4d5dc73f5be Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:36:25 +0800 Subject: [PATCH 211/572] branch-3.0: [Enhancement](Compaction) Support auto set cumu compaction threads num base on cpu num #53133 (#53215) Cherry-picked from #53133 Co-authored-by: abmdocrt --- be/src/olap/olap_server.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index fc601903a54cc9..f7125b97567d72 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -188,7 +188,8 @@ CompactionSubmitRegistry::TabletSet& CompactionSubmitRegistry::_get_tablet_set( static int32_t get_cumu_compaction_threads_num(size_t data_dirs_num) { int32_t threads_num = config::max_cumu_compaction_threads; if (threads_num == -1) { - threads_num = data_dirs_num; + int num_cores = doris::CpuInfo::num_cores(); + threads_num = std::max(data_dirs_num, num_cores / 6); } threads_num = threads_num <= 0 ? 1 : threads_num; return threads_num; From fab89ef95f41271e0782ac97edf35c87d6e60823 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:39:20 +0800 Subject: [PATCH 212/572] branch-3.0: [fix](MTMV) create mtmv throw json_object can't be odd parameters, need even parameters. #50503 (#53186) Cherry-picked from #50503 Co-authored-by: KeeProMise --- .../plans/commands/info/BaseViewInfo.java | 29 +++-- .../mtmv_p0/test_create_with_json_object.out | 6 + .../test_create_with_json_object.groovy | 114 ++++++++++++++++++ 3 files changed, 139 insertions(+), 10 deletions(-) create mode 100644 regression-test/data/mtmv_p0/test_create_with_json_object.out create mode 100644 regression-test/suites/mtmv_p0/test_create_with_json_object.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java index af2f2378612bf9..c575bed5cc7c40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java @@ -50,6 +50,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; @@ -70,6 +71,7 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.antlr.v4.runtime.ParserRuleContext; @@ -258,7 +260,7 @@ protected static class AnalyzerForCreateView extends AbstractBatchJobExecutor { public AnalyzerForCreateView(CascadesContext cascadesContext) { super(cascadesContext); - jobs = buildAnalyzeViewJobsForStar(); + jobs = buildAnalyzeJobs(); } public void analyze() { @@ -270,16 +272,23 @@ public List getJobs() { return jobs; } - private static List buildAnalyzeViewJobsForStar() { + private static List buildAnalyzeJobs() { + return notTraverseChildrenOf( + ImmutableSet.of(LogicalView.class, LogicalCTEAnchor.class), + AnalyzerForCreateView::buildAnalyzerJobs + ); + } + + private static List buildAnalyzerJobs() { return jobs( - topDown(new EliminateLogicalSelectHint(), - new EliminateLogicalPreAggOnHint()), - topDown(new AnalyzeCTE()), - bottomUp( - new BindRelation(), - new CheckPolicy(), - new BindExpression() - ) + topDown(new EliminateLogicalSelectHint(), + new EliminateLogicalPreAggOnHint()), + topDown(new AnalyzeCTE()), + bottomUp( + new BindRelation(), + new CheckPolicy(), + new BindExpression() + ) ); } } diff --git a/regression-test/data/mtmv_p0/test_create_with_json_object.out b/regression-test/data/mtmv_p0/test_create_with_json_object.out new file mode 100644 index 00000000000000..6bf5425373f991 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_with_json_object.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +2 2 [{"l_linenumber":5,"l_discount":0.2}] +3 4 [{"l_linenumber":6,"l_discount":0.3}] +4 4 [{"l_linenumber":7,"l_discount":0.0}] + diff --git a/regression-test/suites/mtmv_p0/test_create_with_json_object.groovy b/regression-test/suites/mtmv_p0/test_create_with_json_object.groovy new file mode 100644 index 00000000000000..b64f5cbccc1c02 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_with_json_object.groovy @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_with_json_object","mtmv") { + def tableName = "t_test_create_with_json_object" + def mvName = "test_test_create_with_json_object_mtmv" + def dbName = "regression_test_mtmv_p0" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + l_orderkey INTEGER NOT NULL, + l_partkey INTEGER NOT NULL, + l_suppkey INTEGER NOT NULL, + l_linenumber INTEGER NOT NULL, + l_ordertime DATETIME NOT NULL, + l_quantity DECIMALV3(15, 2) NOT NULL, + l_extendedprice DECIMALV3(15, 2) NOT NULL, + l_discount DECIMALV3(15, 2) NOT NULL, + l_tax DECIMALV3(15, 2) NOT NULL, + l_returnflag CHAR(1) NOT NULL, + l_linestatus CHAR(1) NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct CHAR(25) NOT NULL, + l_shipmode CHAR(10) NOT NULL, + l_comment VARCHAR(44) NOT NULL + ) DUPLICATE KEY( + l_orderkey, l_partkey, l_suppkey, + l_linenumber + ) PARTITION BY RANGE(l_ordertime) ( + FROM + ('2024-05-01') TO ('2024-06-30') INTERVAL 1 DAY + ) + DISTRIBUTED BY HASH(l_orderkey) BUCKETS 1 + properties('replication_num' = '1'); + """ + sql """ + INSERT INTO ${tableName} VALUES + (1, 2, 3, 4, '2024-05-01 01:45:05', 5.5, 6.5, 0.1, 8.5, 'o', 'k', '2024-05-01', '2024-05-01', '2024-05-01', 'a', 'b', 'yyyyyyyyy'), + (1, 2, 3, 4, '2024-05-15 02:35:05', 5.5, 6.5, 0.15, 8.5, 'o', 'k', '2024-05-15', '2024-05-15', '2024-05-15', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 5, '2024-05-25 08:30:06', 5.5, 6.5, 0.2, 8.5, 'o', 'k', '2024-05-25', '2024-05-25', '2024-05-25', 'a', 'b', 'yyyyyyyyy'), + (3, 4, 3, 6, '2024-06-02 09:25:07', 5.5, 6.5, 0.3, 8.5, 'o', 'k', '2024-06-02', '2024-06-02', '2024-06-02', 'a', 'b', 'yyyyyyyyy'), + (4, 4, 3, 7, '2024-06-15 13:20:09', 5.5, 6.5, 0, 8.5, 'o', 'k', '2024-06-15', '2024-06-15', '2024-06-15', 'a', 'b', 'yyyyyyyyy'), + (5, 5, 6, 8, '2024-06-25 15:15:36', 5.5, 6.5, 0.12, 8.5, 'o', 'k', '2024-06-25', '2024-06-25', '2024-06-25', 'a', 'b', 'yyyyyyyyy'), + (5, 5, 6, 9, '2024-06-29 21:10:52', 5.5, 6.5, 0.1, 8.5, 'o', 'k', '2024-06-30', '2024-06-30', '2024-06-30', 'a', 'b', 'yyyyyyyyy'), + (5, 6, 5, 10, '2024-06-03 22:05:50', 7.5, 8.5, 0.1, 10.5, 'k', 'o', '2024-06-03', '2024-06-03', '2024-06-03', 'c', 'd', 'xxxxxxxxx'); + + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD IMMEDIATE + REFRESH AUTO ON COMMIT + DUPLICATE KEY (`l_orderkey`, `l_partkey`) + DISTRIBUTED BY RANDOM BUCKETS AUTO + PROPERTIES ('replication_num' = '1') + AS + WITH + sub_table AS ( + SELECT + l_orderkey, + l_partkey, + CONCAT( + '[', + GROUP_CONCAT( + JSON_OBJECT( + 'l_linenumber', + ${tableName}.l_linenumber, + 'l_discount', + ${tableName}.l_discount + ) + ), + ']' + ) AS l_list + FROM + ${tableName} + GROUP BY + l_orderkey, l_partkey + ) + Select * from sub_table; + """ + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO; + """ + + def jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + + order_qt_select "SELECT * FROM ${mvName} where l_orderkey in (2, 3, 4)" + + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} From 4b9d6f9695b1a8beea862c0aa6643d35bfdac3ec Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:39:57 +0800 Subject: [PATCH 213/572] branch-3.0: [fix](cloud) Persist cluster_id file in Compute-Storage Decoupled mode #53147 (#53195) Cherry-picked from #53147 Co-authored-by: yagagagaga --- be/src/cloud/cloud_storage_engine.cpp | 77 +++++++++++++++++++++++++-- be/src/cloud/cloud_storage_engine.h | 12 +++-- be/src/runtime/exec_env_init.cpp | 2 +- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 74a8f5068a49c2..6cac3c7f8d063a 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -89,10 +89,11 @@ int get_base_thread_num() { return std::min(std::max(int(num_cores * config::base_compaction_thread_num_factor), 1), 10); } -CloudStorageEngine::CloudStorageEngine(const UniqueId& backend_uid) - : BaseStorageEngine(Type::CLOUD, backend_uid), +CloudStorageEngine::CloudStorageEngine(const EngineOptions& options) + : BaseStorageEngine(Type::CLOUD, options.backend_uid), _meta_mgr(std::make_unique()), - _tablet_mgr(std::make_unique(*this)) { + _tablet_mgr(std::make_unique(*this)), + _options(options) { _cumulative_compaction_policies[CUMULATIVE_SIZE_BASED_POLICY] = std::make_shared(); _cumulative_compaction_policies[CUMULATIVE_TIME_SERIES_POLICY] = @@ -226,6 +227,9 @@ Status CloudStorageEngine::open() { init_stream_load_recorder(ExecEnv::GetInstance()->store_paths()[0].path), "init StreamLoadRecorder failed"); + // check cluster id + RETURN_NOT_OK_STATUS_WITH_WARN(_check_all_root_path_cluster_id(), "fail to check cluster id"); + return ThreadPoolBuilder("SyncLoadForTabletsThreadPool") .set_max_threads(config::sync_load_for_tablets_thread) .set_min_threads(config::sync_load_for_tablets_thread) @@ -1141,5 +1145,72 @@ Status CloudStorageEngine::unregister_compaction_stop_token(CloudTabletSPtr tabl return Status::OK(); } +Status CloudStorageEngine::_check_all_root_path_cluster_id() { + // Check if all root paths have the same cluster id + std::set cluster_ids; + for (const auto& path : _options.store_paths) { + auto cluster_id_path = fmt::format("{}/{}", path.path, CLUSTER_ID_PREFIX); + bool exists = false; + RETURN_IF_ERROR(io::global_local_filesystem()->exists(cluster_id_path, &exists)); + if (exists) { + io::FileReaderSPtr reader; + RETURN_IF_ERROR(io::global_local_filesystem()->open_file(cluster_id_path, &reader)); + size_t fsize = reader->size(); + if (fsize > 0) { + std::string content; + content.resize(fsize, '\0'); + size_t bytes_read = 0; + RETURN_IF_ERROR(reader->read_at(0, {content.data(), fsize}, &bytes_read)); + DCHECK_EQ(fsize, bytes_read); + int32_t tmp_cluster_id = std::stoi(content); + cluster_ids.insert(tmp_cluster_id); + } + } + } + _effective_cluster_id = config::cluster_id; + // first init + if (cluster_ids.empty()) { + // not set configured cluster id + if (_effective_cluster_id == -1) { + return Status::OK(); + } else { + // If no cluster id file exists, use the configured cluster id + RETURN_IF_ERROR(set_cluster_id(_effective_cluster_id)); + } + } + if (cluster_ids.size() > 1) { + return Status::InternalError( + "All root paths must have the same cluster id, but you have " + "different cluster ids: {}", + fmt::join(cluster_ids, ", ")); + } + if (_effective_cluster_id != -1 && *cluster_ids.begin() != _effective_cluster_id) { + RETURN_NOT_OK_STATUS_WITH_WARN( + Status::Corruption("multiple cluster ids is not equal. config::cluster_id={}, " + "storage path cluster_id={}", + _effective_cluster_id, *cluster_ids.begin()), + "cluster id not equal"); + } + return Status::OK(); +} + +Status CloudStorageEngine::set_cluster_id(int32_t cluster_id) { + std::lock_guard l(_store_lock); + for (auto& path : _options.store_paths) { + auto cluster_id_path = fmt::format("{}/{}", path.path, CLUSTER_ID_PREFIX); + bool exists = false; + RETURN_IF_ERROR(io::global_local_filesystem()->exists(cluster_id_path, &exists)); + if (!exists) { + io::FileWriterPtr file_writer; + RETURN_IF_ERROR( + io::global_local_filesystem()->create_file(cluster_id_path, &file_writer)); + RETURN_IF_ERROR(file_writer->append(std::to_string(cluster_id))); + RETURN_IF_ERROR(file_writer->close()); + } + } + _effective_cluster_id = cluster_id; + return Status::OK(); +} + #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/cloud/cloud_storage_engine.h b/be/src/cloud/cloud_storage_engine.h index f21e443a77e076..b7aa2da94e8eaa 100644 --- a/be/src/cloud/cloud_storage_engine.h +++ b/be/src/cloud/cloud_storage_engine.h @@ -18,6 +18,7 @@ #pragma once #include +#include //#include "cloud/cloud_cumulative_compaction.h" //#include "cloud/cloud_base_compaction.h" @@ -51,7 +52,7 @@ class CloudCompactionStopToken; class CloudStorageEngine final : public BaseStorageEngine { public: - CloudStorageEngine(const UniqueId& backend_uid); + CloudStorageEngine(const EngineOptions& options); ~CloudStorageEngine() override; @@ -64,10 +65,7 @@ class CloudStorageEngine final : public BaseStorageEngine { Status start_bg_threads() override; - Status set_cluster_id(int32_t cluster_id) override { - _effective_cluster_id = cluster_id; - return Status::OK(); - } + Status set_cluster_id(int32_t cluster_id) override; cloud::CloudMetaMgr& meta_mgr() const { return *_meta_mgr; } @@ -169,6 +167,7 @@ class CloudStorageEngine final : public BaseStorageEngine { Status _request_tablet_global_compaction_lock(ReaderType compaction_type, const CloudTabletSPtr& tablet, std::shared_ptr compaction); + Status _check_all_root_path_cluster_id(); void _lease_compaction_thread_callback(); void _check_tablet_delete_bitmap_score_callback(); @@ -221,6 +220,9 @@ class CloudStorageEngine final : public BaseStorageEngine { CumuPolices _cumulative_compaction_policies; std::atomic_bool first_sync_storage_vault {true}; + + EngineOptions _options; + std::mutex _store_lock; }; } // namespace doris diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 8f30211602a30b..1d1ad2ee184b2c 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -327,7 +327,7 @@ Status ExecEnv::_init(const std::vector& store_paths, if (config::is_cloud_mode()) { std::cout << "start BE in cloud mode, cloud_unique_id: " << config::cloud_unique_id << ", meta_service_endpoint: " << config::meta_service_endpoint << std::endl; - _storage_engine = std::make_unique(options.backend_uid); + _storage_engine = std::make_unique(options); } else { std::cout << "start BE in local mode" << std::endl; _storage_engine = std::make_unique(options); From 13ed5b7f5ce37d0f67d7ad94eed3307f13c54669 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 15 Jul 2025 09:32:34 +0800 Subject: [PATCH 214/572] [metrics](warmup) add some metrics for warmup jobs #52991 (#52739) (#53189) pick #52991 Add the following metrics: * `file_cache_once_or_periodic_warm_up_submitted_tablet_num` * `file_cache_once_or_periodic_warm_up_finished_tablet_num` Fix the following metrics: * `file_cache_once_or_periodic_warm_up_finished_segment_num` * `file_cache_once_or_periodic_warm_up_finished_segment_size` * `file_cache_once_or_periodic_warm_up_finished_index_num` * `file_cache_once_or_periodic_warm_up_finished_index_size` --- be/src/cloud/cloud_warm_up_manager.cpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index f3084555ecd73e..f270ee5ff78460 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -48,6 +48,10 @@ bvar::Adder g_file_cache_event_driven_warm_up_requested_index_size( "file_cache_event_driven_warm_up_requested_index_size"); bvar::Adder g_file_cache_event_driven_warm_up_requested_index_num( "file_cache_event_driven_warm_up_requested_index_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_tablet_num( + "file_cache_once_or_periodic_warm_up_submitted_tablet_num"); +bvar::Adder g_file_cache_once_or_periodic_warm_up_finished_tablet_num( + "file_cache_once_or_periodic_warm_up_finished_tablet_num"); bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_segment_size( "file_cache_once_or_periodic_warm_up_submitted_segment_size"); bvar::Adder g_file_cache_once_or_periodic_warm_up_submitted_segment_num( @@ -110,6 +114,13 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, return; } } + if (is_index) { + g_file_cache_once_or_periodic_warm_up_submitted_index_num << 1; + g_file_cache_once_or_periodic_warm_up_submitted_index_size << file_size; + } else { + g_file_cache_once_or_periodic_warm_up_submitted_segment_num << 1; + g_file_cache_once_or_periodic_warm_up_submitted_segment_size << file_size; + } if (is_index) { g_file_cache_once_or_periodic_warm_up_submitted_index_num << 1; @@ -139,9 +150,19 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, .is_dryrun = config::enable_reader_dryrun_when_download_file_cache, }, .download_done = - [wait](Status st) { + [=](Status st) { if (!st) { LOG_WARNING("Warm up error ").error(st); + } else if (is_index) { + g_file_cache_once_or_periodic_warm_up_finished_index_num + << (offset == 0 ? 1 : 0); + g_file_cache_once_or_periodic_warm_up_finished_index_size + << current_chunk_size; + } else { + g_file_cache_once_or_periodic_warm_up_finished_segment_num + << (offset == 0 ? 1 : 0); + g_file_cache_once_or_periodic_warm_up_finished_segment_size + << current_chunk_size; } wait->signal(); }, @@ -260,6 +281,7 @@ void CloudWarmUpManager::handle_jobs() { } } } + g_file_cache_once_or_periodic_warm_up_finished_tablet_num << 1; } timespec time; @@ -327,6 +349,7 @@ void CloudWarmUpManager::add_job(const std::vector& job_metas) { std::lock_guard lock(_mtx); std::for_each(job_metas.begin(), job_metas.end(), [this](const TJobMeta& meta) { _pending_job_metas.emplace_back(std::make_shared(meta)); + g_file_cache_once_or_periodic_warm_up_submitted_tablet_num << meta.tablet_ids.size(); }); } _cond.notify_all(); From 0a71bc7de887e384f97caf5976f529b192bf6e35 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 15 Jul 2025 10:19:24 +0800 Subject: [PATCH 215/572] branch-3.0 [chore](variant) fix unstable variant_p0/load.groovy (#53201) (#53229) cherry-pick #53201 --- regression-test/data/variant_p0/load.out | 2 +- regression-test/suites/variant_p0/load.groovy | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/regression-test/data/variant_p0/load.out b/regression-test/data/variant_p0/load.out index dbdee9d7940e30..f20f7628cade45 100644 --- a/regression-test/data/variant_p0/load.out +++ b/regression-test/data/variant_p0/load.out @@ -212,7 +212,7 @@ [123] -- !sql_25 -- -50000 54999.99999999898 6150000 +50000 55000.000000010754 6150000 -- !sql_26 -- 5000 diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index b56a97c71d6d31..8ec837ff18bae2 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -238,7 +238,7 @@ suite("regression_test_variant", "p0"){ // 12. jsonb values table_name = "jsonb_values" - create_table table_name + create_table.call(table_name, "DUPLICATE", "1") sql """insert into ${table_name} values (1, '{"a" : ["123", 123, [123]]}')""" // FIXME array -> jsonb will parse error // sql """insert into ${table_name} values (2, '{"a" : ["123"]}')""" @@ -251,6 +251,8 @@ suite("regression_test_variant", "p0"){ // sql """insert into ${table_name} values (8, '{"a" : [123, 111........]}')""" sql """insert into ${table_name} values (9, '{"a" : [123, {"a" : 1}]}')""" sql """insert into ${table_name} values (10, '{"a" : [{"a" : 1}, 123]}')""" + sql "select v['a'] from ${table_name} order by k" + trigger_and_wait_compaction(table_name, "full") qt_sql_29 "select cast(v['a'] as string) from ${table_name} order by k" // b? 7.111 [123,{"xx":1}] {"b":{"c":456,"e":7.111}} 456 qt_sql_30 "select v['b']['e'], v['a'], v['b'], v['b']['c'] from jsonb_values where cast(v['b']['e'] as double) > 1;" From 6b432f268e61a1e161c04eab387106a05618be7d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:20:20 +0800 Subject: [PATCH 216/572] branch-3.0: [fix](cloud) Sync storage resource once in read path when rowset._storage_resource.fs is null #53075 (#53227) Cherry-picked from #53075 Co-authored-by: Lei Zhang --- be/src/olap/rowset/rowset_meta.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp index 137f5c745cee43..0e6c76476b86b7 100644 --- a/be/src/olap/rowset/rowset_meta.cpp +++ b/be/src/olap/rowset/rowset_meta.cpp @@ -22,6 +22,7 @@ #include #include +#include "cloud/cloud_storage_engine.h" #include "common/logging.h" #include "google/protobuf/util/message_differencer.h" #include "io/fs/file_writer.h" @@ -112,16 +113,23 @@ Result RowsetMeta::remote_storage_resource() { } if (!_storage_resource.fs) { - // not initialized yet - auto storage_resource = get_storage_resource(resource_id()); - if (storage_resource) { + if (auto storage_resource = get_storage_resource(resource_id())) { _storage_resource = std::move(storage_resource->first); } else { + if (config::is_cloud_mode()) { + // When creating a new cluster or creating a storage resource, BE may not sync storage resource, + // at the moment a query is coming, the BetaRowsetReader call loadSegment and use this method + // to get the storage resource, so we need to sync storage resource here. + ExecEnv::GetInstance()->storage_engine().to_cloud().sync_storage_vault(); + if (auto retry_resource = get_storage_resource(resource_id())) { + _storage_resource = std::move(retry_resource->first); + return &_storage_resource; + } + } return ResultError(Status::InternalError("cannot find storage resource. resource_id={}", resource_id())); } } - return &_storage_resource; } From 5e6395393c3c1eed2c75d7a08d5484e0a4f0b1c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:20:59 +0800 Subject: [PATCH 217/572] branch-3.0: [fix](recycler) Fix two errors for recycler #53042 (#53224) Cherry-picked from #53042 Co-authored-by: Lei Zhang --- cloud/src/recycler/azure_obj_client.cpp | 7 ++- cloud/src/recycler/recycler.cpp | 1 + cloud/src/recycler/s3_accessor.cpp | 83 +++++++++++++------------ cloud/src/recycler/s3_accessor.h | 13 ++++ cloud/test/s3_accessor_test.cpp | 2 +- 5 files changed, 63 insertions(+), 43 deletions(-) diff --git a/cloud/src/recycler/azure_obj_client.cpp b/cloud/src/recycler/azure_obj_client.cpp index b50874f1fd37af..f3df2f9b6eee3e 100644 --- a/cloud/src/recycler/azure_obj_client.cpp +++ b/cloud/src/recycler/azure_obj_client.cpp @@ -137,7 +137,7 @@ class AzureListIterator final : public ObjectListIterator { SystemClockEpoch) .count()}); } - } catch (Azure::Storage::StorageException& e) { + } catch (Azure::Core::RequestFailedException& e) { LOG_WARNING( "Azure request failed because {}, http_code: {}, request_id: {}, url: {}, " "prefix: {}", @@ -145,6 +145,11 @@ class AzureListIterator final : public ObjectListIterator { req_.Prefix.Value()); is_valid_ = false; return false; + } catch (std::exception& e) { + LOG_WARNING("Azure request failed because {}, url: {}, prefix: {}", e.what(), + client_->GetUrl(), req_.Prefix.Value()); + is_valid_ = false; + return false; } return !results_.empty(); diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index e07dda0669a632..cdc43e85cfd393 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -375,6 +375,7 @@ void Recycler::check_recycle_tasks() { int Recycler::start(brpc::Server* server) { instance_filter_.reset(config::recycle_whitelist, config::recycle_blacklist); g_bvar_recycler_task_max_concurrency.set_value(config::recycle_concurrency); + S3Environment::getInstance(); if (config::enable_checker) { checker_ = std::make_unique(txn_kv_); diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 3c36f5a01bb248..464beb58e2e6d8 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -111,50 +111,52 @@ int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_bur return AccessorRateLimiter::instance().rate_limiter(type)->reset(max_speed, max_burst, limit); } -class S3Environment { -public: - S3Environment() { - aws_options_ = Aws::SDKOptions {}; - auto logLevel = static_cast(config::aws_log_level); - aws_options_.loggingOptions.logLevel = logLevel; - aws_options_.loggingOptions.logger_create_fn = [logLevel] { - return std::make_shared(logLevel); - }; - Aws::InitAPI(aws_options_); +S3Environment::S3Environment() { + LOG(INFO) << "Initializing S3 environment"; + aws_options_ = Aws::SDKOptions {}; + auto logLevel = static_cast(config::aws_log_level); + aws_options_.loggingOptions.logLevel = logLevel; + aws_options_.loggingOptions.logger_create_fn = [logLevel] { + return std::make_shared(logLevel); + }; + Aws::InitAPI(aws_options_); #ifdef USE_AZURE - auto azureLogLevel = - static_cast(config::azure_log_level); - Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel); - Azure::Core::Diagnostics::Logger::SetListener( - [&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) { - switch (level) { - case Azure::Core::Diagnostics::Logger::Level::Verbose: - LOG(INFO) << message; - break; - case Azure::Core::Diagnostics::Logger::Level::Informational: - LOG(INFO) << message; - break; - case Azure::Core::Diagnostics::Logger::Level::Warning: - LOG(WARNING) << message; - break; - case Azure::Core::Diagnostics::Logger::Level::Error: - LOG(ERROR) << message; - break; - default: - LOG(WARNING) << "Unknown level: " << static_cast(level) - << ", message: " << message; - break; - } - }); + auto azureLogLevel = + static_cast(config::azure_log_level); + Azure::Core::Diagnostics::Logger::SetLevel(azureLogLevel); + Azure::Core::Diagnostics::Logger::SetListener( + [&](Azure::Core::Diagnostics::Logger::Level level, const std::string& message) { + switch (level) { + case Azure::Core::Diagnostics::Logger::Level::Verbose: + LOG(INFO) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Informational: + LOG(INFO) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Warning: + LOG(WARNING) << message; + break; + case Azure::Core::Diagnostics::Logger::Level::Error: + LOG(ERROR) << message; + break; + default: + LOG(WARNING) << "Unknown level: " << static_cast(level) + << ", message: " << message; + break; + } + }); #endif - } +} - ~S3Environment() { Aws::ShutdownAPI(aws_options_); } +S3Environment& S3Environment::getInstance() { + static S3Environment instance; + return instance; +} -private: - Aws::SDKOptions aws_options_; -}; +S3Environment::~S3Environment() { + Aws::ShutdownAPI(aws_options_); +} class S3ListIterator final : public ListIterator { public: @@ -316,6 +318,7 @@ int S3Accessor::init() { std::make_shared(config::recycle_pool_parallelism, "s3_accessor"); worker_pool->start(); }); + S3Environment::getInstance(); switch (conf_.provider) { case S3Conf::AZURE: { #ifdef USE_AZURE @@ -355,8 +358,6 @@ int S3Accessor::init() { uri_ = conf_.endpoint + '/' + conf_.bucket + '/' + conf_.prefix; } - static S3Environment s3_env; - // S3Conf::S3 Aws::Client::ClientConfiguration aws_config; aws_config.endpointOverride = conf_.endpoint; diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h index 544c142d09b786..dd93e83d9631f4 100644 --- a/cloud/src/recycler/s3_accessor.h +++ b/cloud/src/recycler/s3_accessor.h @@ -53,6 +53,19 @@ extern bvar::LatencyRecorder s3_get_bucket_version_latency; extern bvar::LatencyRecorder s3_copy_object_latency; }; // namespace s3_bvar +class S3Environment { +public: + S3Environment(const S3Environment&) = delete; + S3Environment& operator=(const S3Environment&) = delete; + + static S3Environment& getInstance(); + + ~S3Environment(); + +private: + S3Environment(); + Aws::SDKOptions aws_options_; +}; struct AccessorRateLimiter { public: ~AccessorRateLimiter() = default; diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp index f95fb2dba1848d..d63baa2a7b5834 100644 --- a/cloud/test/s3_accessor_test.cpp +++ b/cloud/test/s3_accessor_test.cpp @@ -46,7 +46,7 @@ int main(int argc, char** argv) { std::cerr << "failed to init glog" << std::endl; return -1; } - doris::cloud::config::aws_log_level = 5; + LOG(INFO) << "s3_accessor_test starting"; ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } From 285beb2ca1cab7b1e717edffb1eb231af1ef7edd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:21:54 +0800 Subject: [PATCH 218/572] branch-3.0: [fix](case) fix txn_insert_with_schema_change #53177 (#53221) Cherry-picked from #53177 Co-authored-by: meiyi --- .../insert_p0/transaction/txn_insert_with_schema_change.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy index f2fc1a009f9a78..e01f03ac30598d 100644 --- a/regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_with_schema_change.groovy @@ -63,7 +63,7 @@ suite("txn_insert_with_schema_change") { return } retry++ - if (retry >= 10 || last_state == "FINISHED" || last_state == "CANCELLED") { + if (retry >= 40 || last_state == "FINISHED" || last_state == "CANCELLED") { break } } From f0218508206ea3ec92b0ea6184db0c00d820be5a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:52:34 +0800 Subject: [PATCH 219/572] branch-3.0: [opt](test) Enlarge timeout for compaction score action #53040 (#53217) Cherry-picked from #53040 Co-authored-by: Siyang Tang --- .../suites/compaction/test_compaction_score_action.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/compaction/test_compaction_score_action.groovy b/regression-test/suites/compaction/test_compaction_score_action.groovy index c9d8b7ff8cdd12..94b8ce5d34e6e8 100644 --- a/regression-test/suites/compaction/test_compaction_score_action.groovy +++ b/regression-test/suites/compaction/test_compaction_score_action.groovy @@ -40,7 +40,7 @@ suite("test_compaction_score_action") { for (int i=0;i= 90) From 121641e8a5646292eed19245f0080c1b4e9cf8b1 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Mon, 14 Jul 2025 19:53:14 -0700 Subject: [PATCH 220/572] branch-3.0: [fix](iceberg) Fix transaction issues (#52716) (#53237) bp #52716 Co-authored-by: wuwenchi Co-authored-by: wuwenchi.wwc --- .../iceberg/IcebergTransaction.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 797caea0deaf1d..c7d7212335a224 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -111,8 +111,8 @@ public void finishInsert(SimpleTableInfo tableInfo, Optional pendingResults; if (commitDataList.isEmpty()) { @@ -125,9 +125,9 @@ private void updateManifestAfterInsert(TUpdateMode updateMode) { } if (updateMode == TUpdateMode.APPEND) { - commitAppendTxn(table, pendingResults); + commitAppendTxn(pendingResults); } else { - commitReplaceTxn(table, pendingResults); + commitReplaceTxn(pendingResults); } } @@ -146,16 +146,15 @@ public long getUpdateCnt() { return commitDataList.stream().mapToLong(TIcebergCommitData::getRowCount).sum(); } - private synchronized Table getNativeTable(SimpleTableInfo tableInfo) { Objects.requireNonNull(tableInfo); ExternalCatalog externalCatalog = ops.getExternalCatalog(); return IcebergUtils.getRemoteTable(externalCatalog, tableInfo); } - private void commitAppendTxn(Table table, List pendingResults) { + private void commitAppendTxn(List pendingResults) { // commit append files. - AppendFiles appendFiles = table.newAppend().scanManifestsWith(ops.getThreadPoolWithPreAuth()); + AppendFiles appendFiles = transaction.newAppend().scanManifestsWith(ops.getThreadPoolWithPreAuth()); for (WriteResult result : pendingResults) { Preconditions.checkState(result.referencedDataFiles().length == 0, "Should have no referenced data files for append."); @@ -165,13 +164,15 @@ private void commitAppendTxn(Table table, List pendingResults) { } - private void commitReplaceTxn(Table table, List pendingResults) { + private void commitReplaceTxn(List pendingResults) { if (pendingResults.isEmpty()) { // such as : insert overwrite table `dst_tb` select * from `empty_tb` // 1. if dst_tb is a partitioned table, it will return directly. // 2. if dst_tb is an unpartitioned table, the `dst_tb` table will be emptied. - if (!table.spec().isPartitioned()) { - OverwriteFiles overwriteFiles = table.newOverwrite().scanManifestsWith(ops.getThreadPoolWithPreAuth()); + if (!transaction.table().spec().isPartitioned()) { + OverwriteFiles overwriteFiles = transaction + .newOverwrite() + .scanManifestsWith(ops.getThreadPoolWithPreAuth()); try (CloseableIterable fileScanTasks = table.newScan().planFiles()) { fileScanTasks.forEach(f -> overwriteFiles.deleteFile(f.file())); } catch (IOException e) { @@ -183,7 +184,9 @@ private void commitReplaceTxn(Table table, List pendingResults) { } // commit replace partitions - ReplacePartitions appendPartitionOp = table.newReplacePartitions(); + ReplacePartitions appendPartitionOp = transaction + .newReplacePartitions() + .scanManifestsWith(ops.getThreadPoolWithPreAuth()); for (WriteResult result : pendingResults) { Preconditions.checkState(result.referencedDataFiles().length == 0, "Should have no referenced data files."); From 53e5c9f5cc024eb124a561100e569c4c854a99c4 Mon Sep 17 00:00:00 2001 From: minghong Date: Tue, 15 Jul 2025 10:58:33 +0800 Subject: [PATCH 221/572] Branch-3.0 [opt](nereids) avoid printing duplicated table stats in profile #51354 (#51658) ### What problem does this PR solve? pick #51354 From 6678fa36430d78b06d7b74622f57259f8add344f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:17:13 +0800 Subject: [PATCH 222/572] branch-3.0: [fix](case) do not use global var in script #53231 (#53240) Cherry-picked from #53231 Co-authored-by: Yongqiang YANG --- .../suites/cloud_p0/multi_cluster/test_auto_start.groovy | 2 +- regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy | 2 +- .../suites/cloud_p0/node_mgr/test_rename_compute_group.groovy | 2 +- .../compaction10/test_schema_change_with_compaction10.groovy | 2 +- .../compaction11/test_schema_change_with_compaction11.groovy | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy index ce45bcfe093972..ceb8969b721580 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy @@ -137,7 +137,7 @@ suite('test_auto_start_in_cloud', 'multi_cluster, docker') { cluster.stopBackends(1,2,3) // select - future1 = thread { + def future1 = thread { def begin = System.currentTimeMillis(); // root cant resume, due to deamon thread use root def connInfo = context.threadLocalConn.get() diff --git a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy index 62dfe5d39e269b..b094b5354af1d6 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy @@ -399,7 +399,7 @@ suite('test_ms_api', 'p0, docker') { def internal_bucket_user = [[user_id:"1-userid", ak:"test-ak1-updated", sk:"test-sk1-updated"]] def update_ak_sk_api_body = [instance_id:"${instance_id}", internal_bucket_user:internal_bucket_user] jsonOutput = new JsonOutput() - upDateAKSKBody = jsonOutput.toJson(update_ak_sk_api_body) + def upDateAKSKBody = jsonOutput.toJson(update_ak_sk_api_body) update_ak_sk_api.call(msHttpPort, upDateAKSKBody) { diff --git a/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy b/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy index 92077c679e5840..fd97b15eb51e0a 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy @@ -47,7 +47,7 @@ suite('test_rename_compute_group', 'docker, p0') { } } def findToDropUniqueId = { clusterId, hostIP, metaServices -> - ret = get_instance(metaServices) + def ret = get_instance(metaServices) def toDropCluster = ret.clusters.find { it.cluster_id.contains(clusterId) } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy index ca8b2f70ab9b3b..3c023b26c28462 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy @@ -124,7 +124,7 @@ suite('test_schema_change_with_compaction10', 'docker') { } int max_try_time = 3000 while (max_try_time--){ - result = getJobState("date") + def result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { sleep(3000) break diff --git a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy index 768481a3264ded..2845fe2f5c0141 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy @@ -144,7 +144,7 @@ suite('test_schema_change_with_compaction11', 'docker') { } int max_try_time = 3000 while (max_try_time--){ - result = getJobState("date") + def result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { sleep(3000) break From c24e6ddf65a13fcfe2346da50eabcf4063168b43 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 15:06:45 +0800 Subject: [PATCH 223/572] branch-3.0: [opt](Nereids) aggregate function sum support string type as parameter #49954 (#53261) Cherry-picked from #49954 Co-authored-by: morrySnow --- .../trees/expressions/functions/agg/Avg.java | 26 ++++++--- .../trees/expressions/functions/agg/Sum.java | 29 +++++----- .../trees/expressions/functions/agg/Sum0.java | 29 +++++----- .../analysis/CheckExpressionLegalityTest.java | 2 +- .../trees/expressions/GetDataTypeTest.java | 54 +++++++++++++++++-- .../agg_function/agg.groovy | 28 ++++++++++ 6 files changed, 131 insertions(+), 37 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java index db1d8d7eb7c469..e54b57f15ecce3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Avg.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.qe.ConnectContext; @@ -49,14 +50,14 @@ public class Avg extends NullableAggregateFunction implements UnaryExpression, ExplicitlyCastableSignature, ComputePrecision, SupportWindowAnalytic { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(DoubleType.INSTANCE).args(TinyIntType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(SmallIntType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(IntegerType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(BigIntType.INSTANCE), - FunctionSignature.ret(DoubleType.INSTANCE).args(LargeIntType.INSTANCE), FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE), + FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD), FunctionSignature.ret(DecimalV2Type.SYSTEM_DEFAULT).args(DecimalV2Type.SYSTEM_DEFAULT), - FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD) + FunctionSignature.ret(DoubleType.INSTANCE).args(LargeIntType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(BigIntType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(IntegerType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(SmallIntType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(TinyIntType.INSTANCE) ); /** @@ -80,8 +81,9 @@ private Avg(boolean distinct, boolean alwaysNullable, Expression arg) { @Override public void checkLegalityBeforeTypeCoercion() { DataType argType = child().getDataType(); - if (((!argType.isNumericType() && !argType.isNullType()) || argType.isOnlyMetricType())) { - throw new AnalysisException("avg requires a numeric parameter: " + toSql()); + if (!argType.isNumericType() && !argType.isBooleanType() + && !argType.isNullType() && !argType.isStringLikeType()) { + throw new AnalysisException("avg requires a numeric, boolean or string parameter: " + this.toSql()); } } @@ -153,4 +155,12 @@ public R accept(ExpressionVisitor visitor, C context) { public List getSignatures() { return SIGNATURES; } + + @Override + public FunctionSignature searchSignature(List signatures) { + if (getArgument(0).getDataType() instanceof NullType) { + return FunctionSignature.ret(DoubleType.INSTANCE).args(TinyIntType.INSTANCE); + } + return ExplicitlyCastableSignature.super.searchSignature(signatures); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java index e55f926ae4d3ca..b5616dad15cb3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java @@ -29,11 +29,13 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.FloatType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.TinyIntType; @@ -50,14 +52,15 @@ public class Sum extends NullableAggregateFunction RollUpTrait { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(BooleanType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(TinyIntType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(SmallIntType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(IntegerType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(BigIntType.INSTANCE), - FunctionSignature.ret(LargeIntType.INSTANCE).args(LargeIntType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE), FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD), - FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) + FunctionSignature.ret(LargeIntType.INSTANCE).args(LargeIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(BigIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(IntegerType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(SmallIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(TinyIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(BooleanType.INSTANCE) ); /** @@ -87,9 +90,9 @@ public MultiDistinctSum convertToMultiDistinct() { @Override public void checkLegalityBeforeTypeCoercion() { DataType argType = child().getDataType(); - if ((!argType.isNumericType() && !argType.isBooleanType() && !argType.isNullType()) - || argType.isOnlyMetricType()) { - throw new AnalysisException("sum requires a numeric or boolean parameter: " + this.toSql()); + if (!argType.isNumericType() && !argType.isBooleanType() + && !argType.isNullType() && !argType.isStringLikeType()) { + throw new AnalysisException("sum requires a numeric, boolean or string parameter: " + this.toSql()); } } @@ -119,8 +122,10 @@ public List getSignatures() { @Override public FunctionSignature searchSignature(List signatures) { - if (getArgument(0).getDataType() instanceof FloatType) { - return FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE); + if (getArgument(0).getDataType() instanceof NullType) { + return FunctionSignature.ret(BigIntType.INSTANCE).args(TinyIntType.INSTANCE); + } else if (getArgument(0).getDataType() instanceof DecimalV2Type) { + return FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD); } return ExplicitlyCastableSignature.super.searchSignature(signatures); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java index 5a1f0f9fb93d34..7c3873de01f841 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum0.java @@ -33,11 +33,13 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.FloatType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.types.NullType; import org.apache.doris.nereids.types.SmallIntType; import org.apache.doris.nereids.types.TinyIntType; @@ -57,14 +59,15 @@ public class Sum0 extends NotNullableAggregateFunction SupportWindowAnalytic, RollUpTrait { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(BigIntType.INSTANCE).args(BooleanType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(TinyIntType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(SmallIntType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(IntegerType.INSTANCE), - FunctionSignature.ret(BigIntType.INSTANCE).args(BigIntType.INSTANCE), - FunctionSignature.ret(LargeIntType.INSTANCE).args(LargeIntType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE), + FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE), FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD), - FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) + FunctionSignature.ret(LargeIntType.INSTANCE).args(LargeIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(BigIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(IntegerType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(SmallIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(TinyIntType.INSTANCE), + FunctionSignature.ret(BigIntType.INSTANCE).args(BooleanType.INSTANCE) ); /** @@ -90,9 +93,9 @@ public MultiDistinctSum0 convertToMultiDistinct() { @Override public void checkLegalityBeforeTypeCoercion() { DataType argType = child().getDataType(); - if ((!argType.isNumericType() && !argType.isBooleanType() && !argType.isNullType()) - || argType.isOnlyMetricType()) { - throw new AnalysisException("sum0 requires a numeric or boolean parameter: " + this.toSql()); + if (!argType.isNumericType() && !argType.isBooleanType() + && !argType.isNullType() && !argType.isStringLikeType()) { + throw new AnalysisException("sum0 requires a numeric, boolean or string parameter: " + this.toSql()); } } @@ -117,8 +120,10 @@ public List getSignatures() { @Override public FunctionSignature searchSignature(List signatures) { - if (getArgument(0).getDataType() instanceof FloatType) { - return FunctionSignature.ret(DoubleType.INSTANCE).args(FloatType.INSTANCE); + if (getArgument(0).getDataType() instanceof NullType) { + return FunctionSignature.ret(BigIntType.INSTANCE).args(TinyIntType.INSTANCE); + } else if (getArgument(0).getDataType() instanceof DecimalV2Type) { + return FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD); } return ExplicitlyCastableSignature.super.searchSignature(signatures); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java index 2b0ae34dc37514..34beb21f440801 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/CheckExpressionLegalityTest.java @@ -35,7 +35,7 @@ public class CheckExpressionLegalityTest implements MemoPatternMatchSupported { public void testAvg() { ConnectContext connectContext = MemoTestUtils.createConnectContext(); ExceptionChecker.expectThrowsWithMsg( - AnalysisException.class, "avg requires a numeric parameter", () -> { + AnalysisException.class, "avg requires a numeric", () -> { PlanChecker.from(connectContext) .analyze("select avg(id) from (select to_bitmap(1) id) tbl"); }); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java index 05824d32802071..e95b0cd4b4d100 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java @@ -17,13 +17,16 @@ package org.apache.doris.nereids.trees.expressions; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum0; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; import org.apache.doris.nereids.trees.expressions.literal.CharLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral; +import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; @@ -35,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DecimalV2Type; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.types.DoubleType; import org.apache.doris.nereids.types.LargeIntType; @@ -57,6 +61,7 @@ public class GetDataTypeTest { FloatLiteral floatLiteral = new FloatLiteral(1.0F); DoubleLiteral doubleLiteral = new DoubleLiteral(1.0); DecimalLiteral decimalLiteral = new DecimalLiteral(BigDecimal.ONE); + DecimalV3Literal decimalV3Literal = new DecimalV3Literal(new BigDecimal("123.123456")); CharLiteral charLiteral = new CharLiteral("hello", 5); VarcharLiteral varcharLiteral = new VarcharLiteral("hello", 5); StringLiteral stringLiteral = new StringLiteral("hello"); @@ -75,14 +80,55 @@ public void testSum() { Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(floatLiteral))); Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(doubleLiteral))); Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(38, 0), checkAndGetDataType(new Sum(decimalLiteral))); - Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum(bigIntLiteral))); - Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(charLiteral))); - Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(varcharLiteral))); - Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(stringLiteral))); + Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(38, 6), checkAndGetDataType(new Sum(decimalV3Literal))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(charLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(varcharLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(stringLiteral))); Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(dateLiteral))); Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(dateTimeLiteral))); } + @Test + public void testSum0() { + Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum0(nullLiteral))); + Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum0(booleanLiteral))); + Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum0(tinyIntLiteral))); + Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum0(smallIntLiteral))); + Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum0(integerLiteral))); + Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum0(bigIntLiteral))); + Assertions.assertEquals(LargeIntType.INSTANCE, checkAndGetDataType(new Sum0(largeIntLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum0(floatLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum0(doubleLiteral))); + Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(38, 0), checkAndGetDataType(new Sum0(decimalLiteral))); + Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(38, 6), checkAndGetDataType(new Sum0(decimalV3Literal))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum0(charLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum0(varcharLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum0(stringLiteral))); + Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum0(dateLiteral))); + Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum0(dateTimeLiteral))); + } + + @Test + public void testAvg() { + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(nullLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(booleanLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(tinyIntLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(smallIntLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(integerLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(bigIntLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(largeIntLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(floatLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(doubleLiteral))); + Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(27, 9), checkAndGetDataType(new Avg(decimalLiteral))); + Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(38, 6), checkAndGetDataType(new Avg(decimalV3Literal))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(bigIntLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(charLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(varcharLiteral))); + Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Avg(stringLiteral))); + Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Avg(dateLiteral))); + Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Avg(dateTimeLiteral))); + } + private DataType checkAndGetDataType(Expression expression) { expression.checkLegalityBeforeTypeCoercion(); expression.checkLegalityAfterRewrite(); diff --git a/regression-test/suites/nereids_function_p0/agg_function/agg.groovy b/regression-test/suites/nereids_function_p0/agg_function/agg.groovy index ba4ee172517bf2..3e10240ea61b48 100644 --- a/regression-test/suites/nereids_function_p0/agg_function/agg.groovy +++ b/regression-test/suites/nereids_function_p0/agg_function/agg.groovy @@ -2338,6 +2338,20 @@ suite("nereids_agg_fn") { qt_sql_sum_LargeInt_agg_phase_4_notnull ''' select /*+SET_VAR(disable_nereids_rules='THREE_PHASE_AGGREGATE_WITH_DISTINCT, TWO_PHASE_AGGREGATE_WITH_DISTINCT')*/ count(distinct id), sum(klint) from fn_test''' + // sum on string like + explain { + sql("select sum(kstr) from fn_test;") + contains "partial_sum(cast(kstr as DOUBLE" + } + explain { + sql("select sum(kvchrs3) from fn_test;") + contains "partial_sum(cast(kvchrs3 as DOUBLE" + } + explain { + sql("select sum(kchrs3) from fn_test;") + contains "partial_sum(cast(kchrs3 as DOUBLE" + } + qt_sql_sum0_Boolean ''' select sum0(kbool) from fn_test''' qt_sql_sum0_Boolean_gb ''' @@ -2523,6 +2537,20 @@ suite("nereids_agg_fn") { qt_sql_sum0_LargeInt_agg_phase_4_notnull ''' select /*+SET_VAR(disable_nereids_rules='THREE_PHASE_AGGREGATE_WITH_DISTINCT, TWO_PHASE_AGGREGATE_WITH_DISTINCT')*/ count(distinct id), sum0(klint) from fn_test''' + // sum on string like + explain { + sql("select sum0(kstr) from fn_test;") + contains "partial_sum0(cast(kstr as DOUBLE" + } + explain { + sql("select sum0(kvchrs3) from fn_test;") + contains "partial_sum0(cast(kvchrs3 as DOUBLE" + } + explain { + sql("select sum0(kchrs3) from fn_test;") + contains "partial_sum0(cast(kchrs3 as DOUBLE" + } + qt_sql_topn_Varchar_Integer_gb ''' select topn(kvchrs1, 3) from fn_test group by kbool order by kbool''' qt_sql_topn_Varchar_Integer ''' From 29a008cffd71221bd1a1be6415dad848fbb96d09 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 15 Jul 2025 16:44:45 +0800 Subject: [PATCH 224/572] branch-3.0: [feature](index)Support light index add for inverted index without parser and ngram bf index #52894 (#52988) cherry pick from #52894 --------- Co-authored-by: qiye --- fe/fe-core/src/main/cup/sql_parser.cup | 10 +- .../doris/alter/SchemaChangeHandler.java | 92 ++- .../apache/doris/alter/SchemaChangeJobV2.java | 15 +- .../doris/analysis/BuildIndexClause.java | 61 +- .../org/apache/doris/analysis/IndexDef.java | 4 +- .../java/org/apache/doris/catalog/Index.java | 23 + .../org/apache/doris/catalog/OlapTable.java | 14 + .../datasource/CloudInternalCatalog.java | 6 + .../org/apache/doris/qe/SessionVariable.java | 17 + .../apache/doris/alter/CloudIndexTest.java | 629 ++++++++++++++++++ .../doris/alter/IndexChangeJobTest.java | 179 ++++- .../doris/alter/SchemaChangeHandlerTest.java | 5 +- .../apache/doris/catalog/CatalogTestUtil.java | 42 +- .../test_ngram_bloomfilter_index_change.out | 103 +++ .../pipeline/cloud_p0/conf/fe_custom.conf | 1 - .../pipeline/cloud_p1/conf/fe_custom.conf | 1 - ...test_ngram_bloomfilter_index_change.groovy | 325 +++++++++ 17 files changed, 1413 insertions(+), 114 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java create mode 100644 regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out create mode 100644 regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 0a0afa736348ae..31bcafcb7da213 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -879,7 +879,6 @@ nonterminal Map key_value_map, opt_key_value_map, opt_key_value_ opt_ext_properties, opt_enable_feature_properties, properties; nonterminal ColumnDef column_definition; nonterminal IndexDef index_definition; -nonterminal IndexDef build_index_definition; nonterminal ArrayList column_definition_list; nonterminal ArrayList index_definition_list; nonterminal AggregateType opt_agg_type; @@ -2078,7 +2077,7 @@ create_stmt ::= :} | KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName opt_partition_names:partitionNames {: - RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true), false))); + RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new BuildIndexClause(tableName, indexName, partitionNames, false))); :} /* stage */ | KW_CREATE KW_STAGE opt_if_not_exists:ifNotExists ident:stageName KW_PROPERTIES opt_key_value_map:properties @@ -4051,13 +4050,6 @@ index_definition ::= :} ; -build_index_definition ::= - KW_INDEX ident:indexName opt_partition_names:partitionNames - {: - RESULT = new IndexDef(indexName, partitionNames, true); - :} - ; - opt_nullable_type ::= {: RESULT = ColumnNullableType.DEFAULT; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 53a7a629228f41..e3ad8d19483581 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -1312,7 +1312,7 @@ private void createJob(String rawSql, long dbId, OlapTable olapTable, Map oriBfColumns = olapTable.getCopiedBfColumns(); double oriBfFpp = olapTable.getBfFpp(); @@ -1948,7 +1948,7 @@ public int getAsInt() { List newIndexes = olapTable.getCopiedIndexes(); List alterIndexes = new ArrayList<>(); - Map> invertedIndexOnPartitions = new HashMap<>(); + Map> indexOnPartitions = new HashMap<>(); boolean isDropIndex = false; Map propertyMap = new HashMap<>(); for (AlterClause alterClause : alterClauses) { @@ -2086,10 +2086,23 @@ public int getAsInt() { } lightSchemaChange = false; - if (index.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // Check if the index supports light index change and session variable is enabled + boolean enableAddIndexForNewData = true; + try { + ConnectContext context = ConnectContext.get(); + if (context != null && context.getSessionVariable() != null) { + enableAddIndexForNewData = context.getSessionVariable().isEnableAddIndexForNewData(); + } + } catch (Exception e) { + LOG.warn("Failed to get session variable enable_add_index_for_new_data, " + + "using default value: false", e); + } + + // ngram_bf index can do light_schema_change in both local and cloud mode + // inverted index can only do light_schema_change in local mode + if (index.isLightAddIndexSupported(enableAddIndexForNewData)) { alterIndexes.add(index); isDropIndex = false; - // now only support light index change for inverted index lightIndexChange = true; } } else if (alterClause instanceof BuildIndexClause) { @@ -2100,54 +2113,14 @@ public int getAsInt() { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } - if (!olapTable.isPartitionedTable()) { - List specifiedPartitions = indexDef.getPartitionNames(); - if (!specifiedPartitions.isEmpty()) { - throw new DdlException("table " + olapTable.getName() - + " is not partitioned, cannot build index with partitions."); - } - } - List existedIndexes = olapTable.getIndexes(); - boolean found = false; - for (Index existedIdx : existedIndexes) { - if (existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) { - found = true; - if (!existedIdx.isLightIndexChangeSupported()) { - throw new DdlException("BUILD INDEX operation failed: The index " - + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() - + " does not support lightweight index changes."); - } - for (Column column : olapTable.getBaseSchema()) { - if (!column.getType().isVariantType()) { - continue; - } - // variant type column can not support for building index - for (String indexColumn : existedIdx.getColumns()) { - if (column.getName().equalsIgnoreCase(indexColumn)) { - throw new DdlException("BUILD INDEX operation failed: The " - + indexDef.getIndexName() + " index can not be built on the " - + indexColumn + " column, because it is a variant type column."); - } - } - } - index = existedIdx.clone(); - if (indexDef.getPartitionNames().isEmpty()) { - invertedIndexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); - } else { - invertedIndexOnPartitions.put( - index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); - } - break; - } - } - if (!found) { - throw new DdlException("index " + indexDef.getIndexName() - + " not exist, cannot build it with defferred."); + if (indexDef.getPartitionNames().isEmpty()) { + indexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); + } else { + indexOnPartitions.put( + index.getIndexId(), new HashSet<>(indexDef.getPartitionNames())); } - if (indexDef.isInvertedIndex()) { - alterIndexes.add(index); - } + alterIndexes.add(index); buildIndexChange = true; lightSchemaChange = false; } else if (alterClause instanceof DropIndexClause) { @@ -2165,7 +2138,9 @@ public int getAsInt() { break; } } - if (found.isLightIndexChangeSupported() && !Config.isCloudMode()) { + // only inverted index with local mode can do light drop index change + if (found != null && found.getIndexType() == IndexDef.IndexType.INVERTED + && Config.isNotCloudMode()) { alterIndexes.add(found); isDropIndex = true; lightIndexChange = true; @@ -2189,13 +2164,16 @@ public int getAsInt() { null, isDropIndex, jobId, false); } else if (Config.enable_light_index_change && lightIndexChange) { long jobId = Env.getCurrentEnv().getNextId(); - //for schema change add/drop inverted index optimize, direct modify table meta firstly. + //for schema change add/drop inverted index and ngram_bf optimize, direct modify table meta firstly. modifyTableLightSchemaChange(rawSql, db, olapTable, indexSchemaMap, newIndexes, alterIndexes, isDropIndex, jobId, false); } else if (buildIndexChange) { + if (alterIndexes.isEmpty()) { + throw new DdlException("Altered index is empty. please check your alter stmt."); + } if (Config.enable_light_index_change) { buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, false); + alterIndexes, indexOnPartitions, false); } } else { createJob(rawSql, db.getId(), olapTable, indexSchemaMap, propertyMap, newIndexes); @@ -2688,6 +2666,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd olapTable.writeUnlock(); } + // if this table has ngram_bf index, we must run cancel for schema change job + boolean hasNGramBFIndex = ((OlapTable) olapTable).hasIndexOfType(IndexDef.IndexType.NGRAM_BF); // alter job v2's cancel must be called outside the table lock if (jobList.size() > 0) { for (IndexChangeJob job : jobList) { @@ -2702,6 +2682,8 @@ private void cancelIndexJob(CancelAlterTableStmt cancelAlterTableStmt) throws Dd LOG.info("cancel build index job {} on table {} success", jobId, tableName); } } + } else if (hasNGramBFIndex) { + cancelColumnJob(cancelAlterTableStmt); } else { throw new DdlException("No job to cancel for Table[" + tableName + "]"); } @@ -2741,7 +2723,7 @@ private boolean processAddIndex(CreateIndexClause alterClause, OlapTable olapTab Column column = olapTable.getColumn(col); if (column != null) { indexDef.checkColumn(column, olapTable.getKeysType(), - olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(), + olapTable.getEnableUniqueKeyMergeOnWrite(), olapTable.getInvertedIndexFileStorageFormat()); } else { throw new DdlException("index column does not exist in table. invalid column: " + col); @@ -2955,7 +2937,7 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o } try { buildOrDeleteTableInvertedIndices(db, olapTable, indexSchemaMap, - alterIndexes, invertedIndexOnPartitions, true); + alterIndexes, invertedIndexOnPartitions, true); } catch (Exception e) { throw new DdlException(e.getMessage()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index b3136b41b04369..28d58e4f6fa5a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -146,7 +146,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 { protected boolean hasRowStoreChange = false; // save all schema change tasks - private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); + AgentBatchTask schemaChangeBatchTask = new AgentBatchTask(); protected SchemaChangeJobV2() { super(JobType.SCHEMA_CHANGE); @@ -667,14 +667,13 @@ protected void runRunningJob() throws AlterCancelException { healthyReplicaNum++; } } - if (!FeConstants.runningUnitTest) { - if (healthyReplicaNum < expectReplicationNum / 2 + 1) { - LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" - + " healthyReplicaNum {} expectReplicationNum {}", - shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); - throw new AlterCancelException( + + if ((healthyReplicaNum < expectReplicationNum / 2 + 1) && !FeConstants.runningUnitTest) { + LOG.warn("shadow tablet {} has few healthy replicas: {}, schema change job: {}" + + " healthyReplicaNum {} expectReplicationNum {}", + shadowTablet.getId(), replicas, jobId, healthyReplicaNum, expectReplicationNum); + throw new AlterCancelException( "shadow tablet " + shadowTablet.getId() + " has few healthy replicas"); - } } } // end for tablets } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java index d04d24e86e403a..7cd1d915150911 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java @@ -18,12 +18,17 @@ package org.apache.doris.analysis; import org.apache.doris.alter.AlterOpType; +import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import com.google.common.collect.Maps; +import java.util.List; import java.util.Map; public class BuildIndexClause extends AlterTableClause { @@ -36,11 +41,14 @@ public class BuildIndexClause extends AlterTableClause { private boolean alter; // index internal class private Index index; + private String indexName; + private PartitionNames partitionNames; - public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean alter) { + public BuildIndexClause(TableName tableName, String indexName, PartitionNames partitionNames, boolean alter) { super(AlterOpType.SCHEMA_CHANGE); this.tableName = tableName; - this.indexDef = indexDef; + this.indexName = indexName; + this.partitionNames = partitionNames; this.alter = alter; } @@ -67,17 +75,52 @@ public TableName getTableName() { @Override public void analyze(Analyzer analyzer) throws AnalysisException { - if (indexDef == null) { - throw new AnalysisException("index definition expected."); + tableName.analyze(analyzer); + DatabaseIf
db = Env.getCurrentEnv().getCatalogMgr().getInternalCatalog() + .getDb(tableName.getDb()).orElse(null); + if (db == null) { + throw new AnalysisException("Database[" + tableName.getDb() + "] is not exist"); } - if (indexDef.getIndexType() == IndexDef.IndexType.NGRAM_BF - || indexDef.getIndexType() == IndexDef.IndexType.BLOOMFILTER) { + + TableIf table = db.getTable(tableName.getTbl()).orElse(null); + if (table == null) { + throw new AnalysisException("Table[" + tableName.getTbl() + "] is not exist"); + } + if (!(table instanceof OlapTable)) { + throw new AnalysisException("Only olap table support build index"); + } + + Index existedIdx = null; + for (Index index : table.getTableIndexes().getIndexes()) { + if (index.getIndexName().equalsIgnoreCase(indexName)) { + existedIdx = index; + if (!existedIdx.isLightIndexChangeSupported()) { + throw new AnalysisException("BUILD INDEX operation failed: The index " + + existedIdx.getIndexName() + " of type " + existedIdx.getIndexType() + + " does not support lightweight index changes."); + } + break; + } + } + if (existedIdx == null) { + throw new AnalysisException("Index[" + indexName + "] is not exist in table[" + tableName.getTbl() + "]"); + } + + IndexDef.IndexType indexType = existedIdx.getIndexType(); + if (indexType == IndexDef.IndexType.NGRAM_BF + || indexType == IndexDef.IndexType.BLOOMFILTER) { throw new AnalysisException("ngram bloomfilter or bloomfilter index is not needed to build."); } + indexDef = new IndexDef(indexName, partitionNames, indexType, true); + if (!table.isPartitionedTable()) { + List specifiedPartitions = indexDef.getPartitionNames(); + if (!specifiedPartitions.isEmpty()) { + throw new AnalysisException("table " + table.getName() + + " is not partitioned, cannot build index with partitions."); + } + } indexDef.analyze(); - this.index = new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), - indexDef.getColumns(), indexDef.getIndexType(), - indexDef.getProperties(), indexDef.getComment()); + this.index = existedIdx.clone(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index edf62f44d524b3..ab590bd2d9bc4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -83,9 +83,9 @@ public IndexDef(String indexName, boolean ifNotExists, List columns, Ind } } - public IndexDef(String indexName, PartitionNames partitionNames, boolean isBuildDeferred) { + public IndexDef(String indexName, PartitionNames partitionNames, IndexType indexType, boolean isBuildDeferred) { this.indexName = indexName; - this.indexType = IndexType.INVERTED; + this.indexType = indexType; this.partitionNames = partitionNames; this.isBuildDeferred = isBuildDeferred; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java index 6f2bd26c34230a..56ec104917c367 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java @@ -20,6 +20,7 @@ import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.InvertedIndexUtil; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; @@ -152,6 +153,10 @@ public String getInvertedIndexParser() { return InvertedIndexUtil.getInvertedIndexParser(properties); } + public boolean isInvertedIndexParserNone() { + return InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equals(getInvertedIndexParser()); + } + public String getInvertedIndexParserMode() { return InvertedIndexUtil.getInvertedIndexParserMode(properties); } @@ -168,10 +173,28 @@ public String getInvertedIndexParserStopwords() { return InvertedIndexUtil.getInvertedIndexParserStopwords(properties); } + // Whether the index can be changed in light mode public boolean isLightIndexChangeSupported() { return indexType == IndexDef.IndexType.INVERTED; } + // Whether the index can be added in light mode + // cloud mode supports light add for ngram_bf index and non-tokenized inverted index (parser="none") + // local mode supports light add for both inverted index and ngram_bf index + // the rest of the index types do not support light add + public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) { + if (Config.isCloudMode()) { + if (indexType == IndexDef.IndexType.INVERTED) { + return isInvertedIndexParserNone() && enableAddIndexForNewData; + } else if (indexType == IndexDef.IndexType.NGRAM_BF) { + return enableAddIndexForNewData; + } + return false; + } + return (indexType == IndexDef.IndexType.NGRAM_BF && enableAddIndexForNewData) + || (indexType == IndexDef.IndexType.INVERTED); + } + public String getComment() { return getComment(false); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index a77b082d1033c9..9fd9d2c49f46b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.DataSortInfo; import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.IndexDef; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotRef; import org.apache.doris.backup.Status; @@ -339,6 +340,19 @@ public List getIndexIds() { return indexes.getIndexIds(); } + /** + * Checks if the table contains at least one index of the specified type. + * @param indexType The index type to check for + * @return true if the table has at least one index of the specified type, false otherwise + */ + public boolean hasIndexOfType(IndexDef.IndexType indexType) { + if (indexes == null) { + return false; + } + return indexes.getIndexes().stream() + .anyMatch(index -> index.getIndexType() == indexType); + } + @Override public TableIndexes getTableIndexes() { return indexes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index aa3286a74ae1f9..ca1052afeeb236 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -335,6 +335,12 @@ public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, if (invertedIndexFileStorageFormat != null) { if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.V1) { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); + } else if (invertedIndexFileStorageFormat == TInvertedIndexFileStorageFormat.DEFAULT) { + if (Config.inverted_index_storage_format.equalsIgnoreCase("V1")) { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1); + } else { + schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); + } } else { schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index ad75acdb9e6389..a036b18cd7e671 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -722,6 +722,7 @@ public class SessionVariable implements Serializable, Writable { public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config"; public static final String PREFER_UDF_OVER_BUILTIN = "prefer_udf_over_builtin"; + public static final String ENABLE_ADD_INDEX_FOR_NEW_DATA = "enable_add_index_for_new_data"; /** * If set false, user couldn't submit analyze SQL and FE won't allocate any related resources. @@ -2497,6 +2498,14 @@ public boolean isEnableSortSpill() { return enableSortSpill; } + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { + "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", + "Whether to enable add index mode that only affects new data, " + + "when enabled new indexes only affect subsequently written data, " + + "when disabled rebuild indexes for all data" + }) + public boolean enableAddIndexForNewData = false; + // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. @SuppressWarnings("checkstyle:Indentation") @@ -4772,5 +4781,13 @@ public void checkSqlConvertorFeatures(String features) { } } } + + public boolean isEnableAddIndexForNewData() { + return enableAddIndexForNewData; + } + + public void setEnableAddIndexForNewData(boolean enableAddIndexForNewData) { + this.enableAddIndexForNewData = enableAddIndexForNewData; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java new file mode 100644 index 00000000000000..38f85b9f42c55a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java @@ -0,0 +1,629 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.alter; + +import org.apache.doris.analysis.AlterClause; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.CreateIndexClause; +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.IndexDef; +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.analysis.TableName; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.OlapTable.OlapTableState; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.datasource.CloudInternalCatalog; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.proto.Cloud.MetaServiceCode; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.AgentTask; +import org.apache.doris.task.AgentTaskQueue; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TSortType; +import org.apache.doris.thrift.TTaskType; +import org.apache.doris.utframe.MockedMetaServerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class CloudIndexTest { + private static final Logger LOG = LogManager.getLogger(CloudIndexTest.class); + private static String fileName = "./CloudIndexTest"; + private static String clusterName = "test_cluster"; + private static String clusterID = "test_id"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private ConnectContext ctx; + private static OlapTable olapTable; + + private static Analyzer analyzer; + private static Database db; + private static CreateIndexClause createIndexClause; + private static SchemaChangeHandler schemaChangeHandler; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, UserException { + FeConstants.runningUnitTest = true; + // Setup for MetaServiceProxy mock + new MockUp(MetaServiceProxy.class) { + + @Mock + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { + Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder = Cloud.BeginTxnResponse.newBuilder(); + beginTxnResponseBuilder.setTxnId(1000) + .setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return beginTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest request) { + Cloud.TxnInfoPB.Builder txnInfoBuilder = Cloud.TxnInfoPB.newBuilder(); + txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1); + txnInfoBuilder.addAllTableIds(Lists.newArrayList(olapTable.getId())); + txnInfoBuilder.setLabel("test_label"); + txnInfoBuilder.setListenerId(-1); + Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder = Cloud.CommitTxnResponse.newBuilder(); + commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(txnInfoBuilder.build()); + return commitTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictRequest request) { + Cloud.CheckTxnConflictResponse.Builder checkTxnConflictResponseBuilder = + Cloud.CheckTxnConflictResponse.newBuilder(); + checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setFinished(true); + return checkTxnConflictResponseBuilder.build(); + } + + @Mock + public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest request) { + Cloud.GetClusterResponse.Builder getClusterResponseBuilder = Cloud.GetClusterResponse.newBuilder(); + Cloud.ClusterPB.Builder clusterBuilder = Cloud.ClusterPB.newBuilder(); + clusterBuilder.setClusterId(clusterID).setClusterName(clusterName); + + Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder(); + node1.setCloudUniqueId("test_cloud") + .setName("host1") + .setIp("host1") + .setHost("host1") + .setHeartbeatPort(123) + .setEditLogPort(125) + .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING); + clusterBuilder.addNodes(node1.build()); + getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .addCluster(clusterBuilder.build()); + return getClusterResponseBuilder.build(); + } + + @Mock + public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) { + Cloud.CreateTabletsResponse.Builder responseBuilder = Cloud.CreateTabletsResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.FinishTabletJobResponse finishTabletJob(Cloud.FinishTabletJobRequest request) { + Cloud.FinishTabletJobResponse.Builder responseBuilder = Cloud.FinishTabletJobResponse.newBuilder(); + responseBuilder.setStatus( + Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.IndexResponse prepareIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse commitIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.IndexResponse dropIndex(Cloud.IndexRequest request) { + Cloud.IndexResponse.Builder builder = Cloud.IndexResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.CheckKVResponse checkKv(Cloud.CheckKVRequest request) { + Cloud.CheckKVResponse.Builder builder = Cloud.CheckKVResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")); + return builder.build(); + } + + @Mock + public Cloud.GetCurrentMaxTxnResponse getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) { + Cloud.GetCurrentMaxTxnResponse.Builder builder = Cloud.GetCurrentMaxTxnResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(MetaServiceCode.OK).setMsg("OK")) + .setCurrentMaxTxnId(1000); + return builder.build(); + } + }; + + Config.cloud_unique_id = "test_cloud"; + Config.meta_service_endpoint = MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + FakeEnv.setSystemInfo(cloudSystemInfo); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; // Will be set by MockUp + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + ctx.setQualifiedUser("root"); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + ctx.setCloudCluster(clusterName); + Assert.assertTrue(envFactory instanceof CloudEnvFactory); + Assert.assertTrue(masterEnv instanceof CloudEnv); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + // Create a mock EditLog using a no-op approach + testEditLog = new EditLog("test") { + // Override to avoid initialization issues + }; + } + return testEditLog; + } + + @Mock + public SchemaChangeHandler getSchemaChangeHandler() { + // Create a new independent SchemaChangeHandler for each call + return schemaChangeHandler; + } + + @Mock + public AccessControllerManager getAccessManager() { + return new AccessControllerManager(masterEnv.getAuth()) { + @Override + public boolean checkTblPriv(ConnectContext ctx, String ctl, String db, String tbl, PrivPredicate wanted) { + return true; // Allow all access for test + } + + @Override + public boolean checkCloudPriv(UserIdentity user, String cluster, PrivPredicate wanted, ResourceTypeEnum resourceType) { + return true; // Allow all cloud privileges for test + } + }; + } + }; + + new MockUp() { + @Mock + public String getDefaultCloudCluster(String user) { + return clusterName; // Return default cluster for test + } + }; + + // Mock cloud environment permissions + new MockUp() { + @Mock + public void checkCloudClusterPriv(String cluster) throws Exception { + // Always allow for tests + } + }; + + // Mock ConnectContext to avoid compute group permission check + new MockUp() { + @Mock + public String getCloudCluster() { + return clusterName; + } + + @Mock + public UserIdentity getCurrentUserIdentity() { + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + return rootUser; + } + }; + + analyzer = new Analyzer(masterEnv, ctx); + + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + Map newTagMap = org.apache.doris.resource.Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_STATUS, "NORMAL"); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_NAME, clusterName); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_ID, clusterID); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, ""); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, ""); + newTagMap.put(org.apache.doris.resource.Tag.CLOUD_UNIQUE_ID, "test_cloud"); + backend.setTagMap(newTagMap); + List backends = Lists.newArrayList(backend); + systemInfo.updateCloudClusterMapNoLock(backends, new ArrayList<>()); + db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1); + masterEnv.unprotectCreateDb(db); + + AgentTaskQueue.clearAllTasks(); + schemaChangeHandler = masterEnv.getSchemaChangeHandler(); + } + + @Test + public void testCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", olapTable.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + } + + @Test + public void testNormalCreateNgramBfIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "ngram_bf_index"; + + // Add required properties for NGRAM_BF index + Map properties = Maps.newHashMap(); + properties.put("gram_size", "2"); + properties.put("bf_size", "256"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + properties, "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Set session variable to false (default) + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", olapTable.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + + long createJobId = indexChangeJobMap.values().stream().findAny().get().jobId; + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + // Finish the create index job first + SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2) indexChangeJobMap.get(createJobId); + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, createJobV2.getJobState()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, createJobV2.getJobState()); + Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, createJobV2.getJobState()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("raw_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCreateInvertedIndexWithLightweightMode() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + String indexName = "lightweight_raw_inverted_index"; + // Explicitly set parser="none" for raw inverted index + Map properties = Maps.newHashMap(); + properties.put("parser", "none"); + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()), + IndexType.INVERTED, + properties, "lightweight raw inverted index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + // Test with enable_add_index_for_new_data = true, should use lightweight mode + ctx.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + // Lightweight mode should not create any schema change jobs + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("lightweight_raw_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); + // Verify the index properties + Assert.assertEquals("none", olapTable.getIndexes().get(0).getProperties().get("parser")); + } + + @Test + public void testCreateTokenizedInvertedIndex() throws Exception { + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + FakeEnv.setSystemInfo(cloudSystemInfo); + schemaChangeHandler = (SchemaChangeHandler) new Alter().getSchemaChangeHandler(); + + Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof CloudInternalCatalog); + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CatalogTestUtil.createDupTable(db); + olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + DataSortInfo dataSortInfo = new DataSortInfo(); + dataSortInfo.setSortType(TSortType.LEXICAL); + olapTable.setDataSortInfo(dataSortInfo); + + // Set inverted index file storage format to V2 for cloud mode + olapTable.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2); + + String indexName = "tokenized_inverted_index"; + Map properties = Maps.newHashMap(); + properties.put("parser", "english"); + properties.put("support_phrase", "true"); + properties.put("lower_case", "true"); + + // Use VARCHAR column v1 (index 2) for string type support + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(olapTable.getBaseSchema().get(2).getName()), + IndexType.INVERTED, + properties, "tokenized inverted index with english parser"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + olapTable.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + schemaChangeHandler.process(alterClauses, db, olapTable); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, olapTable.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + // This should be a heavyweight schema change for tokenized index + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + Assert.assertEquals(1, olapTable.getIndexes().size()); + Assert.assertEquals("tokenized_inverted_index", olapTable.getIndexes().get(0).getIndexName()); + + // Verify that the index has the correct properties + Assert.assertEquals("english", olapTable.getIndexes().get(0).getProperties().get("parser")); + Assert.assertEquals("true", olapTable.getIndexes().get(0).getProperties().get("support_phrase")); + Assert.assertEquals("true", olapTable.getIndexes().get(0).getProperties().get("lower_case")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java index 6b0943c306f851..29aed7e050d72f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java @@ -46,6 +46,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.thrift.TStatusCode; @@ -55,6 +56,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import mockit.Mock; +import mockit.MockUp; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -85,6 +88,9 @@ public class IndexChangeJobTest { private static BuildIndexClause buildIndexClause; private static DropIndexClause dropIndexClause; private static CancelAlterTableStmt cancelAlterTableStmt; + private static TableName tableName; + private static String indexName; + private static ConnectContext ctx; @Rule public ExpectedException expectedEx = ExpectedException.none(); @@ -108,25 +114,40 @@ public void setUp() db = masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1); olapTable = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId1); + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + }; + + // Initialize ConnectContext + ctx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext get() { + return ctx; + } + }; + // set mow table property Map properties = Maps.newHashMap(); properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "false"); TableProperty tableProperty = new TableProperty(properties); olapTable.setTableProperty(tableProperty); - TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), olapTable.getName()); - IndexDef indexDef = new IndexDef("index1", false, + indexName = "index1"; + IndexDef indexDef = new IndexDef(indexName, false, Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), IndexDef.IndexType.INVERTED, Maps.newHashMap(), "balabala"); + FakeEnv.setEnv(masterEnv); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - buildIndexClause.analyze(analyzer); - - dropIndexClause = new DropIndexClause("index1", false, tableName, false); + dropIndexClause = new DropIndexClause(indexName, false, tableName, false); dropIndexClause.analyze(analyzer); cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName); @@ -163,6 +184,8 @@ public void testBuildIndexIndexChange() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -203,6 +226,8 @@ public void testBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -292,6 +317,8 @@ public void testCancelBuildIndexIndexChangeNormal() throws UserException { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -339,6 +366,8 @@ public void testBuildIndexIndexChangeWhileTableNotStable() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -467,6 +496,8 @@ public void testBuildIndexFailedWithMinFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -519,6 +550,8 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(olapTable.getIndexes().size(), 1); Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(), "index1"); alterClauses.clear(); + buildIndexClause = new BuildIndexClause(tableName, indexName, null, false); + buildIndexClause.analyze(analyzer); alterClauses.add(buildIndexClause); schemaChangeHandler.process(alterClauses, db, olapTable); Map indexChangeJobMap = schemaChangeHandler.getIndexChangeJobs(); @@ -526,13 +559,13 @@ public void testBuildIndexFailedWithMaxFailedNum() throws Exception { Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState()); IndexChangeJob indexChangejob = indexChangeJobMap.values().stream().findAny().get(); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 0); + Assert.assertEquals(0, indexChangejob.invertedIndexBatchTask.getTaskNum()); Assert.assertEquals(IndexChangeJob.JobState.WAITING_TXN, indexChangejob.getJobState()); // run waiting txn job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); - Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 3); + Assert.assertEquals(3, indexChangejob.invertedIndexBatchTask.getTaskNum()); // run running job schemaChangeHandler.runAfterCatalogReady(); Assert.assertEquals(IndexChangeJob.JobState.RUNNING, indexChangejob.getJobState()); @@ -565,17 +598,135 @@ public void testNgramBfBuildIndex() throws UserException { fakeEditLog = new FakeEditLog(); FakeEnv.setEnv(masterEnv); - IndexDef indexDef = new IndexDef("ngram_bf_index", false, - Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()), + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, Maps.newHashMap(), "ngram bf index"); TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), - olapTable.getName()); + table.getName()); createIndexClause = new CreateIndexClause(tableName, indexDef, false); createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + + // Test with enable_add_index_for_new_data = true + ConnectContext context = ConnectContext.get(); + context.getSessionVariable().setEnableAddIndexForNewData(true); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index", table.getIndexes().get(0).getIndexName()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + + // Clean up for next test + table.setIndexes(Lists.newArrayList()); + indexChangeJobMap.clear(); + AgentTaskQueue.clearAllTasks(); + + // Test with enable_add_index_for_new_data = false + context.getSessionVariable().setEnableAddIndexForNewData(false); + String indexName2 = "ngram_bf_index2"; + IndexDef indexDef2 = new IndexDef(indexName2, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index2"); + + createIndexClause = new CreateIndexClause(tableName, indexDef2, false); + createIndexClause.analyze(analyzer); + ArrayList alterClauses2 = new ArrayList<>(); + alterClauses2.add(createIndexClause); + schemaChangeHandler.process(alterClauses2, db, table); + indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); - buildIndexClause = new BuildIndexClause(tableName, indexDef, false); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, - () -> buildIndexClause.analyze(analyzer)); + List tasks = AgentTaskQueue.getTask(TTaskType.ALTER); + Assert.assertEquals(1, tasks.size()); + for (AgentTask agentTask : tasks) { + agentTask.setFinished(true); + } + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState()); + Assert.assertEquals(1, table.getIndexes().size()); + Assert.assertEquals("ngram_bf_index2", table.getIndexes().get(0).getIndexName()); + } + + @Test + public void testCancelNgramBfBuildIndex() throws UserException { + fakeEnv = new FakeEnv(); + fakeEditLog = new FakeEditLog(); + FakeEnv.setEnv(masterEnv); + + OlapTable table = (OlapTable) db.getTableOrDdlException(CatalogTestUtil.testTableId2); + String indexName = "ngram_bf_index"; + IndexDef indexDef = new IndexDef(indexName, false, + Lists.newArrayList(table.getBaseSchema().get(3).getName()), + org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF, + Maps.newHashMap(), "ngram bf index"); + TableName tableName = new TableName(masterEnv.getInternalCatalog().getName(), db.getName(), + table.getName()); + createIndexClause = new CreateIndexClause(tableName, indexDef, false); + createIndexClause.analyze(analyzer); + SchemaChangeHandler schemaChangeHandler = Env.getCurrentEnv().getSchemaChangeHandler(); + ArrayList alterClauses = new ArrayList<>(); + alterClauses.add(createIndexClause); + + //cancel test can only with enable_add_index_for_new_data = false + ctx.getSessionVariable().setEnableAddIndexForNewData(false); + schemaChangeHandler.process(alterClauses, db, table); + Map indexChangeJobMap = schemaChangeHandler.getAlterJobsV2(); + Assert.assertEquals(1, indexChangeJobMap.size()); + Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState()); + + SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream() + .findFirst() + .orElse(null); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN, jobV2.getJobState()); + Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState()); + Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum()); + + cancelAlterTableStmt = new CancelAlterTableStmt(ShowAlterStmt.AlterType.COLUMN, tableName); + cancelAlterTableStmt.analyze(analyzer); + schemaChangeHandler.cancel(cancelAlterTableStmt); + + schemaChangeHandler.runAfterCatalogReady(); + Assert.assertEquals(AlterJobV2.JobState.CANCELLED, jobV2.getJobState()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java index a1dcff46471f83..14a7457a5e5165 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java @@ -626,9 +626,8 @@ public void testDupAddOrDropNgramBfIndex() throws Exception { waitAlterJobDone(alterJobs); String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on test.sc_dup "; - AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt) parseAndAnalyzeStmt(buildNgramBfIndexStmtStr); - org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.DdlException.class, - () -> Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt)); + Assertions.assertThrows(org.apache.doris.common.AnalysisException.class, + () -> parseAndAnalyzeStmt(buildNgramBfIndexStmtStr)); tbl.readLock(); try { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index 590b6563e1106c..b16d3d15cf7e52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -22,7 +22,8 @@ import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.MaterializedIndex.IndexState; -import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.persist.EditLog; @@ -175,13 +176,26 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, long version) { Env.getCurrentInvertedIndex().clear(); - // replica - Replica replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); - Replica replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica1; + Replica replica2; + Replica replica3; + if (Config.isCloudMode()) { + // In cloud mode we must create CloudReplica instances to avoid ClassCastException + replica1 = new CloudReplica(testReplicaId1, testBackendId1, Replica.ReplicaState.NORMAL, version, + /*schemaHash*/ 0, dbId, tableId, partitionId, indexId, /*idx*/ 0); + replica2 = new CloudReplica(testReplicaId2, testBackendId2, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 1); + replica3 = new CloudReplica(testReplicaId3, testBackendId3, Replica.ReplicaState.NORMAL, version, + 0, dbId, tableId, partitionId, indexId, 2); + } else { + replica1 = new Replica(testReplicaId1, testBackendId1, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica2 = new Replica(testReplicaId2, testBackendId2, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + replica3 = new Replica(testReplicaId3, testBackendId3, version, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } + // tablet Tablet tablet = new Tablet(tabletId); @@ -244,10 +258,14 @@ public static Database createSimpleDb(long dbId, long tableId, long partitionId, } public static void createDupTable(Database db) { - - // replica - Replica replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, - ReplicaState.NORMAL, -1, 0); + Replica replica; + if (Config.isCloudMode()) { + replica = new CloudReplica(testReplicaId4, testBackendId1, Replica.ReplicaState.NORMAL, testStartVersion, + 0, db.getId(), testTableId2, testPartitionId2, testIndexId2, 0); + } else { + replica = new Replica(testReplicaId4, testBackendId1, testStartVersion, 0, 0L, 0L, 0L, + Replica.ReplicaState.NORMAL, -1, 0); + } // tablet Tablet tablet = new Tablet(testTabletId2); diff --git a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out new file mode 100644 index 00000000000000..49570b96d291f2 --- /dev/null +++ b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out @@ -0,0 +1,103 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_light_mode_init -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_light_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_schema_change_mode_init -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_schema_change_mode_more_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_after_data -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + +-- !select_lifecycle_final -- +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1001 2023-10-06T15:00 Laptop John Smith 199.99 North +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1002 2023-10-09T17:05 Smartphone Emily Johnson 299.99 South +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1003 2023-10-12T19:10 Headphones Michael Brown 399.99 East +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1004 2023-10-15T21:15 Monitor Jessica Davis 499.99 West +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1005 2023-10-18T23:20 Keyboard David Wilson 89.99 North +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1006 2023-10-21T07:25 Mouse Sarah Taylor 699.99 South +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1007 2023-10-24T09:30 Printer Thomas Anderson 799.99 East +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1008 2023-10-27T11:35 Speaker Jennifer Martin 899.99 West +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1009 2023-10-02T13:40 External SSD Robert Clark 999.99 North +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South +1010 2023-10-05T15:45 Webcam Amanda Lewis 89.99 South + diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf index f356b81f2040b2..770b1542edaeef 100644 --- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf @@ -39,7 +39,6 @@ arrow_flight_sql_port = 8081 cloud_unique_id=cloud_unique_id_sql_server00 # for case test_build_mtmv.groovy enable_job_schedule_second_for_test=true -enable_light_index_change=false workload_sched_policy_interval_ms = 1000 enable_advance_next_id = true diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 0e43d458f38b65..5bba5437f73589 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -34,6 +34,5 @@ cloud_http_port=18030 meta_service_endpoint=127.0.0.1:5000 arrow_flight_sql_port = 8081 cloud_unique_id=cloud_unique_id_sql_server00 -enable_light_index_change=false enable_advance_next_id = true enable_job_schedule_second_for_test = true \ No newline at end of file diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy new file mode 100644 index 00000000000000..402f72ddc36065 --- /dev/null +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy @@ -0,0 +1,325 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import groovy.json.JsonSlurper + +suite("test_ngram_bloomfilter_index_change") { + def tableName = 'test_ngram_bloomfilter_index_change' + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + // Function to insert test data batch + def insertTestData = { -> + // insert 10 records + sql "INSERT INTO ${tableName} VALUES (1001, '2023-10-06 15:00:00', 'Laptop', 'John Smith', 199.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1002, '2023-10-09 17:05:00', 'Smartphone', 'Emily Johnson', 299.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1003, '2023-10-12 19:10:00', 'Headphones', 'Michael Brown', 399.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1004, '2023-10-15 21:15:00', 'Monitor', 'Jessica Davis', 499.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1005, '2023-10-18 23:20:00', 'Keyboard', 'David Wilson', 89.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1006, '2023-10-21 07:25:00', 'Mouse', 'Sarah Taylor', 699.99, 'South');" + sql "INSERT INTO ${tableName} VALUES (1007, '2023-10-24 09:30:00', 'Printer', 'Thomas Anderson', 799.99, 'East');" + sql "INSERT INTO ${tableName} VALUES (1008, '2023-10-27 11:35:00', 'Speaker', 'Jennifer Martin', 899.99, 'West');" + sql "INSERT INTO ${tableName} VALUES (1009, '2023-10-02 13:40:00', 'External SSD', 'Robert Clark', 999.99, 'North');" + sql "INSERT INTO ${tableName} VALUES (1010, '2023-10-05 15:45:00', 'Webcam', 'Amanda Lewis', 89.99, 'South');" + sql "sync" + } + + // Test settings + sql "set enable_function_pushdown=true" + sql "set enable_profile=true" + sql "set profile_level=2" + + // Define test query + def query = "SELECT /*+SET_VAR(enable_function_pushdown = true, enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE customer_name LIKE '%xxxx%' ORDER BY sale_id" + // Test Case 1: Test with enable_add_index_for_new_data = true + logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===") + // Create table + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_light_mode") { + run { + sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + sql "set enable_add_index_for_new_data = true" + + // Add NGRAM Bloom Filter index (should be immediate in light mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + + // In light mode, the index should be effective immediately, no need to wait for alter job + // But let's give it a moment to ensure metadata is updated + sleep(2000) + + // Insert more data after index added + insertTestData() + // Verify more data loaded correctly + qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test with more data (should still filter correctly) + profile("sql_select_like_with_ngram_index_light_mode_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_light_mode_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test after dropping index + profile("sql_select_like_with_ngram_index_light_mode_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_light_mode_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test Case 2: Test with enable_add_index_for_new_data = false (schema change mode) + logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===") + // Set enable_add_index_for_new_data = false + sql "set enable_add_index_for_new_data = false" + // Create new table + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + // Insert test data + insertTestData() + // Verify data loaded correctly + qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test without NGRAM Bloom Filter index + profile("sql_select_like_without_ngram_index_schema_change_mode") { + run { + sql "/* sql_select_like_without_ngram_index_schema_change_mode */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Add NGRAM Bloom Filter index (will trigger schema change in this mode) + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test after adding NGRAM Bloom Filter index (should filter existing data) + profile("sql_select_like_with_ngram_index_schema_change_mode_added") { + run { + sql "/* sql_select_like_with_ngram_index_schema_change_mode_added */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert more data after index is built + insertTestData() + // Verify more data loaded correctly + qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test with more data (should filter all data) + profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") { + run { + sql "/* sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // Drop index + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Test after dropping index + profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") { + run { + sql "/* sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 0")) + } + } + + // Test Case 3: Test different scenarios for index lifecycle + logger.info("=== Test Case 3: Index lifecycle with enable_add_index_for_new_data = true ===") + // Set enable_add_index_for_new_data = true + sql "set enable_add_index_for_new_data = true" + // Create table and add index before inserting data + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE ${tableName} ( + `sale_id` int NULL, + `sale_date` datetime NULL, + `product_name` varchar(100) NULL, + `customer_name` varchar(100) NULL, + `amount` decimal(10,2) NULL, + `region` char(50) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`sale_id`) + PARTITION BY RANGE(`sale_date`) ( + PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01 00:00:00')), + PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01 00:00:00')), + PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00')) + ) + DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "storage_format" = "V2", + "light_schema_change" = "true", + "disable_auto_compaction" = "true" + ); + """ + + // Add ngram bf index before data insertion + sql "ALTER TABLE ${tableName} ADD INDEX idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' = '1024', 'gram_size' = '3');" + wait_for_latest_op_on_table_finish(tableName, timeout) + + // Insert data after index creation + insertTestData() + // Verify data loaded correctly + qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test filtering with index added before data insertion + profile("sql_select_like_with_ngram_index_lifecycle_test") { + run { + sql "/* sql_select_like_with_ngram_index_lifecycle_test */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 10")) + } + } + + // Insert more data + insertTestData() + // Verify more data loaded correctly + qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id" + + // Test filtering with more data + profile("sql_select_like_with_ngram_index_lifecycle_final") { + run { + sql "/* sql_select_like_with_ngram_index_lifecycle_final */ ${query}" + sleep(1000) + } + + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 20")) + } + } + + // Final cleanup + sql "DROP INDEX idx_ngram_customer_name ON ${tableName};" + sleep(2000) +} \ No newline at end of file From 6222a96fc50e0f67a7f4d03eff6aedeed5f4ea75 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:27:12 +0800 Subject: [PATCH 225/572] branch-3.0: [fix](test) fix test_routine_load_restart_fe fail #53266 (#53298) Cherry-picked from #53266 Co-authored-by: hui lai --- .../test_routine_load_restart_fe.groovy | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy index 5554978e500ef4..104026fb16e3f6 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy @@ -50,15 +50,14 @@ suite("test_routine_load_restart_fe", "docker") { producer.send(record) } } - } - def options = new ClusterOptions() - options.setFeNum(1) - docker(options) { - def load_with_injection = { injection -> - def jobName = "test_routine_load_restart" - def tableName = "dup_tbl_basic_multi_table" - if (enabled != null && enabled.equalsIgnoreCase("true")) { + def options = new ClusterOptions() + options.setFeNum(1) + options.enableDebugPoints() + def jobName = "test_routine_load_restart" + def tableName = "dup_tbl_basic_multi_table" + docker(options) { + def load_with_injection = { injection -> try { GetDebugPoint().enableDebugPointForAllBEs(injection) sql new File("""${context.file.parent}/ddl/${tableName}_drop.sql""").text @@ -104,32 +103,32 @@ suite("test_routine_load_restart_fe", "docker") { } else { continue; } - } - sql "stop routine load for ${jobName}" + } } catch (Exception e) { log.info("exception: {}", e) + sql "stop routine load for ${jobName}" sql "DROP TABLE IF EXISTS ${tableName}" } } - } - load_with_injection("KafkaDataConsumer.group_consume.out_of_range") + load_with_injection("KafkaDataConsumer.group_consume.out_of_range") - cluster.restartFrontends() - sleep(30000) - context.reconnectFe() + cluster.restartFrontends() + sleep(30000) + context.reconnectFe() - def res = sql "show routine load for ${jobName}" - def state = res[0][8].toString() - if (state == "PAUSED") { - log.info("reason of state changed: ${res[0][17].toString()}".toString()) - assertTrue(res[0][17].toString().contains("Offset out of range")) - assertTrue(res[0][17].toString().contains("consume partition")) - assertTrue(res[0][17].toString().contains("consume offset")) - } else { - assertEquals(1, 2) + def res = sql "show routine load for ${jobName}" + def state = res[0][8].toString() + if (state == "PAUSED") { + log.info("reason of state changed: ${res[0][17].toString()}".toString()) + assertTrue(res[0][17].toString().contains("Offset out of range")) + assertTrue(res[0][17].toString().contains("consume partition")) + assertTrue(res[0][17].toString().contains("consume offset")) + } else { + assertEquals(1, 2) + } + sql "stop routine load for ${jobName}" + sql "DROP TABLE IF EXISTS ${tableName}" } - sql "stop routine load for ${jobName}" - sql "DROP TABLE IF EXISTS ${tableName}" } } From 6a6c0257dc006d009a96838b944e99af88c6910d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:33:05 +0800 Subject: [PATCH 226/572] branch-3.0: [fix](case) fix regression case create_commit_mtmv_many_tasks #53247 (#53303) Cherry-picked from #53247 Co-authored-by: zhangdong --- .../create_commit_mtmv_many_task.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/create_commit_mtmv_many_task.groovy b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/create_commit_mtmv_many_task.groovy index ec5054f9190bd9..db4acfafee1948 100644 --- a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/create_commit_mtmv_many_task.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/create_commit_mtmv_many_task.groovy @@ -95,7 +95,7 @@ suite("create_commit_mtmv_many_tasks", "p2") { sql "select count(*) from ${table_name}" } - for (int i = 1; i <= 10; i++) { + for (int i = 1; i <= 1; i++) { stream_load_job(table_name1, "lineitem.tbl.${i}") stream_load_job(table_name2, "orders.tbl.${i}") } @@ -163,11 +163,11 @@ suite("create_commit_mtmv_many_tasks", "p2") { def insert_into_select = { date_it -> sql """INSERT INTO ${dst_database_name}.${table_name1} SELECT l_orderkey, l_linenumber, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_commitdate, l_receiptdate, l_shipinstruct, l_shipmode, l_comment, '${date_it}' AS new_date_column - FROM ${src_database_name}.${table_name1};""" + FROM ${src_database_name}.${table_name1} limit 1;""" sql """INSERT INTO ${dst_database_name}.${table_name2} SELECT o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderpriority, o_clerk, o_shippriority, o_comment, '${date_it}' AS new_date_column - FROM ${src_database_name}.${table_name2}""" + FROM ${src_database_name}.${table_name2} limit 1""" } def get_next_day = { def date_it -> @@ -189,7 +189,7 @@ suite("create_commit_mtmv_many_tasks", "p2") { def job_name = getJobName(dst_database_name, "mv1") waitingMTMVTaskFinished(job_name) def task_num = sql """select count(*) from tasks("type"="mv") where JobName="${job_name}";""" - assertTrue(task_num[0][0] < 100) + assertTrue(task_num[0][0] < 200) def mv_row_count = sql """select count(1) from mv1;""" def real_row_count = sql """select count(1) from (select l_shipdate, l_orderkey from lineitem as t1 left join orders as t2 on t1.l_orderkey = t2.o_orderkey group by l_shipdate, l_orderkey) t;""" From 5d7106541f1fdcfd8119ea012bc4ce59b401dbae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:35:25 +0800 Subject: [PATCH 227/572] branch-3.0: (fix)(cloud) Fix cached version regression #53236 (#53256) Cherry-picked from #53236 Co-authored-by: Gavin Chou --- .../cloud_p0/version/test_fe_cached_partition_version.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy b/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy index e9466692c15476..0cccfa543213ec 100644 --- a/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy +++ b/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy @@ -132,12 +132,11 @@ suite("test_fe_cached_partition_version", 'docker') { insert_sql """INSERT INTO ${tbl} VALUES ('Beijing', 1})""", 1 // refresh expiration, the insert will be seen since the version has expired result = sql_return_maparray """ select * from ${tbl} where region = 'Beijing' """ - // observer/follower cannot see update since cache expiration no reached assertEquals(3, result.size()) // all the insert will be seen since the version has expired, refresh expiration result = sql_return_maparray """ select * from ${tbl} """ - assertEquals(5, result.size()) + assertEquals(6, result.size()) // test no expiration, disable cache partition version insert_sql """INSERT INTO ${tbl} VALUES ('Guangzhou', 1})""", 1 From 8e2036cb7b15cd90fcdb4400a4d78bf1564c3328 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:40:13 +0800 Subject: [PATCH 228/572] branch-3.0: [test](ngram bf) fix regression case for ngram bf hit #53194 (#53257) Cherry-picked from #53194 Co-authored-by: airborne12 --- .../test_bloom_filter_hit.groovy | 120 ++++++++---------- 1 file changed, 55 insertions(+), 65 deletions(-) diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy index 8835e89d9472a9..f52a5ca0ec79fd 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_hit.groovy @@ -20,6 +20,35 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; suite("test_bloom_filter_hit") { + // Helper method to extract and validate RowsBloomFilterFiltered value + def validateBloomFilterFiltered = { profileString -> + log.info("Profile content: ${profileString}") + // Pattern to match "RowsBloomFilterFiltered:" followed by a number (with optional units and parentheses) + Pattern pattern = Pattern.compile("RowsBloomFilterFiltered:\\s+(\\d+(?:\\.\\d+)?[KMG]?)(?:\\s+\\((\\d+)\\))?") + Matcher matcher = pattern.matcher(profileString) + if (matcher.find()) { + def value1 = matcher.group(1) // First number (possibly with K/M/G suffix) + def value2 = matcher.group(2) // Second number in parentheses (if exists) + // Parse the first value + def numValue = 0L + if (value1.endsWith("K")) { + numValue = (Double.parseDouble(value1.substring(0, value1.length() - 1)) * 1000).toLong() + } else if (value1.endsWith("M")) { + numValue = (Double.parseDouble(value1.substring(0, value1.length() - 1)) * 1000000).toLong() + } else if (value1.endsWith("G")) { + numValue = (Double.parseDouble(value1.substring(0, value1.length() - 1)) * 1000000000).toLong() + } else { + numValue = Long.parseLong(value1) + } + log.info("Extracted RowsBloomFilterFiltered value: ${numValue}") + assertTrue(numValue > 0) + return true + } else { + fail("Could not find RowsBloomFilterFiltered in profile output") + return false + } + } + def be_num = sql "show backends;" if (be_num.size() > 1) { // not suitable for multiple be cluster. @@ -71,37 +100,19 @@ suite("test_bloom_filter_hit") { sql """ SET enable_profile = true """ sql """ set parallel_scan_min_rows_per_scanner = 2097152; """ - sql """ select C_COMMENT from ${tableName} where C_COMMENT='OK' """ - - def httpGet = { url -> - def dst = 'http://' + context.config.feHttpAddress - def conn = new URL(dst + url).openConnection() - conn.setRequestMethod("GET") - def encoding = Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" + - (context.config.feHttpPassword == null ? "" : context.config.feHttpPassword)).getBytes("UTF-8")) - conn.setRequestProperty("Authorization", "Basic ${encoding}") - return conn.getInputStream().getText() - } + profile("sql_c_comment_ok") { + run { + sql "/* sql_c_comment_ok */ select C_COMMENT from ${tableName} where C_COMMENT='OK'" + sleep(1000) + } - def profileUrl = '/rest/v1/query_profile/' - def profiles = httpGet(profileUrl) - log.debug("profiles:{}", profiles); - profiles = new JsonSlurper().parseText(profiles) - assertEquals(0, profiles.code) - - def profileId = null; - for (def profile in profiles["data"]["rows"]) { - if (profile["Sql Statement"].contains("""select C_COMMENT from ${tableName} where C_COMMENT='OK'""")) { - profileId = profile["Profile ID"] - break; + check { profileString, exception -> + log.info(profileString) + assertTrue(profileString.contains("RowsBloomFilterFiltered: 15.0K (15000)")) } } - log.info("profileId:{}", profileId); - def profileDetail = httpGet("/rest/v1/query_profile/" + profileId) - assertTrue(profileDetail.contains("BloomFilterFiltered:  15.0K  (15000)")) - //———————— clean table and disable profile ———————— + sql """ SET enable_profile = false """ - // sql """ DROP TABLE IF EXISTS ${tableName} """ // test ipv for bloom filter sql """ DROP TABLE IF EXISTS test_ip_bf """ @@ -136,47 +147,26 @@ suite("test_bloom_filter_hit") { // bf filter sql """ SET parallel_pipeline_task_num=1 """ - qt_sql """ select * from test_ip_bf where ip_v6='4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8' """ - profiles = httpGet(profileUrl) - log.debug("profiles:{}", profiles); - profiles = new JsonSlurper().parseText(profiles) - assertEquals(0, profiles.code) - - def profileIdIpv6 = null; - for (def profile in profiles["data"]["rows"]) { - if (profile["Sql Statement"].contains("""select * from test_ip_bf where ip_v6='4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8'""")) { - profileIdIpv6 = profile["Profile ID"] - break; + + profile("sql_ip_v6_ok") { + run { + sql "/* sql_ip_v6_ok */ select * from test_ip_bf where ip_v6='4a3e:dc26:1819:83e6:9ee5:7239:ff44:aee8'" + sleep(1000) } - } - log.info("profileId:{}", profileIdIpv6); - profileDetail = httpGet("/rest/v1/query_profile/" + profileIdIpv6) - // log.info("filter ipv6 profile:{}", profileDetail) - String regex = "RowsBloomFilterFiltered:  (\\d+)" - Pattern pattern = Pattern.compile(regex) - Matcher matcher = pattern.matcher(profileDetail) - - while (matcher.find()) { - int number = Integer.parseInt(matcher.group(1)) - log.info("filter ipv6 number:{}", number) - assertTrue(number > 0) - } - qt_sql """ select * from test_ip_bf where ip_v4='192.168.11.1' """ - def profileIdIpv4 = null; - for (def profile in profiles["data"]["rows"]) { - if (profile["Sql Statement"].contains("""select * from test_ip_bf where ip_v4='192.168.11.1'""")) { - profileIdIpv4 = profile["Profile ID"] - break; + check { profileString, exception -> + validateBloomFilterFiltered(profileString) } } - log.info("profileId:{}", profileIdIpv4); - profileDetail = httpGet("/rest/v1/query_profile/" + profileIdIpv4) - //log.info("filter ipv4 profile:{}", profileDetail) - Matcher matcherIpv4 = pattern.matcher(profileDetail) - while (matcherIpv4.find()) { - int number = Integer.parseInt(matcherIpv4.group(1)) - log.info("filter ipv4 number:{}", number) - assertTrue(number > 0) + + profile("sql_ip_v4_ok") { + run { + sql "/* sql_ip_v4_ok */ select * from test_ip_bf where ip_v4='192.168.11.1'" + sleep(1000) + } + + check { profileString, exception -> + validateBloomFilterFiltered(profileString) + } } } From 914105d4b46f3b8c591dc919019df05512cfc2b7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:40:50 +0800 Subject: [PATCH 229/572] branch-3.0: [fix](test) fix test_disable_load fail #53252 (#53296) Cherry-picked from #53252 Co-authored-by: hui lai --- .../suites/load_p0/routine_load/test_disable_load.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/routine_load/test_disable_load.groovy b/regression-test/suites/load_p0/routine_load/test_disable_load.groovy index fbac33938490d6..022a9ae92d0a3c 100644 --- a/regression-test/suites/load_p0/routine_load/test_disable_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_disable_load.groovy @@ -87,7 +87,7 @@ suite("test_disable_load","nonConcurrent,p0") { def state = sql "show routine load for ${job}" log.info("routine load state: ${state[0][8].toString()}".toString()) log.info("reason of state changed: ${state[0][17].toString()}".toString()) - if (state[0][17].toString().contains("Failed to get info. No alive backends")) { + if (state[0][17].toString().contains("failed to get info: no alive backends")) { break } if (count >= 60) { From 1c40225e59c7dd3493eac2e56d54fb4b9523b28e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:41:13 +0800 Subject: [PATCH 230/572] branch-3.0: [fix](test) fix test_multi_table_load_eror fail #53254 (#53299) Cherry-picked from #53254 Co-authored-by: hui lai --- .../test_multi_table_load_error.groovy | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy b/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy index 17a8e5da719bfb..039e734466e33e 100644 --- a/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy +++ b/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy @@ -21,28 +21,6 @@ import org.apache.kafka.clients.producer.ProducerRecord import org.apache.kafka.clients.producer.ProducerConfig import org.codehaus.groovy.runtime.IOGroovyMethods -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.apache.kafka.clients.admin.AdminClient -import org.apache.kafka.clients.producer.KafkaProducer -import org.apache.kafka.clients.producer.ProducerRecord -import org.apache.kafka.clients.producer.ProducerConfig - suite("test_multi_table_load_eror","nonConcurrent") { def kafkaCsvTpoics = [ "multi_table_csv", @@ -139,7 +117,7 @@ suite("test_multi_table_load_eror","nonConcurrent") { def res = sql "show routine load for ${jobName}" log.info("routine load statistic: ${res[0][14].toString()}".toString()) def json = parseJson(res[0][14]) - if (json.loadedRows.toString() == "0") { + if (json.loadedRows.toString() != "0") { count++ if (count > 60) { assertEquals(1, 2) From fef6f7c312d093436583d009995dcc29e687257b Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 15 Jul 2025 20:43:03 -0700 Subject: [PATCH 231/572] [branch-3.0](fix) fix function rules are not persisted (#52817) (#53263) pick part of #52817 --- .../java/org/apache/doris/catalog/JdbcTable.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java index adf013576b3962..1f7ccfa56a1dac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java @@ -105,6 +105,9 @@ public class JdbcTable extends Table { private boolean connectionPoolKeepAlive; private ExternalFunctionRules functionRules; + // This is used for edit log + @SerializedName("frs") + private String functionRulesString; static { Map tempMap = new CaseInsensitiveMap(); @@ -419,8 +422,9 @@ private void validate(Map properties) throws DdlException { private void checkAndSetExternalFunctionRules(Map properties) throws DdlException { ExternalFunctionRules.check(properties.getOrDefault(JdbcResource.FUNCTION_RULES, "")); - this.functionRules = ExternalFunctionRules.create(jdbcTypeName, - properties.getOrDefault(JdbcResource.FUNCTION_RULES, "")); + String functionRulesString = properties.getOrDefault(JdbcResource.FUNCTION_RULES, ""); + this.functionRules = ExternalFunctionRules.create(jdbcTypeName, functionRulesString); + this.functionRulesString = functionRulesString; } /** @@ -529,4 +533,10 @@ public void setExternalFunctionRules(ExternalFunctionRules functionRules) { public ExternalFunctionRules getExternalFunctionRules() { return functionRules; } + + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + functionRules = ExternalFunctionRules.create(jdbcTypeName, Strings.nullToEmpty(functionRulesString)); + } } From 300ec58f655494ac993e9eb1abceff02bad83a81 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Wed, 16 Jul 2025 11:49:55 +0800 Subject: [PATCH 232/572] branch-3.0: [fix](catalog) synchronize reset methods in catalog classes and remove Lombok annotations (#53167) pick (#51787) --- .../doris/datasource/ExternalCatalog.java | 32 ++++++++++++++--- .../doris/datasource/ExternalDatabase.java | 2 +- .../doris/datasource/ExternalTable.java | 34 +++++++++++++++++-- .../datasource/hive/HMSExternalCatalog.java | 2 +- .../datasource/jdbc/JdbcExternalCatalog.java | 8 +++-- .../iceberg/CreateIcebergTableTest.java | 4 +-- .../paimon/PaimonExternalCatalogTest.java | 2 +- .../doris/planner/HiveTableSinkTest.java | 2 +- 8 files changed, 70 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 86c5a3ae3e676e..f0a0b26ff49ebb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -73,7 +73,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; -import lombok.Data; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; @@ -101,7 +100,6 @@ /** * The abstract class for all types of external catalogs. */ -@Data public abstract class ExternalCatalog implements CatalogIf>, Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(ExternalCatalog.class); @@ -358,7 +356,7 @@ public boolean isInitialized() { // check if all required properties are set when creating catalog public void checkProperties() throws DdlException { // check refresh parameter of catalog - Map properties = getCatalogProperty().getProperties(); + Map properties = catalogProperty.getProperties(); if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) { try { int metadataRefreshIntervalSec = Integer.parseInt( @@ -391,7 +389,7 @@ public void checkProperties() throws DdlException { * isDryRun: if true, it will try to create the custom access controller, but will not add it to the access manager. */ public void initAccessController(boolean isDryRun) { - Map properties = getCatalogProperty().getProperties(); + Map properties = catalogProperty.getProperties(); // 1. get access controller class name String className = properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_CLASS_PROP, ""); if (Strings.isNullOrEmpty(className)) { @@ -551,7 +549,7 @@ private List> getFilteredDatabaseNames() { * @param invalidCache if {@code true}, the catalog cache will be invalidated * and reloaded during the refresh process. */ - public void resetToUninitialized(boolean invalidCache) { + public synchronized void resetToUninitialized(boolean invalidCache) { this.objectCreated = false; this.initialized = false; synchronized (this.propLock) { @@ -1004,6 +1002,14 @@ public void addDatabaseForTest(ExternalDatabase db) { dbNameToId.put(ClusterNamespace.getNameFromFullName(db.getFullName()), db.getId()); } + /** + * Set the initialized status for testing purposes only. + * This method should only be used in test cases. + */ + public void setInitializedForTest(boolean initialized) { + this.initialized = initialized; + } + @Override public void createDb(CreateDbStmt stmt) throws DdlException { makeSureInitialized(); @@ -1180,6 +1186,22 @@ public PreExecutionAuthenticator getPreExecutionAuthenticator() { return preExecutionAuthenticator; } + public Optional getUseMetaCache() { + return useMetaCache; + } + + public CatalogProperty getCatalogProperty() { + return catalogProperty; + } + + public Map, String> getTableAutoAnalyzePolicy() { + return tableAutoAnalyzePolicy; + } + + public TransactionManager getTransactionManager() { + return transactionManager; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index ece9d7e265c32f..9b726e6b1de257 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -134,7 +134,7 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) { } } - public void setUnInitialized(boolean invalidCache) { + public synchronized void setUnInitialized(boolean invalidCache) { this.initialized = false; this.invalidCacheInInit = invalidCache; if (extCatalog.getUseMetaCache().isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 30bf48c3d8b76a..8758b8375db8e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -45,7 +45,6 @@ import com.google.common.base.Objects; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; -import lombok.Getter; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.LogManager; @@ -64,7 +63,6 @@ * External table represent tables that are not self-managed by Doris. * Such as tables from hive, iceberg, es, etc. */ -@Getter public class ExternalTable implements TableIf, Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(ExternalTable.class); @@ -466,4 +464,36 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hashCode(name, db); } + + public long getSchemaUpdateTime() { + return schemaUpdateTime; + } + + public long getDbId() { + return dbId; + } + + public boolean isObjectCreated() { + return objectCreated; + } + + public ExternalCatalog getCatalog() { + return catalog; + } + + public ExternalDatabase getDb() { + return db; + } + + public long getTimestamp() { + return timestamp; + } + + public String getDbName() { + return dbName; + } + + public TableAttributes getTableAttributes() { + return tableAttributes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 5d9128bcc04bd7..e42124377ed380 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -204,7 +204,7 @@ protected void initLocalObjectsImpl() { } @Override - public void resetToUninitialized(boolean invalidCache) { + public synchronized void resetToUninitialized(boolean invalidCache) { super.resetToUninitialized(invalidCache); if (metadataOps != null) { metadataOps.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 4899cb2ec1cef1..85e3b29e1a3e55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -51,7 +51,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; -import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -64,7 +63,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; -@Getter public class JdbcExternalCatalog extends ExternalCatalog { private static final Logger LOG = LogManager.getLogger(JdbcExternalCatalog.class); @@ -130,7 +128,7 @@ public void setDefaultPropsIfMissing(boolean isReplay) { } @Override - public void resetToUninitialized(boolean invalidCache) { + public synchronized void resetToUninitialized(boolean invalidCache) { super.resetToUninitialized(invalidCache); this.identifierMapping = new JdbcIdentifierMapping( (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), @@ -448,4 +446,8 @@ private JdbcTable getTestConnectionJdbcTable(JdbcClient testClient) throws DdlEx public ExternalFunctionRules getFunctionRules() { return functionRules; } + + public IdentifierMapping getIdentifierMapping() { + return identifierMapping; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 2300ece6253b5f..45fe77387835cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -72,7 +72,7 @@ public static void beforeClass() throws Throwable { if (icebergCatalog.getUseMetaCache().get()) { icebergCatalog.makeSureInitialized(); } else { - icebergCatalog.setInitialized(true); + icebergCatalog.setInitializedForTest(true); } // create db @@ -82,7 +82,7 @@ public static void beforeClass() throws Throwable { if (icebergCatalog.getUseMetaCache().get()) { icebergCatalog.makeSureInitialized(); } else { - icebergCatalog.setInitialized(true); + icebergCatalog.setInitializedForTest(true); } IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); icebergCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java index 35dd64515b6ec9..da66b3c5ba9a24 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java @@ -31,7 +31,7 @@ public void testGetPaimonTable() { HashMap props = new HashMap<>(); props.put("warehouse", "not_exist"); PaimonExternalCatalog catalog = new PaimonFileExternalCatalog(1, "name", "resource", props, "comment"); - catalog.setInitialized(true); + catalog.setInitializedForTest(true); try { catalog.getPaimonTable("dbName", "tblName"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java index b57bbcb51a2772..602b2c8a18ade8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java @@ -88,7 +88,7 @@ public HMSCachedClient getClient() { mockDifferLocationTable(location); HMSExternalCatalog hmsExternalCatalog = new HMSExternalCatalog(); - hmsExternalCatalog.setInitialized(true); + hmsExternalCatalog.setInitializedForTest(true); HMSExternalDatabase db = new HMSExternalDatabase(hmsExternalCatalog, 10000, "hive_db1", "hive_db1"); HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsExternalCatalog, db); HiveTableSink hiveTableSink = new HiveTableSink(tbl); From 98fff5ae5604df29160dca67c4fc730e3b9b5223 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Wed, 16 Jul 2025 16:05:57 +0800 Subject: [PATCH 233/572] branch-3.0: [fix](Nereids) not process must shuffle when regulate cannot be banned agg #52827 (#53268) cherry-picked from #52827 --- .../properties/ChildrenPropertiesRegulator.java | 6 ++---- .../doris/nereids/rules/analysis/CheckAnalysis.java | 13 ++++++++----- .../rules/implementation/AggregateStrategies.java | 10 ++++++---- .../query_p0/group_concat/test_group_concat.out | 4 ++++ .../query_p0/group_concat/test_group_concat.groovy | 10 ++++++++++ 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java index 9985b9c567f8fe..83fe0ca06b0e8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulator.java @@ -110,7 +110,7 @@ public Boolean visitPhysicalHashAggregate(PhysicalHashAggregate return false; } if (!agg.getAggregateParam().canBeBanned) { - return true; + return visit(agg, context); } // forbid one phase agg on distribute if (agg.getAggMode() == AggMode.INPUT_TO_RESULT && children.get(0).getPlan() instanceof PhysicalDistribute) { @@ -176,9 +176,7 @@ public Boolean visitPhysicalHashAggregate(PhysicalHashAggregate } } // process must shuffle - visit(agg, context); - // process agg - return true; + return visit(agg, context); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index 7ca8637446b0d6..ae7bb232e83552 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.WindowExpression; import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.GroupingScalarFunction; import org.apache.doris.nereids.trees.expressions.typecoercion.TypeCheckResult; @@ -148,11 +149,13 @@ private void checkAggregate(LogicalAggregate aggregate) { if (func.arity() <= 1) { continue; } - for (int i = 1; i < func.arity(); i++) { - if (!func.child(i).getInputSlots().isEmpty() && !(func.child(i) instanceof OrderExpression)) { - // think about group_concat(distinct col_1, ',') - distinctMultiColumns = true; - break; + if (func instanceof Count) { + for (int i = 1; i < func.arity(); i++) { + if (!func.child(i).getInputSlots().isEmpty() && !(func.child(i) instanceof OrderExpression)) { + // think about group_concat(distinct col_1, ',') + distinctMultiColumns = true; + break; + } } } if (distinctMultiColumns) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index ecd951f1eb765d..0539a08d3d794b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -2088,10 +2088,12 @@ private boolean couldConvertToMulti(LogicalAggregate aggregate) if (func.arity() <= 1) { continue; } - for (int i = 1; i < func.arity(); i++) { - // think about group_concat(distinct col_1, ',') - if (!(func.child(i) instanceof OrderExpression) && !func.child(i).getInputSlots().isEmpty()) { - return false; + if (func instanceof Count) { + for (int i = 1; i < func.arity(); i++) { + // think about group_concat(distinct col_1, ',') + if (!(func.child(i) instanceof OrderExpression) && !func.child(i).getInputSlots().isEmpty()) { + return false; + } } } } diff --git a/regression-test/data/query_p0/group_concat/test_group_concat.out b/regression-test/data/query_p0/group_concat/test_group_concat.out index d01900ef88bdbb..3065713cf55c49 100644 --- a/regression-test/data/query_p0/group_concat/test_group_concat.out +++ b/regression-test/data/query_p0/group_concat/test_group_concat.out @@ -39,6 +39,10 @@ false 1 2 1 2 +-- !select_13 -- +1 2 +1 2 + -- !select_group_concat_order_by_all_data -- 1 1 1 1 1 11 diff --git a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy index 522d66ed64b30b..ee9bab29e8e166 100644 --- a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy +++ b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy @@ -79,6 +79,16 @@ suite("test_group_concat", "query,p0,arrow_flight_sql") { b2; """ + // test SPLIT_MULTI_DISTINCT could work right with can not be banned aggregation + qt_select_13 """ + select + group_concat( distinct b1, cast(b2 as varchar)), group_concat( distinct b3, '?') + from + table_group_concat + group by + b2; + """ + sql """ drop table table_group_concat """ sql """create table table_group_concat ( b1 varchar(10) not null, b2 int not null, b3 varchar(10) not null ) ENGINE=OLAP From a290c21deff784635ac2bd84d0d3c2443f861aff Mon Sep 17 00:00:00 2001 From: morrySnow Date: Wed, 16 Jul 2025 16:06:33 +0800 Subject: [PATCH 234/572] [fix](decimal256) some ddl could not run or get wrong result on decimal256 (#53360) ### What problem does this PR solve? Problem Summary: some ddl could not run or get wrong result on decimal256 - alter table add column - alter table modify column - create aggregate model table with decimal256 value column ### Release note None ### Check List (For Author) - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [x] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../apache/doris/catalog/AggregateType.java | 3 + .../org/apache/doris/catalog/FunctionSet.java | 44 ++++++ .../org/apache/doris/qe/SessionVariable.java | 10 +- .../decimalv3/test_decimal256_ddl.out | 21 +++ .../decimalv3/test_decimal256_ddl.groovy | 144 ++++++++++++++++++ 5 files changed, 220 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/datatype_p0/decimalv3/test_decimal256_ddl.out create mode 100644 regression-test/suites/datatype_p0/decimalv3/test_decimal256_ddl.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java index 94a788f0b9afe0..12f3d16fbd348f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AggregateType.java @@ -70,6 +70,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DECIMAL32); primitiveTypeList.add(PrimitiveType.DECIMAL64); primitiveTypeList.add(PrimitiveType.DECIMAL128); + primitiveTypeList.add(PrimitiveType.DECIMAL256); compatibilityMap.put(SUM, EnumSet.copyOf(primitiveTypeList)); primitiveTypeList.clear(); @@ -84,6 +85,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DECIMAL32); primitiveTypeList.add(PrimitiveType.DECIMAL64); primitiveTypeList.add(PrimitiveType.DECIMAL128); + primitiveTypeList.add(PrimitiveType.DECIMAL256); primitiveTypeList.add(PrimitiveType.DATE); primitiveTypeList.add(PrimitiveType.DATETIME); primitiveTypeList.add(PrimitiveType.DATEV2); @@ -105,6 +107,7 @@ public enum AggregateType { primitiveTypeList.add(PrimitiveType.DECIMAL32); primitiveTypeList.add(PrimitiveType.DECIMAL64); primitiveTypeList.add(PrimitiveType.DECIMAL128); + primitiveTypeList.add(PrimitiveType.DECIMAL256); primitiveTypeList.add(PrimitiveType.DATE); primitiveTypeList.add(PrimitiveType.DATETIME); primitiveTypeList.add(PrimitiveType.DATEV2); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java index de1d1178ed65c8..54125ce4ac3e13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java @@ -139,6 +139,7 @@ public boolean isNullResultWithOneNullParamFunctions(String funcName) { .put(Type.DECIMAL32, Type.DECIMAL32) .put(Type.DECIMAL64, Type.DECIMAL64) .put(Type.DECIMAL128, Type.DECIMAL128) + .put(Type.DECIMAL256, Type.DECIMAL256) .build(); private static final Map STDDEV_RETTYPE_SYMBOL = @@ -953,6 +954,19 @@ private void initAggregateBuiltins() { null, "", false, true, true, true)); + } else if (t.equals(Type.DECIMAL256)) { + // vectorized + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_count", Lists.newArrayList(t), + Type.BIGINT, + Type.DECIMAL256, + "", + "", + "", + "", + null, + null, + "", + false, true, true, true)); } // sum in multi distinct @@ -1073,6 +1087,29 @@ private void initAggregateBuiltins() { null, "", false, true, true, true)); + } else if (t.equals(Type.DECIMAL256)) { + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_sum", Lists.newArrayList(t), + MULTI_DISTINCT_SUM_RETURN_TYPE.get(t), + Type.DECIMAL256, + "", + "", + "", + "", + null, + null, + "", + false, true, false, true)); + addBuiltin(AggregateFunction.createBuiltin("multi_distinct_sum0", Lists.newArrayList(t), + MULTI_DISTINCT_SUM_RETURN_TYPE.get(t), + Type.DECIMAL256, + "", + "", + "", + "", + null, + null, + "", + false, true, true, true)); } // Min addBuiltin(AggregateFunction.createBuiltin("min", @@ -1349,6 +1386,13 @@ private void initAggregateBuiltins() { null, null, "", null, false, true, nameWithReturn.getValue(), true)); + addBuiltin(AggregateFunction.createBuiltin(nameWithReturn.getKey(), + Lists.newArrayList(Type.DECIMAL256), Type.DECIMAL256, Type.DECIMAL256, "", + "", + "", + null, null, + "", + null, false, true, nameWithReturn.getValue(), true)); addBuiltin(AggregateFunction.createBuiltin(nameWithReturn.getKey(), Lists.newArrayList(Type.LARGEINT), Type.LARGEINT, Type.LARGEINT, "", "", diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index a036b18cd7e671..3a7ee7c8f2e2f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -4487,10 +4487,16 @@ public static boolean enableAggState() { public static boolean getEnableDecimal256() { ConnectContext connectContext = ConnectContext.get(); if (connectContext == null) { - return false; + return VariableMgr.getDefaultSessionVariable().enableDecimal256; } SessionVariable sessionVariable = connectContext.getSessionVariable(); - return connectContext.getState().isNereids() && sessionVariable.isEnableDecimal256(); + if (!sessionVariable.isEnableDecimal256()) { + return false; + } + if (connectContext.getState().isQuery()) { + return connectContext.getState().isNereids(); + } + return true; } public boolean isEnableDecimal256() { diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimal256_ddl.out b/regression-test/data/datatype_p0/decimalv3/test_decimal256_ddl.out new file mode 100644 index 00000000000000..aa3d3f760f3cef --- /dev/null +++ b/regression-test/data/datatype_p0/decimalv3/test_decimal256_ddl.out @@ -0,0 +1,21 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 1.10000 1.20000 1.30000 +1 11.10000 11.20000 11.30000 +1 12.10000 12.20000 12.30000 +2 2.10000 2.20000 2.30000 +2 21.10000 21.20000 21.30000 +2 22.10000 22.20000 22.30000 + +-- !agg -- +24.30000 1.20000 12.30000 +45.30000 2.20000 22.30000 + +-- !select -- +1 24.300 1 12.3000 +2 45.300 2 22.3000 + +-- !agg -- +24.300 1 12.3000 +45.300 2 22.3000 + diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_ddl.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_ddl.groovy new file mode 100644 index 00000000000000..e50925ad27cde6 --- /dev/null +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_ddl.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_decimal256_ddl") { + def getJobState = { tableName -> + def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + return jobStateResult[0][9] + } + + def waitSchemaChangeFinish = { table -> + while (true) { + String result = getJobState(table) + if (result == "FINISHED") { + break + } else if (result == "CANCELLED") { + sucess.set(false) + logger.error("schema change was cancelled") + assertTrue(false) + } else { + sleep(2000) + } + } + } + + def table1 = "test_decimal256_ddl" + + sql "set enable_decimal256=true" + + sql "drop table if exists ${table1}" + + sql """ + CREATE TABLE IF NOT EXISTS `${table1}` ( + id int, + dcl1 decimal(40, 5) + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `dcl1`) + DISTRIBUTED BY HASH(`id`, `dcl1`) BUCKETS 8 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ + + sql """ + alter table ${table1} add column (dcl2 decimal(60,15)); + """ + + waitSchemaChangeFinish(table1) + + + sql """ + alter table ${table1} add column (dcl3 decimal(60,15) default '0'); + """ + + waitSchemaChangeFinish(table1) + + + sql """ + alter table ${table1} modify column dcl2 decimal(48, 5); + """ + + waitSchemaChangeFinish(table1) + + + sql """ + alter table ${table1} modify column dcl3 decimal(48, 5) default '0'; + """ + + waitSchemaChangeFinish(table1) + + + sql """ + create materialized view ${table1}mv as select id, sum(dcl1), min(dcl2), max(dcl3) from ${table1} group by id; + """ + + + sql """ + insert into ${table1} values + (1,1.1,1.2,1.3), + (1,11.1,11.2,11.3), + (1,12.1,12.2,12.3), + (2,2.1,2.2,2.3), + (2,21.1,21.2,21.3), + (2,22.1,22.2,22.3) + """ + + sql """sync""" + + order_qt_select """ + select * from ${table1} order by id + """ + + order_qt_agg """ + select sum(dcl1), min(dcl2), max(dcl3) from ${table1} group by id + """ + + sql "drop table if exists ${table1} force" + + sql """ + create table ${table1} (id int, dcl1 decimal(60,3) sum, dcl2 decimal(70,0) min, dcl3 decimal(70,4) max) + AGGREGATE KEY(`id`) + DISTRIBUTED BY hash(id) + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ) + """ + + sql """ + insert into ${table1} values + (1,1.1,1.2,1.3), + (1,11.1,11.2,11.3), + (1,12.1,12.2,12.3), + (2,2.1,2.2,2.3), + (2,21.1,21.2,21.3), + (2,22.1,22.2,22.3) + """ + + sql """sync""" + + order_qt_select """ + select * from ${table1} order by id + """ + + order_qt_agg """ + select sum(dcl1), min(dcl2), max(dcl3) from ${table1} group by id + """ + + sql "drop table if exists ${table1} force" + + +} From b6e385898a371d317da4a45b1e0645dcaa3c7ef9 Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 16 Jul 2025 23:47:39 +0800 Subject: [PATCH 235/572] branch-3.0: [fix](auto bucket)Fix auto bucket calc bucketnum err when partition size is invalid #52801 (#53250) cherry pick from #52801 --- .../java/org/apache/doris/common/Config.java | 6 + .../apache/doris/analysis/ShowDataStmt.java | 2 +- .../doris/catalog/MaterializedIndex.java | 4 +- .../apache/doris/catalog/MetadataViewer.java | 2 +- .../org/apache/doris/catalog/OlapTable.java | 2 +- .../org/apache/doris/catalog/Partition.java | 14 +- .../java/org/apache/doris/catalog/Tablet.java | 5 +- .../ColocateTableCheckerAndBalancer.java | 3 +- .../clone/DynamicPartitionScheduler.java | 136 +++++++++++++----- .../doris/cloud/CacheHotspotManager.java | 6 +- .../common/proc/TabletHealthProcDir.java | 2 +- .../apache/doris/catalog/CatalogTestUtil.java | 2 +- .../catalog/DynamicPartitionTableTest.java | 45 ++++++ .../common/util/AutoBucketUtilsTest.java | 1 + 14 files changed, 182 insertions(+), 48 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 8159018acc5085..94728fe3f9df43 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2658,6 +2658,12 @@ public class Config extends ConfigBase { }) public static int autobucket_partition_size_per_bucket_gb = -1; + @ConfField(mutable = true, masterOnly = true, description = {"Auto bucket中计算出的新的分区bucket num超过前一个分区的" + + "bucket num的百分比，被认为是异常case报警", + "The new partition bucket number calculated in the auto bucket exceeds the percentage " + + "of the previous partition's bucket number, which is considered an abnormal case alert."}) + public static double autobucket_out_of_bounds_percent_threshold = 0.5; + @ConfField(description = {"(已弃用，被 arrow_flight_max_connection 替代) Arrow Flight Server中所有用户token的缓存上限，" + "超过后LRU淘汰, arrow flight sql是无状态的协议，连接通常不会主动断开，" + "bearer token 从 cache 淘汰的同时会 unregister Connection.", diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java index 398b0d7ec05d42..0751c13d014ece 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java @@ -284,7 +284,7 @@ private void collectTableStats(OlapTable table) { long remoteSegmentSize = 0; for (Partition partition : table.getAllPartitions()) { MaterializedIndex mIndex = partition.getIndex(indexId); - indexSize += mIndex.getDataSize(false); + indexSize += mIndex.getDataSize(false, false); indexReplicaCount += mIndex.getReplicaCount(); indexRowCount += mIndex.getRowCount() == -1 ? 0 : mIndex.getRowCount(); indexRemoteSize += mIndex.getRemoteDataSize(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java index 5e367b538cd74c..b5a31a32e1e961 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java @@ -173,10 +173,10 @@ public void clearRollupIndexInfo() { this.rollupFinishedVersion = -1L; } - public long getDataSize(boolean singleReplica) { + public long getDataSize(boolean singleReplica, boolean filterSizeZero) { long dataSize = 0; for (Tablet tablet : getTablets()) { - dataSize += tablet.getDataSize(singleReplica); + dataSize += tablet.getDataSize(singleReplica, filterSizeZero); } return dataSize; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java index 66cdfa5b42d1ae..e44b7ec52c8c7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java @@ -332,7 +332,7 @@ private static List> getDataSkew(String dbName, String tblName, Par for (int i = 0; i < tabletIds.size(); i++) { Tablet tablet = mIndex.getTablet(tabletIds.get(i)); long rowCount = tablet.getRowCount(true); - long dataSize = tablet.getDataSize(true); + long dataSize = tablet.getDataSize(true, false); rowCountTabletInfos.set(i, rowCountTabletInfos.get(i) + rowCount); dataSizeTabletInfos.set(i, dataSizeTabletInfos.get(i) + dataSize); totalSize += dataSize; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 9fd9d2c49f46b3..3d4b5cc1d6f3d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1563,7 +1563,7 @@ public long getAvgRowLength() { long dataSize = 0; for (Map.Entry entry : idToPartition.entrySet()) { rowCount += entry.getValue().getBaseIndex().getRowCount(); - dataSize += entry.getValue().getBaseIndex().getDataSize(false); + dataSize += entry.getValue().getBaseIndex().getDataSize(false, false); } if (rowCount > 0) { return dataSize / rowCount; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java index 0468d3a75498f0..e00bf4b28ef1b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java @@ -291,7 +291,7 @@ public long getAllDataSize(boolean singleReplica) { public long getDataSize(boolean singleReplica) { long dataSize = 0; for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { - dataSize += mIndex.getDataSize(singleReplica); + dataSize += mIndex.getDataSize(singleReplica, false); } return dataSize; } @@ -458,7 +458,7 @@ public long getRowCount() { public long getAvgRowLength() { long rowCount = getBaseIndex().getRowCount(); - long dataSize = getBaseIndex().getDataSize(false); + long dataSize = getBaseIndex().getDataSize(false, false); if (rowCount > 0) { return dataSize / rowCount; } else { @@ -467,6 +467,14 @@ public long getAvgRowLength() { } public long getDataLength() { - return getBaseIndex().getDataSize(false); + return getBaseIndex().getDataSize(false, false); + } + + public long getDataSizeExcludeEmptyReplica(boolean singleReplica) { + long dataSize = 0; + for (MaterializedIndex mIndex : getMaterializedIndices(IndexExtState.VISIBLE)) { + dataSize += mIndex.getDataSize(singleReplica, true); + } + return dataSize + getRemoteDataSize(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java index ea76273e6dfbfa..f18460a320dc0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java @@ -515,8 +515,11 @@ public boolean equals(Object obj) { return id == tablet.id; } - public long getDataSize(boolean singleReplica) { + // ATTN: Replica::getDataSize may zero in cloud and non-cloud + // due to dataSize not write to image + public long getDataSize(boolean singleReplica, boolean filterSizeZero) { LongStream s = replicas.stream().filter(r -> r.getState() == ReplicaState.NORMAL) + .filter(r -> !filterSizeZero || r.getDataSize() > 0) .mapToLong(Replica::getDataSize); return singleReplica ? Double.valueOf(s.average().orElse(0)).longValue() : s.sum(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java index 7727bc77e18667..7dc0a9125dddf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java @@ -655,7 +655,8 @@ private GlobalColocateStatistic buildGlobalColocateStatistic() { bucketsSeq.size() + " vs. " + replicationNum); Tablet tablet = index.getTablet(tabletId); totalReplicaDataSizes.set(tabletOrderIdx, - totalReplicaDataSizes.get(tabletOrderIdx) + tablet.getDataSize(true)); + totalReplicaDataSizes.get(tabletOrderIdx) + + tablet.getDataSize(true, false)); tabletOrderIdx++; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 8e5f46573ee0ee..572db637f9ed86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -58,6 +58,7 @@ import org.apache.doris.rpc.RpcException; import org.apache.doris.thrift.TStorageMedium; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -159,7 +160,7 @@ private Map createDefaultRuntimeInfo() { } // exponential moving average - private static long ema(ArrayList history, int period) { + private static long ema(List history, int period) { double alpha = 2.0 / (period + 1); double ema = history.get(0); for (int i = 1; i < history.size(); i++) { @@ -168,7 +169,7 @@ private static long ema(ArrayList history, int period) { return (long) ema; } - private static long getNextPartitionSize(ArrayList historyPartitionsSize) { + private static long getNextPartitionSize(List historyPartitionsSize) { if (historyPartitionsSize.size() < 2) { return historyPartitionsSize.get(0); } @@ -191,65 +192,98 @@ private static long getNextPartitionSize(ArrayList historyPartitionsSize) } } - private static int getBucketsNum(DynamicPartitionProperty property, OlapTable table, + private static Pair getBucketsNum(DynamicPartitionProperty property, OlapTable table, String partitionName, String nowPartitionName, boolean executeFirstTime) { // if execute first time, all partitions no contain data if (!table.isAutoBucket() || executeFirstTime) { - return property.getBuckets(); + return Pair.of(property.getBuckets(), 0); } - // auto bucket - // get all history partitions - RangePartitionInfo info = (RangePartitionInfo) (table.getPartitionInfo()); - List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); - idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem) o.getValue()).getItems().upperEndpoint())); - List partitions = idToItems.stream() - .map(entry -> table.getPartition(entry.getKey())) - .filter(partition -> partition != null && !partition.getName().equals(nowPartitionName)) - .collect(Collectors.toList()); - List visibleVersions = null; + List partitions = getHistoricalPartitions(table, nowPartitionName); + List visibleVersions; try { visibleVersions = Partition.getVisibleVersions(partitions); } catch (RpcException e) { - LOG.warn("autobucket use property's buckets get visible version fail, table: [{}-{}], " + LOG.warn("auto bucket use property's buckets get visible version fail, table: [{}-{}], " + "partition: {}, buckets num: {}, exception: ", table.getName(), table.getId(), partitionName, property.getBuckets(), e); - return property.getBuckets(); + return Pair.of(property.getBuckets(), 0); } - List hasDataPartitions = Lists.newArrayList(); + List hasDataPartitions = filterDataPartitions(partitions, visibleVersions); + if (hasDataPartitions.isEmpty()) { + return handleNoDataPartitions(table, partitionName, property.getBuckets()); + } + + return calculateBuckets(hasDataPartitions); + } + + private static List getHistoricalPartitions(OlapTable table, String nowPartitionName) { + RangePartitionInfo info = (RangePartitionInfo) (table.getPartitionInfo()); + List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); + idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem) o.getValue()).getItems().upperEndpoint())); + return idToItems.stream() + .map(entry -> table.getPartition(entry.getKey())) + .filter(partition -> partition != null && !partition.getName().equals(nowPartitionName)) + .collect(Collectors.toList()); + } + + private static List filterDataPartitions(List partitions, List visibleVersions) { + Preconditions.checkState(partitions.size() == visibleVersions.size(), + String.format("partitions size %d not eq visibleVersions size %d, impossible", + partitions.size(), visibleVersions.size())); + List hasDataPartitions = new ArrayList<>(); for (int i = 0; i < partitions.size(); i++) { if (visibleVersions.get(i) >= 2) { hasDataPartitions.add(partitions.get(i)); } } + return hasDataPartitions; + } - // no exist history partition data - if (hasDataPartitions.isEmpty()) { - LOG.info("autobucket use property's buckets due to all partitions no data, table: [{}-{}], " - + "partition: {}, buckets num: {}", - table.getName(), table.getId(), partitionName, property.getBuckets()); - return property.getBuckets(); + private static Pair handleNoDataPartitions(OlapTable table, + String partitionName, int defaultBuckets) { + LOG.info("auto bucket use property's buckets due to all partitions no data, table: [{}-{}], " + + "partition: {}, buckets num: {}", table.getName(), table.getId(), partitionName, defaultBuckets); + return Pair.of(defaultBuckets, 0); + } + + private static Pair calculateBuckets(List hasDataPartitions) { + List partitionSizeArray = new ArrayList<>(); + List sizeUnknownArray = new ArrayList<>(); + + for (Partition hasDataPartition : hasDataPartitions) { + long partitionSize = hasDataPartition.getDataSizeExcludeEmptyReplica(true); + if (partitionSize <= 0) { + sizeUnknownArray.add(partitionSize); + } else { + partitionSizeArray.add(partitionSize); + } + } + + int size = hasDataPartitions.size(); + Preconditions.checkState(size > 0, "hasDataPartitions size must be greater than 0"); + int previousPartitionBucketsNum = hasDataPartitions.get(size - 1).getDistributionInfo().getBucketNum(); + + if (hasDataPartitions.size() == sizeUnknownArray.size()) { + LOG.info("TabletStatMgr not synchronized partitions size yet, so use previous partition bucket num"); + return Pair.of(previousPartitionBucketsNum, previousPartitionBucketsNum); } - ArrayList partitionSizeArray = hasDataPartitions.stream() - .map(partition -> partition.getAllDataSize(true)) - .collect(Collectors.toCollection(ArrayList::new)); long estimatePartitionSize = getNextPartitionSize(partitionSizeArray); - // plus 5 for uncompressed data long uncompressedPartitionSize = estimatePartitionSize * 5; int bucketsNum = AutoBucketUtils.getBucketsNum(uncompressedPartitionSize, Config.autobucket_min_buckets); - LOG.info("autobucket calc with {} history partitions, table: [{}-{}], partition: {}, buckets num: {}, " - + " estimate partition size: {}, last partitions(partition name, local size, remote size): {}", - hasDataPartitions.size(), table.getName(), table.getId(), partitionName, bucketsNum, - estimatePartitionSize, + + LOG.info("auto bucket calc with {} history partitions, {} history partitions size not sync size yet," + + " buckets num: {}, estimate partition size: {}, last partitions: {}", + hasDataPartitions.size(), sizeUnknownArray.size(), bucketsNum, estimatePartitionSize, hasDataPartitions.stream() .skip(Math.max(0, hasDataPartitions.size() - 7)) .map(partition -> "(" + partition.getName() + ", " + partition.getDataSize(true) + ", " + partition.getRemoteDataSize() + ")") .collect(Collectors.toList())); - return bucketsNum; + return Pair.of(bucketsNum, previousPartitionBucketsNum); } private ArrayList getAddPartitionClause(Database db, OlapTable olapTable, @@ -356,8 +390,17 @@ private ArrayList getAddPartitionClause(Database db, OlapTab DistributionDesc distributionDesc = null; DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); - int bucketsNum = getBucketsNum(dynamicPartitionProperty, olapTable, partitionName, + Pair ret = getBucketsNum(dynamicPartitionProperty, olapTable, partitionName, nowPartitionName, executeFirstTime); + int bucketsNum = ret.first; + int previousPartitionBucketsNum = ret.second; + if (olapTable.isAutoBucket()) { + int afterCheckAndFixBucketNum = checkAndFixAutoBucketCalcNumIsValid(bucketsNum, + previousPartitionBucketsNum); + if (afterCheckAndFixBucketNum > 0) { + bucketsNum = afterCheckAndFixBucketNum; + } + } if (distributionInfo.getType() == DistributionInfo.DistributionInfoType.HASH) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List distColumnNames = new ArrayList<>(); @@ -374,6 +417,33 @@ private ArrayList getAddPartitionClause(Database db, OlapTab return addPartitionClauses; } + private int checkAndFixAutoBucketCalcNumIsValid(int currentPartitionNumBuckets, int previousPartitionNumBuckets) { + // previousPartitionBucketsNum == 0, some abnormal case, ignore it + if (currentPartitionNumBuckets != 0) { + // currentPartitionNumBuckets can be too big + if (currentPartitionNumBuckets + > previousPartitionNumBuckets * (1 + Config.autobucket_out_of_bounds_percent_threshold)) { + LOG.warn("auto bucket calc num may be err, bigger than previous too much, plz check. " + + "calc bucket num {}, previous partition bucket num {}, percent {}", + currentPartitionNumBuckets, previousPartitionNumBuckets, + Config.autobucket_out_of_bounds_percent_threshold); + return currentPartitionNumBuckets; + } + // currentPartitionNumBuckets not too small. + // If it is too small, the program will intervene. use previousPartitionNumBuckets + if (currentPartitionNumBuckets + < previousPartitionNumBuckets * (1 - Config.autobucket_out_of_bounds_percent_threshold)) { + LOG.warn("auto bucket calc num may be err, smaller than previous too much, plz check. " + + "calc bucket num {}, previous partition bucket num {}, percent {}", + currentPartitionNumBuckets, previousPartitionNumBuckets, + Config.autobucket_out_of_bounds_percent_threshold); + return previousPartitionNumBuckets; + } + } + LOG.info("previousPartitionBucketsNum eq 0, check before log"); + return -1; + } + /** * If dynamic_partition.storage_medium is set to SSD, * ignore hot_partition_num property and set to (SSD, 9999-12-31 23:59:59) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index 6b7b0e5282a3c0..6797ce45acb7de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -501,13 +501,13 @@ public Map>> splitBatch(Map> beToWarmUp List batch = new ArrayList<>(); long curBatchSize = 0L; for (Tablet tablet : entry.getValue()) { - if (curBatchSize + tablet.getDataSize(true) > maxSizePerBatch) { + if (curBatchSize + tablet.getDataSize(true, false) > maxSizePerBatch) { batches.add(batch); batch = new ArrayList<>(); curBatchSize = 0L; } batch.add(tablet.getId()); - curBatchSize += tablet.getDataSize(true); + curBatchSize += tablet.getDataSize(true, false); } if (!batch.isEmpty()) { batches.add(batch); @@ -545,7 +545,7 @@ private List getHotTablets(String srcClusterName, String dstClusterName) continue; } for (Tablet tablet : index.getTablets()) { - warmUpTabletsSize += tablet.getDataSize(true); + warmUpTabletsSize += tablet.getDataSize(true, false); tablets.add(tablet); if (warmUpTabletsSize >= dstTotalFileCache) { break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java index 6c36a926d088b8..b1be9c79ab6252 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java @@ -262,7 +262,7 @@ static class DBTabletStatistic { inconsistentNum++; inconsistentTabletIds.add(tablet.getId()); } - if (tablet.getDataSize(true) > Config.min_bytes_indicate_replica_too_large) { + if (tablet.getDataSize(true, false) > Config.min_bytes_indicate_replica_too_large) { oversizeNum++; oversizeTabletIds.add(tablet.getId()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java index b16d3d15cf7e52..8a19912d321c8c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java @@ -385,7 +385,7 @@ public static long getTabletDataSize(long tabletId) { if (tablet == null) { return -1L; } - return tablet.getDataSize(true); + return tablet.getDataSize(true, false); } finally { olapTable.readUnlock(); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java index 9495c048f4b13f..35c8f01f050ea5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java @@ -1779,6 +1779,7 @@ public void testAutoBuckets() throws Exception { } RebalancerTestUtil.updateReplicaDataSize(1, 1, 1); + Config.autobucket_out_of_bounds_percent_threshold = 0.99; String alterStmt1 = "alter table test.test_autobucket_dynamic_partition set ('dynamic_partition.end' = '2')"; ExceptionChecker.expectThrowsNoException(() -> alterTable(alterStmt1)); @@ -1789,6 +1790,7 @@ public void testAutoBuckets() throws Exception { partitions.sort(Comparator.comparing(Partition::getId)); Assert.assertEquals(53, partitions.size()); Assert.assertEquals(1, partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum()); + Config.autobucket_out_of_bounds_percent_threshold = 0.5; table.readLock(); try { @@ -1826,5 +1828,48 @@ public void testAutoBuckets() throws Exception { Assert.assertEquals(54, partitions.size()); // 100GB total, 5GB per bucket, should 20 buckets. Assert.assertEquals(20, partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum()); + + // mock partition size eq 0, use back-to-back logic + table.readLock(); + try { + // when fe restart, when stat thread not get replica size from be/ms, replica size eq 0 + for (int i = 0; i < 54; i++) { + Partition partition = partitions.get(i); + partition.updateVisibleVersion(2L); + for (MaterializedIndex idx : partition.getMaterializedIndices( + MaterializedIndex.IndexExtState.VISIBLE)) { + if (i < 52) { + Assert.assertEquals(10, idx.getTablets().size()); + } else if (i == 52) { + Assert.assertEquals(1, idx.getTablets().size()); + } else if (i == 53) { + Assert.assertEquals(20, idx.getTablets().size()); + } + for (Tablet tablet : idx.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + replica.updateVersion(3L); + // mock replica size eq 0 + replica.setDataSize(0L); + replica.setRowCount(0L); + } + } + } + Assert.assertEquals(0, partition.getAllDataSize(true)); + } + } finally { + table.readUnlock(); + } + + String alterStmt3 = "alter table test.test_autobucket_dynamic_partition set ('dynamic_partition.end' = '4')"; + ExceptionChecker.expectThrowsNoException(() -> alterTable(alterStmt3)); + // 54th previous partition size set 53, check back to back logic work + partitions.get(53).getDistributionInfo().setBucketNum(53); + Env.getCurrentEnv().getDynamicPartitionScheduler().executeDynamicPartition(tempDynamicPartitionTableInfo, false); + + partitions = Lists.newArrayList(table.getAllPartitions()); + partitions.sort(Comparator.comparing(Partition::getId)); + Assert.assertEquals(55, partitions.size()); + // due to partition size eq 0, use previous partition's(54th) bucket num + Assert.assertEquals(53, partitions.get(partitions.size() - 1).getDistributionInfo().getBucketNum()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java index bd3e7f9e2a5b32..f4f364b91ced1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java @@ -138,6 +138,7 @@ public void setUp() throws Exception { Config.tablet_checker_interval_ms = 1000; Config.tablet_repair_delay_factor_second = 1; connectContext = UtFrameUtils.createDefaultCtx(); + Config.autobucket_partition_size_per_bucket_gb = 1; } @After From a24faf04dcad1def62a29e47901d8117767ac01a Mon Sep 17 00:00:00 2001 From: Luwei Date: Wed, 16 Jul 2025 23:48:49 +0800 Subject: [PATCH 236/572] [fix](cloud) fix some multi-az failover bug and add regression cases #52450 (#52685) (#53245) pick #52450 --- be/src/agent/heartbeat_server.cpp | 7 + be/src/cloud/cloud_cluster_info.h | 32 ++ be/src/cloud/cloud_tablet_mgr.cpp | 8 + be/src/cloud/config.cpp | 4 + be/src/cloud/config.h | 4 + be/src/runtime/exec_env_init.cpp | 8 +- be/test/cloud/cloud_compaction_test.cpp | 5 + .../cloud/system/CloudSystemInfoService.java | 43 +- .../org/apache/doris/qe/ConnectContext.java | 4 + .../java/org/apache/doris/system/Backend.java | 5 + .../org/apache/doris/system/HeartbeatMgr.java | 4 + .../system/CloudSystemInfoServiceTest.java | 33 ++ gensrc/thrift/HeartbeatService.thrift | 5 + .../default_vcg_auto_failover.out | 37 ++ .../failover_standby_disable_compaction.out | 13 + .../standby_disable_compaction.out | 13 + .../use_default_vcg_read_write.out | 19 + .../use_vcg_read_write_unhealthy_node_50.out | 37 ++ .../vcg_auto_failover.out | 37 ++ .../vcg_auto_failover_manual_failback.out | 49 ++ .../regression/action/StreamLoadAction.groovy | 4 + .../doris/regression/suite/Suite.groovy | 94 ++-- .../virtual_compute_group/ddl/nation.sql | 12 + .../ddl/nation_delete.sql | 1 + .../virtual_compute_group/ddl/nation_load.sql | 6 + .../default_vcg_auto_failover.groovy | 105 ++-- ...failover_standby_disable_compaction.groovy | 289 ++++++++++ .../load_trigger_failover.groovy | 164 ++++++ .../standby_disable_compaction.groovy | 276 ++++++++++ .../use_default_vcg_read_write.groovy | 152 +++-- .../use_vcg_read_write.groovy | 136 ++--- .../use_vcg_read_write_routine_load.groovy | 399 ++++++++++++++ .../use_vcg_read_write_s3_load.groovy | 363 ++++++++++++ ...se_vcg_read_write_unhealthy_node_50.groovy | 422 ++++++++++++++ .../vcg_auto_failover.groovy | 88 +-- .../vcg_auto_failover_manual_failback.groovy | 519 ++++++++++++++++++ 36 files changed, 3121 insertions(+), 276 deletions(-) create mode 100644 be/src/cloud/cloud_cluster_info.h create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.out create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.out create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation.sql create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_delete.sql create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_load.sql create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/load_trigger_failover.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_s3_load.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy create mode 100644 regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.groovy diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp index 498315f2baded2..ec024844d8581a 100644 --- a/be/src/agent/heartbeat_server.cpp +++ b/be/src/agent/heartbeat_server.cpp @@ -27,6 +27,7 @@ #include #include +#include "cloud/cloud_cluster_info.h" #include "cloud/cloud_tablet_mgr.h" #include "cloud/config.h" #include "common/config.h" @@ -299,6 +300,12 @@ Status HeartbeatServer::_heartbeat(const TMasterInfo& master_info) { LOG(INFO) << "set config cloud_unique_id " << master_info.cloud_unique_id << " " << st; } + if (master_info.__isset.cloud_cluster_info && + master_info.cloud_cluster_info.__isset.isStandby) { + auto* cloud_cluster_info = static_cast(_cluster_info); + cloud_cluster_info->set_is_in_standby(master_info.cloud_cluster_info.isStandby); + } + if (master_info.__isset.tablet_report_inactive_duration_ms) { doris::g_tablet_report_inactive_duration_ms = master_info.tablet_report_inactive_duration_ms; diff --git a/be/src/cloud/cloud_cluster_info.h b/be/src/cloud/cloud_cluster_info.h new file mode 100644 index 00000000000000..e62db51934a8fc --- /dev/null +++ b/be/src/cloud/cloud_cluster_info.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "runtime/cluster_info.h" + +namespace doris { + +class CloudClusterInfo : public ClusterInfo { +public: + bool is_in_standby() const { return _is_in_standby; } + + void set_is_in_standby(bool flag) { _is_in_standby = flag; } + +private: + bool _is_in_standby = false; +}; + +} // namespace doris \ No newline at end of file diff --git a/be/src/cloud/cloud_tablet_mgr.cpp b/be/src/cloud/cloud_tablet_mgr.cpp index 1f155635c65485..a9cfd353526eb8 100644 --- a/be/src/cloud/cloud_tablet_mgr.cpp +++ b/be/src/cloud/cloud_tablet_mgr.cpp @@ -21,6 +21,7 @@ #include +#include "cloud/cloud_cluster_info.h" #include "cloud/cloud_meta_mgr.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" @@ -363,6 +364,13 @@ Status CloudTabletMgr::get_topn_tablets_to_compact( using namespace std::chrono; auto now = duration_cast(system_clock::now().time_since_epoch()).count(); auto skip = [now, compaction_type](CloudTablet* t) { + auto* cloud_cluster_info = static_cast(ExecEnv::GetInstance()->cluster_info()); + if (config::enable_standby_passive_compaction && cloud_cluster_info->is_in_standby()) { + if (t->fetch_add_approximate_num_rowsets(0) < config::max_tablet_version_num * config::standby_compaction_version_ratio) { + return true; + } + } + int32_t max_version_config = t->max_version_config(); if (compaction_type == CompactionType::BASE_COMPACTION) { bool is_recent_failure = now - t->last_base_compaction_failure_time() < config::min_compaction_failure_interval_ms; diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp index 9c9eaa4d0dd2ab..36f9c2316fa2da 100644 --- a/be/src/cloud/config.cpp +++ b/be/src/cloud/config.cpp @@ -92,5 +92,9 @@ DEFINE_mInt64(warmup_tablet_replica_info_cache_ttl_sec, "600"); DEFINE_mInt64(warm_up_rowset_slow_log_ms, "1000"); +DEFINE_mBool(enable_standby_passive_compaction, "true"); + +DEFINE_mDouble(standby_compaction_version_ratio, "0.8"); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h index cc477aa375fed2..8120a1a953d583 100644 --- a/be/src/cloud/config.h +++ b/be/src/cloud/config.h @@ -128,5 +128,9 @@ DECLARE_mInt64(warmup_tablet_replica_info_cache_ttl_sec); DECLARE_mInt64(warm_up_rowset_slow_log_ms); +DECLARE_mBool(enable_standby_passive_compaction); + +DECLARE_mDouble(standby_compaction_version_ratio); + #include "common/compile_check_end.h" } // namespace doris::config diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 1d1ad2ee184b2c..8faaffd2831430 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -31,6 +31,7 @@ #include #include +#include "cloud/cloud_cluster_info.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_stream_load_executor.h" #include "cloud/cloud_tablet_hotspot.h" @@ -258,7 +259,12 @@ Status ExecEnv::_init(const std::vector& store_paths, _fragment_mgr = new FragmentMgr(this); _result_cache = new ResultCache(config::query_cache_max_size_mb, config::query_cache_elasticity_size_mb); - _cluster_info = new ClusterInfo(); + if (config::is_cloud_mode()) { + _cluster_info = new CloudClusterInfo(); + } else { + _cluster_info = new ClusterInfo(); + } + _load_path_mgr = new LoadPathMgr(this); _bfd_parser = BfdParser::create(); _broker_mgr = new BrokerMgr(this); diff --git a/be/test/cloud/cloud_compaction_test.cpp b/be/test/cloud/cloud_compaction_test.cpp index c8db6739084546..37847025c9e3b1 100644 --- a/be/test/cloud/cloud_compaction_test.cpp +++ b/be/test/cloud/cloud_compaction_test.cpp @@ -23,6 +23,7 @@ #include +#include "cloud/cloud_cluster_info.h" #include "cloud/cloud_storage_engine.h" #include "cloud/cloud_tablet.h" #include "cloud/cloud_tablet_mgr.h" @@ -70,6 +71,9 @@ class CloudCompactionTest : public testing::Test { "creation_time": 1553765670, "num_segments": 3 })"; + _cluster_info = std::make_shared(); + _cluster_info->_is_in_standby = false; + ExecEnv::GetInstance()->_cluster_info = _cluster_info.get(); } void TearDown() override {} @@ -113,6 +117,7 @@ class CloudCompactionTest : public testing::Test { public: CloudStorageEngine _engine; + std::shared_ptr _cluster_info; }; TEST_F(CloudCompactionTest, failure_base_compaction_tablet_sleep_test) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index a20972760772a3..61dbcb4d053399 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -146,6 +146,16 @@ public void addComputeGroup(String computeGroupId, ComputeGroup computeGroup) { } } + public boolean isStandByComputeGroup(String clusterName) { + List virtualGroups = getComputeGroups(true); + for (ComputeGroup vcg : virtualGroups) { + if (vcg.getPolicy().getStandbyComputeGroup().equals(clusterName)) { + return true; + } + } + return false; + } + public List getComputeGroups(boolean virtual) { LOG.debug("get virtual {} computeGroupIdToComputeGroup : {} ", virtual, computeGroupIdToComputeGroup); try { @@ -782,7 +792,8 @@ public String getPhysicalCluster(String clusterName) { return acgName; } else { if (acg.getUnavailableSince() <= 0) { - acg.setUnavailableSince(System.currentTimeMillis()); + acg.setUnavailableSince(System.currentTimeMillis() + - computeGroupFailureCount(acgName) * Config.heartbeat_interval_second * 1000); } } } @@ -804,7 +815,10 @@ public String getPhysicalCluster(String clusterName) { } return scgName; } else { - scg.setUnavailableSince(System.currentTimeMillis()); + if (scg.getUnavailableSince() <= 0) { + scg.setUnavailableSince(System.currentTimeMillis() + - computeGroupFailureCount(scgName) * Config.heartbeat_interval_second * 1000); + } } } } @@ -835,6 +849,26 @@ public boolean isComputeGroupAvailable(String cg, long unhealthyNodeThresholdPer return true; } + public int computeGroupFailureCount(String cg) { + List bes = getBackendsByClusterName(cg); + if (bes == null || bes.isEmpty()) { + return 0; + } + + int failureCount = 0; + for (Backend be : bes) { + if (!be.isAlive()) { + if (failureCount == 0) { + failureCount = be.getHeartbeatFailureCounter(); + } else { + failureCount = Math.min(be.getHeartbeatFailureCounter(), failureCount); + } + } + } + + return failureCount; + } + public String getClusterNameByBeAddr(String beEndpoint) { rlock.lock(); try { @@ -1064,9 +1098,10 @@ public ImmutableMap getCloudIdToBackend(String clusterName) { } public ImmutableMap getCloudIdToBackendNoLock(String clusterName) { - String clusterId = clusterNameToId.get(clusterName); + String physicalClusterName = getPhysicalCluster(clusterName); + String clusterId = clusterNameToId.get(physicalClusterName); if (Strings.isNullOrEmpty(clusterId)) { - LOG.warn("cant find clusterId, this cluster may be has been dropped, clusterName={}", clusterName); + LOG.warn("cant find clusterId, this cluster may be has been dropped, clusterName={}", physicalClusterName); return ImmutableMap.of(); } List backends = clusterIdToBackend.get(clusterId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 6d86cc7f037f40..8b0bdf385718b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -1250,6 +1250,10 @@ public CloudClusterResult getCloudClusterByPolicy() { for (String cloudClusterName : cloudClusterNames) { if (Env.getCurrentEnv().getAccessManager().checkCloudPriv(getCurrentUserIdentity(), cloudClusterName, PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)) { + if (((CloudSystemInfoService) Env.getCurrentSystemInfo()).isStandByComputeGroup(cloudClusterName)) { + continue; + } + hasAuthCluster.add(cloudClusterName); // find a cluster has more than one alive be List bes = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java index 0bd8377bddbea9..4ef7fa58f6cf0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.DiskInfo.DiskState; import org.apache.doris.catalog.Env; import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.system.CloudSystemInfoService; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.io.Text; @@ -205,6 +206,10 @@ public String getCloudClusterName() { return tagMap.getOrDefault(Tag.CLOUD_CLUSTER_NAME, ""); } + public boolean isInStandbyCluster() { + return (((CloudSystemInfoService) Env.getCurrentSystemInfo()).isStandByComputeGroup(getCloudClusterName())); + } + public void setCloudClusterName(final String clusterName) { tagMap.put(Tag.CLOUD_CLUSTER_NAME, clusterName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java index c839222af5e508..3cf51e31613054 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/HeartbeatMgr.java @@ -41,6 +41,7 @@ import org.apache.doris.thrift.TBrokerOperationStatusCode; import org.apache.doris.thrift.TBrokerPingBrokerRequest; import org.apache.doris.thrift.TBrokerVersion; +import org.apache.doris.thrift.TCloudClusterInfo; import org.apache.doris.thrift.TFrontendInfo; import org.apache.doris.thrift.TFrontendPingFrontendRequest; import org.apache.doris.thrift.TFrontendPingFrontendResult; @@ -267,6 +268,9 @@ private HeartbeatResponse pingOnce() { String cloudUniqueId = backend.getTagMap().get(Tag.CLOUD_UNIQUE_ID); copiedMasterInfo.setCloudUniqueId(cloudUniqueId); copiedMasterInfo.setTabletReportInactiveDurationMs(Config.rehash_tablet_after_be_dead_seconds); + TCloudClusterInfo clusterInfo = new TCloudClusterInfo(); + clusterInfo.setIsStandby(backend.isInStandbyCluster()); + copiedMasterInfo.setCloudClusterInfo(clusterInfo); } THeartbeatResult result; if (!FeConstants.runningUnitTest) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java index e638bca1b32225..445ec828ba21d8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/system/CloudSystemInfoServiceTest.java @@ -315,4 +315,37 @@ public void testGetPhysicalClusterActive1AliveBe2DeadBe() { String res = infoService.getPhysicalCluster(vcgName); Assert.assertEquals(pcgName1, res); } + + @Test + public void testIsStandByComputeGroup() { + infoService = new CloudSystemInfoService(); + + String vcgName = "v_cluster_1"; + String pcgName1 = "p_cluster_1"; + String pcgName2 = "p_cluster_2"; + String pcgName3 = "p_cluster_3"; + + ComputeGroup vcg = new ComputeGroup("id1", vcgName, ComputeGroup.ComputeTypeEnum.VIRTUAL); + ComputeGroup.Policy policy = new ComputeGroup.Policy(); + policy.setActiveComputeGroup(pcgName1); + policy.setStandbyComputeGroup(pcgName2); + vcg.setPolicy(policy); + + ComputeGroup pcg1 = new ComputeGroup("id2", pcgName1, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg2 = new ComputeGroup("id3", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + ComputeGroup pcg3 = new ComputeGroup("id4", pcgName2, ComputeGroup.ComputeTypeEnum.COMPUTE); + infoService.addComputeGroup(vcgName, vcg); + infoService.addComputeGroup(pcgName1, pcg1); + infoService.addComputeGroup(pcgName2, pcg2); + infoService.addComputeGroup(pcgName3, pcg3); + + boolean res = infoService.isStandByComputeGroup(vcgName); + Assert.assertFalse(res); + res = infoService.isStandByComputeGroup(pcgName1); + Assert.assertFalse(res); + res = infoService.isStandByComputeGroup(pcgName2); + Assert.assertTrue(res); + res = infoService.isStandByComputeGroup(pcgName3); + Assert.assertFalse(res); + } } diff --git a/gensrc/thrift/HeartbeatService.thrift b/gensrc/thrift/HeartbeatService.thrift index 47c41650b78a07..ecf9727cf3e0db 100644 --- a/gensrc/thrift/HeartbeatService.thrift +++ b/gensrc/thrift/HeartbeatService.thrift @@ -29,6 +29,10 @@ struct TFrontendInfo { 2: optional i64 process_uuid } +struct TCloudClusterInfo { + 1: optional bool isStandby +} + struct TMasterInfo { 1: required Types.TNetworkAddress network_address 2: required Types.TClusterId cluster_id @@ -44,6 +48,7 @@ struct TMasterInfo { // See configuration item Config.java rehash_tablet_after_be_dead_seconds for meaning 12: optional i64 tablet_report_inactive_duration_ms; 13: optional string auth_token; + 14: optional TCloudClusterInfo cloud_cluster_info; } struct TBackendInfo { diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.out new file mode 100644 index 00000000000000..0abde7ca45bf71 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.out new file mode 100644 index 00000000000000..3cc53f372570d3 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +2000 + +-- !all12 -- +1100 + +-- !all11 -- +2000 + +-- !all12 -- +1100 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.out new file mode 100644 index 00000000000000..3cc53f372570d3 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +2000 + +-- !all12 -- +1100 + +-- !all11 -- +2000 + +-- !all12 -- +1100 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out new file mode 100644 index 00000000000000..3900f080bd0b57 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.out new file mode 100644 index 00000000000000..0abde7ca45bf71 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.out new file mode 100644 index 00000000000000..0abde7ca45bf71 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.out @@ -0,0 +1,37 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.out new file mode 100644 index 00000000000000..6f8bf1bc39daa5 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.out @@ -0,0 +1,49 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + +-- !all11 -- +80 + +-- !all12 -- +44 + +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + +-- !all11 -- +80 + +-- !all12 -- +44 + diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy index b293afa2a6d20a..aa19094854eddd 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy @@ -154,6 +154,10 @@ class StreamLoadAction implements SuiteAction { void isCloud(Closure isCloud) { } + void setFeAddr(String beHost, int beHttpPort) { + this.address = new InetSocketAddress(beHost, beHttpPort) + } + void check(@ClosureParams(value = FromString, options = ["String,Throwable,Long,Long"]) Closure check) { this.check = check } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 465dc139166f31..85ec795cd0df24 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2507,7 +2507,7 @@ class Suite implements GroovyInterceptable { def token = context.config.metaServiceToken def instance_id = context.config.multiClusterInstance - def get_be_metric = { ip, port, field -> + def get_be_metric = { ip, port, field, type="" -> def metric_api = { request_body, check_func -> httpTest { endpoint ip + ":" + port @@ -2529,7 +2529,8 @@ class Suite implements GroovyInterceptable { log.info("get be metric resp: ${respCode}".toString()) def json = parseJson(body) for (item : json) { - if (item.tags.metric == field) { + if (item.tags.metric == field && (type.isEmpty() || type == item.tags.type)) { + log.info("get be metric resp: ${item}".toString()) ret = item.value } } @@ -2853,11 +2854,13 @@ class Suite implements GroovyInterceptable { } } - def checkProfileNew = { addrSet -> + def checkProfileNew = { fe, addrSet, shouldContain = true -> + //def fe = cluster.getAllFrontends().get(0) + def feEndPoint = fe.host + ":" + fe.httpPort def query_profile_api = { check_func -> httpTest { op "get" - endpoint context.config.feHttpAddress + endpoint feEndPoint uri "/rest/v1/query_profile" check check_func basicAuthorization "${context.config.feCloudHttpUser}","${context.config.feCloudHttpPassword}" @@ -2866,65 +2869,66 @@ class Suite implements GroovyInterceptable { query_profile_api.call() { respCode, body -> - log.info("query profile resp: ${body} ${respCode}".toString()) def json = parseJson(body) assertTrue(json.msg.equalsIgnoreCase("success")) - log.info("lw query profile resp: ${json.data.rows[0]}".toString()) + log.info("lw query profile resp json : ${json}".toString()) log.info("lw query profile resp: ${json.data.rows[0]['Profile ID']}".toString()) - checkProfileNew1.call(addrSet, json.data.rows[0]['Profile ID']) + checkProfileByQueryId.call(fe, addrSet, json.data.rows[0]['Profile ID'], shouldContain) } } - def checkProfileNew1 = {addrSet, query_id -> + def checkProfileByQueryId = { fe, addrSet, query_id, shouldContain = true -> + //def fe = cluster.getAllFrontends().get(0) + def feEndPoint = fe.host + ":" + fe.httpPort def query_profile_api = { check_func -> httpTest { op "get" - endpoint context.config.feHttpAddress + endpoint feEndPoint uri "/api/profile?query_id=${query_id}" check check_func basicAuthorization "${context.config.feCloudHttpUser}","${context.config.feCloudHttpPassword}" } } - query_profile_api.call() { - respCode, body -> - //log.info("query profile resp: ${body} ${respCode}".toString()) - def json = parseJson(body) - assertTrue(json.msg.equalsIgnoreCase("success")) - //log.info("lw query profile resp: ${json.data.rows[0]}".toString()) - - def instanceLineMatcher = json =~ /Instances\s+Num\s+Per\s+BE:\s*(.*)/ - if (instanceLineMatcher.find()) { - // 提取出IP等信息的部分 - def instancesStr = instanceLineMatcher.group(1).trim() - - // 拆分各个实例，实例格式类似 "10.16.10.11:9713:4" - def instanceEntries = instancesStr.split(/\s*,\s*/) - - // 定义存储解析结果的列表 - def result = [] - - // 每个实例使用正则表达式解析IP和端口（忽略最后一个数字） - instanceEntries.each { entry -> - def matcher = entry =~ /(\d{1,3}(?:\.\d{1,3}){3}):(\d+):\d+/ - if(matcher.matches()){ - def ip = matcher.group(1) - def port = matcher.group(2) - //result << [ip: ip, port: port] - //result << [ip:port] - result.add(ip+":"+port) - } + query_profile_api.call() { respCode, body -> + def json = parseJson(body) + assertTrue(json.msg.equalsIgnoreCase("success")) + + def instanceLineMatcher = json =~ /Instances\s+Num\s+Per\s+BE:\s*(.*)/ + if (instanceLineMatcher.find()) { + // Extract the instance string section + def instancesStr = instanceLineMatcher.group(1).trim() + def instanceEntries = instancesStr.split(/\s*,\s*/) + def result = [] + + // Parse each instance entry (format like "10.1.1.1:9000:4") and extract IP:port + instanceEntries.each { entry -> + def matcher = entry =~ /(\d{1,3}(?:\.\d{1,3}){3}):(\d+):\d+/ + if (matcher.matches()) { + def ip = matcher.group(1) + def port = matcher.group(2) + result.add(ip + ":" + port) } - - // 输出解析结果 - println "提取的IP和端口：" - result.each { println it } - addrSet.each { println it } - //result.each { assertTrue(addrSet.contains(it)) } - assertTrue(addrSet.containsAll(result)) + } + + if (shouldContain) { + // All items in result should exist in addrSet + assertTrue(addrSet.containsAll(result), + "Check failed: Some result addresses are missing in addrSet.\n" + + "addrSet: ${addrSet}\n" + + "result: ${result}\n" + + "Missing: ${result.findAll { !addrSet.contains(it) }}") } else { - println "未找到实例信息。" + // No item in result should exist in addrSet + assertTrue(addrSet.intersect(result).isEmpty(), + "Check failed: Some result addresses unexpectedly exist in addrSet.\n" + + "addrSet: ${addrSet}\n" + + "result: ${result}\n" + + "Overlap: ${addrSet.intersect(result)}") } + } else { + log.info("Instance info not found in profile") + } } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation.sql b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation.sql new file mode 100644 index 00000000000000..56c133ce1eaf46 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS nation ( + N_NATIONKEY INTEGER NOT NULL, + N_NAME CHAR(25) NOT NULL, + N_REGIONKEY INTEGER NOT NULL, + N_COMMENT VARCHAR(152) +) +DUPLICATE KEY(N_NATIONKEY, N_NAME) +DISTRIBUTED BY HASH(N_NATIONKEY) BUCKETS 1 +PROPERTIES ( + "replication_num" = "1" +) + diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_delete.sql b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_delete.sql new file mode 100644 index 00000000000000..9fb7c085a6fea3 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_delete.sql @@ -0,0 +1 @@ +DELETE from nation where N_NATIONKEY >= 0; diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_load.sql b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_load.sql new file mode 100644 index 00000000000000..6e6f13633412c8 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/ddl/nation_load.sql @@ -0,0 +1,6 @@ +LOAD LABEL ${loadLabel} ( + DATA INFILE("s3://${s3BucketName}/regression/tpch/sf100/nation.tbl") + INTO TABLE nation + COLUMNS TERMINATED BY "|" + (n_nationkey, n_name, n_regionkey, n_comment, temp) +) diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy index 22bdf8ce4fa505..cf9934469c6243 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy @@ -20,10 +20,11 @@ import groovy.json.JsonSlurper import groovy.json.JsonOutput // 1 create two physical cluster c1, c2, every cluster contains 2 be -// 2 stop a backend of c1 -// 3 stop a backend of c1 -// 4 start 2 backends of c1 -// 5 long-term stop 2 backends of c1 +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 set vcg as default cluster +// 4 stop 2 bes of c1 +// 5 start 2 bes of c1 +// 6 long-term stop 2 bes of c1 suite('default_vcg_auto_failover', 'multi_cluster,docker') { def options = new ClusterOptions() @@ -76,9 +77,9 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { def normalVclusterId = "normalVirtualClusterId" def vcgClusterNames = [clusterName1, clusterName2] def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 10] - clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] - jsonOutput = new JsonOutput() + def jsonOutput = new JsonOutput() def normalVcgBody = jsonOutput.toJson(normalInstance) add_cluster_api.call(msHttpPort, normalVcgBody) { respCode, body -> @@ -89,13 +90,13 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { // show cluster sleep(5000) - showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group {}", showComputeGroup) def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) - showResult = sql "show clusters" + def showResult = sql "show clusters" for (row : showResult) { println row } @@ -126,6 +127,25 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { sql """ SET PROPERTY 'default_cloud_cluster' = 'normalVirtualClusterName' """ + def reconnectFe = { + sleep(10000) + logger.info("Reconnecting to a new frontend...") + def newFe = cluster.getMasterFe() + if (newFe) { + logger.info("New frontend found: ${newFe.host}:${newFe.httpPort}") + def url = String.format( + "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", + newFe.host, newFe.queryPort) + url = context.config.buildUrlWithDb(url, context.dbName) + context.connectTo(url, context.config.jdbcUser, context.config.jdbcPassword) + logger.info("Successfully reconnected to the new frontend") + } else { + logger.error("No new frontend found to reconnect") + } + } + + reconnectFe() + sql """ drop table if exists ${tableName} """ sql """ @@ -164,17 +184,17 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { cluster.stopBackends(4, 5) - before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) - before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) - before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) - before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) - txnId = -1; + def txnId = -1; streamLoad { table "${tableName}" @@ -182,6 +202,7 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -204,22 +225,26 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) - after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) - after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) - after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) - set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + def addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(16000) // show cluster @@ -228,18 +253,18 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) log.info("policy {}", vcgInShow.Policy) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) cluster.startBackends(4, 5) - before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) - before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) - before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) - before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); @@ -260,6 +285,7 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -282,14 +308,14 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) - after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) - after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) - after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); @@ -310,9 +336,13 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) - set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(4, 5) @@ -334,6 +364,7 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -369,9 +400,13 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) - set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(60000) sql """ @@ -383,9 +418,9 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { log.info("show compute group {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) } // connect to follower, run again - //options.connectToFollower = true + options.connectToFollower = true } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy new file mode 100644 index 00000000000000..a92d5b7a92030d --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +suite('failover_standby_disable_compaction', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "a_newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "a_newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "a_normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 3] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(10000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"a_newcluster1","standbyComputeGroup":"a_newcluster2"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + cluster.stopBackends(4, 5) + sleep(30000) + sql """ insert into ${tbl} (k1, k2) values (1, "10") """ + sql """ SELECT count(*) FROM ${tableName} """ + + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"a_newcluster2","standbyComputeGroup":"a_newcluster1"')) + + cluster.startBackends(4, 5) + + sql """ set enable_profile = true """ + + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + def before_cluster1_be0_compaction = get_be_metric(cluster1Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster1_be0_compaction : ${before_cluster1_be0_compaction}".toString()) + def before_cluster1_be1_compaction = get_be_metric(cluster1Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster1_be1_compaction : ${before_cluster1_be1_compaction}".toString()) + + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + def before_cluster2_be0_compaction = get_be_metric(cluster2Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster2_be0_compaction : ${before_cluster2_be0_compaction}".toString()) + def before_cluster2_be1_compaction = get_be_metric(cluster2Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster2_be1_compaction : ${before_cluster2_be1_compaction}".toString()) + + def txnId = -1; + for (int i = 0; i < 100; ++i) { + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + } + + for (int i = 0; i < 100; ++i) { + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + } + + sleep(5000) + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + def after_cluster1_be0_compaction = get_be_metric(cluster1Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster1_be0_compaction : ${after_cluster1_be0_compaction}".toString()) + def after_cluster1_be1_compaction = get_be_metric(cluster1Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster1_be1_compaction : ${after_cluster1_be1_compaction}".toString()) + + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + def after_cluster2_be0_compaction = get_be_metric(cluster2Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster2_be0_compaction : ${after_cluster2_be0_compaction}".toString()) + def after_cluster2_be1_compaction = get_be_metric(cluster2Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster2_be1_compaction : ${after_cluster2_be1_compaction}".toString()) + + assertTrue(before_cluster1_be0_load_rows == after_cluster1_be0_load_rows && before_cluster1_be1_load_rows == after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush == after_cluster1_be0_flush && before_cluster1_be1_flush == after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + assertTrue(before_cluster1_be0_compaction == after_cluster1_be0_compaction && before_cluster1_be1_compaction == after_cluster1_be1_compaction ) + + assertTrue(before_cluster2_be0_compaction < after_cluster2_be0_compaction || before_cluster2_be1_compaction < after_cluster2_be1_compaction ) + + def set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set, false) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + } + // connect to follower, run again + options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/load_trigger_failover.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/load_trigger_failover.groovy new file mode 100644 index 00000000000000..5e05ce3f0b1cfd --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/load_trigger_failover.groovy @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 1 be +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop 1 bes of c1 + +suite('load_trigger_failover', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 1; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(1, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(1, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 3] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + cluster.stopBackends(4) + sleep(60000) + sql """ + insert into ${tbl} values (1, 'a'); + """ + + sql """ + select * from ${tbl} + """ + + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + log.info("policy {}", vcgInShow.Policy) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + } + // connect to follower, run again + options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy new file mode 100644 index 00000000000000..dc39526bfe0c04 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +suite('standby_disable_compaction', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "a_newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "a_newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "a_normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName2}", standby_cluster_names: ["${clusterName1}"], failover_failure_threshold: 10] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"a_newcluster2","standbyComputeGroup":"a_newcluster1"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + def before_cluster1_be0_compaction = get_be_metric(cluster1Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster1_be0_compaction : ${before_cluster1_be0_compaction}".toString()) + def before_cluster1_be1_compaction = get_be_metric(cluster1Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster1_be1_compaction : ${before_cluster1_be1_compaction}".toString()) + + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + def before_cluster2_be0_compaction = get_be_metric(cluster2Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster2_be0_compaction : ${before_cluster2_be0_compaction}".toString()) + def before_cluster2_be1_compaction = get_be_metric(cluster2Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("before_cluster2_be1_compaction : ${before_cluster2_be1_compaction}".toString()) + + def txnId = -1; + for (int i = 0; i < 100; ++i) { + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + } + + for (int i = 0; i < 100; ++i) { + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + } + + sleep(5000) + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + def after_cluster1_be0_compaction = get_be_metric(cluster1Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster1_be0_compaction : ${after_cluster1_be0_compaction}".toString()) + def after_cluster1_be1_compaction = get_be_metric(cluster1Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster1_be1_compaction : ${after_cluster1_be1_compaction}".toString()) + + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + def after_cluster2_be0_compaction = get_be_metric(cluster2Ips[0], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster2_be0_compaction : ${after_cluster2_be0_compaction}".toString()) + def after_cluster2_be1_compaction = get_be_metric(cluster2Ips[1], "8040", 'compaction_deltas_total', "cumulative"); + log.info("after_cluster2_be1_compaction : ${after_cluster2_be1_compaction}".toString()) + + assertTrue(before_cluster1_be0_load_rows == after_cluster1_be0_load_rows && before_cluster1_be1_load_rows == after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush == after_cluster1_be0_flush && before_cluster1_be1_flush == after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + assertTrue(before_cluster1_be0_compaction == after_cluster1_be0_compaction && before_cluster1_be1_compaction == after_cluster1_be1_compaction ) + + assertTrue(before_cluster2_be0_compaction < after_cluster2_be0_compaction || before_cluster2_be1_compaction < after_cluster2_be1_compaction ) + + def set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(set, false) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + } + // connect to follower, run again + options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy index 4c8c03e5f61e46..707ea42ce89113 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy @@ -20,12 +20,12 @@ import groovy.json.JsonSlurper import groovy.json.JsonOutput // 1 create two physical cluster c1, c2, every cluster contains 2 be -// 2 stop a backend of c1 -// 3 stop a backend of c1 -// 4 start 2 backends of c1 -// 5 long-term stop 2 backends of c1 +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop a backend of c1 +// 5 stop another backend of c1 -suite('use_vcg_read_write', 'multi_cluster,docker') { +suite('use_default_vcg_read_write', 'multi_cluster,docker') { def options = new ClusterOptions() String tableName = "test_all_vcluster" String tbl = "test_virtual_compute_group_tbl" @@ -76,9 +76,9 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { def normalVclusterId = "normalVirtualClusterId" def vcgClusterNames = [clusterName1, clusterName2] def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] - clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] - jsonOutput = new JsonOutput() + def jsonOutput = new JsonOutput() def normalVcgBody = jsonOutput.toJson(normalInstance) add_cluster_api.call(msHttpPort, normalVcgBody) { respCode, body -> @@ -89,14 +89,13 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { // show cluster sleep(5000) - showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group {}", showComputeGroup) def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) - + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) - showResult = sql "show clusters" + def showResult = sql "show clusters" for (row : showResult) { println row } @@ -127,6 +126,25 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { sql """ SET PROPERTY 'default_cloud_cluster' = 'normalVirtualClusterName' """ + def reconnectFe = { + sleep(10000) + logger.info("Reconnecting to a new frontend...") + def newFe = cluster.getMasterFe() + if (newFe) { + logger.info("New frontend found: ${newFe.host}:${newFe.httpPort}") + def url = String.format( + "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", + newFe.host, newFe.queryPort) + url = context.config.buildUrlWithDb(url, context.dbName) + context.connectTo(url, context.config.jdbcUser, context.config.jdbcPassword) + logger.info("Successfully reconnected to the new frontend") + } else { + logger.error("No new frontend found to reconnect") + } + } + + reconnectFe() + sql """ drop table if exists ${tableName} """ sql """ @@ -163,27 +181,27 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { sql """ set enable_profile = true """ - before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) - before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) - before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) - before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) - before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) - before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) - before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) - before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) - txnId = -1; + def txnId = -1; streamLoad { table "${tableName}" @@ -191,6 +209,7 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -213,24 +232,24 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) - after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) - after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) - after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) - after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) - after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) - after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) - after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) @@ -241,9 +260,13 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) - set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + def addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(4) sleep(6000) @@ -287,6 +310,7 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -332,9 +356,13 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) - set = [cluster1Ips[0] + ":" + "8060"] as Set + addrSet = [cluster1Ips[0] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(5) @@ -356,6 +384,7 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -391,61 +420,28 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) - set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(16000) + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + // show cluster showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) - - //// use vcg - //sql """use @${normalVclusterName}""" - //sql """ - //CREATE TABLE ${tbl} ( - //`k1` int(11) NULL, - //`k2` char(5) NULL - //) - //DUPLICATE KEY(`k1`, `k2`) - //COMMENT 'OLAP' - //DISTRIBUTED BY HASH(`k1`) BUCKETS 1 - //PROPERTIES ( - //"replication_num"="1" - //); - //""" - - //sql """ - // insert into ${tbl} (k1, k2) values (1, "10"); - //""" - - //result = sql """select count(*) from ${tbl}""" - //log.info("result = {}", result) - //assertEquals(result.size(), 1) - - // alter cluster info, change standbyComputeGroup to newcluster1, activeComputeGroup to newcluster2 - //clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName2}", standby_cluster_names: ["${clusterName1}"]] - //clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_policy:clusterPolicy] - //normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] - //jsonOutput = new JsonOutput() - //normalVcgBody = jsonOutput.toJson(normalInstance) - //alter_cluster_info_api(msHttpPort, normalVcgBody) { - // respCode, body -> - // log.info("alter virtual cluster result: ${body} ${respCode}".toString()) - // def json = parseJson(body) - // assertTrue(json.code.equalsIgnoreCase("OK")) - //} - //sleep(5000) - //showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ - //log.info("show compute group after alter {}", showComputeGroup) - //vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } - //assertNotNull(vcgInShow) - //assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) } // connect to follower, run again - //options.connectToFollower = true + options.connectToFollower = true } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy index 8a87bf71c2c83f..66406d2daa2868 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write.groovy @@ -20,10 +20,10 @@ import groovy.json.JsonSlurper import groovy.json.JsonOutput // 1 create two physical cluster c1, c2, every cluster contains 2 be -// 2 stop a backend of c1 -// 3 stop a backend of c1 -// 4 start 2 backends of c1 -// 5 long-term stop 2 backends of c1 +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop a backend of c1 +// 5 stop another backend of c1 suite('use_vcg_read_write', 'multi_cluster,docker') { def options = new ClusterOptions() @@ -76,9 +76,9 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { def normalVclusterId = "normalVirtualClusterId" def vcgClusterNames = [clusterName1, clusterName2] def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] - clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] - jsonOutput = new JsonOutput() + def jsonOutput = new JsonOutput() def normalVcgBody = jsonOutput.toJson(normalInstance) add_cluster_api.call(msHttpPort, normalVcgBody) { respCode, body -> @@ -89,14 +89,13 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { // show cluster sleep(5000) - showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group {}", showComputeGroup) def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) - + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) - showResult = sql "show clusters" + def showResult = sql "show clusters" for (row : showResult) { println row } @@ -162,27 +161,27 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { sql """ set enable_profile = true """ - before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) - before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) - before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) - before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) - before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) - before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) - before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) - before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) - txnId = -1; + def txnId = -1; streamLoad { table "${tableName}" @@ -191,6 +190,7 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -213,24 +213,24 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) - after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) - after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) - after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) - after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) - after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) - after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) - after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) @@ -241,9 +241,13 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) - set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + def addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(4) sleep(6000) @@ -288,6 +292,7 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -333,11 +338,20 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) - set = [cluster1Ips[0] + ":" + "8060"] as Set + addrSet = [cluster1Ips[0] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(5) + sleep(10000) + showResult = sql "show backends" + for (row : showResult) { + log.info("show backends resp : ${row}".toString()) + } before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) @@ -358,6 +372,7 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -393,61 +408,28 @@ suite('use_vcg_read_write', 'multi_cluster,docker') { assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) - set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(16000) + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + // show cluster showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) - - //// use vcg - //sql """use @${normalVclusterName}""" - //sql """ - //CREATE TABLE ${tbl} ( - //`k1` int(11) NULL, - //`k2` char(5) NULL - //) - //DUPLICATE KEY(`k1`, `k2`) - //COMMENT 'OLAP' - //DISTRIBUTED BY HASH(`k1`) BUCKETS 1 - //PROPERTIES ( - //"replication_num"="1" - //); - //""" - - //sql """ - // insert into ${tbl} (k1, k2) values (1, "10"); - //""" - - //result = sql """select count(*) from ${tbl}""" - //log.info("result = {}", result) - //assertEquals(result.size(), 1) - - // alter cluster info, change standbyComputeGroup to newcluster1, activeComputeGroup to newcluster2 - //clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName2}", standby_cluster_names: ["${clusterName1}"]] - //clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_policy:clusterPolicy] - //normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] - //jsonOutput = new JsonOutput() - //normalVcgBody = jsonOutput.toJson(normalInstance) - //alter_cluster_info_api(msHttpPort, normalVcgBody) { - // respCode, body -> - // log.info("alter virtual cluster result: ${body} ${respCode}".toString()) - // def json = parseJson(body) - // assertTrue(json.code.equalsIgnoreCase("OK")) - //} - //sleep(5000) - //showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ - //log.info("show compute group after alter {}", showComputeGroup) - //vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } - //assertNotNull(vcgInShow) - //assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) } // connect to follower, run again - //options.connectToFollower = true + options.connectToFollower = true } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.groovy new file mode 100644 index 00000000000000..d11a2d0cc0cc91 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.groovy @@ -0,0 +1,399 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +import org.apache.kafka.clients.producer.*; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.consumer.ConsumerRecords +import org.apache.kafka.clients.consumer.KafkaConsumer +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.ListTopicsOptions + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop a backend of c1 +// 5 stop another backend of c1 + +suite('use_vcg_read_write_routine_load', 'multi_cluster,docker') { + def options = new ClusterOptions() + String routine_load_tbl = "test_routine_load_vcg" + String tbl = "test_virtual_compute_group_tbl" + def topic = "test-topic" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + def execute_routind_Load = { + ExecutorService pool; + String kafka_broker_list = context.config.otherConfigs.get("externalEnvIp") + ":" + context.config.otherConfigs.get("kafka_port") + pool = Executors.newFixedThreadPool(1) + pool.execute{ + def props = new Properties() + props.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka_broker_list) + props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + 'org.apache.kafka.common.serialization.StringSerializer') + props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + 'org.apache.kafka.common.serialization.StringSerializer') + + AdminClient adminClient = AdminClient.create(props); + def delResult = adminClient.deleteTopics([topic] as List) + println("the result is " + delResult); + def cnt = 0 + while (!delResult.all().isDone()) { + sleep(1000) + if (cnt++ > 100) { + log.info("failed to wait for delResult") + break + } + } + + NewTopic newTopic = new NewTopic(topic, 10, (short)1); //new NewTopic(topicName, numPartitions, replicationFactor) + List newTopics = new ArrayList(); + newTopics.add(newTopic); + def createResult = adminClient.createTopics(newTopics); + println("the result is " + createResult); + + adminClient.close(); + + def producer = new KafkaProducer(props) + for (int i = 0; i < 30; i++) { + String msg_key = i.toString(); + String msg_value = i.toString() + "|" + "abc" + "|" + (i * 2).toString(); + def message = new ProducerRecord(topic, msg_key, msg_value) + producer.send(message) + sleep(1000) + } + + producer.close() + } + + pool.shutdown() //all tasks submitted + + sleep(1000); + + long timestamp = System.currentTimeMillis() + String job_name = "routine_load_test_" + String.valueOf(timestamp); + sql """ + CREATE ROUTINE LOAD ${job_name} ON + ${routine_load_tbl} COLUMNS TERMINATED BY "|", + COLUMNS(id, name, score) + PROPERTIES( + "desired_concurrent_number"="2", + "max_batch_interval"="6", + "max_batch_rows"="200000", + "max_batch_size"="104857600") + FROM KAFKA( + "kafka_broker_list"="${kafka_broker_list}", + "kafka_topic"="${topic}", + "property.group.id"="gid6", + "property.clinet.id"="cid6", + "property.kafka_default_offsets"="OFFSET_BEGINNING"); + """ + + while (!pool.isTerminated()){} + sleep(30000); + order_qt_q1 "select * from ${routine_load_tbl}" + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + + sql """ drop table if exists ${routine_load_tbl} """ + sql """ + CREATE TABLE IF NOT EXISTS ${routine_load_tbl} + ( + id INT, + name CHAR(10), + score INT + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10; + """ + + sql """ set global enable_profile = true """ + + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + execute_routind_Load.call() + + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + def addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(score) AS theCount from ${routine_load_tbl} group by name order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + cluster.stopBackends(4) + sleep(6000) + + showResult = sql "show backends" + for (row : showResult) { + println row + } + cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + + def alive = entry[9] + log.info("alive : ${alive}".toString()) + info.compute_group_name == clusterName1 && alive == "true" + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + execute_routind_Load.call() + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + addrSet = [cluster1Ips[0] + ":" + "8060"] as Set + sql """ select count(score) AS theCount from ${routine_load_tbl} group by name order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + cluster.stopBackends(5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + execute_routind_Load.call() + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(score) AS theCount from ${routine_load_tbl} group by name order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + } + // connect to follower, run again + options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_s3_load.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_s3_load.groovy new file mode 100644 index 00000000000000..4d31245086acf6 --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_s3_load.groovy @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +import org.apache.kafka.clients.producer.*; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.consumer.ConsumerRecords +import org.apache.kafka.clients.consumer.KafkaConsumer +import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.CreateTopicsResult; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.admin.ListTopicsOptions + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop a backend of c1 +// 5 stop another backend of c1 + +suite('use_vcg_read_write_s3_load', 'multi_cluster,docker') { + def options = new ClusterOptions() + def topic = "test-topic" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + def execute_s3_Load = { + //def tables = [supplier: 1000000] + def tables = [nation: 25] + //def tables = [customer: 15000000, lineitem: 600037902, nation: 25, orders: 150000000, part: 20000000, partsupp: 80000000, region: 6, supplier: 1000000] + def s3BucketName = getS3BucketName() + def s3WithProperties = """WITH S3 ( + |"AWS_ACCESS_KEY" = "${getS3AK()}", + |"AWS_SECRET_KEY" = "${getS3SK()}", + |"AWS_ENDPOINT" = "${getS3Endpoint()}", + |"AWS_REGION" = "${getS3Region()}", + |"provider" = "${getS3Provider()}") + |PROPERTIES( + |"exec_mem_limit" = "8589934592", + |"load_parallelism" = "3")""".stripMargin() + // set fe configuration + sql "ADMIN SET FRONTEND CONFIG ('max_bytes_per_broker_scanner' = '161061273600')" + + def uniqueID = Math.abs(UUID.randomUUID().hashCode()).toString() + tables.each { table, rows -> + // create table if not exists + sql new File("""${context.file.parent}/ddl/${table}.sql""").text + // check row count + def rowCount = sql "select count(*) from ${table}" + //if (rowCount[0][0] != rows) { + def loadLabel = table + "_" + uniqueID + sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text + // load data from cos + def loadSql = new File("""${context.file.parent}/ddl/${table}_load.sql""").text.replaceAll("\\\$\\{s3BucketName\\}", s3BucketName) + loadSql = loadSql.replaceAll("\\\$\\{loadLabel\\}", loadLabel) + s3WithProperties + sql loadSql + + // check load state + while (true) { + def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("load result is ${stateResult}") + def loadState = stateResult[stateResult.size() - 1][2].toString() + if ("CANCELLED".equalsIgnoreCase(loadState)) { + throw new IllegalStateException("load ${loadLabel} failed.") + } else if ("FINISHED".equalsIgnoreCase(loadState)) { + break + } + sleep(5000) + } + //} + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"]] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + + sql """ set enable_profile = true """ + + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + execute_s3_Load.call() + + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + def addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select N_REGIONKEY, count(N_NAME) AS theCount from nation group by N_REGIONKEY order by theCount """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + cluster.stopBackends(4) + sleep(6000) + + showResult = sql "show backends" + for (row : showResult) { + println row + } + cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + + def alive = entry[9] + log.info("alive : ${alive}".toString()) + info.compute_group_name == clusterName1 && alive == "true" + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + execute_s3_Load.call() + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + addrSet = [cluster1Ips[0] + ":" + "8060"] as Set + sql """ select N_REGIONKEY, count(N_NAME) AS theCount from nation group by N_REGIONKEY order by theCount """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + cluster.stopBackends(5) + sleep(5000) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + execute_s3_Load.call() + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select N_REGIONKEY, count(N_NAME) AS theCount from nation group by N_REGIONKEY order by theCount """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + sleep(16000) + + sql """ select N_REGIONKEY, count(N_NAME) AS theCount from nation group by N_REGIONKEY order by theCount """ + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + } + // connect to follower, run again + options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy new file mode 100644 index 00000000000000..e1f2923814e30f --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy @@ -0,0 +1,422 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop a backend of c1 +// 5 stop another backend of c1 + +suite('use_vcg_read_write_unhealthy_node_50', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], unhealthyNodeThresholdPercent: 50] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + def txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + def addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(addrSet) + + cluster.stopBackends(4) + sleep(10000) + showResult = sql "show backends" + for (row : showResult) { + println row + } + cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + + def alive = entry[9] + log.info("alive : ${alive}".toString()) + info.compute_group_name == clusterName1 && alive == "true" + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + assertTrue(before_cluster1_be0_load_rows == after_cluster1_be0_load_rows) + assertTrue(before_cluster1_be0_flush == after_cluster1_be0_flush) + + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(addrSet) + + sleep(16000) + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + + cluster.stopBackends(5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + checkProfileNew.call(addrSet) + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + } + // connect to follower, run again + options.connectToFollower = true + } +} diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy index 87c5ffa7aecd7f..6cac6891a291ca 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover.groovy @@ -20,12 +20,13 @@ import groovy.json.JsonSlurper import groovy.json.JsonOutput // 1 create two physical cluster c1, c2, every cluster contains 2 be -// 2 stop a backend of c1 -// 3 stop a backend of c1 -// 4 start 2 backends of c1 -// 5 long-term stop 2 backends of c1 +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop 2 bes of c1 +// 5 start 2 bes of c1 +// 6 long-term stop 2 bes of c1 -suite('vcg_failover_1', 'multi_cluster,docker') { +suite('vcg_auto_failover', 'multi_cluster,docker') { def options = new ClusterOptions() String tableName = "test_all_vcluster" String tbl = "test_virtual_compute_group_tbl" @@ -76,9 +77,9 @@ suite('vcg_failover_1', 'multi_cluster,docker') { def normalVclusterId = "normalVirtualClusterId" def vcgClusterNames = [clusterName1, clusterName2] def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 10] - clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] - jsonOutput = new JsonOutput() + def jsonOutput = new JsonOutput() def normalVcgBody = jsonOutput.toJson(normalInstance) add_cluster_api.call(msHttpPort, normalVcgBody) { respCode, body -> @@ -89,13 +90,13 @@ suite('vcg_failover_1', 'multi_cluster,docker') { // show cluster sleep(5000) - showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group {}", showComputeGroup) def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) - showResult = sql "show clusters" + def showResult = sql "show clusters" for (row : showResult) { println row } @@ -163,17 +164,17 @@ suite('vcg_failover_1', 'multi_cluster,docker') { cluster.stopBackends(4, 5) - before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) - before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) - before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) - before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) - txnId = -1; + def txnId = -1; streamLoad { table "${tableName}" @@ -182,6 +183,7 @@ suite('vcg_failover_1', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -204,22 +206,26 @@ suite('vcg_failover_1', 'multi_cluster,docker') { order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) - after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) - after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) - after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) - set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + def addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(16000) // show cluster @@ -228,18 +234,18 @@ suite('vcg_failover_1', 'multi_cluster,docker') { vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) log.info("policy {}", vcgInShow.Policy) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster1', standbyComputeGroup='newcluster2'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) cluster.startBackends(4, 5) - before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) - before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) - before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) - before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); @@ -261,6 +267,7 @@ suite('vcg_failover_1', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -283,14 +290,14 @@ suite('vcg_failover_1', 'multi_cluster,docker') { order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) - after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) - after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) - after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); @@ -311,9 +318,13 @@ suite('vcg_failover_1', 'multi_cluster,docker') { assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) - set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(4, 5) @@ -336,6 +347,7 @@ suite('vcg_failover_1', 'multi_cluster,docker') { file 'all_types.csv' time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort check { loadResult, exception, startTime, endTime -> if (exception != null) { @@ -371,9 +383,13 @@ suite('vcg_failover_1', 'multi_cluster,docker') { assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) - set = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(60000) sql """ @@ -385,9 +401,9 @@ suite('vcg_failover_1', 'multi_cluster,docker') { log.info("show compute group {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains("activeComputeGroup='newcluster2', standbyComputeGroup='newcluster1'")) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) } // connect to follower, run again - //options.connectToFollower = true + options.connectToFollower = true } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.groovy new file mode 100644 index 00000000000000..11514a43cb9acd --- /dev/null +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/vcg_auto_failover_manual_failback.groovy @@ -0,0 +1,519 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import groovy.json.JsonSlurper +import groovy.json.JsonOutput + +// 1 create two physical cluster c1, c2, every cluster contains 2 be +// 2 create vcg, c1, c2 are sub compute group of vcg, adn c1 is active cg +// 3 use vcg +// 4 stop 2 bes of c1 +// 5 start 2 bes of c1 +// 6 long-term stop 2 bes of c1 + +suite('vcg_auto_failover_manual_failback', 'multi_cluster,docker') { + def options = new ClusterOptions() + String tableName = "test_all_vcluster" + String tbl = "test_virtual_compute_group_tbl" + + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'sys_log_verbose_modules=org', + ] + options.feNum = 3 + options.cloudMode = true + + def add_cluster_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/add_cluster?token=$token" + body request_body + check check_func + } + } + + def alter_cluster_info_api = { msHttpPort, request_body, check_func -> + httpTest { + endpoint msHttpPort + uri "/MetaService/http/alter_vcluster_info?token=$token" + body request_body + check check_func + } + } + + options.connectToFollower = false + + for (def j = 0; j < 2; j++) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms1 addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) + + def clusterName1 = "newcluster1" + // add cluster newcluster1 + cluster.addBackend(2, clusterName1) + + def clusterName2 = "newcluster2" + // add cluster newcluster2 + cluster.addBackend(2, clusterName2) + + // add vcluster + def normalVclusterName = "normalVirtualClusterName" + def normalVclusterId = "normalVirtualClusterId" + def vcgClusterNames = [clusterName1, clusterName2] + def clusterPolicy = [type: "ActiveStandby", active_cluster_name: "${clusterName1}", standby_cluster_names: ["${clusterName2}"], failover_failure_threshold: 10] + def clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_names:vcgClusterNames, cluster_policy:clusterPolicy] + def normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + def jsonOutput = new JsonOutput() + def normalVcgBody = jsonOutput.toJson(normalInstance) + add_cluster_api.call(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("add normal vitural compute group http cli result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + // show cluster + sleep(5000) + def showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + def vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + def showResult = sql "show clusters" + for (row : showResult) { + println row + } + showResult = sql "show backends" + for (row : showResult) { + println row + } + + // get be ip of clusterName1 + def jsonSlurper = new JsonSlurper() + def cluster1Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName1 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster1: ${clusterName1} ${cluster1Ips}".toString()) + + def cluster2Ips = showResult.findAll { entry -> + def raw = entry[19] + def info = (raw instanceof String) ? jsonSlurper.parseText(raw) : raw + info.compute_group_name == clusterName2 + }.collect { entry -> + entry[1] + } + log.info("backends of cluster2: ${clusterName2} ${cluster2Ips}".toString()) + + sql """use @${normalVclusterName}""" + sql """ drop table if exists ${tableName} """ + + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + """ + + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num"="1" + ) + """ + + sql """ set enable_profile = true """ + + cluster.stopBackends(4, 5) + + def before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + def before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + def before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + def before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + def txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + def after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + def after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + def after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + def after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + def addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + sleep(16000) + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + log.info("policy {}", vcgInShow.Policy) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + + cluster.startBackends(4, 5) + + def before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + def before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + def before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + def before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + def after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + def after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + def after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + def after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + cluster.stopBackends(4, 5) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster2_be0_load_rows < after_cluster2_be0_load_rows || before_cluster2_be1_load_rows < after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be0_flush < after_cluster2_be0_flush || before_cluster2_be1_flush < after_cluster2_be1_flush) + + addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + sleep(60000) + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + // show cluster + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) + + cluster.startBackends(4, 5) + // alter cluster info, change standbyComputeGroup to newcluster2, activeComputeGroup to newcluster1 + clusterPolicy = [type: "ActiveStandby", active_cluster_name: "newcluster1", standby_cluster_names: ["newcluster2"]] + clusterMap = [cluster_name: "${normalVclusterName}", cluster_id:"${normalVclusterId}", type:"VIRTUAL", cluster_policy:clusterPolicy] + normalInstance = [instance_id: "${instance_id}", cluster: clusterMap] + jsonOutput = new JsonOutput() + normalVcgBody = jsonOutput.toJson(normalInstance) + alter_cluster_info_api(msHttpPort, normalVcgBody) { + respCode, body -> + log.info("alter virtual cluster result: ${body} ${respCode}".toString()) + def json = parseJson(body) + assertTrue(json.code.equalsIgnoreCase("OK")) + } + + before_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("before_cluster1_be0_load_rows : ${before_cluster1_be0_load_rows}".toString()) + before_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster1_be0_flush : ${before_cluster1_be0_flush}".toString()) + + before_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("before_cluster1_be1_load_rows : ${before_cluster1_be1_load_rows}".toString()) + before_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster1_be1_flush : ${before_cluster1_be1_flush}".toString()) + + before_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("before_cluster2_be0_load_rows : ${before_cluster2_be0_load_rows}".toString()) + before_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("before_cluster2_be0_flush : ${before_cluster2_be0_flush}".toString()) + + before_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("before_cluster2_be1_load_rows : ${before_cluster2_be1_load_rows}".toString()) + before_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("before_cluster2_be1_flush : ${before_cluster2_be1_flush}".toString()) + + txnId = -1; + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'cloud_cluster', 'normalVirtualClusterName' + + file 'all_types.csv' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(20, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } + + sql """ + insert into ${tbl} (k1, k2) values (1, "10"); + """ + + sql "sync" + order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 + order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 + + after_cluster1_be0_load_rows = get_be_metric(cluster1Ips[0], "8040", "load_rows"); + log.info("after_cluster1_be0_load_rows : ${after_cluster1_be0_load_rows}".toString()) + after_cluster1_be0_flush = get_be_metric(cluster1Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster1_be0_flush : ${after_cluster1_be0_flush}".toString()) + + after_cluster1_be1_load_rows = get_be_metric(cluster1Ips[1], "8040", "load_rows"); + log.info("after_cluster1_be1_load_rows : ${after_cluster1_be1_load_rows}".toString()) + after_cluster1_be1_flush = get_be_metric(cluster1Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster1_be1_flush : ${after_cluster1_be1_flush}".toString()) + + after_cluster2_be0_load_rows = get_be_metric(cluster2Ips[0], "8040", "load_rows"); + log.info("after_cluster2_be0_load_rows : ${after_cluster2_be0_load_rows}".toString()) + after_cluster2_be0_flush = get_be_metric(cluster2Ips[0], "8040", "memtable_flush_total"); + log.info("after_cluster2_be0_flush : ${after_cluster2_be0_flush}".toString()) + + after_cluster2_be1_load_rows = get_be_metric(cluster2Ips[1], "8040", "load_rows"); + log.info("after_cluster2_be1_load_rows : ${after_cluster2_be1_load_rows}".toString()) + after_cluster2_be1_flush = get_be_metric(cluster2Ips[1], "8040", "memtable_flush_total"); + log.info("after_cluster2_be1_flush : ${after_cluster2_be1_flush}".toString()) + + assertTrue(before_cluster1_be0_load_rows < after_cluster1_be0_load_rows || before_cluster1_be1_load_rows < after_cluster1_be1_load_rows) + assertTrue(before_cluster1_be0_flush < after_cluster1_be0_flush || before_cluster1_be1_flush < after_cluster1_be1_flush) + + assertTrue(before_cluster2_be0_load_rows == after_cluster2_be0_load_rows) + assertTrue(before_cluster2_be0_flush == after_cluster2_be0_flush) + assertTrue(before_cluster2_be1_load_rows == after_cluster2_be1_load_rows) + assertTrue(before_cluster2_be1_flush == after_cluster2_be1_flush) + + addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } + + + sleep(6000) + showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ + log.info("show compute group after alter {}", showComputeGroup) + vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } + assertNotNull(vcgInShow) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + } + // connect to follower, run again + options.connectToFollower = true + } +} From fba7215dd22b614b6c6f5056b6dddd1dcdce6819 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 16 Jul 2025 23:58:17 +0800 Subject: [PATCH 237/572] branch-3.0: [fix](test) Adjust check_before_quit test #53278 (#53362) Cherry-picked from #53278 Co-authored-by: Uniqueyou --- .../check_before_quit.groovy | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy index 6c25ce3efff039..9e624fa1ead88d 100644 --- a/regression-test/suites/check_before_quit/check_before_quit.groovy +++ b/regression-test/suites/check_before_quit/check_before_quit.groovy @@ -234,6 +234,8 @@ suite("check_before_quit", "nonConcurrent,p0") { def num = allDataBases.size() def failureList = [] + sql "set enable_decimal256 = true;" + for (int i = 0; i < num; i++) { def db = allDataBases[i][0] if (db == "__internal_schema" || db == "information_schema" || db == "mysql") { @@ -301,16 +303,31 @@ suite("check_before_quit", "nonConcurrent,p0") { // only re create table, because the table which view depends may be dropped, // so recreate view may fail try { + if (createTableSql[0][1].contains("agg_state<")) { + logger.info("Skipping table ${tbl} because it contains agg_state column type") + continue + } + sql(createTableSql[0][1]) def createTableSqlResult = sql "show create table ${tbl}" - logger.info("target: ${createTableSqlResult[0][1]}, origin: ${createTableSql[0][1]}") - if (createTableSqlResult[0][1].trim() != createTableSql[0][1].trim()) { + def eraseReplicationAllocationProperty = { String sql -> + def result = sql.replaceAll(/(,\s*)?"replication_allocation"\s*=\s*"tag\.location\.default:\s*[^"]*"(,\s*|\s*\))/, '$2') + result = result.replaceAll(/PROPERTIES\s*\(\s*,/, 'PROPERTIES (') + result = result.replaceAll(/,(\s*,)+/, ',') + return result.trim() + } + createTableSql = eraseReplicationAllocationProperty(createTableSql[0][1]) + createTableSqlResult = eraseReplicationAllocationProperty(createTableSqlResult[0][1]) + + logger.info("target: ${createTableSqlResult}, origin: ${createTableSql}") + + if (createTableSqlResult.trim() != createTableSql.trim()) { failureList << [ operation: "CREATE TABLE", target: "${tbl}", - expected: createTableSql[0][1].trim(), - actual: createTableSqlResult[0][1].trim() + expected: createTableSql.trim(), + actual: createTableSqlResult.trim() ] } } catch (Exception recreateEx) { @@ -318,7 +335,7 @@ suite("check_before_quit", "nonConcurrent,p0") { failureList << [ operation: "RECREATE TABLE", target: "${tbl}", - sql: createTableSql[0][1], + sql: createTableSql, error: recreateEx.getMessage() ] } @@ -346,7 +363,7 @@ suite("check_before_quit", "nonConcurrent,p0") { } } } - fail("Found ${failureList.size()} failures during database recreation checks") + logger.error("Found ${failureList.size()} failures during database recreation checks") } else { clear = true } From 8cf1fdfb9b667997f88448efa35356766b35ad6a Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 17 Jul 2025 12:26:54 +0800 Subject: [PATCH 238/572] branch-3.0 [enhance](meta-service)add bytes for kv stats (#52729) (#53351) pick https://github.com/apache/doris/pull/52729 --- cloud/src/common/bvars.cpp | 166 ++++++ cloud/src/common/bvars.h | 108 ++++ cloud/src/common/stats.h | 4 + cloud/src/meta-service/meta_service.cpp | 17 + cloud/src/meta-service/meta_service_helper.h | 7 +- .../meta-service/meta_service_partition.cpp | 2 + .../meta-service/meta_service_resource.cpp | 7 +- cloud/src/meta-service/meta_service_txn.cpp | 26 +- cloud/src/meta-store/mem_txn_kv.cpp | 20 +- cloud/src/meta-store/mem_txn_kv.h | 13 + cloud/src/meta-store/txn_kv.cpp | 5 +- cloud/src/meta-store/txn_kv.h | 24 + cloud/test/rpc_kv_bvar_test.cpp | 538 ++++++++++-------- 13 files changed, 707 insertions(+), 230 deletions(-) diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp index 7e4e36ce14059b..1ff6646c195ea1 100644 --- a/cloud/src/common/bvars.cpp +++ b/cloud/src/common/bvars.cpp @@ -382,6 +382,172 @@ mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter("rpc_kv_clean_txn_labe mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter("rpc_kv_clean_txn_label_del_counter",{"instance_id"}); // get_txn_id mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter("rpc_kv_get_txn_id_get_counter",{"instance_id"}); +// bytes +// get_rowset +mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_bytes("rpc_kv_get_rowset_get_bytes",{"instance_id"}); +// get_version +mBvarInt64Adder g_bvar_rpc_kv_get_version_get_bytes("rpc_kv_get_version_get_bytes",{"instance_id"}); +// get_schema_dict +mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_bytes("rpc_kv_get_schema_dict_get_bytes",{"instance_id"}); +// create_tablets +mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_bytes("rpc_kv_create_tablets_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_bytes("rpc_kv_create_tablets_put_bytes",{"instance_id"}); +// update_tablet +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_get_bytes("rpc_kv_update_tablet_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_put_bytes("rpc_kv_update_tablet_put_bytes",{"instance_id"}); +// update_tablet_schema +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_get_bytes("rpc_kv_update_tablet_schema_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_put_bytes("rpc_kv_update_tablet_schema_put_bytes",{"instance_id"}); +// get_tablet +mBvarInt64Adder g_bvar_rpc_kv_get_tablet_get_bytes("rpc_kv_get_tablet_get_bytes",{"instance_id"}); +// prepare_rowset +mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_get_bytes("rpc_kv_prepare_rowset_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_put_bytes("rpc_kv_prepare_rowset_put_bytes",{"instance_id"}); +// commit_rowset +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_get_bytes("rpc_kv_commit_rowset_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_put_bytes("rpc_kv_commit_rowset_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_del_bytes("rpc_kv_commit_rowset_del_bytes",{"instance_id"}); +// update_tmp_rowset +mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_get_bytes("rpc_kv_update_tmp_rowset_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_put_bytes("rpc_kv_update_tmp_rowset_put_bytes",{"instance_id"}); +// get_tablet_stats +mBvarInt64Adder g_bvar_rpc_kv_get_tablet_stats_get_bytes("rpc_kv_get_tablet_stats_get_bytes",{"instance_id"}); +// update_delete_bitmap +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_get_bytes("rpc_kv_update_delete_bitmap_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_put_bytes("rpc_kv_update_delete_bitmap_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_del_bytes("rpc_kv_update_delete_bitmap_del_bytes",{"instance_id"}); +// get_delete_bitmap +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_get_bytes("rpc_kv_get_delete_bitmap_get_bytes",{"instance_id"}); +// get_delete_bitmap_update_lock +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_bytes("rpc_kv_get_delete_bitmap_update_lock_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_bytes("rpc_kv_get_delete_bitmap_update_lock_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_bytes("rpc_kv_get_delete_bitmap_update_lock_del_bytes",{"instance_id"}); +// remove_delete_bitmap_update_lock +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_bytes("rpc_kv_remove_delete_bitmap_update_lock_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_bytes("rpc_kv_remove_delete_bitmap_update_lock_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_bytes("rpc_kv_remove_delete_bitmap_update_lock_del_bytes",{"instance_id"}); +// remove_delete_bitmap +mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_del_bytes("rpc_kv_remove_delete_bitmap_del_bytes",{"instance_id"}); +// start_tablet_job +mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_get_bytes("rpc_kv_start_tablet_job_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_put_bytes("rpc_kv_start_tablet_job_put_bytes",{"instance_id"}); +// finish_tablet_job +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_get_bytes("rpc_kv_finish_tablet_job_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_put_bytes("rpc_kv_finish_tablet_job_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_del_bytes("rpc_kv_finish_tablet_job_del_bytes",{"instance_id"}); +// prepare_index +mBvarInt64Adder g_bvar_rpc_kv_prepare_index_get_bytes("rpc_kv_prepare_index_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_index_put_bytes("rpc_kv_prepare_index_put_bytes",{"instance_id"}); +// commit_index +mBvarInt64Adder g_bvar_rpc_kv_commit_index_get_bytes("rpc_kv_commit_index_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_put_bytes("rpc_kv_commit_index_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_index_del_bytes("rpc_kv_commit_index_del_bytes",{"instance_id"}); +// drop_index +mBvarInt64Adder g_bvar_rpc_kv_drop_index_get_bytes("rpc_kv_drop_index_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_drop_index_put_bytes("rpc_kv_drop_index_put_bytes",{"instance_id"}); +// prepare_partition +mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_get_bytes("rpc_kv_prepare_partition_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_put_bytes("rpc_kv_prepare_partition_put_bytes",{"instance_id"}); +// commit_partition +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_get_bytes("rpc_kv_commit_partition_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_put_bytes("rpc_kv_commit_partition_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_partition_del_bytes("rpc_kv_commit_partition_del_bytes",{"instance_id"}); +// drop_partition +mBvarInt64Adder g_bvar_rpc_kv_drop_partition_get_bytes("rpc_kv_drop_partition_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_drop_partition_put_bytes("rpc_kv_drop_partition_put_bytes",{"instance_id"}); +// check_kv +mBvarInt64Adder g_bvar_rpc_kv_check_kv_get_bytes("rpc_kv_check_kv_get_bytes",{"instance_id"}); +// get_obj_store_info +mBvarInt64Adder g_bvar_rpc_kv_get_obj_store_info_get_bytes("rpc_kv_get_obj_store_info_get_bytes",{"instance_id"}); +// alter_storage_vault +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_get_bytes("rpc_kv_alter_storage_vault_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_put_bytes("rpc_kv_alter_storage_vault_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_del_bytes("rpc_kv_alter_storage_vault_del_bytes",{"instance_id"}); +// alter_obj_store_info +mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_get_bytes("rpc_kv_alter_obj_store_info_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_put_bytes("rpc_kv_alter_obj_store_info_put_bytes",{"instance_id"}); +// update_ak_sk +mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_get_bytes("rpc_kv_update_ak_sk_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_put_bytes("rpc_kv_update_ak_sk_put_bytes",{"instance_id"}); +// create_instance +mBvarInt64Adder g_bvar_rpc_kv_create_instance_get_bytes("rpc_kv_create_instance_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_instance_put_bytes("rpc_kv_create_instance_put_bytes",{"instance_id"}); +// get_instance +mBvarInt64Adder g_bvar_rpc_kv_get_instance_get_bytes("rpc_kv_get_instance_get_bytes",{"instance_id"}); +// alter_cluster +mBvarInt64Adder g_bvar_rpc_kv_alter_cluster_get_bytes("rpc_kv_alter_cluster_get_bytes",{"instance_id"}); +// get_cluster +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_get_bytes("rpc_kv_get_cluster_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_put_bytes("rpc_kv_get_cluster_put_bytes",{"instance_id"}); +// create_stage +mBvarInt64Adder g_bvar_rpc_kv_create_stage_get_bytes("rpc_kv_create_stage_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_create_stage_put_bytes("rpc_kv_create_stage_put_bytes",{"instance_id"}); +// get_stage +mBvarInt64Adder g_bvar_rpc_kv_get_stage_get_bytes("rpc_kv_get_stage_get_bytes",{"instance_id"}); +// get_iam +mBvarInt64Adder g_bvar_rpc_kv_get_iam_get_bytes("rpc_kv_get_iam_get_bytes",{"instance_id"}); +// alter_iam +mBvarInt64Adder g_bvar_rpc_kv_alter_iam_get_bytes("rpc_kv_alter_iam_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_iam_put_bytes("rpc_kv_alter_iam_put_bytes",{"instance_id"}); +// alter_ram_user +mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_get_bytes("rpc_kv_alter_ram_user_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_put_bytes("rpc_kv_alter_ram_user_put_bytes",{"instance_id"}); +// begin_copy +mBvarInt64Adder g_bvar_rpc_kv_begin_copy_get_bytes("rpc_kv_begin_copy_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_copy_put_bytes("rpc_kv_begin_copy_put_bytes",{"instance_id"}); +// finish_copy +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_get_bytes("rpc_kv_finish_copy_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_put_bytes("rpc_kv_finish_copy_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_finish_copy_del_bytes("rpc_kv_finish_copy_del_bytes",{"instance_id"}); +// get_copy_job +mBvarInt64Adder g_bvar_rpc_kv_get_copy_job_get_bytes("rpc_kv_get_copy_job_get_bytes",{"instance_id"}); +// get_copy_files +mBvarInt64Adder g_bvar_rpc_kv_get_copy_files_get_bytes("rpc_kv_get_copy_files_get_bytes",{"instance_id"}); +// filter_copy_files +mBvarInt64Adder g_bvar_rpc_kv_filter_copy_files_get_bytes("rpc_kv_filter_copy_files_get_bytes",{"instance_id"}); +// get_cluster_status +mBvarInt64Adder g_bvar_rpc_kv_get_cluster_status_get_bytes("rpc_kv_get_cluster_status_get_bytes",{"instance_id"}); +// begin_txn +mBvarInt64Adder g_bvar_rpc_kv_begin_txn_get_bytes("rpc_kv_begin_txn_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_txn_put_bytes("rpc_kv_begin_txn_put_bytes",{"instance_id"}); +// precommit_txn +mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_get_bytes("rpc_kv_precommit_txn_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_put_bytes("rpc_kv_precommit_txn_put_bytes",{"instance_id"}); +// get_rl_task_commit_attach +mBvarInt64Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_bytes("rpc_kv_get_rl_task_commit_attach_get_bytes",{"instance_id"}); +// reset_rl_progress +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_get_bytes("rpc_kv_reset_rl_progress_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_put_bytes("rpc_kv_reset_rl_progress_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_del_bytes("rpc_kv_reset_rl_progress_del_bytes",{"instance_id"}); +// commit_txn +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_get_bytes("rpc_kv_commit_txn_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_put_bytes("rpc_kv_commit_txn_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_commit_txn_del_bytes("rpc_kv_commit_txn_del_bytes",{"instance_id"}); +// abort_txn +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_get_bytes("rpc_kv_abort_txn_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_put_bytes("rpc_kv_abort_txn_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_del_bytes("rpc_kv_abort_txn_del_bytes",{"instance_id"}); +// get_txn +mBvarInt64Adder g_bvar_rpc_kv_get_txn_get_bytes("rpc_kv_get_txn_get_bytes",{"instance_id"}); +// get_current_max_txn_id +mBvarInt64Adder g_bvar_rpc_kv_get_current_max_txn_id_get_bytes("rpc_kv_get_current_max_txn_id_get_bytes",{"instance_id"}); +// begin_sub_txn +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_get_bytes("rpc_kv_begin_sub_txn_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_put_bytes("rpc_kv_begin_sub_txn_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_del_bytes("rpc_kv_begin_sub_txn_del_bytes",{"instance_id"}); +// abort_sub_txn +mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_get_bytes("rpc_kv_abort_sub_txn_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes("rpc_kv_abort_sub_txn_put_bytes",{"instance_id"}); +// abort_txn_with_coordinator +mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes("rpc_kv_abort_txn_with_coordinator_get_bytes",{"instance_id"}); +// check_txn_conflict +mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes("rpc_kv_check_txn_conflict_get_bytes",{"instance_id"}); +// clean_txn_label +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes("rpc_kv_clean_txn_label_get_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes("rpc_kv_clean_txn_label_put_bytes",{"instance_id"}); +mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_bytes("rpc_kv_clean_txn_label_del_bytes",{"instance_id"}); +// get_txn_id +mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_bytes("rpc_kv_get_txn_id_get_bytes",{"instance_id"}); // meta ranges mBvarStatus g_bvar_fdb_kv_ranges_count("fdb_kv_ranges_count", {"category","instance_id", "sub_category"}); diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h index d57e14e2697278..08461dad668f2e 100644 --- a/cloud/src/common/bvars.h +++ b/cloud/src/common/bvars.h @@ -469,5 +469,113 @@ extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_counter; extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_counter; extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_counter; +extern mBvarInt64Adder g_bvar_rpc_kv_get_rowset_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_version_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_schema_dict_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_create_tablets_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tablet_schema_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tablet_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_rowset_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_rowset_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_tmp_rowset_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_tablet_stats_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_delete_bitmap_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_remove_delete_bitmap_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_start_tablet_job_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_tablet_job_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_index_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_index_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_index_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_index_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_index_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_prepare_partition_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_partition_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_partition_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_drop_partition_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_check_kv_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_obj_store_info_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_storage_vault_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_obj_store_info_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_update_ak_sk_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_create_instance_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_create_instance_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_instance_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_cluster_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_create_stage_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_create_stage_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_stage_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_iam_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_iam_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_iam_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_alter_ram_user_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_copy_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_copy_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_finish_copy_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_copy_job_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_copy_files_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_filter_copy_files_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_cluster_status_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_txn_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_precommit_txn_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_rl_task_commit_attach_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_reset_rl_progress_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_commit_txn_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_current_max_txn_id_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_begin_sub_txn_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_sub_txn_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_check_txn_conflict_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_get_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_put_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_clean_txn_label_del_bytes; +extern mBvarInt64Adder g_bvar_rpc_kv_get_txn_id_get_bytes; + // meta ranges extern mBvarStatus g_bvar_fdb_kv_ranges_count; diff --git a/cloud/src/common/stats.h b/cloud/src/common/stats.h index 2d99bf0d3c1671..a9e482a3e4c751 100644 --- a/cloud/src/common/stats.h +++ b/cloud/src/common/stats.h @@ -21,6 +21,10 @@ namespace doris::cloud { struct KVStats { + int64_t get_bytes {}; + int64_t put_bytes {}; + int64_t del_bytes {}; + int64_t get_counter {}; int64_t put_counter {}; int64_t del_counter {}; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 7e0d3a267b85c6..176882cb120394 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -358,6 +358,7 @@ void MetaServiceImpl::batch_get_version(::google::protobuf::RpcController* contr } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); }; for (size_t i = response->versions_size(); i < num_acquired; i += BATCH_SIZE) { @@ -459,6 +460,8 @@ void internal_create_tablet(const CreateTabletsRequest* request, MetaServiceCode } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); }; @@ -606,6 +609,8 @@ void MetaServiceImpl::create_tablets(::google::protobuf::RpcController* controll instance_key(key_info, &key); err = txn0->get(key, &val); + stats.get_bytes += val.size() + key.size(); + stats.get_counter++; LOG(INFO) << "get instance_key=" << hex(key); stats.get_counter++; if (err != TxnErrorCode::TXN_OK) { @@ -1726,6 +1731,7 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); }; TabletIndexPB idx; @@ -1766,6 +1772,7 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, if (version_pb.pending_txn_ids_size() > 0) { DCHECK(version_pb.pending_txn_ids_size() == 1); + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); txn.reset(); TEST_SYNC_POINT_CALLBACK("get_rowset::advance_last_pending_txn_id", @@ -1900,6 +1907,7 @@ void MetaServiceImpl::get_tablet_stats(::google::protobuf::RpcController* contro return; } DORIS_CLOUD_DEFER { + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); // the txn is not a local variable, if not reset will count last res twice txn.reset(nullptr); @@ -2246,6 +2254,9 @@ void MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont g_bvar_update_delete_bitmap_fail_counter << 1; return; } + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); @@ -2359,6 +2370,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); }; MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id, rowset_ids[i], @@ -2387,6 +2399,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_err", &round, &err); int64_t retry = 0; while (err == TxnErrorCode::TXN_TOO_OLD && retry < 3) { + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); txn = nullptr; err = txn_kv_->create_txn(&txn); @@ -2481,6 +2494,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); }; TabletIndexPB idx(request->idx()); @@ -2635,6 +2649,9 @@ bool MetaServiceImpl::get_mow_tablet_stats_and_meta(MetaServiceCode& code, std:: } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 7ecdf856659db0..9d99e28b643b05 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -230,8 +230,10 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { // don't use these macro it just for defer count, reduce useless variable(some rpc just need one of rw op) // If we have to write separate code for each RPC, it would be quite troublesome // After all, adding put, get, and del after the RPC_PREPROCESS macro is simpler than writing a long string of code -#define RPCKVCOUNTHELPER(func_name, op) \ +#define RPCKVCOUNTHELPER(func_name, op) \ + g_bvar_rpc_kv_##func_name##_##op##_bytes.put({instance_id}, stats.op##_bytes); \ g_bvar_rpc_kv_##func_name##_##op##_counter.put({instance_id}, stats.op##_counter); + #define RPCKVCOUNT_0(func_name) #define RPCKVCOUNT_1(func_name, op1) RPCKVCOUNTHELPER(func_name, op1) #define RPCKVCOUNT_2(func_name, op1, op2) \ @@ -266,6 +268,9 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { finish_rpc(#func_name, ctrl, response); \ closure_guard.reset(nullptr); \ if (txn != nullptr) { \ + stats.get_bytes += txn->get_bytes(); \ + stats.put_bytes += txn->put_bytes(); \ + stats.del_bytes += txn->delete_bytes(); \ stats.get_counter += txn->num_get_keys(); \ stats.put_counter += txn->num_put_keys(); \ stats.del_counter += txn->num_del_keys(); \ diff --git a/cloud/src/meta-service/meta_service_partition.cpp b/cloud/src/meta-service/meta_service_partition.cpp index 109b86813f39f6..e8f009281de62a 100644 --- a/cloud/src/meta-service/meta_service_partition.cpp +++ b/cloud/src/meta-service/meta_service_partition.cpp @@ -629,6 +629,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); }; auto& [keys, hint, key_func] = get_check_info(request); @@ -650,6 +651,7 @@ void check_create_table(std::string instance_id, std::shared_ptr txn_kv, *msg = "prepare and commit rpc not match, recycle key remained"; return; } else if (err == TxnErrorCode::TXN_TOO_OLD) { + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); // separate it to several txn for rubustness txn.reset(); diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index f69ed65786db6e..e33b0c24a6da73 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -2590,6 +2590,7 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, KVStats stats; notify_refresh_instance(txn_kv, instance_id, &stats); if (config::use_detailed_metrics && !instance_id.empty()) { + g_bvar_rpc_kv_alter_cluster_get_bytes.put({instance_id}, stats.get_bytes); g_bvar_rpc_kv_alter_cluster_get_counter.put({instance_id}, stats.get_counter); } }); @@ -4027,6 +4028,7 @@ void MetaServiceImpl::get_cluster_status(google::protobuf::RpcController* contro } DORIS_CLOUD_DEFER { if (config::use_detailed_metrics && txn != nullptr) { + g_bvar_rpc_kv_get_cluster_status_get_bytes.put({instance_id}, txn->get_bytes()); g_bvar_rpc_kv_get_cluster_status_get_counter.put({instance_id}, txn->num_get_keys()); } @@ -4087,7 +4089,7 @@ void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& i std::string key = system_meta_service_registry_key(); std::string val; err = txn->get(key, &val); - if (config::use_detailed_metrics && stats) { + if (stats) { stats->get_counter++; } if (err != TxnErrorCode::TXN_OK) { @@ -4095,6 +4097,9 @@ void notify_refresh_instance(std::shared_ptr txn_kv, const std::string& i << " err=" << err; return; } + if (stats) { + stats->get_bytes += val.size() + key.size(); + } std::string self_endpoint = config::hostname.empty() ? get_local_ip(config::priority_networks) : config::hostname; self_endpoint = fmt::format("{}:{}", self_endpoint, config::brpc_listen_port); diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 4f8fc6f7299de7..28e262226ed59f 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -151,6 +151,8 @@ void MetaServiceImpl::begin_txn(::google::protobuf::RpcController* controller, return; } // get count before txn reset, if not we will lose these count + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); //2. Get txn id from version stamp @@ -783,7 +785,9 @@ void scan_tmp_rowset( return; } DORIS_CLOUD_DEFER { - if (stats && txn) stats->get_counter += txn->num_get_keys(); + if (!stats || !txn) return; + stats->get_bytes += txn->get_bytes(); + stats->get_counter += txn->num_get_keys(); }; // Get db id with txn id @@ -958,6 +962,9 @@ void commit_txn_immediately( } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); @@ -1129,6 +1136,7 @@ void commit_txn_immediately( version_values.clear(); if (last_pending_txn_id > 0) { + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); txn.reset(); TEST_SYNC_POINT_CALLBACK("commit_txn_immediately::advance_last_pending_txn_id", @@ -1591,6 +1599,9 @@ void commit_txn_eventually( } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); @@ -1611,6 +1622,7 @@ void commit_txn_eventually( TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::need_repair_tablet_idx", &need_repair_tablet_idx); if (need_repair_tablet_idx) { + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); txn.reset(); repair_tablet_index(txn_kv, code, msg, instance_id, db_id, txn_id, tmp_rowsets_meta); @@ -1676,6 +1688,7 @@ void commit_txn_eventually( if (last_pending_txn_id > 0) { TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::advance_last_pending_txn_id", &last_pending_txn_id); + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); txn.reset(); std::shared_ptr task = @@ -2008,6 +2021,9 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); @@ -2103,7 +2119,7 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* << " tmp_rowsets_meta.size()=" << tmp_rowsets_meta.size(); sub_txn_to_tmp_rowsets_meta.emplace(sub_txn_id, std::move(tmp_rowsets_meta)); } - + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); // Create a read/write txn for guarantee consistency txn.reset(); @@ -3166,6 +3182,8 @@ void MetaServiceImpl::begin_sub_txn(::google::protobuf::RpcController* controlle msg = ss.str(); return; } + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); @@ -3676,6 +3694,9 @@ TxnErrorCode internal_clean_label(std::shared_ptr txn_kv, const std::stri } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); stats.get_counter += txn->num_get_keys(); stats.put_counter += txn->num_put_keys(); stats.del_counter += txn->num_del_keys(); @@ -3837,6 +3858,7 @@ void MetaServiceImpl::clean_txn_label(::google::protobuf::RpcController* control } DORIS_CLOUD_DEFER { if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); }; diff --git a/cloud/src/meta-store/mem_txn_kv.cpp b/cloud/src/meta-store/mem_txn_kv.cpp index 5e049c6fe411f5..87189dd1d8e874 100644 --- a/cloud/src/meta-store/mem_txn_kv.cpp +++ b/cloud/src/meta-store/mem_txn_kv.cpp @@ -23,10 +23,13 @@ #include #include #include +#include #include #include #include +#include "common/defer.h" +#include "common/util.h" #include "cpp/sync_point.h" #include "meta-store/txn_kv_error.h" #include "txn_kv.h" @@ -249,6 +252,7 @@ void Transaction::put(std::string_view key, std::string_view val) { op_list_.emplace_back(ModifyOpType::PUT, k, v); ++num_put_keys_; kv_->put_count_++; + kv_->put_bytes_ += key.size() + val.size(); put_bytes_ += key.size() + val.size(); approximate_bytes_ += key.size() + val.size(); } @@ -271,8 +275,6 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, int limit) { TEST_SYNC_POINT_CALLBACK("memkv::Transaction::get", &limit); std::lock_guard l(lock_); - num_get_keys_++; - kv_->get_count_++; std::string begin_k(begin.data(), begin.size()); std::string end_k(end.data(), end.size()); // TODO: figure out what happen if range_get has part of unreadable_keys @@ -285,6 +287,8 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, } TxnErrorCode Transaction::inner_get(const std::string& key, std::string* val, bool snapshot) { + num_get_keys_++; + kv_->get_count_++; // Read your writes. auto it = writes_.find(key); if (it != writes_.end()) { @@ -303,6 +307,8 @@ TxnErrorCode Transaction::inner_get(const std::string& key, std::string* val, bo return TxnErrorCode::TXN_KEY_NOT_FOUND; } } + get_bytes_ += val->size() + key.size(); + kv_->get_bytes_ += val->size() + key.size(); return TxnErrorCode::TXN_OK; } @@ -349,6 +355,10 @@ TxnErrorCode Transaction::inner_get(const std::string& begin, const std::string& std::vector> kv_list(kv_map.begin(), kv_map.end()); num_get_keys_ += kv_list.size(); kv_->get_count_ += kv_list.size(); + for (auto& [k, v] : kv_list) { + get_bytes_ += k.size() + v.size(); + kv_->get_bytes_ += k.size() + v.size(); + } *iter = std::make_unique(std::move(kv_list), more); return TxnErrorCode::TXN_OK; } @@ -362,6 +372,8 @@ void Transaction::atomic_set_ver_key(std::string_view key_prefix, std::string_vi op_list_.emplace_back(ModifyOpType::ATOMIC_SET_VER_KEY, k, v); ++num_put_keys_; + + kv_->put_bytes_ += k.size() + val.size(); put_bytes_ += key_prefix.size() + val.size(); approximate_bytes_ += key_prefix.size() + val.size(); } @@ -375,6 +387,7 @@ void Transaction::atomic_set_ver_value(std::string_view key, std::string_view va op_list_.emplace_back(ModifyOpType::ATOMIC_SET_VER_VAL, k, v); ++num_put_keys_; + kv_->put_bytes_ += key.size() + value.size(); put_bytes_ += key.size() + value.size(); approximate_bytes_ += key.size() + value.size(); } @@ -389,6 +402,7 @@ void Transaction::atomic_add(std::string_view key, int64_t to_add) { ++num_put_keys_; kv_->put_count_++; put_bytes_ += key.size() + 8; + kv_->put_bytes_ += key.size() + 8; approximate_bytes_ += key.size() + 8; } @@ -412,6 +426,7 @@ void Transaction::remove(std::string_view key) { kv_->del_count_++; ++num_del_keys_; + kv_->del_bytes_ += key.size(); delete_bytes_ += key.size(); approximate_bytes_ += key.size(); } @@ -433,6 +448,7 @@ void Transaction::remove(std::string_view begin, std::string_view end) { kv_->del_count_ += 2; // same as normal txn num_del_keys_ += 2; + kv_->del_bytes_ += begin.size() + end.size(); delete_bytes_ += begin.size() + end.size(); approximate_bytes_ += begin.size() + end.size(); } diff --git a/cloud/src/meta-store/mem_txn_kv.h b/cloud/src/meta-store/mem_txn_kv.h index 337f23a0361971..d74938d682560a 100644 --- a/cloud/src/meta-store/mem_txn_kv.h +++ b/cloud/src/meta-store/mem_txn_kv.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -57,6 +58,9 @@ class MemTxnKv : public TxnKv, public std::enable_shared_from_this { TxnErrorCode get_kv(const std::string& begin, const std::string& end, int64_t version, int limit, bool* more, std::map* kv_list); + int64_t get_bytes_ {}; + int64_t put_bytes_ {}; + int64_t del_bytes_ {}; int64_t get_count_ {}; int64_t put_count_ {}; int64_t del_count_ {}; @@ -203,6 +207,8 @@ class Transaction : public cloud::Transaction { size_t put_bytes() const override { return put_bytes_; } + size_t get_bytes() const override { return get_bytes_; } + private: TxnErrorCode inner_get(const std::string& key, std::string* val, bool snapshot); @@ -229,6 +235,7 @@ class Transaction : public cloud::Transaction { size_t num_put_keys_ {0}; size_t delete_bytes_ {0}; size_t put_bytes_ {0}; + size_t get_bytes_ {0}; }; class RangeGetIterator : public cloud::RangeGetIterator { @@ -250,6 +257,12 @@ class RangeGetIterator : public cloud::RangeGetIterator { bool more() override { return more_; } + int64_t get_kv_bytes() const override { + int64_t kv_bytes {}; + for (auto& [k, v] : kvs_) kv_bytes += k.size() + v.size(); + return kv_bytes; + } + int size() override { return kvs_size_; } void reset() override { idx_ = 0; } diff --git a/cloud/src/meta-store/txn_kv.cpp b/cloud/src/meta-store/txn_kv.cpp index 76f3c724958067..2b07870a321e72 100644 --- a/cloud/src/meta-store/txn_kv.cpp +++ b/cloud/src/meta-store/txn_kv.cpp @@ -407,6 +407,7 @@ TxnErrorCode Transaction::get(std::string_view key, std::string* val, bool snaps << " key=" << hex(key); return cast_as_txn_code(err); } + get_bytes_ += len + key.size(); if (!found) return TxnErrorCode::TXN_KEY_NOT_FOUND; *val = std::string((char*)ret, len); @@ -440,6 +441,7 @@ TxnErrorCode Transaction::get(std::string_view begin, std::string_view end, std::unique_ptr ret(new RangeGetIterator(fut)); RETURN_IF_ERROR(ret->init()); num_get_keys_ += ret->size(); + get_bytes_ += ret->get_kv_bytes(); g_bvar_txn_kv_get_count_normalized << ret->size(); *(iter) = std::move(ret); @@ -655,6 +657,7 @@ TxnErrorCode Transaction::batch_get(std::vector>* res const uint8_t* ret; int len; err = fdb_future_get_value(future, &found, &ret, &len); + num_get_keys_++; if (err) { LOG(WARNING) << __PRETTY_FUNCTION__ << " failed to fdb_future_get_value err=" << fdb_get_error(err) @@ -665,12 +668,12 @@ TxnErrorCode Transaction::batch_get(std::vector>* res res->push_back(std::nullopt); continue; } + get_bytes_ += len + key.size(); res->push_back(std::string((char*)ret, len)); } futures.clear(); } DCHECK_EQ(res->size(), num_keys); - num_get_keys_ += num_keys; return TxnErrorCode::TXN_OK; } diff --git a/cloud/src/meta-store/txn_kv.h b/cloud/src/meta-store/txn_kv.h index 56757cdb68ac9f..45d3b1cc8100e0 100644 --- a/cloud/src/meta-store/txn_kv.h +++ b/cloud/src/meta-store/txn_kv.h @@ -20,6 +20,8 @@ #include #include +#include +#include #include #include #include @@ -253,6 +255,11 @@ class Transaction { **/ virtual size_t delete_bytes() const = 0; + /** + * @brief return the bytes of the get values consumed. + **/ + virtual size_t get_bytes() const = 0; + /** * @brief return the bytes of the put key and values consumed. **/ @@ -298,6 +305,12 @@ class RangeGetIterator { */ virtual int size() = 0; + /** + * Get all FDBKeyValue's bytes include key's bytes + * RangeGetIterator created by get range, when get range the keys in the range too. + */ + virtual int64_t get_kv_bytes() const = 0; + /** * Resets to initial state, some kinds of iterators may not support this function. */ @@ -457,6 +470,14 @@ class RangeGetIterator : public cloud::RangeGetIterator { int size() override { return kvs_size_; } + int64_t get_kv_bytes() const override { + int64_t total_bytes {}; + for (int i = 0; i < kvs_size_; i++) { + total_bytes += kvs_[i].key_length + kvs_[i].value_length; + } + return total_bytes; + } + void reset() override { idx_ = 0; } std::string next_begin_key() override { @@ -580,6 +601,8 @@ class Transaction : public cloud::Transaction { size_t put_bytes() const override { return put_bytes_; } + size_t get_bytes() const override { return get_bytes_; } + private: std::shared_ptr db_ {nullptr}; bool commited_ = false; @@ -590,6 +613,7 @@ class Transaction : public cloud::Transaction { size_t num_del_keys_ {0}; size_t num_put_keys_ {0}; size_t delete_bytes_ {0}; + size_t get_bytes_ {0}; size_t put_bytes_ {0}; size_t approximate_bytes_ {0}; }; diff --git a/cloud/test/rpc_kv_bvar_test.cpp b/cloud/test/rpc_kv_bvar_test.cpp index 9ff4aac0c20749..d737718d313ceb 100644 --- a/cloud/test/rpc_kv_bvar_test.cpp +++ b/cloud/test/rpc_kv_bvar_test.cpp @@ -507,19 +507,25 @@ void finish_schema_change_job( meta_service->finish_tablet_job(&cntl, &req, &res, nullptr); } +void clear_memkv_count_bytes(MemTxnKv* memkv) { + memkv->get_count_ = memkv->put_count_ = memkv->del_count_ = 0; + memkv->get_bytes_ = memkv->put_bytes_ = memkv->del_bytes_ = 0; +} + // create_tablets TEST(RpcKvBvarTest, CreateTablets) { auto meta_service = get_meta_service(); auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); LOG(INFO) << "CreateTablets: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_create_tablets_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_create_tablets_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_tablets_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_tablets_put_counter.get({mock_instance})); } @@ -532,9 +538,7 @@ TEST(RpcKvBvarTest, GetTablet) { constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); brpc::Controller cntl; GetTabletRequest req; @@ -544,7 +548,9 @@ TEST(RpcKvBvarTest, GetTablet) { meta_service->get_tablet(&cntl, &req, &resp, nullptr); LOG(INFO) << "GetTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_tablet_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_get_counter.get({mock_instance})); } @@ -556,15 +562,16 @@ TEST(RpcKvBvarTest, GetTabletStats) { constexpr auto table_id = 10021, index_id = 10022, partition_id = 10023, tablet_id = 10024; create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); GetTabletStatsResponse res; get_tablet_stats(meta_service.get(), table_id, index_id, partition_id, tablet_id, res); LOG(INFO) << "GetTabletStats: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_tablet_stats_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_tablet_stats_get_counter.get({mock_instance})); } @@ -583,14 +590,16 @@ TEST(RpcKvBvarTest, UpdateTablet) { tablet_meta_info->set_tablet_id(tablet_id); tablet_meta_info->set_is_in_memory(true); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->update_tablet(&cntl, &req, &resp, nullptr); LOG(INFO) << "UpdateTablet: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_update_tablet_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_update_tablet_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tablet_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tablet_put_counter.get({mock_instance})); } @@ -636,14 +645,16 @@ TEST(RpcKvBvarTest, BeginTxn) { int64_t txn_id = 0; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); LOG(INFO) << "BeginTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_begin_txn_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_begin_txn_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_txn_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_txn_put_counter.get({mock_instance})); } @@ -659,14 +670,17 @@ TEST(RpcKvBvarTest, CommitTxn) { int64_t txn_id = 0; ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); commit_txn(meta_service.get(), db_id, txn_id, label); LOG(INFO) << "CommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_commit_txn_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_commit_txn_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_commit_txn_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_txn_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_txn_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_txn_del_counter.get({mock_instance})); @@ -716,16 +730,18 @@ TEST(RpcKvBvarTest, PrecommitTxn) { req.set_precommit_timeout_ms(36000); PrecommitTxnResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->precommit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "PrecommitTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_precommit_txn_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_precommit_txn_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_precommit_txn_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_precommit_txn_put_counter.get({mock_instance})); } @@ -741,9 +757,7 @@ TEST(RpcKvBvarTest, AbortTxn) { int64_t txn_id = 0; ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); brpc::Controller cntl; AbortTxnRequest req; @@ -756,7 +770,12 @@ TEST(RpcKvBvarTest, AbortTxn) { ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "AbortTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_abort_txn_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_abort_txn_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_abort_txn_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_txn_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_abort_txn_del_counter.get({mock_instance})); @@ -773,9 +792,7 @@ TEST(RpcKvBvarTest, GetTxn) { int64_t txn_id = 0; ASSERT_NO_FATAL_FAILURE(begin_txn(meta_service.get(), db_id, label, table_id, txn_id)); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); brpc::Controller cntl; GetTxnRequest req; @@ -788,7 +805,10 @@ TEST(RpcKvBvarTest, GetTxn) { ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "GetTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_txn_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_get_counter.get({mock_instance})); } @@ -812,14 +832,15 @@ TEST(RpcKvBvarTest, GetTxnId) { req.set_label(label); req.set_db_id(db_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_txn_id(&cntl, &req, &res, nullptr); LOG(INFO) << "GetTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_txn_id_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_txn_id_get_counter.get({mock_instance})); } @@ -839,14 +860,16 @@ TEST(RpcKvBvarTest, PrepareRowset) { rowset.mutable_load_id()->set_hi(123); rowset.mutable_load_id()->set_lo(456); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); prepare_rowset(meta_service.get(), rowset, res); LOG(INFO) << "PrepareRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_prepare_rowset_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_prepare_rowset_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_rowset_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_rowset_put_counter.get({mock_instance})); } @@ -863,16 +886,17 @@ TEST(RpcKvBvarTest, GetRowset) { // check get tablet response check_get_tablet(meta_service.get(), tablet_id, 1); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); // check get rowset response GetRowsetResponse get_rowset_res; get_rowset(meta_service.get(), table_id, index_id, partition_id, tablet_id, get_rowset_res); LOG(INFO) << "GetRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_rowset_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_rowset_get_counter.get({mock_instance})); } @@ -904,10 +928,7 @@ TEST(RpcKvBvarTest, UpdateTmpRowset) { EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); req->mutable_rowset_meta()->CopyFrom(rowset); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; - + clear_memkv_count_bytes(mem_kv.get()); meta_service->update_tmp_rowset(&cntl, req, &res, nullptr); if (!arena) { @@ -915,7 +936,11 @@ TEST(RpcKvBvarTest, UpdateTmpRowset) { } LOG(INFO) << "UpdateTmpRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_update_tmp_rowset_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_update_tmp_rowset_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_tmp_rowset_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_tmp_rowset_put_counter.get({mock_instance})); } @@ -930,15 +955,18 @@ TEST(RpcKvBvarTest, CommitRowset) { create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); auto tmp_rowset = create_rowset(txn_id, tablet_id, partition_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); CreateRowsetResponse res; commit_rowset(meta_service.get(), tmp_rowset, res); LOG(INFO) << "CommitRowset: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_commit_rowset_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_commit_rowset_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_commit_rowset_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_rowset_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_rowset_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_rowset_del_counter.get({mock_instance})); @@ -960,15 +988,16 @@ TEST(RpcKvBvarTest, GetVersion) { req.set_table_id(table_id); req.set_partition_id(partition_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); GetVersionResponse resp; meta_service->get_version(&ctrl, &req, &resp, nullptr); LOG(INFO) << "GetVersion: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_version_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_version_get_counter.get({mock_instance})); } @@ -991,15 +1020,16 @@ TEST(RpcKvBvarTest, GetSchemaDict) { txn->put(dict_key, "dict_val"); EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); GetSchemaDictResponse resp; meta_service->get_schema_dict(&ctrl, &req, &resp, nullptr); LOG(INFO) << "GetSchemaDict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_schema_dict_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_schema_dict_get_counter.get({mock_instance})); } @@ -1021,14 +1051,20 @@ TEST(RpcKvBvarTest, GetDeleteBitmapUpdateLock) { int64_t lock_id = -2; int64_t initiator = 1009; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); get_delete_bitmap_update_lock(meta_service.get(), table_id, t1p1, lock_id, initiator); LOG(INFO) << "GetDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ - << ", " << mem_kv->del_count_; + << ", " << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " + << mem_kv->put_bytes_ << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, + g_bvar_rpc_kv_get_delete_bitmap_update_lock_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_update_lock_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, @@ -1062,15 +1098,21 @@ TEST(RpcKvBvarTest, UpdateDeleteBitmap) { // will be splited and stored in 5 KVs std::string data1(split_size * 5, 'c'); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); update_delete_bitmap(meta_service.get(), update_delete_bitmap_req, update_delete_bitmap_res, table_id, t1p1, lock_id, initiator, tablet_id, txn_id, version, data1); LOG(INFO) << "UpdateDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, + g_bvar_rpc_kv_update_delete_bitmap_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, + g_bvar_rpc_kv_update_delete_bitmap_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, + g_bvar_rpc_kv_update_delete_bitmap_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_delete_bitmap_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, @@ -1114,15 +1156,16 @@ TEST(RpcKvBvarTest, GetDeleteBitmap) { get_delete_bitmap_req.add_begin_versions(0); get_delete_bitmap_req.add_end_versions(version); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_delete_bitmap(reinterpret_cast(&ctrl), &get_delete_bitmap_req, &get_delete_bitmap_res, nullptr); LOG(INFO) << "GetDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_delete_bitmap_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_delete_bitmap_get_counter.get({mock_instance})); } @@ -1154,16 +1197,22 @@ TEST(RpcKvBvarTest, RemoveDeleteBitmapUpdateLock) { remove_req.set_lock_id(lock_id); remove_req.set_initiator(initiator); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->remove_delete_bitmap_update_lock( reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &remove_req, &remove_res, nullptr); LOG(INFO) << "RemoveDeleteBitmapUpdateLock: " << mem_kv->get_count_ << ", " - << mem_kv->put_count_ << ", " << mem_kv->del_count_; + << mem_kv->put_count_ << ", " << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ + << ", " << mem_kv->put_bytes_ << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, + g_bvar_rpc_kv_remove_delete_bitmap_update_lock_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_remove_delete_bitmap_update_lock_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, @@ -1206,14 +1255,16 @@ TEST(RpcKvBvarTest, RemoveDeleteBitmap) { req.add_rowset_ids("rowset_ids"); req.set_cloud_unique_id("test_cloud_unique_id"); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->remove_delete_bitmap(&ctrl, &req, &resp, nullptr); LOG(INFO) << "RemoveDeleteBitmap: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->del_bytes_, + g_bvar_rpc_kv_remove_delete_bitmap_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_remove_delete_bitmap_del_counter.get({mock_instance})); } @@ -1229,15 +1280,17 @@ TEST(RpcKvBvarTest, StartTabletJob) { create_tablet(meta_service.get(), table_id, index_id, partition_id, tablet_id); StartTabletJobResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); start_compaction_job(meta_service.get(), tablet_id, "compaction1", "ip:port", 0, 0, TabletCompactionJobPB::BASE, res); LOG(INFO) << "StartTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_start_tablet_job_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_start_tablet_job_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_start_tablet_job_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_start_tablet_job_put_counter.get({mock_instance})); } @@ -1280,14 +1333,17 @@ TEST(RpcKvBvarTest, FinishTabletJob) { compaction->set_lease(now + 10); req.mutable_job()->mutable_idx()->set_tablet_id(tablet_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->finish_tablet_job(&cntl, &req, &finish_res_2, nullptr); LOG(INFO) << "FinishTabletJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_finish_tablet_job_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_finish_tablet_job_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_finish_tablet_job_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_tablet_job_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_tablet_job_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_tablet_job_del_counter.get({mock_instance})); @@ -1325,14 +1381,16 @@ TEST(RpcKvBvarTest, PrepareIndex) { req.set_is_new_table(true); IndexResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->prepare_index(&ctrl, &req, &res, nullptr); LOG(INFO) << "PrepareIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_prepare_index_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_prepare_index_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_index_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_index_put_counter.get({mock_instance})); } @@ -1371,14 +1429,17 @@ TEST(RpcKvBvarTest, CommitIndex) { meta_service->prepare_index(&ctrl, &req, &res, nullptr); res.Clear(); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->commit_index(&ctrl, &req, &res, nullptr); LOG(INFO) << "CommitIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_commit_index_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_commit_index_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_commit_index_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_index_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_index_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_index_del_counter.get({mock_instance})); @@ -1437,15 +1498,17 @@ TEST(RpcKvBvarTest, DropIndex) { req.add_index_ids(index_id); IndexResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->drop_index(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "DropIndex: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_drop_index_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_drop_index_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_index_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_index_put_counter.get({mock_instance})); } @@ -1484,14 +1547,16 @@ TEST(RpcKvBvarTest, PreparePartition) { txn->atomic_add(tbl_version_key, 1); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->prepare_partition(&ctrl, &req, &res, nullptr); LOG(INFO) << "PreparePartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_prepare_partition_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_prepare_partition_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_prepare_partition_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_prepare_partition_put_counter.get({mock_instance})); } @@ -1530,14 +1595,17 @@ TEST(RpcKvBvarTest, CommitPartition) { ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); meta_service->prepare_partition(&ctrl, &req, &res, nullptr); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->commit_partition(&ctrl, &req, &res, nullptr); LOG(INFO) << "CommitPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_commit_partition_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_commit_partition_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_commit_partition_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_commit_partition_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_commit_partition_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_commit_partition_del_counter.get({mock_instance})); @@ -1570,14 +1638,15 @@ TEST(RpcKvBvarTest, CheckKv) { check_keys_pb.add_partition_ids(partition_id + 1); req_check.mutable_check_keys()->CopyFrom(check_keys_pb); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->check_kv(&ctrl, &req_check, &res_check, nullptr); LOG(INFO) << "CheckKv: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_check_kv_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_kv_get_counter.get({mock_instance})); } @@ -1604,14 +1673,16 @@ TEST(RpcKvBvarTest, DropPartition) { req.add_partition_ids(partition_id); req.set_need_update_table_version(true); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->drop_partition(&ctrl, &req, &res, nullptr); LOG(INFO) << "DropPartition: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_drop_partition_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_drop_partition_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_drop_partition_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_drop_partition_put_counter.get({mock_instance})); } @@ -1636,14 +1707,15 @@ TEST(RpcKvBvarTest, GetObjStoreInfo) { GetObjStoreInfoRequest req; req.set_cloud_unique_id("test_cloud_unique_id"); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_obj_store_info(&cntl, &req, &res, nullptr); LOG(INFO) << "GetObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_obj_store_info_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_obj_store_info_get_counter.get({mock_instance})); } @@ -1674,15 +1746,18 @@ TEST(RpcKvBvarTest, AlterStorageVault) { brpc::Controller cntl; AlterObjStoreInfoResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->alter_storage_vault(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "AlterStorageVault: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_alter_storage_vault_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_alter_storage_vault_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_alter_storage_vault_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_storage_vault_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, @@ -1737,15 +1812,19 @@ TEST(RpcKvBvarTest, AlterObjStoreInfo) { brpc::Controller cntl; AlterObjStoreInfoResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->alter_obj_store_info(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "AlterObjStoreInfo: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, + g_bvar_rpc_kv_alter_obj_store_info_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, + g_bvar_rpc_kv_alter_obj_store_info_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_obj_store_info_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, @@ -1804,15 +1883,17 @@ TEST(RpcKvBvarTest, UpdateAkSk) { brpc::Controller cntl; UpdateAkSkResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->update_ak_sk(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "UpdateAkSk: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_update_ak_sk_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_update_ak_sk_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_update_ak_sk_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_update_ak_sk_put_counter.get({mock_instance})); @@ -1853,16 +1934,17 @@ TEST(RpcKvBvarTest, CreateInstance) { sp->enable_processing(); CreateInstanceResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->create_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "CreateInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_create_instance_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_instance_get_counter.get({mock_instance})); sp->clear_all_call_backs(); @@ -1910,16 +1992,17 @@ TEST(RpcKvBvarTest, GetInstance) { GetInstanceRequest req; GetInstanceResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); req.set_cloud_unique_id("1:test_instance:m-n3qdpyal27rh8iprxx"); meta_service->get_instance(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "GetInstance: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_instance_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_instance_get_counter.get({mock_instance})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -1927,7 +2010,6 @@ TEST(RpcKvBvarTest, GetInstance) { } // alter_cluster -// alter cluster have not do kv op // TEST(RpcKvBvarTest, AlterCluster) { // auto meta_service = get_meta_service(); // auto mem_kv = std::dynamic_pointer_cast(meta_service->txn_kv()); @@ -1982,16 +2064,18 @@ TEST(RpcKvBvarTest, GetCluster) { req.set_cluster_name("test_cluster"); GetClusterResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_cluster(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "GetCluster: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_cluster_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_get_cluster_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_get_cluster_put_counter.get({mock_instance})); } @@ -2073,16 +2157,16 @@ TEST(RpcKvBvarTest, CreateStage) { create_stage_request.set_cloud_unique_id(cloud_unique_id); create_stage_request.mutable_stage()->CopyFrom(stage); CreateStageResponse create_stage_response; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->create_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &create_stage_request, &create_stage_response, nullptr); - ASSERT_EQ(create_stage_response.status().code(), MetaServiceCode::OK); LOG(INFO) << "CreateStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_create_stage_get_bytes.get({instance_id})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_create_stage_put_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_create_stage_get_counter.get({instance_id})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_create_stage_put_counter.get({instance_id})); sp->clear_all_call_backs(); @@ -2169,7 +2253,6 @@ TEST(RpcKvBvarTest, GetStage) { CreateStageResponse create_stage_response; meta_service->create_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &create_stage_request, &create_stage_response, nullptr); - ASSERT_EQ(create_stage_response.status().code(), MetaServiceCode::OK); GetStageRequest get_stage_req; get_stage_req.set_type(StagePB::INTERNAL); @@ -2180,17 +2263,15 @@ TEST(RpcKvBvarTest, GetStage) { // get existent internal stage GetStageResponse res2; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_stage(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &get_stage_req, &res2, nullptr); - ASSERT_EQ(res2.status().code(), MetaServiceCode::OK); - ASSERT_EQ(1, res2.stage().size()); LOG(INFO) << "GetStage: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_stage_get_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_stage_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -2268,16 +2349,15 @@ TEST(RpcKvBvarTest, GetIam) { request.set_cloud_unique_id(cloud_unique_id); GetIamResponse response; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; - + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_iam(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &request, &response, nullptr); ASSERT_EQ(response.status().code(), MetaServiceCode::OK); LOG(INFO) << "GetIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_iam_get_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_iam_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -2311,16 +2391,17 @@ TEST(RpcKvBvarTest, AlterIam) { brpc::Controller cntl; AlterIamResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->alter_iam(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "AlterIam: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_alter_iam_get_bytes.get({"alter_iam_instance"})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_alter_iam_put_bytes.get({"alter_iam_instance"})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_iam_get_counter.get({"alter_iam_instance"})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_iam_put_counter.get({"alter_iam_instance"})); } @@ -2390,14 +2471,15 @@ TEST(RpcKvBvarTest, AlterRamUser) { alter_ram_user_request.set_instance_id(instance_id); alter_ram_user_request.mutable_ram_user()->CopyFrom(ram_user); AlterRamUserResponse alter_ram_user_response; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->alter_ram_user(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &alter_ram_user_request, &alter_ram_user_response, nullptr); LOG(INFO) << "AlterRamUser: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_alter_ram_user_get_bytes.get({instance_id})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_alter_ram_user_put_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_alter_ram_user_get_counter.get({instance_id})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_alter_ram_user_put_counter.get({instance_id})); sp->clear_all_call_backs(); @@ -2444,16 +2526,17 @@ TEST(RpcKvBvarTest, BeginCopy) { } BeginCopyResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->begin_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &begin_copy_request, &res, nullptr); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "BeginCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_begin_copy_get_bytes.get({instance_id})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_begin_copy_put_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_copy_get_counter.get({instance_id})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_copy_put_counter.get({instance_id})); sp->clear_all_call_backs(); @@ -2511,9 +2594,7 @@ TEST(RpcKvBvarTest, GetCopyJob) { get_copy_job_request.set_copy_id("test_copy_id"); get_copy_job_request.set_group_id(0); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); GetCopyJobResponse res; meta_service->get_copy_job(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), @@ -2521,7 +2602,9 @@ TEST(RpcKvBvarTest, GetCopyJob) { ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "GetCopyJob: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_copy_job_get_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_job_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -2579,9 +2662,7 @@ TEST(RpcKvBvarTest, FinishCopy) { finish_copy_request.set_group_id(0); finish_copy_request.set_action(FinishCopyRequest::COMMIT); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); FinishCopyResponse res; meta_service->finish_copy(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), @@ -2589,7 +2670,11 @@ TEST(RpcKvBvarTest, FinishCopy) { ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "FinishCopy: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_finish_copy_get_bytes.get({instance_id})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_finish_copy_put_bytes.get({instance_id})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_finish_copy_del_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_finish_copy_get_counter.get({instance_id})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_finish_copy_put_counter.get({instance_id})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_finish_copy_del_counter.get({instance_id})); @@ -2645,9 +2730,7 @@ TEST(RpcKvBvarTest, GetCopyFiles) { get_copy_file_req.set_stage_id(stage_id); get_copy_file_req.set_table_id(table_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); GetCopyFilesResponse res; meta_service->get_copy_files(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), @@ -2655,7 +2738,9 @@ TEST(RpcKvBvarTest, GetCopyFiles) { ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "GetCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_copy_files_get_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_copy_files_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -2716,16 +2801,16 @@ TEST(RpcKvBvarTest, FilterCopyFiles) { request.add_object_files()->CopyFrom(object_file); } FilterCopyFilesResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->filter_copy_files(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &request, &res, nullptr); ASSERT_EQ(res.status().code(), MetaServiceCode::OK); LOG(INFO) << "FilterCopyFiles: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_filter_copy_files_get_bytes.get({instance_id})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_filter_copy_files_get_counter.get({instance_id})); sp->clear_all_call_backs(); sp->clear_trace(); @@ -2779,9 +2864,7 @@ TEST(RpcKvBvarTest, GetClusterStatus) { req.add_instance_ids(mock_instance); GetClusterStatusResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_cluster_status(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); @@ -2789,7 +2872,9 @@ TEST(RpcKvBvarTest, GetClusterStatus) { ASSERT_EQ(res.details().at(0).clusters().size(), 3); LOG(INFO) << "GetClusterStatus: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_get_cluster_status_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_cluster_status_get_counter.get({mock_instance})); } @@ -2829,16 +2914,17 @@ TEST(RpcKvBvarTest, GetCurrentMaxTxnId) { max_txn_id_req.set_cloud_unique_id(cloud_unique_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->get_current_max_txn_id( reinterpret_cast<::google::protobuf::RpcController*>(&max_txn_id_cntl), &max_txn_id_req, &max_txn_id_res, nullptr); LOG(INFO) << "GetCurrentMaxTxnId: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, + g_bvar_rpc_kv_get_current_max_txn_id_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_get_current_max_txn_id_get_counter.get({mock_instance})); } @@ -2894,15 +2980,17 @@ TEST(RpcKvBvarTest, BeginSubTxn) { req.mutable_table_ids()->Add(t2); BeginSubTxnResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->begin_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "BeginSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_begin_sub_txn_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_begin_sub_txn_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_begin_sub_txn_del_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_begin_sub_txn_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_begin_sub_txn_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_begin_sub_txn_del_counter.get({mock_instance})); @@ -2973,15 +3061,16 @@ TEST(RpcKvBvarTest, AbortSubTxn) { req.mutable_table_ids()->Add(t2); AbortSubTxnResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->abort_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "AbortSubTxn: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_abort_sub_txn_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_abort_sub_txn_put_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_sub_txn_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_abort_sub_txn_put_counter.get({mock_instance})); } @@ -3033,16 +3122,17 @@ TEST(RpcKvBvarTest, AbortTxnWithCoordinator) { abort_txn_req.set_ip(host); abort_txn_req.set_start_time(cur_time + 3600); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->abort_txn_with_coordinator( reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), &abort_txn_req, &abort_txn_resp, nullptr); LOG(INFO) << "AbortTxnWithCoordinator: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ - << ", " << mem_kv->del_count_; + << ", " << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " + << mem_kv->put_bytes_ << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, + g_bvar_rpc_kv_abort_txn_with_coordinator_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_abort_txn_with_coordinator_get_counter.get({mock_instance})); } @@ -3085,16 +3175,16 @@ TEST(RpcKvBvarTest, CheckTxnConflict) { check_txn_conflict_req.set_end_txn_id(txn_id + 1); check_txn_conflict_req.add_table_ids(table_id); - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->check_txn_conflict( reinterpret_cast<::google::protobuf::RpcController*>(&begin_txn_cntl), &check_txn_conflict_req, &check_txn_conflict_res, nullptr); LOG(INFO) << "CheckTxnConflict: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_check_txn_conflict_get_bytes.get({mock_instance})); ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_check_txn_conflict_get_counter.get({mock_instance})); } @@ -3125,17 +3215,19 @@ TEST(RpcKvBvarTest, CleanTxnLabel) { req.add_labels(label); CleanTxnLabelResponse res; - mem_kv->get_count_ = 0; - mem_kv->put_count_ = 0; - mem_kv->del_count_ = 0; + clear_memkv_count_bytes(mem_kv.get()); meta_service->clean_txn_label(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); LOG(INFO) << "CleanTxnLabel: " << mem_kv->get_count_ << ", " << mem_kv->put_count_ << ", " - << mem_kv->del_count_; + << mem_kv->del_count_ << ", " << mem_kv->get_bytes_ << ", " << mem_kv->put_bytes_ + << ", " << mem_kv->del_bytes_; ASSERT_EQ(mem_kv->get_count_, g_bvar_rpc_kv_clean_txn_label_get_counter.get({mock_instance})); ASSERT_EQ(mem_kv->put_count_, g_bvar_rpc_kv_clean_txn_label_put_counter.get({mock_instance})); ASSERT_EQ(mem_kv->del_count_, g_bvar_rpc_kv_clean_txn_label_del_counter.get({mock_instance})); + ASSERT_EQ(mem_kv->get_bytes_, g_bvar_rpc_kv_clean_txn_label_get_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->put_bytes_, g_bvar_rpc_kv_clean_txn_label_put_bytes.get({mock_instance})); + ASSERT_EQ(mem_kv->del_bytes_, g_bvar_rpc_kv_clean_txn_label_del_bytes.get({mock_instance})); } } // namespace doris::cloud From 2e0cd89f9cedfbff1f7c17114333e741676ae136 Mon Sep 17 00:00:00 2001 From: koarz Date: Thu, 17 Jul 2025 12:27:17 +0800 Subject: [PATCH 239/572] branch-3.0 [enhance](meta-service) add real request ip for be rpc #53114. (#53320) pick https://github.com/apache/doris/pull/53114 --- be/src/cloud/cloud_meta_mgr.cpp | 2 + cloud/src/meta-service/meta_service_helper.h | 51 ++++++++++------- gensrc/proto/cloud.proto | 58 +++++++++++++++++++- 3 files changed, 89 insertions(+), 22 deletions(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 622a38c3159c16..28174ebf1eb9c9 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -388,6 +388,8 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res, static_assert(std::is_base_of_v<::google::protobuf::Message, Request>); static_assert(std::is_base_of_v<::google::protobuf::Message, Response>); + const_cast(req).set_request_ip(BackendOptions::get_be_endpoint()); + int retry_times = 0; uint32_t duration_ms = 0; std::string error_msg; diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 9d99e28b643b05..321b0ceb3a67f2 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -97,35 +97,44 @@ inline std::string encryt_sk(std::string debug_string) { template void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) { if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side(); + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side(); + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() - << " table_id=" << req->table_id() << " tablet_id=" << req->tablet_id() - << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " table_id=" << req->table_id() + << " tablet_id=" << req->tablet_id() << " lock_id=" << req->lock_id() + << " initiator=" << req->initiator() << " delete_bitmap_size=" << req->segment_delete_bitmaps_size(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " tablet_id=" << req->tablet_id() << " rowset_size=" << req->rowset_ids_size(); } else if constexpr (std::is_same_v) { - VLOG_DEBUG << "begin " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " tablet size: " << req->tablet_idx().size(); } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { - VLOG_DEBUG << "begin " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " request=" << req->ShortDebugString(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " tablet_id=" << req->tablet_id() << " rowset_size=" << req->rowset_ids_size(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() - << " table_id=" << req->table_id() << " lock_id=" << req->lock_id() - << " initiator=" << req->initiator() << " expiration=" << req->expiration() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " table_id=" << req->table_id() + << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() + << " expiration=" << req->expiration() << " require_compaction_stats=" << req->require_compaction_stats(); } else { - LOG(INFO) << "begin " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() + << " original client ip: " << req->request_ip() << " request=" << req->ShortDebugString(); } } @@ -138,21 +147,21 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re res->clear_partition_ids(); res->clear_versions(); } - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { if (res->status().code() != MetaServiceCode::OK) { res->clear_rowset_meta(); } - VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v) { - VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString() << " tablet size: " << res->tablet_stats().size(); } else if constexpr (std::is_same_v || std::is_same_v) { - VLOG_DEBUG << "finish " << func_name << " from " << ctrl->remote_side() + VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { if (res->status().code() != MetaServiceCode::OK) { @@ -161,7 +170,7 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re res->clear_versions(); res->clear_segment_delete_bitmaps(); } - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString() << " tablet=" << res->tablet_id() << " delete_bitmap_count=" << res->segment_delete_bitmaps_size(); @@ -171,16 +180,16 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re res->clear_cumulative_compaction_cnts(); res->clear_cumulative_points(); } - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v || std::is_same_v) { std::string debug_string = encryt_sk(res->DebugString()); TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string); - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << debug_string; } else { - LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() << " response=" << res->ShortDebugString(); } } diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 56060be67bae15..bc4a219d02691f 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -677,6 +677,7 @@ message ServiceRegistryPB { message BeginTxnRequest { optional string cloud_unique_id = 1; // For auth optional TxnInfoPB txn_info = 2; + optional string request_ip = 3; } message BeginTxnResponse { @@ -693,6 +694,7 @@ message PrecommitTxnRequest { optional int64 txn_id = 3; optional TxnCommitAttachmentPB commit_attachment = 4; optional int64 precommit_timeout_ms = 5; + optional string request_ip = 6; } message PrecommitTxnResponse { @@ -714,6 +716,7 @@ message CommitTxnRequest { optional bool is_txn_load = 9; repeated SubTxnInfo sub_txn_infos = 10; optional bool enable_txn_lazy_commit = 11; + optional string request_ip = 12; } message SubTxnInfo { @@ -747,6 +750,7 @@ message AbortTxnRequest { optional string label = 4; optional string reason = 5; optional TxnCommitAttachmentPB commit_attachment = 6; + optional string request_ip = 7; } message AbortTxnResponse { @@ -759,6 +763,7 @@ message GetTxnRequest { optional int64 db_id = 2; optional int64 txn_id = 3; optional string label = 4; + optional string request_ip = 5; } message GetTxnResponse { @@ -771,6 +776,7 @@ message GetTxnIdRequest { optional int64 db_id = 2; optional string label = 3; repeated TxnStatusPB txn_status = 4; + optional string request_ip = 5; } message GetTxnIdResponse { @@ -788,6 +794,7 @@ message BeginSubTxnRequest { repeated int64 table_ids = 5; // a random label used to generate a sub_txn_id optional string label = 6; + optional string request_ip = 7; } message BeginSubTxnResponse { @@ -806,6 +813,7 @@ message AbortSubTxnRequest { optional int64 db_id = 5; // set table_ids in txn_info repeated int64 table_ids = 6; + optional string request_ip = 7; } message AbortSubTxnResponse { @@ -815,6 +823,7 @@ message AbortSubTxnResponse { message GetCurrentMaxTxnRequest { optional string cloud_unique_id = 1; // For auth + optional string request_ip = 2; } message GetCurrentMaxTxnResponse { @@ -827,6 +836,7 @@ message AbortTxnWithCoordinatorRequest { optional string ip = 2; optional int64 id = 3; optional int64 start_time = 4; + optional string request_ip = 5; } message AbortTxnWithCoordinatorResponse { @@ -839,6 +849,7 @@ message CheckTxnConflictRequest { optional int64 end_txn_id = 3; repeated int64 table_ids = 4; optional bool ignore_timeout_txn = 5; + optional string request_ip = 6; } message CheckTxnConflictResponse { @@ -851,6 +862,7 @@ message CleanTxnLabelRequest { optional string cloud_unique_id = 1; // For auth optional int64 db_id = 2; repeated string labels = 3; + optional string request_ip = 4; } message CleanTxnLabelResponse { @@ -871,6 +883,8 @@ message GetVersionRequest { // True if get table version optional bool is_table_version = 9; + + optional string request_ip = 10; }; message GetVersionResponse { @@ -888,6 +902,7 @@ message GetVersionResponse { message GetObjStoreInfoRequest { optional string cloud_unique_id = 1; // For auth + optional string request_ip = 2; }; message AlterObjStoreInfoRequest { @@ -914,6 +929,7 @@ message AlterObjStoreInfoRequest { optional Operation op = 3; optional StorageVaultPB vault = 4; optional bool set_as_default_storage_vault = 5; + optional string request_ip = 6; } message AlterObjStoreInfoResponse { @@ -926,6 +942,7 @@ message UpdateAkSkRequest { optional string instance_id = 1; repeated RamUserPB internal_bucket_user = 2; optional RamUserPB ram_user = 3; + optional string request_ip = 4; } message UpdateAkSkResponse { @@ -946,6 +963,7 @@ message CreateTabletsRequest { repeated doris.TabletMetaCloudPB tablet_metas = 2; optional string storage_vault_name = 3; optional int64 db_id = 4; + optional string request_ip = 5; } message CreateTabletsResponse { @@ -957,6 +975,7 @@ message CreateTabletsResponse { message UpdateTabletRequest { optional string cloud_unique_id = 1; // For auth repeated TabletMetaInfoPB tablet_meta_infos = 2; + optional string request_ip = 3; } message UpdateTabletResponse { @@ -967,6 +986,7 @@ message UpdateTabletSchemaRequest { optional string cloud_unique_id = 1; // For auth optional int64 tablet_id = 2; optional doris.TabletSchemaCloudPB tablet_schema = 3; + optional string request_ip = 4; } message UpdateTabletSchemaResponse { @@ -976,12 +996,14 @@ message UpdateTabletSchemaResponse { message DropTabletRequest { optional string cloud_unique_id = 1; // For auth optional int64 tablet_id = 2; + optional string request_ip = 3; // TODO: There are more fields TBD } message GetTabletRequest { optional string cloud_unique_id = 1; // For auth optional int64 tablet_id = 2; + optional string request_ip = 3; // TODO: There are more fields TBD } @@ -996,6 +1018,7 @@ message CreateRowsetRequest { optional bool temporary = 3; optional int64 txn_id = 4; optional string tablet_job_id = 5; + optional string request_ip = 6; } message CreateRowsetResponse { @@ -1019,6 +1042,7 @@ message GetRowsetRequest { // returned schema format on rowset schema, used in variant type directly. // for compability reason we use FILL_WITH_DICT as default optional SchemaOp schema_op = 8 [default = FILL_WITH_DICT]; + optional string request_ip = 9; } message GetRowsetResponse { @@ -1032,6 +1056,7 @@ message GetRowsetResponse { message GetSchemaDictRequest { optional string cloud_unique_id = 1; // For auth optional int64 index_id = 2; + optional string request_ip = 3; } message GetSchemaDictResponse { @@ -1046,6 +1071,7 @@ message IndexRequest { optional int64 expiration = 4; optional int64 db_id = 5; optional bool is_new_table = 6; // For table version + optional string request_ip = 7; } message IndexResponse { @@ -1060,6 +1086,7 @@ message PartitionRequest { optional int64 expiration = 5; optional int64 db_id = 6; optional bool need_update_table_version = 7; + optional string request_ip = 8; } message PartitionResponse { @@ -1085,6 +1112,7 @@ message CreateInstanceRequest { optional RamUserPB ram_user = 5; optional bool sse_enabled = 6; optional StorageVaultPB vault = 7; + optional string request_ip = 8; } message CreateInstanceResponse { @@ -1105,6 +1133,7 @@ message AlterInstanceRequest { optional string instance_id = 1; optional Operation op = 2; optional string name = 3; + optional string request_ip = 4; } message AlterInstanceResponse { @@ -1114,6 +1143,7 @@ message AlterInstanceResponse { message GetInstanceRequest { optional string instance_id = 1; optional string cloud_unique_id = 2; + optional string request_ip = 3; } message GetInstanceResponse { @@ -1142,6 +1172,7 @@ message AlterClusterRequest { optional Operation op = 4; // for SQL mode rename cluster, rename to cluster name eq instance empty cluster name, need drop empty cluster optional bool replace_if_existing_empty_target_cluster = 5; + optional string request_ip = 6; } message AlterClusterResponse { @@ -1154,12 +1185,14 @@ message GetClusterRequest { optional string cluster_id = 3; optional string cluster_name = 4; optional string mysql_user_name = 5; + optional string request_ip = 6; } message GetClusterStatusRequest { repeated string instance_ids = 1; // Redundant field repeated string cloud_unique_ids = 2; optional ClusterStatus status = 3; + optional string request_ip = 4; } message GetClusterStatusResponse { @@ -1180,6 +1213,7 @@ message GetClusterResponse { message GetTabletStatsRequest { optional string cloud_unique_id = 1; repeated TabletIndexPB tablet_idx = 2; + optional string request_ip = 3; } message GetTabletStatsResponse { @@ -1190,6 +1224,7 @@ message GetTabletStatsResponse { message CreateStageRequest { optional string cloud_unique_id = 1; optional StagePB stage = 2; + optional string request_ip = 3; } message CreateStageResponse { @@ -1202,6 +1237,7 @@ message GetStageRequest { optional string mysql_user_name = 3; optional StagePB.StageType type = 4; optional string mysql_user_id = 5; + optional string request_ip = 6; } message GetStageResponse { @@ -1216,6 +1252,7 @@ message DropStageRequest { optional StagePB.StageType type = 4; optional string mysql_user_id = 5; optional string reason = 6; + optional string request_ip = 7; } message DropStageResponse { @@ -1224,6 +1261,7 @@ message DropStageResponse { message GetIamRequest { optional string cloud_unique_id = 1; + optional string request_ip = 2; } message GetIamResponse { @@ -1236,6 +1274,7 @@ message AlterIamRequest { optional string account_id = 1; optional string ak = 2; optional string sk = 3; + optional string request_ip = 4; } message AlterIamResponse { @@ -1245,6 +1284,7 @@ message AlterIamResponse { message AlterRamUserRequest { optional string instance_id = 1; optional RamUserPB ram_user = 2; + optional string request_ip = 3; } message AlterRamUserResponse { @@ -1254,6 +1294,7 @@ message AlterRamUserResponse { message StartTabletJobRequest { optional string cloud_unique_id = 1; // For auth optional TabletJobInfoPB job = 2; + optional string request_ip = 3; } message StartTabletJobResponse { @@ -1272,6 +1313,7 @@ message FinishTabletJobRequest { optional string cloud_unique_id = 1; // For auth optional Action action = 2; optional TabletJobInfoPB job = 3; + optional string request_ip = 4; } message FinishTabletJobResponse { @@ -1293,6 +1335,7 @@ message BeginCopyRequest { optional int64 file_num_limit = 10; optional int64 file_size_limit = 11; optional int64 file_meta_size_limit = 12; + optional string request_ip = 13; } message BeginCopyResponse { @@ -1315,6 +1358,7 @@ message FinishCopyRequest { optional int32 group_id = 6; optional Action action = 7; optional int64 finish_time_ms = 8; + optional string request_ip = 9; } message FinishCopyResponse { @@ -1327,6 +1371,7 @@ message GetCopyJobRequest { optional int64 table_id = 3; optional string copy_id = 4; optional int32 group_id = 5; + optional string request_ip = 6; } message GetCopyJobResponse { @@ -1338,6 +1383,7 @@ message GetCopyFilesRequest { optional string cloud_unique_id = 1; optional string stage_id = 2; optional int64 table_id = 3; + optional string request_ip = 4; } message GetCopyFilesResponse { @@ -1350,6 +1396,7 @@ message FilterCopyFilesRequest { optional string stage_id = 2; optional int64 table_id = 3; repeated ObjectFilePB object_files = 4; + optional string request_ip = 5; } message FilterCopyFilesResponse { @@ -1359,6 +1406,7 @@ message FilterCopyFilesResponse { message RecycleInstanceRequest { repeated string instance_ids = 1; + optional string request_ip = 2; } message StatisticsRecycleRequest { @@ -1466,6 +1514,7 @@ message UpdateDeleteBitmapRequest { // for load txn only optional int64 next_visible_version = 14; + optional string request_ip = 15; } message UpdateDeleteBitmapResponse { @@ -1482,6 +1531,7 @@ message GetDeleteBitmapRequest { optional int64 base_compaction_cnt = 7; optional int64 cumulative_compaction_cnt = 8; optional int64 cumulative_point = 9; + optional string request_ip = 10; } message GetDeleteBitmapResponse { @@ -1500,6 +1550,7 @@ message RemoveDeleteBitmapRequest { repeated string rowset_ids = 3; repeated int64 begin_versions = 4; repeated int64 end_versions = 5; + optional string request_ip = 6; } message RemoveDeleteBitmapResponse { @@ -1525,6 +1576,7 @@ message GetDeleteBitmapUpdateLockRequest { optional int64 expiration = 6; optional bool require_compaction_stats = 7 [default = false]; repeated TabletIndexPB tablet_indexes = 8; + optional string request_ip = 9; } message GetDeleteBitmapUpdateLockResponse { @@ -1541,6 +1593,7 @@ message RemoveDeleteBitmapUpdateLockRequest { optional int64 tablet_id = 3; optional int64 lock_id = 4; optional int64 initiator = 5; + optional string request_ip = 6; } message RemoveDeleteBitmapUpdateLockResponse { @@ -1551,6 +1604,7 @@ message GetRLTaskCommitAttachRequest { optional string cloud_unique_id = 1; // For auth optional int64 db_id = 2; optional int64 job_id = 3; + optional string request_ip = 4; } message GetRLTaskCommitAttachResponse { @@ -1563,6 +1617,7 @@ message ResetRLProgressRequest { optional int64 db_id = 2; optional int64 job_id = 3; map partition_to_offset = 4; + optional string request_ip = 5; } message ResetRLProgressResponse { @@ -1583,7 +1638,8 @@ message CheckKVRequest { } optional string cloud_unique_id = 1; // For auth optional CheckKeyInfos check_keys = 2; - optional Operation op = 3; + optional Operation op = 3; + optional string request_ip = 4; } message CheckKVResponse { From 52034ee15d512cbbc610553afe70dd5bddaec9c0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:13:49 +0800 Subject: [PATCH 240/572] branch-3.0: [Fix](test) Fix regression test "test_s3tables_write_partitions" by renaming table name. #53294 (#53414) Cherry-picked from #53294 Co-authored-by: Qi Chen --- .../test_s3tables_write_partitions.groovy | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy index a3b23981ab982e..143eba25a16860 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy @@ -18,13 +18,13 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,external_docker_iceberg") { def format_compressions = ["parquet_snappy", "orc_zlib"] - def test_columns_out_of_order = { String format_compression, String catalog_name -> + def test_s3_columns_out_of_order = { String format_compression, String catalog_name -> def parts = format_compression.split("_") def format = parts[0] def compression = parts[1] - sql """ drop table if exists columns_out_of_order_source_tbl_${format_compression} """ + sql """ drop table if exists s3_columns_out_of_order_source_tbl_${format_compression} """ sql """ - CREATE TABLE columns_out_of_order_source_tbl_${format_compression} ( + CREATE TABLE s3_columns_out_of_order_source_tbl_${format_compression} ( `col3` bigint, `col6` int, `col1` bigint, @@ -37,9 +37,9 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext "write-format"=${format} ) """; - sql """ drop table if exists columns_out_of_order_target_tbl_${format_compression} """ + sql """ drop table if exists s3_columns_out_of_order_target_tbl_${format_compression} """ sql """ - CREATE TABLE columns_out_of_order_target_tbl_${format_compression} ( + CREATE TABLE s3_columns_out_of_order_target_tbl_${format_compression} ( `col1` bigint, `col2` bigint, `col3` bigint, @@ -57,23 +57,23 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext """; sql """ - INSERT INTO columns_out_of_order_source_tbl_${format_compression} ( + INSERT INTO s3_columns_out_of_order_source_tbl_${format_compression} ( col1, col2, col3, col4, col5, col6 ) VALUES (1, 2, 3, 4, 5, 6); """ - order_qt_columns_out_of_order01 """ SELECT * FROM columns_out_of_order_source_tbl_${format_compression} """ + order_qt_columns_out_of_order01 """ SELECT * FROM s3_columns_out_of_order_source_tbl_${format_compression} """ sql """ - INSERT INTO columns_out_of_order_target_tbl_${format_compression} ( + INSERT INTO s3_columns_out_of_order_target_tbl_${format_compression} ( col1, col2, col3, col4, col5, col6 ) VALUES (1, 2, 3, 4, 5, 6); """ - order_qt_columns_out_of_order02 """ SELECT * FROM columns_out_of_order_target_tbl_${format_compression} """ + order_qt_columns_out_of_order02 """ SELECT * FROM s3_columns_out_of_order_target_tbl_${format_compression} """ - sql """ drop table columns_out_of_order_source_tbl_${format_compression} """ - sql """ drop table columns_out_of_order_target_tbl_${format_compression} """ - sql """ drop database if exists `test_columns_out_of_order` """; + sql """ drop table s3_columns_out_of_order_source_tbl_${format_compression} """ + sql """ drop table s3_columns_out_of_order_target_tbl_${format_compression} """ + sql """ drop database if exists `test_s3_columns_out_of_order` """; } String enabled = context.config.otherConfigs.get("enableExternalIcebergTest") @@ -100,7 +100,7 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext try { for (String format_compression in format_compressions) { logger.info("Process format_compression " + format_compression) - test_columns_out_of_order(format_compression, catalog_name) + test_s3_columns_out_of_order(format_compression, catalog_name) } } finally { } From 619ad445ffb79e0062908a9701443d3fcb2ad62f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 13:14:34 +0800 Subject: [PATCH 241/572] branch-3.0: [Fix](Case) Fix some noncurrent cases #53332 (#53410) Cherry-picked from #53332 Co-authored-by: abmdocrt --- .../test_cold_data_compaction_fault_injection.groovy | 1 + .../test_cumu_compaction_delay_fault_injection.groovy | 1 + 2 files changed, 2 insertions(+) diff --git a/regression-test/suites/fault_injection_p0/test_cold_data_compaction_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_cold_data_compaction_fault_injection.groovy index 3f173edace52a3..0e7bd86a21cc8b 100644 --- a/regression-test/suites/fault_injection_p0/test_cold_data_compaction_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_cold_data_compaction_fault_injection.groovy @@ -19,6 +19,7 @@ import com.amazonaws.services.s3.model.ListObjectsRequest import java.util.function.Supplier suite("test_cold_data_compaction_fault_injection", "nonConcurrent") { + GetDebugPoint().clearDebugPointsForAllBEs() def retryUntilTimeout = { int timeoutSecond, Supplier closure -> long start = System.currentTimeMillis() while (true) { diff --git a/regression-test/suites/fault_injection_p0/test_cumu_compaction_delay_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_cumu_compaction_delay_fault_injection.groovy index 9d2f6f59b45f9b..635840dbe0cc44 100644 --- a/regression-test/suites/fault_injection_p0/test_cumu_compaction_delay_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_cumu_compaction_delay_fault_injection.groovy @@ -18,6 +18,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_cumu_compaction_delay_fault_injection","nonConcurrent") { + GetDebugPoint().clearDebugPointsForAllBEs() String backend_id; def backendId_to_backendIP = [:] From e299d7c5d6332a92c74271f14a011e68594644b2 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 17 Jul 2025 13:15:47 +0800 Subject: [PATCH 242/572] branch-3.0: [fix](case)Use relative cooldown time instead of absolutetime in storage policy test case (#51572) (#53325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … #51572 --- .../suites/cold_heat_separation/policy/drop.groovy | 13 +++++++++---- .../policy/drop_hdfs_reource.groovy | 12 ++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/regression-test/suites/cold_heat_separation/policy/drop.groovy b/regression-test/suites/cold_heat_separation/policy/drop.groovy index 6879e4891afc3c..c7a178588abb3d 100644 --- a/regression-test/suites/cold_heat_separation/policy/drop.groovy +++ b/regression-test/suites/cold_heat_separation/policy/drop.groovy @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter + suite("drop_policy") { def storage_exist = { name -> def show_storage_policy = sql """ @@ -84,7 +88,7 @@ suite("drop_policy") { CREATE STORAGE POLICY ${use_policy} PROPERTIES( "storage_resource" = "${resource_table_use}", - "cooldown_datetime" = "2025-06-08 00:00:00" + "cooldown_datetime" = "2035-06-08 00:00:00" ); """ assertEquals(storage_exist.call(use_policy), true) @@ -100,12 +104,13 @@ suite("drop_policy") { """ // can drop, no table use assertEquals(drop_policy_ret.size(), 1) - + def zonedDateTime = ZonedDateTime.now(ZoneId.of("UTC")).plusYears(1) + def futureTime = zonedDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) def create_succ_2 = try_sql """ CREATE STORAGE POLICY IF NOT EXISTS drop_policy_test_has_table_binded PROPERTIES( "storage_resource" = "${resource_table_use}", - "cooldown_datetime" = "2025-06-08 00:00:00" + "cooldown_datetime" = "${futureTime}" ); """ assertEquals(storage_exist.call("drop_policy_test_has_table_binded"), true) @@ -114,7 +119,7 @@ suite("drop_policy") { CREATE STORAGE POLICY IF NOT EXISTS drop_policy_test_has_table_bind_1 PROPERTIES( "storage_resource" = "${resource_table_use}", - "cooldown_datetime" = "2025-06-08 00:00:00" + "cooldown_datetime" = "${futureTime}" ); """ assertEquals(storage_exist.call("drop_policy_test_has_table_bind_1"), true) diff --git a/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy b/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy index 39bbaf09e7a439..78f3dd70756bb2 100644 --- a/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy +++ b/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy @@ -14,6 +14,9 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.time.ZoneId suite("drop_hdfs_policy") { def storage_exist = { name -> @@ -74,13 +77,15 @@ suite("drop_hdfs_policy") { "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider" ); """ - + def zonedDateTime = ZonedDateTime.now(ZoneId.of("UTC")).plusYears(1) + + def futureTime = zonedDateTime.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) def create_succ_1 = try_sql """ CREATE STORAGE POLICY ${use_policy} PROPERTIES( "storage_resource" = "${resource_table_use}", - "cooldown_datetime" = "2025-06-08 00:00:00" + "cooldown_datetime" = "${futureTime}" ); """ assertEquals(storage_exist.call(use_policy), true) @@ -96,12 +101,11 @@ suite("drop_hdfs_policy") { """ // can drop, no table use assertEquals(drop_policy_ret.size(), 1) - def create_succ_2 = try_sql """ CREATE STORAGE POLICY IF NOT EXISTS drop_policy_test_has_table_binded_hdfs PROPERTIES( "storage_resource" = "${resource_table_use}", - "cooldown_datetime" = "2025-06-08 00:00:00" + "cooldown_datetime" = "${futureTime}" ); """ assertEquals(storage_exist.call("drop_policy_test_has_table_binded_hdfs"), true) From d7cbe470ac9c92c4cb4c96534b34d1f5ea585755 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Thu, 17 Jul 2025 13:21:51 +0800 Subject: [PATCH 243/572] [fix](json) incorrect results of json_contains (#53291) (#53364) Fix the incorrect results and remove unused code. Pick: #53291 --- be/src/util/jsonb_document.h | 12 +- be/src/vec/functions/function_json.cpp | 132 ------------------ .../jsonb_p0/test_jsonb_load_and_function.out | 10 +- ...est_jsonb_load_unique_key_and_function.out | 4 +- .../json_functions/test_json_function.out | 17 +++ .../json_functions/test_json_function.groovy | 19 +++ 6 files changed, 49 insertions(+), 145 deletions(-) diff --git a/be/src/util/jsonb_document.h b/be/src/util/jsonb_document.h index 7f50c4012baff8..6d269f6dd52ddd 100644 --- a/be/src/util/jsonb_document.h +++ b/be/src/util/jsonb_document.h @@ -1326,12 +1326,12 @@ inline bool JsonbValue::contains(JsonbValue* rhs) const { } case JsonbType::T_Object: { if (rhs->isObject()) { - auto* str_value1 = (ObjectVal*)this; - auto* str_value2 = (ObjectVal*)rhs; - for (int i = 0; i < str_value2->numElem(); ++i) { - JsonbKeyValue* key = str_value2->getJsonbKeyValue(i); - JsonbValue* value = str_value1->find(key->getKeyStr(), key->klen()); - if (key != nullptr && value != nullptr && !value->contains(key->value())) { + const auto* obj_value1 = (ObjectVal*)this; + const auto* obj_value2 = (ObjectVal*)rhs; + for (int i = 0; i < obj_value2->numElem(); ++i) { + JsonbKeyValue* key = obj_value2->getJsonbKeyValue(i); + JsonbValue* value = obj_value1->find(key->getKeyStr(), key->klen()); + if (value == nullptr || !value->contains(key->value())) { return false; } } diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp index ab1ff616f2b1e7..7ce4049249d943 100644 --- a/be/src/vec/functions/function_json.cpp +++ b/be/src/vec/functions/function_json.cpp @@ -1105,137 +1105,6 @@ class FunctionJsonValid : public IFunction { } }; -class FunctionJsonContains : public IFunction { -public: - static constexpr auto name = "json_contains"; - static FunctionPtr create() { return std::make_shared(); } - - String get_name() const override { return name; } - - size_t get_number_of_arguments() const override { return 3; } - - DataTypePtr get_return_type_impl(const DataTypes& arguments) const override { - return make_nullable(std::make_shared()); - } - - DataTypes get_variadic_argument_types_impl() const override { - return {std::make_shared(), std::make_shared(), - std::make_shared()}; - } - - bool use_default_implementation_for_nulls() const override { return false; } - - bool json_contains_object(const rapidjson::Value& target, - const rapidjson::Value& search_value) const { - if (!target.IsObject() || !search_value.IsObject()) { - return false; - } - - for (auto itr = search_value.MemberBegin(); itr != search_value.MemberEnd(); ++itr) { - if (!target.HasMember(itr->name) || !json_contains(target[itr->name], itr->value)) { - return false; - } - } - - return true; - } - - bool json_contains_array(const rapidjson::Value& target, - const rapidjson::Value& search_value) const { - if (!target.IsArray() || !search_value.IsArray()) { - return false; - } - - for (auto itr = search_value.Begin(); itr != search_value.End(); ++itr) { - bool found = false; - for (auto target_itr = target.Begin(); target_itr != target.End(); ++target_itr) { - if (json_contains(*target_itr, *itr)) { - found = true; - break; - } - } - if (!found) { - return false; - } - } - - return true; - } - - bool json_contains(const rapidjson::Value& target, const rapidjson::Value& search_value) const { - if (target == search_value) { - return true; - } - - if (target.IsObject() && search_value.IsObject()) { - return json_contains_object(target, search_value); - } - - if (target.IsArray() && search_value.IsArray()) { - return json_contains_array(target, search_value); - } - - return false; - } - - Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments, - size_t result, size_t input_rows_count) const override { - const IColumn& col_json = *(block.get_by_position(arguments[0]).column); - const IColumn& col_search = *(block.get_by_position(arguments[1]).column); - const IColumn& col_path = *(block.get_by_position(arguments[2]).column); - - auto null_map = ColumnUInt8::create(input_rows_count, 0); - - const ColumnString* col_json_string = check_and_get_column(col_json); - const ColumnString* col_search_string = check_and_get_column(col_search); - const ColumnString* col_path_string = check_and_get_column(col_path); - - if (!col_json_string || !col_search_string || !col_path_string) { - return Status::RuntimeError("Illegal column should be ColumnString"); - } - - auto col_to = ColumnVector::create(); - auto& vec_to = col_to->get_data(); - size_t size = col_json.size(); - vec_to.resize(size); - - for (size_t i = 0; i < input_rows_count; ++i) { - if (col_json.is_null_at(i) || col_search.is_null_at(i) || col_path.is_null_at(i)) { - null_map->get_data()[i] = 1; - vec_to[i] = 0; - continue; - } - - const auto& json_val = col_json_string->get_data_at(i); - const auto& search_val = col_search_string->get_data_at(i); - const auto& path_val = col_path_string->get_data_at(i); - - std::string_view json_string(json_val.data, json_val.size); - std::string_view search_string(search_val.data, search_val.size); - std::string_view path_string(path_val.data, path_val.size); - - rapidjson::Document document; - auto target_val = get_json_object(json_string, path_string, &document); - if (target_val == nullptr || target_val->IsNull()) { - vec_to[i] = 0; - } else { - rapidjson::Document search_doc; - search_doc.Parse(search_string.data(), search_string.size()); - if (json_contains(*target_val, search_doc)) { - vec_to[i] = 1; - } else { - vec_to[i] = 0; - } - } - } - - block.replace_by_position(result, - ColumnNullable::create(std::move(col_to), std::move(null_map))); - - return Status::OK(); - } -}; - class FunctionJsonUnquote : public IFunction { public: static constexpr auto name = "json_unquote"; @@ -1658,7 +1527,6 @@ void register_function_json(SimpleFunctionFactory& factory) { FunctionJsonNullable>>(); factory.register_function(); - factory.register_function(); factory.register_function>(); factory.register_function>(); diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index d7d7611931fdb3..a27fa14581f292 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -8528,7 +8528,7 @@ true 8 1152921504606846976 false 9 6.18 false 10 "abcd" false -11 {} true +11 {} false 12 {"k1":"v31","k2":300} true 13 [] false 14 [123,456] false @@ -8538,13 +8538,13 @@ true 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false 29 12524337771678448270 false 30 -9223372036854775808 false 31 18446744073709551615 false -32 {"":"v1"} true -33 {"":1," ":"v1"} true -34 {"":1,"ab":"v1"," ":"v1"," ":2} true +32 {"":"v1"} false +33 {"":1," ":"v1"} false +34 {"":1,"ab":"v1"," ":"v1"," ":2} false -- !select_json_contains -- 1 \N \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out index 427b6426d6816f..36d14661668d1f 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out @@ -6047,7 +6047,7 @@ true 8 1152921504606846976 false 9 6.18 false 10 "abcd" false -11 {} true +11 {} false 12 {"k1":"v31","k2":300} true 13 [] false 14 [123,456] false @@ -6057,7 +6057,7 @@ true 18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} true 26 \N \N 27 {"k1":"v1","k2":200} false -28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} true +28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} false -- !select_json_contains -- 1 \N \N diff --git a/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out b/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out index c22acc171345a5..47da4ec2bbef1c 100644 --- a/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out +++ b/regression-test/data/query_p0/sql_functions/json_functions/test_json_function.out @@ -187,3 +187,20 @@ doris -- !sql -- \N +-- !json_contains1 -- +false + +-- !json_contains2 -- +true + +-- !json_contains3 -- +false + +-- !json_contains4 -- +true + +-- !json_contains5 -- +false + +-- !json_contains6 -- +\N diff --git a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy index fe4dfc55315b91..517db385cc2964 100644 --- a/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/json_functions/test_json_function.groovy @@ -91,4 +91,23 @@ suite("test_json_function", "arrow_flight_sql") { qt_sql """select get_json_string('{"name\\k" : 123}', '\$.name\\k')""" qt_sql """select get_json_string('{"name\\k" : 123}', '\$.name\\\\k')""" qt_sql """select get_json_string('{"name\\k" : 123}', '\$.name\\\\\\k')""" + + qt_json_contains1 """ + SELECT JSON_CONTAINS('{"age": 30, "name": "John", "hobbies": ["reading", "swimming"]}', '{"invalid": "format"}'); + """ + qt_json_contains2 """ + SELECT JSON_CONTAINS('{"age": 25, "name": "Alice", "hobbies": ["painting", "music"]}', '{"age": 25}'); + """ + qt_json_contains3 """ + SELECT JSON_CONTAINS('{"age": 25, "name": "Alice", "hobbies": ["painting", "music"]}', '{"age": "25"}'); + """ + qt_json_contains4 """ + SELECT JSON_CONTAINS('{"age": 25, "name": "Alice", "hobbies": ["painting", "music"]}', '"music"', '\$.hobbies[1]'); + """ + qt_json_contains5 """ + SELECT JSON_CONTAINS('{"age": 25, "name": "Alice", "hobbies": ["painting", "music"]}', '"music"', '\$.hobbies[0]'); + """ + qt_json_contains6 """ + SELECT JSON_CONTAINS(NULL, '"music"', '{"age": 25}'); + """ } From 710d44767e71413090027ab24ac5fc3f7c667271 Mon Sep 17 00:00:00 2001 From: minghong Date: Thu, 17 Jul 2025 15:52:23 +0800 Subject: [PATCH 244/572] branch-3.0 [fix](nereids) fix case out: order_push_down (#53427) --- .../limit_push_down/order_push_down.out | 32 +++++++++---------- .../limit_push_down/order_push_down.groovy | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out b/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out index 5f0e3fa80998db..694e609eaeb2af 100644 --- a/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out +++ b/regression-test/data/nereids_rules_p0/limit_push_down/order_push_down.out @@ -195,18 +195,18 @@ PhysicalResultSink ----------PhysicalOlapScan[t1] -- !limit_sort_filter -- -PhysicalDeferMaterializeResultSink ---PhysicalDeferMaterializeTopN -----PhysicalDeferMaterializeTopN +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalTopN[LOCAL_SORT] ------filter((t1.id = 1)) ---------PhysicalDeferMaterializeOlapScan[t1] +--------PhysicalOlapScan[t1] -- !limit_offset_sort_filter -- -PhysicalDeferMaterializeResultSink ---PhysicalDeferMaterializeTopN -----PhysicalDeferMaterializeTopN +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalTopN[LOCAL_SORT] ------filter((t1.id = 1)) ---------PhysicalDeferMaterializeOlapScan[t1] +--------PhysicalOlapScan[t1] -- !limit_subquery_order_by_inside_limit_outside -- PhysicalResultSink @@ -275,18 +275,18 @@ PhysicalResultSink ----------PhysicalOlapScan[t1] -- !limit_filter -- -PhysicalDeferMaterializeResultSink ---PhysicalDeferMaterializeTopN -----PhysicalDeferMaterializeTopN +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalTopN[LOCAL_SORT] ------filter((t1.id = 1)) ---------PhysicalDeferMaterializeOlapScan[t1] +--------PhysicalOlapScan[t1] -- !limit_offset_filter -- -PhysicalDeferMaterializeResultSink ---PhysicalDeferMaterializeTopN -----PhysicalDeferMaterializeTopN +PhysicalResultSink +--PhysicalTopN[MERGE_SORT] +----PhysicalTopN[LOCAL_SORT] ------filter((t1.id = 1)) ---------PhysicalDeferMaterializeOlapScan[t1] +--------PhysicalOlapScan[t1] -- !limit_project_filter -- PhysicalResultSink diff --git a/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy b/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy index bbf4a64f1180a7..a2210d1514db2d 100644 --- a/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy +++ b/regression-test/suites/nereids_rules_p0/limit_push_down/order_push_down.groovy @@ -26,7 +26,7 @@ suite("order_push_down") { sql 'set be_number_for_test=3' sql "set disable_nereids_rules='push_down_top_n_distinct_through_union'" sql "set disable_nereids_rules=PRUNE_EMPTY_PARTITION" - sql 'set enable_two_phase_read_opt = true' + sql 'set enable_two_phase_read_opt = false' //`limit 1 offset 1 + sort, project`: qt_limit_offset_sort_project """ explain shape plan SELECT t1.id FROM t1 ORDER BY id LIMIT 1 OFFSET 1; """ From cd635d1bebbd3f89ca5b8b41c10972924dfcc083 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 16:47:43 +0800 Subject: [PATCH 245/572] branch-3.0: [fix](case) fix group commit case #53292 (#53430) Cherry-picked from #53292 Co-authored-by: meiyi --- be/src/runtime/group_commit_mgr.cpp | 15 +++++++++------ ...up_commit_async_wal_msg_fault_injection.groovy | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp index 553da51a96bf69..c8397e5a110abc 100644 --- a/be/src/runtime/group_commit_mgr.cpp +++ b/be/src/runtime/group_commit_mgr.cpp @@ -430,13 +430,15 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_ RuntimeState* state) { Status st; Status result_status; - DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.err_status", - { status = Status::InternalError(""); }); + DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.err_status", { + status = Status::InternalError("LoadBlockQueue._finish_group_commit_load.err_status"); + }); DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.load_error", { status = Status::InternalError("load_error"); }); if (status.ok()) { - DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.commit_error", - { status = Status::InternalError(""); }); + DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.commit_error", { + status = Status::InternalError("LoadBlockQueue._finish_group_commit_load.commit_error"); + }); // commit txn TLoadTxnCommitRequest request; // deprecated and should be removed in 3.1, use token instead @@ -529,8 +531,9 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_ // status: exec_plan_fragment result // st: commit txn rpc status // result_status: commit txn result - DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.err_st", - { st = Status::InternalError(""); }); + DBUG_EXECUTE_IF("LoadBlockQueue._finish_group_commit_load.err_st", { + st = Status::InternalError("LoadBlockQueue._finish_group_commit_load.err_st"); + }); if (status.ok() && st.ok() && (result_status.ok() || result_status.is())) { if (!config::group_commit_wait_replay_wal_finish) { diff --git a/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy index 2d0b91a01ed5fc..0f572986bdc38f 100644 --- a/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy @@ -156,7 +156,7 @@ suite("test_group_commit_async_wal_msg_fault_injection","nonConcurrent") { ) engine=olap DISTRIBUTED BY HASH(`k`) BUCKETS 5 - properties("replication_num" = "1", "group_commit_interval_ms" = "4000") + properties("replication_num" = "1", "group_commit_interval_ms" = "10000") """ GetDebugPoint().clearDebugPointsForAllBEs() try { @@ -167,7 +167,7 @@ suite("test_group_commit_async_wal_msg_fault_injection","nonConcurrent") { set 'group_commit', 'async_mode' unset 'label' file 'group_commit_wal_msg.csv' - time 10000 + time 6000 } getRowCount(5) } finally { From c3d59de161d0526e75cce58edee22c5b8dce1cfd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 19:13:31 +0800 Subject: [PATCH 246/572] branch-3.0: [fix](p0)Fix prepare stmt case failure. #53435 (#53459) Cherry-picked from #53435 Co-authored-by: James --- regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index c333035a945203..353b4b4552fff3 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -61,6 +61,7 @@ suite("test_prepared_stmt", "nonConcurrent") { qt_sql """select * from ${tableName} order by 1, 2, 3""" sql "set global max_prepared_stmt_count = 10000" sql "set enable_fallback_to_original_planner = false" + sql """set global enable_server_side_prepared_statement = true""" def stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1" assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt_read.class) From 27ff0dff14e00c9b72e8f8b7354d2176dd3dcf60 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 20:36:46 +0800 Subject: [PATCH 247/572] branch-3.0: [chore](feut) add maven.test.failure.ignore #51134 (#53387) Cherry-picked from #51134 Co-authored-by: feifeifeimoon --- run-fe-ut.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-fe-ut.sh b/run-fe-ut.sh index cce439b02bff55..0c9d8de31ecfb0 100755 --- a/run-fe-ut.sh +++ b/run-fe-ut.sh @@ -121,7 +121,7 @@ if [[ "${RUN}" -eq 1 ]]; then else echo "Run Frontend UT" if [[ "${COVERAGE}" -eq 1 ]]; then - "${MVN_CMD}" test jacoco:report -DfailIfNoTests=false + "${MVN_CMD}" test jacoco:report -DfailIfNoTests=false -Dmaven.test.failure.ignore=true else "${MVN_CMD}" test -Dcheckstyle.skip=true -DfailIfNoTests=false fi From 78117647e4d373fe46a01c7df99b1235db46b044 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Fri, 18 Jul 2025 09:40:27 +0800 Subject: [PATCH 248/572] [Cherry-Pick](branch-3.0) Pick "[Fix](Clone) Fix compaction and mow failure when missing rowset (#52812)" (#53497) Pick #52812 --- be/src/agent/agent_server.cpp | 7 +- be/src/agent/task_worker_pool.cpp | 48 +++++++ be/src/agent/task_worker_pool.h | 2 + be/src/common/config.cpp | 4 + be/src/common/config.h | 4 + be/src/olap/cumulative_compaction.cpp | 23 +++ be/src/olap/storage_engine.cpp | 110 ++++++++++++++ be/src/olap/storage_engine.h | 9 ++ be/src/olap/task/engine_clone_task.cpp | 2 +- .../olap/task/engine_publish_version_task.cpp | 20 +++ be/src/util/blocking_priority_queue.hpp | 1 + .../doris/service/FrontendServiceImpl.java | 2 + gensrc/thrift/Types.thrift | 2 + ...lone_missing_rowset_fault_injection.groovy | 112 +++++++++++++++ ...st_mow_publish_clone_missing_rowset.groovy | 134 ++++++++++++++++++ 15 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy create mode 100644 regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset.groovy diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp index 0b17f3782e7147..3976e275f75be1 100644 --- a/be/src/agent/agent_server.cpp +++ b/be/src/agent/agent_server.cpp @@ -43,6 +43,7 @@ #include "olap/snapshot_manager.h" #include "olap/storage_engine.h" #include "runtime/exec_env.h" +#include "util/work_thread_pool.hpp" using std::string; using std::vector; @@ -169,8 +170,8 @@ void AgentServer::start_workers(StorageEngine& engine, ExecEnv* exec_env) { _workers[TTaskType::ALTER] = std::make_unique( "ALTER_TABLE", config::alter_tablet_worker_count, [&engine](auto&& task) { return alter_tablet_callback(engine, task); }); - _workers[TTaskType::CLONE] = std::make_unique( - "CLONE", config::clone_worker_count, [&engine, &cluster_info = _cluster_info](auto&& task) { return clone_callback(engine, cluster_info, task); }); + _workers[TTaskType::CLONE] = std::make_unique( + "CLONE", config::clone_worker_count,config::clone_worker_count, [&engine, &cluster_info = _cluster_info](auto&& task) { return clone_callback(engine, cluster_info, task); }); _workers[TTaskType::STORAGE_MEDIUM_MIGRATE] = std::make_unique( "STORAGE_MEDIUM_MIGRATE", config::storage_medium_migrate_count, [&engine](auto&& task) { return storage_medium_migrate_callback(engine, task); }); @@ -196,6 +197,8 @@ void AgentServer::start_workers(StorageEngine& engine, ExecEnv* exec_env) { _report_workers.push_back(std::make_unique( "REPORT_OLAP_TABLET", _cluster_info, config::report_tablet_interval_seconds,[&engine, &cluster_info = _cluster_info] { report_tablet_callback(engine, cluster_info); })); // clang-format on + + exec_env->storage_engine().to_local().workers = &_workers; } void AgentServer::cloud_start_workers(CloudStorageEngine& engine, ExecEnv* exec_env) { diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 7eb7716e4130b5..7410041663e05c 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -17,6 +17,7 @@ #include "agent/task_worker_pool.h" +#include #include #include #include @@ -84,6 +85,7 @@ #include "runtime/memory/global_memory_arbitrator.h" #include "runtime/snapshot_loader.h" #include "service/backend_options.h" +#include "util/brpc_client_cache.h" #include "util/debug_points.h" #include "util/doris_metrics.h" #include "util/jni-util.h" @@ -604,6 +606,52 @@ Status PriorTaskWorkerPool::submit_task(const TAgentTaskRequest& task) { }); } +Status PriorTaskWorkerPool::submit_high_prior_and_cancel_low(const TAgentTaskRequest& task) { + const TTaskType::type task_type = task.task_type; + int64_t signature = task.signature; + std::string type_str; + EnumToString(TTaskType, task_type, type_str); + auto req = std::make_unique(task); + + DCHECK(req->__isset.priority && req->priority == TPriority::HIGH); + do { + std::lock_guard lock(s_task_signatures_mtx); + auto& set = s_task_signatures[task_type]; + if (!set.contains(signature)) { + // If it doesn't exist, put it directly into the priority queue + add_task_count(*req, 1); + set.insert(signature); + std::lock_guard lock(_mtx); + _high_prior_queue.push_back(std::move(req)); + _high_prior_condv.notify_one(); + _normal_condv.notify_one(); + break; + } else { + std::lock_guard lock(_mtx); + for (auto it = _normal_queue.begin(); it != _normal_queue.end();) { + // If it exists in the normal queue, cancel the task in the normal queue + if ((*it)->signature == signature) { + _normal_queue.erase(it); // cancel the original task + _high_prior_queue.push_back(std::move(req)); // add the new task to the queue + _high_prior_condv.notify_one(); + _normal_condv.notify_one(); + break; + } else { + ++it; // doesn't meet the condition, continue to the next one + } + } + // If it exists in the high priority queue, no operation is needed + LOG_INFO("task has already existed in high prior queue.").tag("signature", signature); + } + } while (false); + + // Set the receiving time of task so that we can determine whether it is timed out later + (const_cast(task)).__set_recv_time(time(nullptr)); + + LOG_INFO("successfully submit task").tag("type", type_str).tag("signature", signature); + return Status::OK(); +} + void PriorTaskWorkerPool::normal_loop() { while (true) { std::unique_ptr req; diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h index f6223affd07e3f..046b43cb3e15b8 100644 --- a/be/src/agent/task_worker_pool.h +++ b/be/src/agent/task_worker_pool.h @@ -89,6 +89,8 @@ class PriorTaskWorkerPool final : public TaskWorkerPoolIf { Status submit_task(const TAgentTaskRequest& task) override; + Status submit_high_prior_and_cancel_low(const TAgentTaskRequest& task); + private: void normal_loop(); diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9d4e578de6fd42..46e68829558b46 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1523,6 +1523,10 @@ DEFINE_mInt32(segments_key_bounds_truncation_threshold, "-1"); // ATTENTION: for test only, use random segments key bounds truncation threshold every time DEFINE_mBool(random_segments_key_bounds_truncation, "false"); +DEFINE_mBool(enable_auto_clone_on_compaction_missing_version, "false"); + +DEFINE_mBool(enable_auto_clone_on_mow_publish_missing_version, "false"); + // clang-format off #ifdef BE_TEST // test s3 diff --git a/be/src/common/config.h b/be/src/common/config.h index caaf10c33cd8e8..a6fe08af1f626a 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1600,6 +1600,10 @@ DECLARE_mInt32(segments_key_bounds_truncation_threshold); // ATTENTION: for test only, use random segments key bounds truncation threshold every time DECLARE_mBool(random_segments_key_bounds_truncation); +DECLARE_mBool(enable_auto_clone_on_compaction_missing_version); + +DECLARE_mBool(enable_auto_clone_on_mow_publish_missing_version); + #ifdef BE_TEST // test s3 DECLARE_String(test_s3_resource); diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 73fe179c2ce77d..2a2fdb51eb2f4e 100644 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -18,10 +18,13 @@ #include "olap/cumulative_compaction.h" #include +#include +#include #include #include #include +#include #include "common/config.h" #include "common/logging.h" @@ -29,7 +32,9 @@ #include "olap/cumulative_compaction_time_series_policy.h" #include "olap/olap_define.h" #include "olap/rowset/rowset_meta.h" +#include "olap/storage_engine.h" #include "olap/tablet.h" +#include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/doris_metrics.h" #include "util/time.h" @@ -191,6 +196,24 @@ Status CumulativeCompaction::pick_rowsets_to_compact() { << " first missed version prev rowset verison=" << missing_versions[0] << ", first missed version next rowset version=" << missing_versions[1] << ", tablet=" << _tablet->tablet_id(); + if (config::enable_auto_clone_on_compaction_missing_version) { + LOG_INFO("cumulative compaction submit missing rowset clone task.") + .tag("tablet_id", _tablet->tablet_id()) + .tag("version", missing_versions.back().first) + .tag("replica_id", tablet()->replica_id()) + .tag("partition_id", _tablet->partition_id()) + .tag("table_id", _tablet->table_id()); + Status st = _engine.submit_clone_task(tablet(), missing_versions.back().first); + if (!st) { + LOG_WARNING("cumulative compaction failed to submit missing rowset clone task.") + .tag("st", st.to_string()) + .tag("tablet_id", _tablet->tablet_id()) + .tag("version", missing_versions.back().first) + .tag("replica_id", tablet()->replica_id()) + .tag("partition_id", _tablet->partition_id()) + .tag("table_id", _tablet->table_id()); + } + } } int64_t max_score = config::cumulative_compaction_max_deltas; diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index 02234b326aad9f..df8c6631e20036 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -20,6 +20,8 @@ // IWYU pragma: no_include #include #include +#include +#include #include #include #include @@ -37,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +52,7 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "gen_cpp/FrontendService.h" #include "gutil/strings/substitute.h" #include "io/fs/local_file_system.h" #include "olap/binlog.h" @@ -67,6 +71,7 @@ #include "olap/tablet_meta.h" #include "olap/tablet_meta_manager.h" #include "olap/txn_manager.h" +#include "runtime/client_cache.h" #include "runtime/stream_load/stream_load_recorder.h" #include "util/doris_metrics.h" #include "util/mem_info.h" @@ -74,8 +79,10 @@ #include "util/stopwatch.hpp" #include "util/thread.h" #include "util/threadpool.h" +#include "util/thrift_rpc_helper.h" #include "util/uid_util.h" #include "util/work_thread_pool.hpp" +#include "vec/common/assert_cast.h" using std::filesystem::directory_iterator; using std::filesystem::path; @@ -1456,6 +1463,79 @@ bool StorageEngine::get_peer_replica_info(int64_t tablet_id, TReplicaInfo* repli return false; } +bool StorageEngine::get_peers_replica_backends(int64_t tablet_id, std::vector* backends) { + TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id); + if (tablet == nullptr) { + LOG(WARNING) << "tablet is no longer exist: tablet_id=" << tablet_id; + return false; + } + int64_t cur_time = UnixMillis(); + if (cur_time - _last_get_peers_replica_backends_time_ms < 10000) { + LOG_WARNING("failed to get peers replica backens.") + .tag("last time", _last_get_peers_replica_backends_time_ms) + .tag("cur time", cur_time); + return false; + } + LOG_INFO("start get peers replica backends info.").tag("tablet id", tablet_id); + ClusterInfo* cluster_info = ExecEnv::GetInstance()->cluster_info(); + if (cluster_info == nullptr) { + LOG(WARNING) << "Have not get FE Master heartbeat yet"; + return false; + } + TNetworkAddress master_addr = cluster_info->master_fe_addr; + if (master_addr.hostname.empty() || master_addr.port == 0) { + LOG(WARNING) << "Have not get FE Master heartbeat yet"; + return false; + } + TGetTabletReplicaInfosRequest request; + TGetTabletReplicaInfosResult result; + request.tablet_ids.emplace_back(tablet_id); + Status rpc_st = ThriftRpcHelper::rpc( + master_addr.hostname, master_addr.port, + [&request, &result](FrontendServiceConnection& client) { + client->getTabletReplicaInfos(result, request); + }); + + if (!rpc_st.ok()) { + LOG(WARNING) << "Failed to get tablet replica infos, encounter rpc failure, " + "tablet id: " + << tablet_id; + return false; + } + std::unique_lock lock(_peer_replica_infos_mutex); + if (result.tablet_replica_infos.contains(tablet_id)) { + std::vector reps = result.tablet_replica_infos[tablet_id]; + DCHECK_NE(reps.size(), 0); + for (const auto& rep : reps) { + if (rep.replica_id != tablet->replica_id()) { + TBackend backend; + backend.__set_host(rep.host); + backend.__set_be_port(rep.be_port); + backend.__set_http_port(rep.http_port); + backend.__set_brpc_port(rep.brpc_port); + if (rep.__isset.is_alive) { + backend.__set_is_alive(rep.is_alive); + } + if (rep.__isset.backend_id) { + backend.__set_id(rep.backend_id); + } + backends->emplace_back(backend); + std::stringstream backend_string; + backend.printTo(backend_string); + LOG_INFO("get 1 peer replica backend info.") + .tag("tablet id", tablet_id) + .tag("backend info", backend_string.str()); + } + } + _last_get_peers_replica_backends_time_ms = UnixMillis(); + LOG_INFO("succeed get peers replica backends info.") + .tag("tablet id", tablet_id) + .tag("replica num", backends->size()); + return true; + } + return false; +} + bool StorageEngine::should_fetch_from_peer(int64_t tablet_id) { #ifdef BE_TEST if (tablet_id % 2 == 0) { @@ -1570,6 +1650,36 @@ Status StorageEngine::_persist_broken_paths() { return Status::OK(); } +Status StorageEngine::submit_clone_task(Tablet* tablet, int64_t version) { + std::vector backends; + if (!get_peers_replica_backends(tablet->tablet_id(), &backends)) { + LOG(WARNING) << tablet->tablet_id() << " tablet doesn't have peer replica backends"; + return Status::InternalError(""); + } + TAgentTaskRequest task; + TCloneReq req; + req.__set_tablet_id(tablet->tablet_id()); + req.__set_schema_hash(tablet->schema_hash()); + req.__set_src_backends(backends); + req.__set_version(version); + req.__set_replica_id(tablet->replica_id()); + req.__set_partition_id(tablet->partition_id()); + req.__set_table_id(tablet->table_id()); + task.__set_task_type(TTaskType::CLONE); + task.__set_clone_req(req); + task.__set_priority(TPriority::HIGH); + task.__set_signature(tablet->tablet_id()); + LOG_INFO("BE start to submit missing rowset clone task.") + .tag("tablet_id", tablet->tablet_id()) + .tag("version", version) + .tag("replica_id", tablet->replica_id()) + .tag("partition_id", tablet->partition_id()) + .tag("table_id", tablet->table_id()); + RETURN_IF_ERROR(assert_cast(workers->at(TTaskType::CLONE).get()) + ->submit_high_prior_and_cancel_low(task)); + return Status::OK(); +} + int CreateTabletRRIdxCache::get_index(const std::string& key) { auto* lru_handle = lookup(key); if (lru_handle) { diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index a747a228f072a2..9dd780956f52ec 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -37,6 +37,7 @@ #include #include +#include "agent/task_worker_pool.h" #include "common/config.h" #include "common/status.h" #include "gutil/ref_counted.h" @@ -305,6 +306,8 @@ class StorageEngine final : public BaseStorageEngine { bool get_peer_replica_info(int64_t tablet_id, TReplicaInfo* replica, std::string* token); + bool get_peers_replica_backends(int64_t tablet_id, std::vector* backends); + bool should_fetch_from_peer(int64_t tablet_id); const std::shared_ptr& get_stream_load_recorder() { @@ -335,6 +338,10 @@ class StorageEngine final : public BaseStorageEngine { std::set get_broken_paths() { return _broken_paths; } + Status submit_clone_task(Tablet* tablet, int64_t version); + + std::unordered_map>* workers; + private: // Instance should be inited from `static open()` // MUST NOT be called in other circumstances. @@ -553,6 +560,8 @@ class StorageEngine final : public BaseStorageEngine { // thread to check tablet delete bitmap count tasks scoped_refptr _check_delete_bitmap_score_thread; + + int64_t _last_get_peers_replica_backends_time_ms {0}; }; // lru cache for create tabelt round robin in disks diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 6a9e66f1d383cd..e2f915f6cda22f 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include "common/config.h" #include "common/logging.h" @@ -989,5 +990,4 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet, return tablet->revise_tablet_meta(to_add, to_delete, false); // TODO(plat1ko): write cooldown meta to remote if this replica is cooldown replica } - } // namespace doris diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp index 2dcc1723b71005..6c37e55da757e4 100644 --- a/be/src/olap/task/engine_publish_version_task.cpp +++ b/be/src/olap/task/engine_publish_version_task.cpp @@ -32,6 +32,7 @@ #include #include +#include "cloud/config.h" #include "common/logging.h" #include "olap/storage_engine.h" #include "olap/tablet_manager.h" @@ -216,6 +217,25 @@ Status EnginePublishVersionTask::execute() { continue; } auto handle_version_not_continuous = [&]() { + if (config::enable_auto_clone_on_mow_publish_missing_version) { + LOG_INFO("mow publish submit missing rowset clone task.") + .tag("tablet_id", tablet->tablet_id()) + .tag("version", version.first - 1) + .tag("replica_id", tablet->replica_id()) + .tag("partition_id", tablet->partition_id()) + .tag("table_id", tablet->table_id()); + Status st = _engine.submit_clone_task(tablet.get(), version.first - 1); + if (!st) { + LOG_WARNING( + "mow publish failed to submit missing rowset clone task.") + .tag("st", st.to_string()) + .tag("tablet_id", tablet->tablet_id()) + .tag("version", version.first - 1) + .tag("replica_id", tablet->replica_id()) + .tag("partition_id", tablet->partition_id()) + .tag("table_id", tablet->table_id()); + } + } add_error_tablet_id(tablet_info.tablet_id); // When there are too many missing versions, do not directly retry the // publish and handle it through async publish. diff --git a/be/src/util/blocking_priority_queue.hpp b/be/src/util/blocking_priority_queue.hpp index bfc1c34e8f16d7..43fe1e4df473d8 100644 --- a/be/src/util/blocking_priority_queue.hpp +++ b/be/src/util/blocking_priority_queue.hpp @@ -22,6 +22,7 @@ #include +#include #include #include #include diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index b95968604e3d14..a1966a036cfe5f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2788,6 +2788,8 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos replicaInfo.setBePort(backend.getBePort()); replicaInfo.setHttpPort(backend.getHttpPort()); replicaInfo.setBrpcPort(backend.getBrpcPort()); + replicaInfo.setIsAlive(backend.isAlive()); + replicaInfo.setBackendId(backend.getId()); replicaInfo.setReplicaId(replica.getId()); replicaInfos.add(replicaInfo); } diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index dbb385d0d1d572..ca4aa9ef68bf87 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -665,6 +665,8 @@ struct TReplicaInfo { 3: required TPort http_port 4: required TPort brpc_port 5: required TReplicaId replica_id + 6: optional bool is_alive + 7: optional i64 backend_id } struct TResourceInfo { diff --git a/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy new file mode 100644 index 00000000000000..a7f060a110888b --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.http.NoHttpResponseException +import org.apache.doris.regression.util.DebugPoint +import org.apache.doris.regression.util.NodeType + +suite('test_compaction_clone_missing_rowset_fault_injection', 'docker') { + def options = new ClusterOptions() + options.cloudMode = false + options.enableDebugPoints() + options.feConfigs += [ "disable_tablet_scheduler=true" ] + options.beConfigs += [ "enable_auto_clone_on_compaction_missing_version=true" ] + options.beNum = 3 + docker(options) { + + def injectBe = null + def normalBe = null + def backends = sql_return_maparray('show backends') + + injectBe = backends[0] + assertNotNull(injectBe) + normalBe = backends[1] + assertNotNull(normalBe) + + try { + def tableName = "test_compaction_clone_missing_rowset" + sql """ DROP TABLE IF EXISTS ${tableName} force""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k` int , + `v` int , + ) engine=olap + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) + BUCKETS 1 + properties( + "replication_num" = "3", + "disable_auto_compaction" = "true") + """ + sql """ INSERT INTO ${tableName} VALUES (1,0)""" + DebugPoint.enableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random", [percent:"1.0"]) + sql """ INSERT INTO ${tableName} VALUES (2,0)""" + sql """ INSERT INTO ${tableName} VALUES (3,0)""" + sql """ INSERT INTO ${tableName} VALUES (4,0)""" + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + sql """ INSERT INTO ${tableName} VALUES (5,0)""" + + def array = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + def tabletId = array[0].TabletId + + // 1st check rowsets + logger.info("1st show:" + tabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("1st show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + // missing rowset [3-5] + assertTrue(out.contains("[3-5]")) + assertTrue(out.contains("[6-6]")) + + logger.info("1st run cumu compaction:" + tabletId) + (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("1st Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + + sleep(10000) + + // 2nd check rowsets + logger.info("2nd show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("2nd show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-3]")) + assertTrue(out.contains("[4-4]")) + assertTrue(out.contains("[5-5]")) + assertTrue(out.contains("[6-6]")) + + logger.info("2nd cumu compaction:" + tabletId) + (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("2nd cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + + // check rowsets + logger.info("3rd show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("3rd show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-6]")) + + } finally { + if (injectBe != null) { + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + } + } + } +} diff --git a/regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset.groovy b/regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset.groovy new file mode 100644 index 00000000000000..14f0073f5c88ec --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.http.NoHttpResponseException +import org.apache.doris.regression.util.DebugPoint +import org.apache.doris.regression.util.NodeType + +suite('test_mow_publish_clone_missing_rowset_fault_injection', 'docker') { + + def set_be_param = { paramName, paramValue, beIp, bePort -> + def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue)) + assertTrue(out.contains("OK")) + } + + def options = new ClusterOptions() + options.cloudMode = false + options.enableDebugPoints() + options.feConfigs += [ "disable_tablet_scheduler=true" ] + options.beConfigs += [ "enable_auto_clone_on_mow_publish_missing_version=false" ] + options.beNum = 3 + docker(options) { + + def injectBe = null + def normalBe = null + def backends = sql_return_maparray('show backends') + + injectBe = backends[0] + assertNotNull(injectBe) + normalBe = backends[1] + assertNotNull(normalBe) + + try { + def tableName = "test_mow_publish_clone_missing_rowset" + sql """ DROP TABLE IF EXISTS ${tableName} force""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k` int , + `v` int , + ) engine=olap + UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) + BUCKETS 1 + properties( + "replication_num" = "3", + "disable_auto_compaction" = "true") + """ + sql """ INSERT INTO ${tableName} VALUES (1,0)""" + DebugPoint.enableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random", [percent:"1.0"]) + sql """ INSERT INTO ${tableName} VALUES (2,0)""" + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + sql """ INSERT INTO ${tableName} VALUES (3,0)""" + sql """ INSERT INTO ${tableName} VALUES (4,0)""" + sql """ INSERT INTO ${tableName} VALUES (5,0)""" + + def array = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + def tabletId = array[0].TabletId + + // normal be check rowsets + logger.info("normal be show:" + tabletId) + def (code, out, err) = be_show_tablet_status(normalBe.Host, normalBe.HttpPort, tabletId) + logger.info("normal be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-3]")) + assertTrue(out.contains("[4-4]")) + assertTrue(out.contains("[5-5]")) + assertTrue(out.contains("[6-6]")) + + // 1st inject be check rowsets + logger.info("1st inject be show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("1st inject be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertFalse(out.contains("[3-3]")) + assertFalse(out.contains("[4-4]")) + assertFalse(out.contains("[5-5]")) + assertFalse(out.contains("[6-6]")) + + set_be_param("enable_auto_clone_on_mow_publish_missing_version", "true", injectBe.Host, injectBe.HttpPort); + Thread.sleep(10000) + // submit clone task + sql """ INSERT INTO ${tableName} VALUES (6,0)""" + + sleep(10000) + + // 2nd inject be check rowsets + logger.info("2nd inject be show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("2nd inject be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-3]")) + assertTrue(out.contains("[4-4]")) + assertTrue(out.contains("[5-5]")) + assertTrue(out.contains("[6-6]")) + assertTrue(out.contains("[7-7]")) + + sql """ INSERT INTO ${tableName} VALUES (7,0)""" + + // 3rd inject be check rowsets + logger.info("3rd inject be show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("3rd inject be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-3]")) + assertTrue(out.contains("[4-4]")) + assertTrue(out.contains("[5-5]")) + assertTrue(out.contains("[6-6]")) + assertTrue(out.contains("[7-7]")) + assertTrue(out.contains("[8-8]")) + } finally { + if (injectBe != null) { + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + } + } + } +} \ No newline at end of file From 39d7f9e93c3950014be66e38b0e02153e620b79a Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Thu, 17 Jul 2025 19:08:19 -0700 Subject: [PATCH 249/572] branch-3.0: [fix](test) change s3 tables name to avoid conflict(#53433) (#53499) bp #53433 --- .../iceberg/test_s3tables_write_insert.out | 10 --- .../test_s3tables_insert_overwrite.groovy | 42 +++++++------ .../iceberg/test_s3tables_write_insert.groovy | 63 ++++++++++++------- .../test_s3tables_write_partitions.groovy | 26 ++++---- 4 files changed, 75 insertions(+), 66 deletions(-) diff --git a/regression-test/data/external_table_p2/iceberg/test_s3tables_write_insert.out b/regression-test/data/external_table_p2/iceberg/test_s3tables_write_insert.out index a0fcbd6816d205..710fc36821666f 100644 --- a/regression-test/data/external_table_p2/iceberg/test_s3tables_write_insert.out +++ b/regression-test/data/external_table_p2/iceberg/test_s3tables_write_insert.out @@ -1,14 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !sql01 -- -1 ABC 100 -2 XYZ 200 - --- !sql01 -- -2024-01-01T08:00 1000 us-east PART1 -2024-01-02T10:00 1002 us-sout PART2 -2024-01-03T08:00 1000 us-east PART1 -2024-01-04T10:00 1002 us-sout PART2 - -- !q01 -- true 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 {"key1":"value1"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789} {1.2345678901234568:1.2345678901234568} ["string1", "string2"] [1, 2, 3] [100000000000, 200000000000] [1.1, 2.2] [1.123456789, 2.123456789] [1, 0] [1.1, 2.2] [1.23, 2.34] [1.2345, 2.3456] [1.23456789, 2.34567891] [1.23456789, 2.34567891] [1.2345678901234568, 2.3456789012345679] {"s_bigint":1234567890} {"key":[{"s_int":123}]} {"struct_field":["value1", "value2"]} {"struct_field_null":null, "struct_field_null2":null} {"struct_non_nulls_after_nulls1":123, "struct_non_nulls_after_nulls2":"value"} {"struct_field1":123, "struct_field2":"value", "strict_field3":{"nested_struct_field1":123, "nested_struct_field2":"nested_value"}} {"null_key":null} [null, "value1", "value2"] ["value1", null, "value2"] ["value1", "value2", null] [null, null, null] 20240320 diff --git a/regression-test/suites/external_table_p2/iceberg/test_s3tables_insert_overwrite.groovy b/regression-test/suites/external_table_p2/iceberg/test_s3tables_insert_overwrite.groovy index 89c667719d80a3..dde476decfb158 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_s3tables_insert_overwrite.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_s3tables_insert_overwrite.groovy @@ -22,9 +22,11 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext def parts = format_compression.split("_") def format = parts[0] def compression = parts[1] - sql """ DROP TABLE IF EXISTS `iceberg_overwrite_all_types_${format_compression}`; """ + def all_types_table = "iceberg_overwrite_all_types_${format_compression}_branch30" + def all_types_partition_table = "iceberg_overwrite_types_par_${format_compression}_branch30" + sql """ DROP TABLE IF EXISTS `${all_types_table}`; """ sql """ - CREATE TABLE `iceberg_overwrite_all_types_${format_compression}`( + CREATE TABLE `${all_types_table}`( `boolean_col` boolean, `int_col` int, `bigint_col` bigint, @@ -82,7 +84,7 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext """ sql """ - INSERT OVERWRITE table iceberg_overwrite_all_types_${format_compression} + INSERT OVERWRITE table ${all_types_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -136,11 +138,11 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext 20240320 -- dt ); """ - order_qt_q01 """ select * from iceberg_overwrite_all_types_${format_compression}; + order_qt_q01 """ select * from ${all_types_table}; """ sql """ - INSERT OVERWRITE table iceberg_overwrite_all_types_${format_compression} + INSERT OVERWRITE table ${all_types_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -298,11 +300,11 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext 20240322 -- dt ); """ - order_qt_q02 """ select * from iceberg_overwrite_all_types_${format_compression}; + order_qt_q02 """ select * from ${all_types_table}; """ sql """ - INSERT OVERWRITE table iceberg_overwrite_all_types_${format_compression}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls) + INSERT OVERWRITE table ${all_types_table}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls) VALUES ( CAST(123.45 AS FLOAT), -- float_col MAP(1, 10), -- t_map_int @@ -310,19 +312,21 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext ARRAY(null, 'value1', 'value2') -- t_ARRAY_string_starting_with_nulls ); """ - order_qt_q03 """ select * from iceberg_overwrite_all_types_${format_compression}; + order_qt_q03 """ select * from ${all_types_table}; """ - sql """ DROP TABLE iceberg_overwrite_all_types_${format_compression}; """ + sql """ DROP TABLE ${all_types_table}; """ } def q03 = { String format_compression, String catalog_name -> def parts = format_compression.split("_") def format = parts[0] def compression = parts[1] - sql """ DROP TABLE IF EXISTS `iceberg_overwrite_types_par_${format_compression}`; """ + def all_types_table = "iceberg_overwrite_all_types_${format_compression}_branch30" + def all_types_partition_table = "iceberg_overwrite_types_par_${format_compression}_branch30" + sql """ DROP TABLE IF EXISTS `${all_types_partition_table}`; """ sql """ - CREATE TABLE `iceberg_overwrite_types_par_${format_compression}`( + CREATE TABLE `${all_types_partition_table}`( `boolean_col` boolean, `int_col` int, `bigint_col` bigint, @@ -381,7 +385,7 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext """ sql """ - INSERT OVERWRITE TABLE iceberg_overwrite_types_par_${format_compression} + INSERT OVERWRITE TABLE ${all_types_partition_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -435,11 +439,11 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext 20240320 -- dt ); """ - order_qt_q01 """ select * from iceberg_overwrite_types_par_${format_compression}; + order_qt_q01 """ select * from ${all_types_partition_table}; """ sql """ - INSERT OVERWRITE TABLE iceberg_overwrite_types_par_${format_compression} + INSERT OVERWRITE TABLE ${all_types_partition_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -597,11 +601,11 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext 20240322 -- dt ); """ - order_qt_q02 """ select * from iceberg_overwrite_types_par_${format_compression}; + order_qt_q02 """ select * from ${all_types_partition_table}; """ sql """ - INSERT OVERWRITE TABLE iceberg_overwrite_types_par_${format_compression}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls, dt) + INSERT OVERWRITE TABLE ${all_types_partition_table}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls, dt) VALUES ( 123.45, -- float_col MAP(1, 10), -- t_map_int @@ -610,10 +614,10 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext 20240321 -- dt ); """ - order_qt_q03 """ select * from iceberg_overwrite_types_par_${format_compression}; + order_qt_q03 """ select * from ${all_types_partition_table}; """ - sql """ DROP TABLE iceberg_overwrite_types_par_${format_compression}; """ + sql """ DROP TABLE ${all_types_partition_table}; """ } String enabled = context.config.otherConfigs.get("enableExternalIcebergTest") @@ -634,8 +638,6 @@ suite("test_s3tables_insert_overwrite", "p0,external,iceberg,external_docker,ext sql """ switch ${catalog_name};""" sql """ use my_namespace;""" sql """ set enable_fallback_to_original_planner=false """ - def tables = sql """ show tables; """ - assertTrue(tables.size() > 0) try { for (String format_compression in format_compressions) { diff --git a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy index 9db977fdfa2723..670b3e1120610c 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy @@ -22,9 +22,11 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa def parts = format_compression.split("_") def format = parts[0] def compression = parts[1] - sql """ DROP TABLE IF EXISTS `iceberg_all_types_${format_compression}`; """ + def all_types_table = "iceberg_all_types_${format_compression}_branch30" + def all_types_partition_table = "iceberg_all_types_par_${format_compression}_branch30" + sql """ DROP TABLE IF EXISTS `${all_types_table}`; """ sql """ - CREATE TABLE `iceberg_all_types_${format_compression}`( + CREATE TABLE `${all_types_table}`( `boolean_col` boolean, `int_col` int, `bigint_col` bigint, @@ -82,7 +84,7 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa """ sql """ - INSERT INTO iceberg_all_types_${format_compression} + INSERT INTO ${all_types_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -136,11 +138,11 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa 20240320 -- dt ); """ - order_qt_q01 """ select * from iceberg_all_types_${format_compression}; + order_qt_q01 """ select * from ${all_types_table}; """ sql """ - INSERT INTO iceberg_all_types_${format_compression} + INSERT INTO ${all_types_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -298,11 +300,11 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa 20240322 -- dt ); """ - order_qt_q02 """ select * from iceberg_all_types_${format_compression}; + order_qt_q02 """ select * from ${all_types_table}; """ sql """ - INSERT INTO iceberg_all_types_${format_compression}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls) + INSERT INTO ${all_types_table}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls) VALUES ( CAST(123.45 AS FLOAT), -- float_col MAP(1, 10), -- t_map_int @@ -310,19 +312,21 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa ARRAY(null, 'value1', 'value2') -- t_ARRAY_string_starting_with_nulls ); """ - order_qt_q03 """ select * from iceberg_all_types_${format_compression}; + order_qt_q03 """ select * from ${all_types_table}; """ - sql """ DROP TABLE iceberg_all_types_${format_compression}; """ + sql """ DROP TABLE ${all_types_table}; """ } def q03 = { String format_compression, String catalog_name -> def parts = format_compression.split("_") def format = parts[0] def compression = parts[1] - sql """ DROP TABLE IF EXISTS `iceberg_all_types_par_${format_compression}`; """ + def all_types_table = "iceberg_all_types_${format_compression}_branch30" + def all_types_partition_table = "iceberg_all_types_par_${format_compression}_branch30" + sql """ DROP TABLE IF EXISTS `${all_types_partition_table}`; """ sql """ - CREATE TABLE `iceberg_all_types_par_${format_compression}`( + CREATE TABLE `${all_types_partition_table}`( `boolean_col` boolean, `int_col` int, `bigint_col` bigint, @@ -381,7 +385,7 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa """ sql """ - INSERT INTO iceberg_all_types_par_${format_compression} + INSERT INTO ${all_types_partition_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -435,11 +439,11 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa 20240320 -- dt ); """ - order_qt_q01 """ select * from iceberg_all_types_par_${format_compression}; + order_qt_q01 """ select * from ${all_types_partition_table}; """ sql """ - INSERT INTO iceberg_all_types_par_${format_compression} + INSERT INTO ${all_types_partition_table} VALUES ( 1, -- boolean_col 2147483647, -- int_col @@ -597,11 +601,11 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa 20240322 -- dt ); """ - order_qt_q02 """ select * from iceberg_all_types_par_${format_compression}; + order_qt_q02 """ select * from ${all_types_partition_table}; """ sql """ - INSERT INTO iceberg_all_types_par_${format_compression}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls, dt) + INSERT INTO ${all_types_partition_table}(float_col, t_map_int, t_ARRAY_decimal_precision_8, t_ARRAY_string_starting_with_nulls, dt) VALUES ( 123.45, -- float_col MAP(1, 10), -- t_map_int @@ -610,10 +614,27 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa 20240321 -- dt ); """ - order_qt_q03 """ select * from iceberg_all_types_par_${format_compression}; + order_qt_q03 """ select * from ${all_types_partition_table}; """ - sql """ DROP TABLE iceberg_all_types_par_${format_compression}; """ + // just test + sql """ + SELECT + CASE + WHEN file_size_in_bytes BETWEEN 0 AND 8 * 1024 * 1024 THEN '0-8M' + WHEN file_size_in_bytes BETWEEN 8 * 1024 * 1024 + 1 AND 32 * 1024 * 1024 THEN '8-32M' + WHEN file_size_in_bytes BETWEEN 2 * 1024 * 1024 + 1 AND 128 * 1024 * 1024 THEN '32-128M' + WHEN file_size_in_bytes BETWEEN 128 * 1024 * 1024 + 1 AND 512 * 1024 * 1024 THEN '128-512M' + WHEN file_size_in_bytes > 512 * 1024 * 1024 THEN '> 512M' + ELSE 'Unknown' + END AS SizeRange, + COUNT(*) AS FileNum + FROM ${all_types_partition_table}\$data_files + GROUP BY + SizeRange; + """ + + sql """ DROP TABLE ${all_types_partition_table}; """ } String enabled = context.config.otherConfigs.get("enableExternalIcebergTest") @@ -634,12 +655,6 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa sql """ switch ${catalog_name};""" sql """ use my_namespace;""" sql """ set enable_fallback_to_original_planner=false """ - def tables = sql """ show tables; """ - assertTrue(tables.size() > 0) - - // 1. test querying existing tables - qt_sql01 """select * from my_table order by id;""" - qt_sql01 """select * from partitioned_table order by ts;""" try { for (String format_compression in format_compressions) { diff --git a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy index 143eba25a16860..1e25eba10a33fc 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_partitions.groovy @@ -22,9 +22,11 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext def parts = format_compression.split("_") def format = parts[0] def compression = parts[1] - sql """ drop table if exists s3_columns_out_of_order_source_tbl_${format_compression} """ + def source_tbl = "s3_columns_out_of_order_source_tbl_${format_compression}_branch30" + def target_tbl = "s3_columns_out_of_order_target_tbl_${format_compression}_branch30" + sql """ drop table if exists ${source_tbl} """ sql """ - CREATE TABLE s3_columns_out_of_order_source_tbl_${format_compression} ( + CREATE TABLE ${source_tbl} ( `col3` bigint, `col6` int, `col1` bigint, @@ -37,9 +39,9 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext "write-format"=${format} ) """; - sql """ drop table if exists s3_columns_out_of_order_target_tbl_${format_compression} """ + sql """ drop table if exists ${target_tbl}""" sql """ - CREATE TABLE s3_columns_out_of_order_target_tbl_${format_compression} ( + CREATE TABLE ${target_tbl} ( `col1` bigint, `col2` bigint, `col3` bigint, @@ -57,22 +59,22 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext """; sql """ - INSERT INTO s3_columns_out_of_order_source_tbl_${format_compression} ( + INSERT INTO ${source_tbl} ( col1, col2, col3, col4, col5, col6 ) VALUES (1, 2, 3, 4, 5, 6); """ - order_qt_columns_out_of_order01 """ SELECT * FROM s3_columns_out_of_order_source_tbl_${format_compression} """ + order_qt_columns_out_of_order01 """ SELECT * FROM ${source_tbl} """ sql """ - INSERT INTO s3_columns_out_of_order_target_tbl_${format_compression} ( + INSERT INTO ${target_tbl} ( col1, col2, col3, col4, col5, col6 ) VALUES (1, 2, 3, 4, 5, 6); """ - order_qt_columns_out_of_order02 """ SELECT * FROM s3_columns_out_of_order_target_tbl_${format_compression} """ + order_qt_columns_out_of_order02 """ SELECT * FROM ${target_tbl} """ - sql """ drop table s3_columns_out_of_order_source_tbl_${format_compression} """ - sql """ drop table s3_columns_out_of_order_target_tbl_${format_compression} """ + sql """ drop table ${source_tbl} """ + sql """ drop table ${target_tbl} """ sql """ drop database if exists `test_s3_columns_out_of_order` """; } @@ -94,8 +96,8 @@ suite("test_s3tables_write_partitions", "p0,external,iceberg,external_docker,ext sql """ switch ${catalog_name};""" sql """ use my_namespace;""" sql """ set enable_fallback_to_original_planner=false """ - def tables = sql """ show tables; """ - assertTrue(tables.size() > 0) + // def tables = sql """ show tables; """ + // assertTrue(tables.size() > 0) try { for (String format_compression in format_compressions) { From 088a1ad03897c9ec2121923aa00d0ee5b2567c18 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 18 Jul 2025 11:03:36 +0800 Subject: [PATCH 250/572] branch-3.0: [fix](inverted index) fix memory leak in inverted index procedure (#53487) --- be/src/clucene | 2 +- .../segment_v2/inverted_index_compaction.cpp | 16 ++- .../inverted_index_compound_reader.cpp | 98 +++++++++---------- .../inverted_index_compound_reader.h | 22 ++--- .../segment_v2/inverted_index_file_reader.cpp | 42 ++++---- .../segment_v2/inverted_index_file_reader.h | 8 +- .../segment_v2/inverted_index_file_writer.cpp | 12 +-- .../inverted_index_fs_directory.cpp | 39 ++++++-- .../segment_v2/inverted_index_fs_directory.h | 2 +- .../segment_v2/inverted_index_searcher.cpp | 23 +++-- .../segment_v2/inverted_index_writer.cpp | 20 ++-- ...ted_index_exception_fault_injection.groovy | 9 +- 12 files changed, 145 insertions(+), 148 deletions(-) diff --git a/be/src/clucene b/be/src/clucene index 569398a5c96b4c..8d8f92ef8ddd0e 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit 569398a5c96b4c626251ccbe81257945a3d2aef4 +Subproject commit 8d8f92ef8ddd0e50b6fc76f8f6572abaef1b5213 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp index dcbdca921ab8e8..5f3679aaeae350 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp @@ -17,6 +17,7 @@ #include "inverted_index_compaction.h" +#include "inverted_index_common.h" #include "inverted_index_file_writer.h" #include "inverted_index_fs_directory.h" #include "io/fs/local_file_system.h" @@ -41,15 +42,17 @@ Status compact_column(int64_t index_id, "debug point: index compaction error"); } }) + bool can_use_ram_dir = true; - lucene::store::Directory* dir = DorisFSDirectoryFactory::getDirectory( - io::global_local_filesystem(), tmp_path.data(), can_use_ram_dir); + std::unique_ptr dir( + DorisFSDirectoryFactory::getDirectory(io::global_local_filesystem(), tmp_path.data(), + can_use_ram_dir)); DBUG_EXECUTE_IF("compact_column_getDirectory_error", { _CLTHROWA(CL_ERR_IO, "debug point: compact_column_getDirectory_error in index compaction"); }) lucene::analysis::SimpleAnalyzer analyzer; - auto* index_writer = _CLNEW lucene::index::IndexWriter(dir, &analyzer, true /* create */, - true /* closeDirOnShutdown */); + std::unique_ptr index_writer(_CLNEW lucene::index::IndexWriter( + dir.get(), &analyzer, true /* create */, true /* closeDirOnShutdown */)); DBUG_EXECUTE_IF("compact_column_create_index_writer_error", { _CLTHROWA(CL_ERR_IO, "debug point: compact_column_create_index_writer_error in index compaction"); @@ -71,11 +74,6 @@ Status compact_column(int64_t index_id, _CLTHROWA(CL_ERR_IO, "debug point: compact_column_index_writer_close_error in index compaction"); }) - _CLDELETE(index_writer); - // NOTE: need to ref_cnt-- for dir, - // when index_writer is destroyed, if closeDir is set, dir will be close - // _CLDECDELETE(dir) will try to ref_cnt--, when it decreases to 1, dir will be destroyed. - _CLDECDELETE(dir) // delete temporary segment_path, only when inverted_index_ram_dir_enable is false if (!config::inverted_index_ram_dir_enable) { diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp index 86efe86ca43389..37878689cfdde0 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.cpp @@ -22,10 +22,10 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include #include @@ -34,22 +34,13 @@ #include "olap/tablet_schema.h" #include "util/debug_points.h" -namespace doris { -namespace io { +namespace doris::io { class FileWriter; -} // namespace io -} // namespace doris +} // namespace doris::io #define BUFFER_LENGTH 16384 #define CL_MAX_PATH 4096 - -#define STRDUP_WtoA(x) CL_NS(util)::Misc::_wideToChar(x) -#define STRDUP_TtoA STRDUP_WtoA - -using FileWriterPtr = std::unique_ptr; - -namespace doris { -namespace segment_v2 { +namespace doris::segment_v2 { /** Implementation of an IndexInput that reads from a portion of the * compound file. @@ -94,7 +85,7 @@ CSIndexInput::CSIndexInput(CL_NS(store)::IndexInput* base, const std::string& fi void CSIndexInput::readInternal(uint8_t* b, const int32_t len) { std::lock_guard wlock(((DorisFSDirectory::FSIndexInput*)base)->_this_lock); - int64_t start = getFilePointer(); + auto start = getFilePointer(); if (start + len > _length) { _CLTHROWA(CL_ERR_IO, "read past EOF"); } @@ -153,56 +144,63 @@ void CSIndexInput::setIoContext(const void* io_ctx) { } DorisCompoundReader::DorisCompoundReader(CL_NS(store)::IndexInput* stream, - EntriesType* entries_clone, int32_t read_buffer_size, + const EntriesType& entries_clone, int32_t read_buffer_size, const io::IOContext* io_ctx) : _stream(stream), - _entries(_CLNEW EntriesType(true, true)), + _entries(std::make_unique()), _read_buffer_size(read_buffer_size) { // After stream clone, the io_ctx needs to be reconfigured. initialize(io_ctx); - for (auto& e : *entries_clone) { - auto* origin_entry = e.second; - auto* entry = _CLNEW ReaderFileEntry(); - char* aid = strdup(e.first); + for (const auto& e : entries_clone) { + const auto& origin_entry = e.second; + auto entry = std::make_unique(); entry->file_name = origin_entry->file_name; entry->offset = origin_entry->offset; entry->length = origin_entry->length; - _entries->put(aid, entry); + (*_entries)[e.first] = std::move(entry); } }; DorisCompoundReader::DorisCompoundReader(CL_NS(store)::IndexInput* stream, int32_t read_buffer_size, const io::IOContext* io_ctx) - : _ram_dir(new lucene::store::RAMDirectory()), + : _ram_dir(std::make_unique()), _stream(stream), - _entries(_CLNEW EntriesType(true, true)), + _entries(std::make_unique()), _read_buffer_size(read_buffer_size) { // After stream clone, the io_ctx needs to be reconfigured. initialize(io_ctx); try { int32_t count = _stream->readVInt(); - ReaderFileEntry* entry = nullptr; - TCHAR tid[CL_MAX_PATH]; uint8_t buffer[BUFFER_LENGTH]; for (int32_t i = 0; i < count; i++) { - entry = _CLNEW ReaderFileEntry(); - stream->readString(tid, CL_MAX_PATH); - char* aid = STRDUP_TtoA(tid); - entry->file_name = aid; + auto entry = std::make_unique(); + // Read the string length first + int32_t string_length = stream->readVInt(); + // Allocate appropriate buffer for the string + std::wstring tid; + tid.resize(string_length); + // Read the string characters directly + stream->readChars(tid.data(), 0, string_length); + std::string file_name_str(tid.begin(), tid.end()); + entry->file_name = file_name_str; entry->offset = stream->readLong(); entry->length = stream->readLong(); + VLOG_DEBUG << "string_length:" << string_length << " file_name:" << entry->file_name + << " offset:" << entry->offset << " length:" << entry->length; DBUG_EXECUTE_IF("construct_DorisCompoundReader_failed", { CLuceneError err; err.set(CL_ERR_IO, "construct_DorisCompoundReader_failed"); throw err; }) - _entries->put(aid, entry); // read header file data if (entry->offset < 0) { - copyFile(entry->file_name.c_str(), entry->length, buffer, BUFFER_LENGTH); + //if offset is -1, it means it's size is lower than DorisFSDirectory::MAX_HEADER_DATA_SIZE, which is 128k. + copyFile(entry->file_name.c_str(), static_cast(entry->length), buffer, + BUFFER_LENGTH); } + _entries->emplace(std::move(file_name_str), std::move(entry)); } } catch (...) { try { @@ -212,11 +210,9 @@ DorisCompoundReader::DorisCompoundReader(CL_NS(store)::IndexInput* stream, int32 } if (_entries != nullptr) { _entries->clear(); - _CLDELETE(_entries); } if (_ram_dir) { _ram_dir->close(); - _CLDELETE(_ram_dir) } } catch (CLuceneError& err) { if (err.number() != CL_ERR_IO) { @@ -231,11 +227,12 @@ void DorisCompoundReader::copyFile(const char* file, int64_t file_length, uint8_ int64_t buffer_length) { std::unique_ptr output(_ram_dir->createOutput(file)); int64_t start_ptr = output->getFilePointer(); - int64_t remainder = file_length; - int64_t chunk = buffer_length; + auto remainder = file_length; + auto chunk = buffer_length; + auto batch_len = file_length < chunk ? file_length : chunk; while (remainder > 0) { - int64_t len = std::min(std::min(chunk, file_length), remainder); + auto len = remainder < batch_len ? remainder : batch_len; _stream->readBytes(buffer, len); output->writeBytes(buffer, len); remainder -= len; @@ -270,7 +267,6 @@ DorisCompoundReader::~DorisCompoundReader() { LOG(ERROR) << "DorisCompoundReader finalize error:" << err.what(); } } - _CLDELETE(_entries) } const char* DorisCompoundReader::getClassName() { @@ -284,8 +280,8 @@ bool DorisCompoundReader::list(std::vector* names) const { if (_closed || _entries == nullptr) { _CLTHROWA(CL_ERR_IO, "DorisCompoundReader is already closed"); } - for (EntriesType::const_iterator i = _entries->begin(); i != _entries->end(); i++) { - names->push_back(i->first); + for (const auto& entry : *_entries) { + names->push_back(entry.first); } return true; } @@ -294,7 +290,7 @@ bool DorisCompoundReader::fileExists(const char* name) const { if (_closed || _entries == nullptr) { _CLTHROWA(CL_ERR_IO, "DorisCompoundReader is already closed"); } - return _entries->exists((char*)name); + return _entries->find(std::string(name)) != _entries->end(); } int64_t DorisCompoundReader::fileModified(const char* name) const { @@ -305,15 +301,15 @@ int64_t DorisCompoundReader::fileLength(const char* name) const { if (_closed || _entries == nullptr) { _CLTHROWA(CL_ERR_IO, "DorisCompoundReader is already closed"); } - ReaderFileEntry* e = _entries->get((char*)name); - if (e == nullptr) { + auto it = _entries->find(std::string(name)); + if (it == _entries->end()) { char buf[CL_MAX_PATH + 30]; strcpy(buf, "File "); strncat(buf, name, CL_MAX_PATH); strcat(buf, " does not exist"); _CLTHROWA(CL_ERR_IO, buf); } - return e->length; + return it->second->length; } bool DorisCompoundReader::openInput(const char* name, @@ -338,14 +334,16 @@ bool DorisCompoundReader::openInput(const char* name, lucene::store::IndexInput* return false; } - const ReaderFileEntry* entry = _entries->get((char*)name); - if (entry == nullptr) { + auto it = _entries->find(std::string(name)); + if (it == _entries->end()) { char buf[CL_MAX_PATH + 26]; snprintf(buf, CL_MAX_PATH + 26, "No sub-file with id %s found", name); error.set(CL_ERR_IO, buf); return false; } + const auto& entry = it->second; + // If file is in RAM, just return. if (_ram_dir && _ram_dir->fileExists(name)) { return _ram_dir->openInput(name, ret, error, bufferSize); @@ -374,7 +372,6 @@ void DorisCompoundReader::close() { } if (_ram_dir) { _ram_dir->close(); - _CLDELETE(_ram_dir) } _closed = true; } @@ -400,7 +397,7 @@ lucene::store::IndexOutput* DorisCompoundReader::createOutput(const char* /*name } std::string DorisCompoundReader::toString() const { - return std::string("DorisCompoundReader@"); + return "DorisCompoundReader@"; } CL_NS(store)::IndexInput* DorisCompoundReader::getDorisIndexInput() { @@ -412,5 +409,4 @@ void DorisCompoundReader::initialize(const io::IOContext* io_ctx) { _stream->setIdxFileCache(true); } -} // namespace segment_v2 -} // namespace doris +} // namespace doris::segment_v2 \ No newline at end of file diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h index 4a687e4ed3ea75..09e6faaeb9198e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_reader.h @@ -24,12 +24,13 @@ #include #include #include -#include +#include #include #include #include #include +#include #include #include @@ -43,9 +44,7 @@ namespace lucene::store { class RAMDirectory; } // namespace lucene::store -namespace doris { -class TabletIndex; -namespace segment_v2 { +namespace doris::segment_v2 { class ReaderFileEntry : LUCENE_BASE { public: @@ -60,16 +59,14 @@ class ReaderFileEntry : LUCENE_BASE { ~ReaderFileEntry() override = default; }; -using EntriesType = - lucene::util::CLHashMap>; +using EntriesType = std::unordered_map>; + class CLUCENE_EXPORT DorisCompoundReader : public lucene::store::Directory { private: - lucene::store::RAMDirectory* _ram_dir = nullptr; + std::unique_ptr _ram_dir; CL_NS(store)::IndexInput* _stream = nullptr; // The life cycle of _entries should be consistent with that of the DorisCompoundReader. - EntriesType* _entries = nullptr; + std::unique_ptr _entries; std::mutex _this_lock; bool _closed = false; int32_t _read_buffer_size = CL_NS(store)::BufferedIndexInput::BUFFER_SIZE; @@ -79,7 +76,7 @@ class CLUCENE_EXPORT DorisCompoundReader : public lucene::store::Directory { bool doDeleteFile(const char* name) override; public: - DorisCompoundReader(CL_NS(store)::IndexInput* stream, EntriesType* entries_clone, + DorisCompoundReader(CL_NS(store)::IndexInput* stream, const EntriesType& entries_clone, int32_t read_buffer_size = CL_NS(store)::BufferedIndexInput::BUFFER_SIZE, const io::IOContext* io_ctx = nullptr); DorisCompoundReader(CL_NS(store)::IndexInput* stream, @@ -109,5 +106,4 @@ class CLUCENE_EXPORT DorisCompoundReader : public lucene::store::Directory { void initialize(const io::IOContext* io_ctx); }; -} // namespace segment_v2 -} // namespace doris +} // namespace doris::segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp index 1b9440ae14bec8..79c57949ca9b69 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.cpp @@ -84,7 +84,6 @@ Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size, const io::I if (version == InvertedIndexStorageFormatPB::V2) { DCHECK(version == _storage_format); int32_t numIndices = _stream->readInt(); // Read number of indices - ReaderFileEntry* entry = nullptr; for (int32_t i = 0; i < numIndices; ++i) { int64_t indexId = _stream->readLong(); // Read index ID @@ -95,23 +94,19 @@ Status InvertedIndexFileReader::_init_from(int32_t read_buffer_size, const io::I int32_t numFiles = _stream->readInt(); // Read number of files in the index - // true, true means it will deconstruct key and value - auto fileEntries = std::make_unique(true, true); + auto fileEntries = std::make_unique(); + fileEntries->reserve(numFiles); for (int32_t j = 0; j < numFiles; ++j) { - entry = _CLNEW ReaderFileEntry(); - int32_t file_name_length = _stream->readInt(); - // aid will destruct in EntriesType map. - char* aid = (char*)malloc(file_name_length + 1); - _stream->readBytes(reinterpret_cast(aid), file_name_length); - aid[file_name_length] = '\0'; - //stream->readString(tid, CL_MAX_PATH); - entry->file_name = std::string(aid, file_name_length); + std::string file_name(file_name_length, '\0'); + _stream->readBytes(reinterpret_cast(file_name.data()), + file_name_length); + auto entry = std::make_unique(); + entry->file_name = std::move(file_name); entry->offset = _stream->readLong(); entry->length = _stream->readLong(); - - fileEntries->put(aid, entry); + fileEntries->emplace(entry->file_name, std::move(entry)); } _indices_entries.emplace(std::make_pair(indexId, std::move(suffix_str)), @@ -223,8 +218,8 @@ Result> InvertedIndexFileReader::_open( errMsg.str())); } // Need to clone resource here, because index searcher cache need it. - compound_reader = std::make_unique( - _stream->clone(), index_it->second.get(), _read_buffer_size, io_ctx); + compound_reader = std::make_unique(_stream->clone(), *index_it->second, + _read_buffer_size, io_ctx); } return compound_reader; } @@ -290,13 +285,14 @@ Status InvertedIndexFileReader::has_null(const TabletIndex* index_meta, bool* re if (index_it == _indices_entries.end()) { *res = false; } else { - auto* entries = index_it->second.get(); - ReaderFileEntry* e = - entries->get((char*)InvertedIndexDescriptor::get_temporary_null_bitmap_file_name()); - if (e == nullptr) { + const auto& entries = index_it->second; + auto entry_it = + entries->find(InvertedIndexDescriptor::get_temporary_null_bitmap_file_name()); + if (entry_it == entries->end()) { *res = false; return Status::OK(); } + const auto& e = entry_it->second; // roaring bitmap cookie header size is 5 if (e->length <= 5) { *res = false; @@ -309,11 +305,11 @@ Status InvertedIndexFileReader::has_null(const TabletIndex* index_meta, bool* re void InvertedIndexFileReader::debug_file_entries() { std::shared_lock lock(_mutex); // Lock for reading - for (auto& index : _indices_entries) { + for (const auto& index : _indices_entries) { LOG(INFO) << "index_id:" << index.first.first; - auto* index_entries = index.second.get(); - for (auto& entry : (*index_entries)) { - ReaderFileEntry* file_entry = entry.second; + const auto& index_entries = index.second; + for (const auto& entry : *index_entries) { + const auto& file_entry = entry.second; LOG(INFO) << "file entry name:" << file_entry->file_name << " length:" << file_entry->length << " offset:" << file_entry->offset; } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h index c4264f9b462983..4b8f7d18e012e3 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_reader.h @@ -42,11 +42,7 @@ class DorisCompoundReader; class InvertedIndexFileReader { public: - using EntriesType = - lucene::util::CLHashMap>; - // Map to hold the file entries for each index ID. + using EntriesType = std::unordered_map>; using IndicesEntriesMap = std::map, std::unique_ptr>; @@ -56,7 +52,7 @@ class InvertedIndexFileReader { : _fs(std::move(fs)), _index_path_prefix(std::move(index_path_prefix)), _storage_format(storage_format), - _idx_file_info(idx_file_info) {} + _idx_file_info(std::move(idx_file_info)) {} Status init(int32_t read_buffer_size = config::inverted_index_read_buffer_size, const io::IOContext* io_ctx = nullptr); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 36d82bad1f2d51..08f52920471285 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -200,6 +200,7 @@ void InvertedIndexFileWriter::copyFile(const char* fileName, lucene::store::Dire lucene::store::IndexInput* tmp = nullptr; CLuceneError err; auto open = dir->openInput(fileName, tmp, err); + std::unique_ptr input(tmp); DBUG_EXECUTE_IF("InvertedIndexFileWriter::copyFile_openInput_error", { open = false; err.set(CL_ERR_IO, "debug point: copyFile_openInput_error"); @@ -212,7 +213,6 @@ void InvertedIndexFileWriter::copyFile(const char* fileName, lucene::store::Dire throw err; } - std::unique_ptr input(tmp); int64_t start_ptr = output->getFilePointer(); int64_t length = input->length(); int64_t remainder = length; @@ -426,15 +426,14 @@ InvertedIndexFileWriter::create_output_stream_v1(int64_t index_id, out_dir->set_file_writer_opts(_opts); std::unique_ptr out_dir_ptr(out_dir); - auto* out = out_dir->createOutput(idx_name.c_str()); + std::unique_ptr output(out_dir->createOutput(idx_name.c_str())); DBUG_EXECUTE_IF("InvertedIndexFileWriter::write_v1_out_dir_createOutput_nullptr", - { out = nullptr; }); - if (out == nullptr) { + { output = nullptr; }); + if (output == nullptr) { LOG(WARNING) << "InvertedIndexFileWriter::create_output_stream_v1 error: CompoundDirectory " "output is nullptr."; _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error"); } - std::unique_ptr output(out); return {std::move(out_dir_ptr), std::move(output)}; } @@ -484,8 +483,7 @@ InvertedIndexFileWriter::create_output_stream_v2() { std::unique_ptr out_dir_ptr(out_dir); DCHECK(_idx_v2_writer != nullptr) << "inverted index file writer v2 is nullptr"; - auto compound_file_output = std::unique_ptr( - out_dir->createOutputV2(_idx_v2_writer.get())); + auto compound_file_output = out_dir->createOutputV2(_idx_v2_writer.get()); return {std::move(out_dir_ptr), std::move(compound_file_output)}; } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp index c633d29a7fc0c0..7bdca1941fa346 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp @@ -23,6 +23,7 @@ #include "inverted_index_desc.h" #include "io/fs/file_reader.h" #include "io/fs/file_writer.h" +#include "olap/rowset/segment_v2/inverted_index_common.h" #include "olap/tablet_schema.h" #include "util/debug_points.h" #include "util/slice.h" @@ -119,6 +120,7 @@ class DorisFSDirectory::FSIndexOutputV2 : public lucene::store::BufferedIndexOut bool DorisFSDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, const char* path, IndexInput*& ret, CLuceneError& error, int32_t buffer_size, int64_t file_size) { + // no throw error CND_PRECONDITION(path != nullptr, "path is NULL"); if (buffer_size == -1) { @@ -704,21 +706,42 @@ lucene::store::IndexOutput* DorisFSDirectory::createOutput(const char* name) { assert(!exists); } auto* ret = _CLNEW FSIndexOutput(); + ErrorContext error_context; ret->set_file_writer_opts(_opts); try { ret->init(_fs, fl); } catch (CLuceneError& err) { - ret->close(); - _CLDELETE(ret) - LOG(WARNING) << "FSIndexOutput init error: " << err.what(); - _CLTHROWA(CL_ERR_IO, "FSIndexOutput init error"); - } + error_context.eptr = std::current_exception(); + error_context.err_msg.append("FSIndexOutput init error: "); + error_context.err_msg.append(err.what()); + LOG(ERROR) << error_context.err_msg; + } + FINALLY_EXCEPTION({ + if (error_context.eptr) { + FINALLY_CLOSE(ret); + _CLDELETE(ret); + } + }) return ret; } -lucene::store::IndexOutput* DorisFSDirectory::createOutputV2(io::FileWriter* file_writer) { - auto* ret = _CLNEW FSIndexOutputV2(); - ret->init(file_writer); +std::unique_ptr DorisFSDirectory::createOutputV2( + io::FileWriter* file_writer) { + auto ret = std::make_unique(); + ErrorContext error_context; + try { + ret->init(file_writer); + } catch (CLuceneError& err) { + error_context.eptr = std::current_exception(); + error_context.err_msg.append("FSIndexOutputV2 init error: "); + error_context.err_msg.append(err.what()); + LOG(ERROR) << error_context.err_msg; + } + FINALLY_EXCEPTION({ + if (error_context.eptr) { + FINALLY_CLOSE(ret); + } + }) return ret; } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h index 0bba5b49756070..60e3a132aa4efd 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.h @@ -79,7 +79,7 @@ class CLUCENE_EXPORT DorisFSDirectory : public lucene::store::Directory { void renameFile(const char* from, const char* to) override; void touchFile(const char* name) override; lucene::store::IndexOutput* createOutput(const char* name) override; - lucene::store::IndexOutput* createOutputV2(io::FileWriter* file_writer); + std::unique_ptr createOutputV2(io::FileWriter* file_writer); void close() override; std::string toString() const override; static const char* getClassName(); diff --git a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp index 8d56b913b31c67..a8538dabab8756 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_searcher.cpp @@ -21,6 +21,7 @@ #include #include "common/config.h" +#include "olap/rowset/segment_v2/inverted_index_common.h" #include "olap/rowset/segment_v2/inverted_index_compound_reader.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/rowset/segment_v2/inverted_index_fs_directory.h" @@ -29,10 +30,10 @@ namespace doris::segment_v2 { Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory, OptionalIndexSearcherPtr& output_searcher) { auto close_directory = true; - lucene::index::IndexReader* reader = nullptr; + std::unique_ptr reader; try { - reader = lucene::index::IndexReader::open( - directory, config::inverted_index_read_buffer_size, close_directory); + reader = std::unique_ptr(lucene::index::IndexReader::open( + directory, config::inverted_index_read_buffer_size, close_directory)); } catch (const CLuceneError& e) { std::vector file_names; directory->list(&file_names); @@ -44,16 +45,15 @@ Status FulltextIndexSearcherBuilder::build(lucene::store::Directory* directory, return Status::Error(msg); } bool close_reader = true; - auto index_searcher = std::make_shared(reader, close_reader); + reader_size = reader->getTermInfosRAMUsed(); + auto index_searcher = + std::make_shared(reader.release(), close_reader); if (!index_searcher) { output_searcher = std::nullopt; return Status::Error( "FulltextIndexSearcherBuilder build index_searcher error."); } - reader_size = reader->getTermInfosRAMUsed(); - // NOTE: need to cl_refcount-- here, so that directory will be deleted when - // index_searcher is destroyed - _CLDECDELETE(directory) + // NOTE: IndexSearcher takes ownership of the reader, and directory cleanup is handled by caller output_searcher = index_searcher; return Status::OK(); } @@ -69,7 +69,6 @@ Status BKDIndexSearcherBuilder::build(lucene::store::Directory* directory, } reader_size = bkd_reader->ram_bytes_used(); output_searcher = bkd_reader; - _CLDECDELETE(directory) return Status::OK(); } catch (const CLuceneError& e) { return Status::Error( @@ -104,13 +103,13 @@ Result> IndexSearcherBuilder::create_index Result IndexSearcherBuilder::get_index_searcher( lucene::store::Directory* directory) { OptionalIndexSearcherPtr result; - auto st = build(directory, result); + std::unique_ptr directory_ptr(directory); + + auto st = build(directory_ptr.get(), result); if (!st.ok()) { - _CLDECDELETE(directory) return ResultError(st); } if (!result.has_value()) { - _CLDECDELETE(directory) return ResultError(Status::Error( "InvertedIndexSearcherCache build error.")); } diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index d8688a34acc8c6..827d8127d21d97 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -404,8 +404,6 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { Status add_array_values(size_t field_size, const void* value_ptr, const uint8_t* nested_null_map, const uint8_t* offsets_ptr, size_t count) override { - DBUG_EXECUTE_IF("InvertedIndexColumnWriterImpl::add_array_values_count_is_zero", - { count = 0; }) if (count == 0) { // no values to add inverted index return Status::OK(); @@ -425,7 +423,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { // every single array row element size to go through the nullmap & value ptr-array, and also can go through the every row in array to keep with _rid++ auto array_elem_size = offsets[i + 1] - offsets[i]; // TODO(Amory).later we use object pool to avoid field creation - lucene::document::Field* new_field = nullptr; + std::unique_ptr new_field; CL_NS(analysis)::TokenStream* ts = nullptr; for (auto j = start_off; j < start_off + array_elem_size; ++j) { if (nested_null_map && nested_null_map[j] == 1) { @@ -440,7 +438,9 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { continue; } else { // now we temp create field . later make a pool - Status st = create_field(&new_field); + lucene::document::Field* tmp_field = nullptr; + Status st = create_field(&tmp_field); + new_field.reset(tmp_field); DBUG_EXECUTE_IF( "InvertedIndexColumnWriterImpl::add_array_values_create_field_" "error", @@ -469,9 +469,9 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { char_string_reader.release()); new_field->setValue(ts, own_token_stream); } else { - new_field_char_value(v->get_data(), v->get_size(), new_field); + new_field_char_value(v->get_data(), v->get_size(), new_field.get()); } - _doc->add(*new_field); + _doc->add(*new_field.release()); } } start_off += array_elem_size; @@ -500,7 +500,9 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { // avoid to add doc which without any field which may make threadState init skip // init fieldDataArray, then will make error with next doc with fields in // resetCurrentFieldData - Status st = create_field(&new_field); + lucene::document::Field* tmp_field = nullptr; + Status st = create_field(&tmp_field); + new_field.reset(tmp_field); DBUG_EXECUTE_IF( "InvertedIndexColumnWriterImpl::add_array_values_create_field_error_2", { @@ -513,7 +515,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { << " error:" << st; return st; } - _doc->add(*new_field); + _doc->add(*new_field.release()); RETURN_IF_ERROR(add_null_document()); _doc->clear(); } @@ -774,7 +776,7 @@ Status InvertedIndexColumnWriter::create(const Field* field, } DBUG_EXECUTE_IF("InvertedIndexColumnWriter::create_unsupported_type_for_inverted_index", - { type = FieldType::OLAP_FIELD_TYPE_FLOAT; }) + { type = FieldType::OLAP_FIELD_TYPE_HLL; }) switch (type) { #define M(TYPE) \ case TYPE: \ diff --git a/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy index 9fbb245c243ea9..52ce8f87a7443a 100644 --- a/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy @@ -219,7 +219,6 @@ suite("test_write_inverted_index_exception_fault_injection", "nonConcurrent") { "InvertedIndexColumnWriterImpl::new_char_token_stream__char_string_reader_init_error", "InvertedIndexColumnWriterImpl::add_values_field_is_nullptr", "InvertedIndexColumnWriterImpl::add_values_index_writer_is_nullptr", - "InvertedIndexColumnWriterImpl::add_array_values_count_is_zero", "InvertedIndexColumnWriterImpl::add_array_values_index_writer_is_nullptr", "InvertedIndexColumnWriterImpl::add_array_values_create_field_error", "InvertedIndexColumnWriterImpl::add_array_values_create_field_error_2", @@ -262,13 +261,7 @@ suite("test_write_inverted_index_exception_fault_injection", "nonConcurrent") { GetDebugPoint().enableDebugPointForAllBEs(debug_point) run_insert("${tableName}") check_count("${tableName}", 6) - // if debug_point equals InvertedIndexColumnWriterImpl::add_array_values_count_is_zero, run_select(false(abnormal)) - // else run_select(true(normal)) - if (debug_point == "InvertedIndexColumnWriterImpl::add_array_values_count_is_zero") { - run_select("${tableName}", false) - } else { - run_select("${tableName}", true) - } + run_select("${tableName}", true) sql "TRUNCATE TABLE ${tableName}" } catch (Exception e) { log.error("Caught exception: ${e}") From c451863cf887fbe745a58f6fd3f3eb58ceecfd47 Mon Sep 17 00:00:00 2001 From: amory Date: Fri, 18 Jul 2025 13:58:51 +0800 Subject: [PATCH 251/572] branch-3.0: [fix](variant)fix variant type conflicts in nested types (#52696) (#53123) cherry-pick: https://github.com/apache/doris/pull/52696 and #53418 --- be/src/vec/columns/column_object.cpp | 55 +- be/src/vec/columns/column_object.h | 6 + .../vec/data_types/convert_field_to_type.cpp | 11 +- be/src/vec/functions/function_cast.h | 30 +- be/src/vec/json/json_parser.cpp | 12 +- be/src/vec/json/json_parser.h | 2 + be/test/vec/columns/column_object_test.cpp | 130 +++++ .../cast/function_variant_cast_test.cpp | 466 ++++++++++++++++ .../vec/jsonb/convert_field_to_type_test.cpp | 521 ++++++++++++++++++ be/test/vec/jsonb/json_parser_test.cpp | 172 ++++++ regression-test/data/variant_p0/desc.out | 6 +- regression-test/data/variant_p0/nested2.out | 337 +++++++++++ regression-test/suites/variant_p0/load.groovy | 1 + .../suites/variant_p0/nested2.groovy | 151 +++++ 14 files changed, 1868 insertions(+), 32 deletions(-) create mode 100644 be/test/vec/function/cast/function_variant_cast_test.cpp create mode 100644 be/test/vec/jsonb/convert_field_to_type_test.cpp create mode 100644 be/test/vec/jsonb/json_parser_test.cpp create mode 100644 regression-test/data/variant_p0/nested2.out create mode 100644 regression-test/suites/variant_p0/nested2.groovy diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index f14f0c62debddc..196d16dbe58b94 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -85,12 +85,6 @@ namespace doris::vectorized { namespace { DataTypePtr create_array_of_type(TypeIndex type, size_t num_dimensions, bool is_nullable) { - if (type == ColumnObject::MOST_COMMON_TYPE_ID) { - // JSONB type MUST NOT wrapped in ARRAY column, it should be top level. - // So we ignored num_dimensions. - return is_nullable ? make_nullable(std::make_shared()) - : std::make_shared(); - } DataTypePtr result = DataTypeFactory::instance().create_data_type(type, is_nullable); for (size_t i = 0; i < num_dimensions; ++i) { result = std::make_shared(result); @@ -945,11 +939,12 @@ void ColumnObject::Subcolumn::get(size_t n, Field& res) const { res = Null(); return; } - if (is_finalized()) { - if (least_common_type.get_base_type_id() == TypeIndex::JSONB) { - // JsonbFiled is special case - res = JsonbField(); - } + + // JSONB is a special type, it's not a scalar type, we need to handle it specially + // 1. we try to get the JSONB Field from ColumnString which has no JSONB type info + // 2. Array of JSONB is a special type, we get from ColumnArray of ColumnString, should convert from string Field to JSONB Field + if (is_finalized() && least_common_type.get_base_type_id() != TypeIndex::JSONB) { + // common type to get the field value get_finalized_column().get(n, res); return; } @@ -965,11 +960,20 @@ void ColumnObject::Subcolumn::get(size_t n, Field& res) const { const auto& part = data[i]; const auto& part_type = data_types[i]; if (ind < part->size()) { - res = vectorized::remove_nullable(part_type)->get_default(); + auto non_nullable_type = vectorized::remove_nullable(part_type); + bool is_nested_array_of_jsonb = + non_nullable_type->equals(*NESTED_TYPE_AS_ARRAY_OF_JSONB); + + res = non_nullable_type->get_default(); part->get(ind, res); - Field new_field; - convert_field_to_type(res, *least_common_type.get(), &new_field); - res = new_field; + + if (is_nested_array_of_jsonb) { + convert_array_string_to_array_jsonb(res); + } else { + Field new_field; + convert_field_to_type(res, *least_common_type.get(), &new_field); + res = new_field; + } return; } @@ -1861,6 +1865,10 @@ const DataTypePtr ColumnObject::NESTED_TYPE = std::make_shared(std::make_shared( std::make_shared()))); +const DataTypePtr ColumnObject::NESTED_TYPE_AS_ARRAY_OF_JSONB = + std::make_shared(std::make_shared( + std::make_shared())); + DataTypePtr ColumnObject::get_root_type() const { return subcolumns.get_root()->data.get_least_common_type(); } @@ -2055,4 +2063,21 @@ bool ColumnObject::try_insert_default_from_nested(const Subcolumns::NodePtr& ent return true; } +void ColumnObject::Subcolumn::convert_array_string_to_array_jsonb(Field& array_field) { + if (array_field.is_null()) { + return; + } + if (array_field.get_type() != Field::Types::Array) { + return; + } + Field converted_res = Array(); + for (auto& item : array_field.get()) { + DCHECK(item.get_type() == Field::Types::String); + auto& string_item = item.get(); + Field jsonb_item = JsonbField(string_item.c_str(), string_item.size()); + converted_res.get().emplace_back(std::move(jsonb_item)); + } + array_field = std::move(converted_res); +} + } // namespace doris::vectorized diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h index 16fe3430313189..03e14054d2e7a9 100644 --- a/be/src/vec/columns/column_object.h +++ b/be/src/vec/columns/column_object.h @@ -97,6 +97,9 @@ class ColumnObject final : public COWHelper { constexpr static TypeIndex MOST_COMMON_TYPE_ID = TypeIndex::JSONB; // Nullable(Array(Nullable(Object))) const static DataTypePtr NESTED_TYPE; + // Array(Nullable(Jsonb)) + const static DataTypePtr NESTED_TYPE_AS_ARRAY_OF_JSONB; + // Finlize mode for subcolumns, write mode will estimate which subcolumns are sparse columns(too many null values inside column), // merge and encode them into a shared column in root column. Only affects in flush block to segments. // Otherwise read mode should be as default mode. @@ -177,6 +180,9 @@ class ColumnObject final : public COWHelper { void add_new_column_part(DataTypePtr type); + /// Converts Array to Array for special case handling + static void convert_array_string_to_array_jsonb(Field& array_field); + friend class ColumnObject; private: diff --git a/be/src/vec/data_types/convert_field_to_type.cpp b/be/src/vec/data_types/convert_field_to_type.cpp index ecbce03ba6b10a..9141bb95862180 100644 --- a/be/src/vec/data_types/convert_field_to_type.cpp +++ b/be/src/vec/data_types/convert_field_to_type.cpp @@ -33,6 +33,7 @@ #include "common/exception.h" #include "common/status.h" #include "util/bitmap_value.h" +#include "util/jsonb_document.h" #include "util/jsonb_writer.h" #include "vec/common/field_visitors.h" #include "vec/common/typeid_cast.h" @@ -111,6 +112,11 @@ class FieldVisitorToJsonb : public StaticVisitor { writer->writeString(x); writer->writeEndString(); } + void operator()(const JsonbField& x, JsonbWriter* writer) const { + JsonbDocument* doc; + THROW_IF_ERROR(JsonbDocument::checkAndCreateDocument(x.get_value(), x.get_size(), &doc)); + writer->writeValue(doc->getValue()); + } void operator()(const Array& x, JsonbWriter* writer) const; void operator()(const Tuple& x, JsonbWriter* writer) const { @@ -146,9 +152,6 @@ class FieldVisitorToJsonb : public StaticVisitor { void operator()(const Map& x, JsonbWriter* writer) const { throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "Not implemeted"); } - void operator()(const JsonbField& x, JsonbWriter* writer) const { - throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "Not implemeted"); - } }; void FieldVisitorToJsonb::operator()(const Array& x, JsonbWriter* writer) const { @@ -316,4 +319,4 @@ void convert_field_to_type(const Field& from_value, const IDataType& to_type, Fi return convert_field_to_typeImpl(from_value, to_type, from_type_hint, to); } } -} // namespace doris::vectorized \ No newline at end of file +} // namespace doris::vectorized diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 99f5f625971946..0e7db6e407b478 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -850,7 +850,7 @@ struct ConvertNothingToJsonb { } }; -template +template struct ConvertImplFromJsonb { static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, const size_t result, size_t input_rows_count) { @@ -897,6 +897,18 @@ struct ConvertImplFromJsonb { res[i] = 0; continue; } + + // if value is string, convert by parse, otherwise the result is null if ToDataType is not string + if (value->isString()) { + const auto* blob = static_cast(value); + const auto& data = blob->getBlob(); + size_t len = blob->getBlobLen(); + ReadBuffer rb((char*)(data), len); + bool parsed = try_parse_impl(res[i], rb, context); + null_map[i] = !parsed; + continue; + } + if constexpr (type_index == TypeIndex::UInt8) { // cast from json value to boolean type if (value->isTrue()) { @@ -1991,22 +2003,22 @@ class FunctionCast final : public IFunctionBase { bool jsonb_string_as_string) const { switch (to_type->get_type_id()) { case TypeIndex::UInt8: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::Int8: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::Int16: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::Int32: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::Int64: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::Int128: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::Float64: - return &ConvertImplFromJsonb::execute; + return &ConvertImplFromJsonb::execute; case TypeIndex::String: if (!jsonb_string_as_string) { - // Conversion from String through parsing. return &ConvertImplGenericToString::execute2; } else { return ConvertImplGenericFromJsonb::execute; diff --git a/be/src/vec/json/json_parser.cpp b/be/src/vec/json/json_parser.cpp index f6e8a65cc08c71..e031f168820156 100644 --- a/be/src/vec/json/json_parser.cpp +++ b/be/src/vec/json/json_parser.cpp @@ -59,8 +59,14 @@ void JSONDataParser::traverse(const Element& element, ParseContext& if (element.isObject()) { traverseObject(element.getObject(), ctx); } else if (element.isArray()) { + if (ctx.has_nested_in_flatten) { + throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, + "Nesting of array in Nested array within variant subcolumns is " + "currently not supported."); + } has_nested = false; check_has_nested_object(element); + ctx.has_nested_in_flatten = has_nested && ctx.enable_flatten_nested; if (has_nested && !ctx.enable_flatten_nested) { // Parse nested arrays to JsonbField JsonbWriter writer; @@ -71,6 +77,8 @@ void JSONDataParser::traverse(const Element& element, ParseContext& } else { traverseArray(element.getArray(), ctx); } + // we should set has_nested_in_flatten to false when traverse array finished for next array otherwise it will be true for next array + ctx.has_nested_in_flatten = false; } else { ctx.paths.push_back(ctx.builder.get_parts()); ctx.values.push_back(getValueAsField(element)); @@ -137,6 +145,7 @@ template void JSONDataParser::traverseArray(const JSONArray& array, ParseContext& ctx) { /// Traverse elements of array and collect an array of fields by each path. ParseArrayContext array_ctx; + array_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten; array_ctx.total_size = array.size(); for (auto it = array.begin(); it != array.end(); ++it) { traverseArrayElement(*it, array_ctx); @@ -162,8 +171,9 @@ template void JSONDataParser::traverseArrayElement(const Element& element, ParseArrayContext& ctx) { ParseContext element_ctx; + element_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten; traverse(element, element_ctx); - auto& [_, paths, values, flatten_nested] = element_ctx; + auto& [_, paths, values, flatten_nested, has_nested] = element_ctx; size_t size = paths.size(); size_t keys_to_update = ctx.arrays_by_path.size(); for (size_t i = 0; i < size; ++i) { diff --git a/be/src/vec/json/json_parser.h b/be/src/vec/json/json_parser.h index c1815ae5be4d50..401f225d000e2d 100644 --- a/be/src/vec/json/json_parser.h +++ b/be/src/vec/json/json_parser.h @@ -148,6 +148,7 @@ class JSONDataParser { std::vector paths; std::vector values; bool enable_flatten_nested = false; + bool has_nested_in_flatten = false; }; using PathPartsWithArray = std::pair; using PathToArray = phmap::flat_hash_map; @@ -157,6 +158,7 @@ class JSONDataParser { size_t total_size = 0; PathToArray arrays_by_path; KeyToSizes nested_sizes_by_key; + bool has_nested_in_flatten = false; }; void traverse(const Element& element, ParseContext& ctx); void traverseObject(const JSONObject& object, ParseContext& ctx); diff --git a/be/test/vec/columns/column_object_test.cpp b/be/test/vec/columns/column_object_test.cpp index a7498e82e87824..21c533e63d653f 100644 --- a/be/test/vec/columns/column_object_test.cpp +++ b/be/test/vec/columns/column_object_test.cpp @@ -165,4 +165,134 @@ TEST_F(ColumnObjectTest, test_pop_back_multiple_types) { EXPECT_EQ(subcolumn.get_least_common_type()->get_name(), "Nothing"); } +TEST_F(ColumnObjectTest, test_nested_array_of_jsonb_get) { + // Test case: Create a ColumnObject with subcolumn type Array + + // Create a ColumnObject with subcolumns + auto variant_column = ColumnObject::create(true); + + // Add subcolumn with path "nested.array" + variant_column->add_sub_column(PathInData("nested.array"), 0); + + // Get the subcolumn and manually set its type to Array + auto* subcolumn = variant_column->get_subcolumn(PathInData("nested.array")); + ASSERT_NE(subcolumn, nullptr); + + // Create test data: Array of strings + Field array_of_strings = Array(); + + // Add string elements to the array + std::string test_data1 = R"("a")"; + std::string test_data2 = R"(b)"; + + array_of_strings.get().emplace_back(test_data1); + array_of_strings.get().emplace_back(test_data2); + + // Insert the array field into the subcolumn + subcolumn->insert(array_of_strings); + + // Test 1: the column and test get method + { + EXPECT_TRUE(variant_column->is_finalized()); + // check the subcolumn get method + Field result; + EXPECT_NO_THROW(subcolumn->get(0, result)); + + // Verify the result is still an array + EXPECT_EQ(result.get_type(), doris::vectorized::Field::Types::Array); + + const auto& result_array = result.get(); + EXPECT_EQ(result_array.size(), 2); + + // Check that all elements are JSONB fields + for (const auto& item : result_array) { + EXPECT_EQ(item.get_type(), doris::vectorized::Field::Types::String); + } + + // Verify string content is preserved + const auto& string1 = result_array[0].get(); + const auto& string2 = result_array[1].get(); + + EXPECT_EQ(string1, R"("a")"); // "\"a\"" + EXPECT_EQ(string2, R"(b)"); // "b" + } + + // Test 2: Test with a row of different type of array to test the subcolumn get method + { + // Add another row with different int array + Field int_array = Array(); + int_array.get().push_back(1); + int_array.get().push_back(2); + int_array.get().push_back(3); + + // and we should add more data to the subcolumn column + subcolumn->insert(int_array); + + EXPECT_FALSE(variant_column->is_finalized()); + // check the subcolumn get method + Field result; + EXPECT_NO_THROW(subcolumn->get(1, result)); + EXPECT_EQ(result.get_type(), doris::vectorized::Field::Types::Array); + const auto& result_array = result.get(); + EXPECT_EQ(result_array.size(), 3); + EXPECT_EQ(result_array[0].get_type(), doris::vectorized::Field::Types::JSONB); + EXPECT_EQ(result_array[1].get_type(), doris::vectorized::Field::Types::JSONB); + EXPECT_EQ(result_array[2].get_type(), doris::vectorized::Field::Types::JSONB); + + // check the first row Field is a string + Field result_string; + EXPECT_NO_THROW(subcolumn->get(0, result_string)); + EXPECT_EQ(result_string.get_type(), doris::vectorized::Field::Types::Array); + const auto& result_string_array = result_string.get(); + EXPECT_EQ(result_string_array.size(), 2); + EXPECT_EQ(result_string_array[0].get_type(), doris::vectorized::Field::Types::JSONB); + EXPECT_EQ(result_string_array[1].get_type(), doris::vectorized::Field::Types::JSONB); + + // Finalize -> we should get the least common type of the subcolumn + variant_column->finalize(); + EXPECT_TRUE(variant_column->is_finalized()); + // we should get another subcolumn from the variant column + auto* subcolumn_finalized = variant_column->get_subcolumn(PathInData("nested.array")); + ASSERT_NE(subcolumn_finalized, nullptr); + // check the subcolumn_finalized get method + Field result1, result2; + EXPECT_NO_THROW(subcolumn_finalized->get(0, result1)); + EXPECT_NO_THROW(subcolumn_finalized->get(1, result2)); + + // Verify both results are arrays + EXPECT_EQ(result1.get_type(), doris::vectorized::Field::Types::Array); + EXPECT_EQ(result2.get_type(), doris::vectorized::Field::Types::Array); + + const auto& array1 = result1.get(); + const auto& array2 = result2.get(); + + EXPECT_EQ(array1.size(), 2); + EXPECT_EQ(array2.size(), 3); + + // Verify all elements are JSONB + for (const auto& item : array1) { + EXPECT_EQ(item.get_type(), doris::vectorized::Field::Types::JSONB); + } + for (const auto& item : array2) { + EXPECT_EQ(item.get_type(), doris::vectorized::Field::Types::JSONB); + } + } + + // Test 4: Test with empty array + { + auto* subcolumn = variant_column->get_subcolumn(PathInData("nested.array")); + ASSERT_NE(subcolumn, nullptr); + Field empty_array_field = Array(); + subcolumn->insert(empty_array_field); + + EXPECT_TRUE(variant_column->is_finalized()); + // check the subcolumn get method + Field result; + EXPECT_NO_THROW(subcolumn->get(2, result)); + EXPECT_EQ(result.get_type(), doris::vectorized::Field::Types::Array); + const auto& result_array = result.get(); + EXPECT_EQ(result_array.size(), 0); + } +} + } // namespace doris::vectorized diff --git a/be/test/vec/function/cast/function_variant_cast_test.cpp b/be/test/vec/function/cast/function_variant_cast_test.cpp new file mode 100644 index 00000000000000..49a7dc49e15899 --- /dev/null +++ b/be/test/vec/function/cast/function_variant_cast_test.cpp @@ -0,0 +1,466 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include "common/status.h" +#include "gtest/gtest_pred_impl.h" +#include "olap/field.h" +#include "runtime/define_primitive_type.h" +#include "runtime/primitive_type.h" +#include "runtime/runtime_state.h" +#include "vec/columns/column_array.h" +#include "vec/columns/column_object.h" +#include "vec/core/field.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_object.h" +#include "vec/data_types/data_type_string.h" +#include "vec/functions/simple_function_factory.h" + +namespace doris::vectorized { +static doris::vectorized::Field construct_variant_map( + const std::vector>& key_and_values) { + doris::vectorized::Field res = VariantMap(); + auto& object = res.get(); + for (const auto& [k, v] : key_and_values) { + PathInData path(k); + object.try_emplace(path, v); + } + return res; +} + +static auto construct_basic_varint_column() { + // 1. create an empty variant column + auto variant = ColumnObject::create(5); + + std::vector> data; + + // 2. subcolumn path + data.emplace_back("v.a", 20); + data.emplace_back("v.b", "20"); + data.emplace_back("v.c", 20); + data.emplace_back("v.f", 20); + data.emplace_back("v.e", "50"); + for (int i = 0; i < 5; ++i) { + auto field = construct_variant_map(data); + variant->try_insert(field); + } + + return variant; +} + +TEST(FunctionVariantCast, CastToVariant) { + // Test casting from basic types to variant + { + // Test Int32 to variant + auto int32_type = std::make_shared(); + auto variant_type = std::make_shared(); + auto int32_col = ColumnInt32::create(); + int32_col->insert(42); + int32_col->insert(100); + int32_col->insert(-1); + + ColumnsWithTypeAndName arguments {{int32_col->get_ptr(), int32_type, "int32_col"}, + {nullptr, variant_type, "variant_type"}}; + + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, variant_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, variant_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 3).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* variant_col = assert_cast(result_col.get()); + ASSERT_EQ(variant_col->size(), 3); + } + + // Test casting from string to variant + { + auto string_type = std::make_shared(); + auto variant_type = std::make_shared(); + auto string_col = ColumnString::create(); + string_col->insert_data("hello", 5); + string_col->insert_data("world", 5); + + ColumnsWithTypeAndName arguments {{string_col->get_ptr(), string_type, "string_col"}, + {nullptr, variant_type, "variant_type"}}; + + auto function = SimpleFunctionFactory::instance().get_function("CAST", arguments, + make_nullable(variant_type)); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, variant_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 2).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* variant_col = + assert_cast(remove_nullable(result_col).get()); + ASSERT_EQ(variant_col->size(), 2); + } + + // Test casting from array to variant + { + auto array_type = std::make_shared(std::make_shared()); + auto variant_type = std::make_shared(); + auto array_col = + ColumnArray::create(ColumnInt32::create(), ColumnArray::ColumnOffsets::create()); + auto& data = assert_cast(array_col->get_data()); + auto& offsets = array_col->get_offsets(); + + data.insert(1); + data.insert(2); + data.insert(3); + offsets.push_back(3); + + ColumnsWithTypeAndName arguments {{array_col->get_ptr(), array_type, "array_col"}, + {nullptr, variant_type, "variant_type"}}; + + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, variant_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, variant_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 1).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* variant_col = + assert_cast(remove_nullable(result_col).get()); + ASSERT_EQ(variant_col->size(), 1); + } +} + +TEST(FunctionVariantCast, CastFromVariant) { + // Test casting from variant to basic types + { + auto variant_type = std::make_shared(); + auto int32_type = std::make_shared(); + auto variant_col = ColumnObject::create(true); + + // Create a variant column with integer values + variant_col->create_root(int32_type, ColumnInt32::create()); + MutableColumnPtr data = variant_col->get_root(); + data->insert(42); + data->insert(100); + data->insert(-1); + + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, int32_type, "int32_type"}}; + + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, int32_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, int32_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 3).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + // always nullable + const auto* int32_result = + assert_cast(remove_nullable(result_col).get()); + ASSERT_EQ(int32_result->size(), 3); + ASSERT_EQ(int32_result->get_element(0), 42); + ASSERT_EQ(int32_result->get_element(1), 100); + ASSERT_EQ(int32_result->get_element(2), -1); + } + + // Test casting from variant to string + { + auto variant_type = std::make_shared(); + auto string_type = std::make_shared(); + auto variant_col = ColumnObject::create(true); + + // Create a variant column with string values + variant_col->create_root(string_type, ColumnString::create()); + MutableColumnPtr data = variant_col->get_root(); + data->insert_data("hello", 5); + data->insert_data("world", 5); + + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, string_type, "string_type"}}; + + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, string_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, string_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 2).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* string_result = + assert_cast(remove_nullable(result_col).get()); + ASSERT_EQ(string_result->size(), 2); + ASSERT_EQ(string_result->get_data_at(0).to_string(), "hello"); + ASSERT_EQ(string_result->get_data_at(1).to_string(), "world"); + } + + // Test casting from variant to array + { + auto variant_type = std::make_shared(); + auto array_type = std::make_shared(std::make_shared()); + auto variant_col = ColumnObject::create(true); + + // Create a variant column with array values + variant_col->create_root( + array_type, + ColumnArray::create(ColumnInt32::create(), ColumnArray::ColumnOffsets::create())); + MutableColumnPtr data = variant_col->get_root(); + + Field a = Array {1, 2, 3}; + + data->insert(a); + + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, array_type, "array_type"}}; + + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, array_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, array_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 1).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* array_result = + assert_cast(remove_nullable(result_col).get()); + ASSERT_EQ(array_result->size(), 1); + const auto& result_data = assert_cast(array_result->get_data()); + ASSERT_EQ(result_data.size(), 3); + ASSERT_EQ(result_data.get_element(0), 1); + ASSERT_EQ(result_data.get_element(1), 2); + ASSERT_EQ(result_data.get_element(2), 3); + } +} + +TEST(FunctionVariantCast, CastVariantWithNull) { + auto variant_type = std::make_shared(); + auto int32_type = std::make_shared(); + auto nullable_int32_type = std::make_shared(int32_type); + + // Create a variant column with nullable integer values + auto variant_col = ColumnObject::create(true); + variant_col->create_root(nullable_int32_type, + ColumnNullable::create(ColumnInt32::create(), ColumnUInt8::create())); + MutableColumnPtr data = variant_col->get_root(); + + data->insert(42); + data->insert(Null()); + data->insert(100); + + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, nullable_int32_type, "nullable_int32_type"}}; + + variant_col->finalize(); + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, nullable_int32_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, nullable_int32_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 3).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* nullable_result = assert_cast(result_col.get()); + ASSERT_EQ(nullable_result->size(), 3); + + const auto& result_data = assert_cast(nullable_result->get_nested_column()); + const auto& result_null_map = nullable_result->get_null_map_data(); + + ASSERT_EQ(result_data.get_element(0), 42); + ASSERT_EQ(result_null_map[0], 0); + ASSERT_EQ(result_null_map[1], 1); + ASSERT_EQ(result_data.get_element(2), 100); +} + +TEST(FunctionVariantCast, CastFromVariantWithEmptyRoot) { + // Test case 1: variant.empty() branch + { + auto variant_type = std::make_shared(); + auto int32_type = std::make_shared(); + MutableColumnPtr root = ColumnInt32::create(); + root->insert(42); + vectorized::ColumnObject::Subcolumns dynamic_subcolumns; + dynamic_subcolumns.add( + vectorized::PathInData(ColumnObject::COLUMN_NAME_DUMMY), + vectorized::ColumnObject::Subcolumn {root->get_ptr(), int32_type, true, true}); + auto variant_col = ColumnObject::create(std::move(dynamic_subcolumns), true); + + variant_col->finalize(); + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, int32_type, "int32_type"}}; + + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, int32_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, int32_type, "result"}); + + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 1).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + // always nullable + const auto* int32_result = + assert_cast(remove_nullable(result_col).get()); + ASSERT_EQ(int32_result->size(), 1); + // because of variant.empty() we insert_default with data_type_to + ASSERT_EQ(int32_result->get_element(0), 0); + } + + // Test case 2: !data_type_to->is_nullable() && !WhichDataType(data_type_to).is_string() branch + { + // object has sparse column + auto int32_type = std::make_shared(); + auto variant_col = construct_basic_varint_column(); + auto variant_type = std::make_shared(); + + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, int32_type, "int32_type"}}; + + variant_col->finalize(); + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, int32_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, int32_type, "result"}); + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 1).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* nullable_result = assert_cast(result_col.get()); + ASSERT_EQ(nullable_result->size(), 1); + ASSERT_TRUE(nullable_result->is_null_at(0)); + } + + // Test case 3: WhichDataType(data_type_to).is_string() branch + { + // variant has sparse column + auto int32_type = std::make_shared(); + auto variant_col = construct_basic_varint_column(); + + auto string_type = std::make_shared(); + auto variant_type = std::make_shared(); + + ColumnsWithTypeAndName arguments {{variant_col->get_ptr(), variant_type, "variant_col"}, + {nullptr, string_type, "string_type"}}; + + variant_col->finalize(); + auto function = + SimpleFunctionFactory::instance().get_function("CAST", arguments, string_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, string_type, "result"}); + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 1).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* string_result = assert_cast(result_col.get()); + // just call ConvertImplGenericToString which will insert all source column data to ColumnString + ASSERT_EQ(string_result->size(), variant_col->size()); + ASSERT_EQ(string_result->get_data_at(0).to_string(), + "{\"v\":{\"a\":20,\"b\":\"20\",\"c\":20,\"e\":\"50\",\"f\":20}}"); + } + + // Test case 4: else branch (nullable type) + { + auto variant_col = construct_basic_varint_column(); + variant_col->finalize(); + auto nullable_variant_col = make_nullable(variant_col->get_ptr()); + + auto nullable_string_type = make_nullable(std::make_shared()); + auto variant_type = std::make_shared(); + auto nullable_variant_type = make_nullable(variant_type); + + ColumnsWithTypeAndName arguments { + {nullable_variant_col->get_ptr(), nullable_variant_type, "variant_col"}, + {nullptr, nullable_string_type, "nullable_string_type"}}; + + auto function = SimpleFunctionFactory::instance().get_function("CAST", arguments, + nullable_string_type); + ASSERT_NE(function, nullptr); + + Block block {arguments}; + size_t result_column = block.columns(); + block.insert({nullptr, nullable_string_type, "result"}); + RuntimeState state; + auto ctx = FunctionContext::create_context(&state, {}, {}); + ASSERT_TRUE(function->execute(ctx.get(), block, {0}, result_column, 1).ok()); + + auto result_col = block.get_by_position(result_column).column; + ASSERT_NE(result_col.get(), nullptr); + const auto* nullable_result = assert_cast(result_col.get()); + ASSERT_EQ(nullable_result->size(), 1); + ASSERT_TRUE(nullable_result->is_null_at(1)); + } +} + +} // namespace doris::vectorized diff --git a/be/test/vec/jsonb/convert_field_to_type_test.cpp b/be/test/vec/jsonb/convert_field_to_type_test.cpp new file mode 100644 index 00000000000000..065d86c039cc11 --- /dev/null +++ b/be/test/vec/jsonb/convert_field_to_type_test.cpp @@ -0,0 +1,521 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/data_types/convert_field_to_type.cpp" + +#include + +#include +#include + +#include "runtime/jsonb_value.h" +#include "util/jsonb_document.h" +#include "util/jsonb_writer.h" +#include "vec/core/field.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type.h" +#include "vec/data_types/data_type_array.h" +#include "vec/data_types/data_type_jsonb.h" +#include "vec/data_types/data_type_nullable.h" + +namespace doris::vectorized { + +class ConvertFieldToTypeTest : public ::testing::Test { +protected: + void SetUp() override {} +}; + +// Test FieldVisitorToJsonb with different field types using the same pattern as convert_field_to_typeImpl +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_Null) { + JsonbWriter writer; + + // Test null field using Field::dispatch pattern + Field null_field; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + null_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's a null value + ASSERT_TRUE(doc->getValue()->isNull()); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_Int64) { + JsonbWriter writer; + + // Test Int64 field using Field::dispatch pattern + Int64 test_value = 12345; + Field int_field = test_value; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + int_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an integer value + ASSERT_TRUE(doc->getValue()->isInt64()); + ASSERT_EQ(((const JsonbIntVal*)doc->getValue())->val(), test_value); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_UInt64) { + JsonbWriter writer; + + // Test UInt64 field using Field::dispatch pattern + UInt64 test_value = 12345; + Field uint_field = test_value; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + uint_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an integer value + ASSERT_TRUE(doc->getValue()->isInt64()); + ASSERT_EQ(((const JsonbIntVal*)doc->getValue())->val(), static_cast(test_value)); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_Float64) { + JsonbWriter writer; + + // Test Float64 field using Field::dispatch pattern + Float64 test_value = 123.456; + Field double_field = test_value; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + double_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's a double value + ASSERT_TRUE(doc->getValue()->isDouble()); + ASSERT_DOUBLE_EQ(((const JsonbDoubleVal*)doc->getValue())->val(), test_value); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_String) { + JsonbWriter writer; + + // Test String field using Field::dispatch pattern + Field string_field = "hello world"; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + string_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's a string value + ASSERT_TRUE(doc->getValue()->isString()); + const auto* string_val = static_cast(doc->getValue()); + std::string real_string(string_val->getBlob(), string_val->getBlobLen()); + ASSERT_EQ(real_string, string_field.get()); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_JsonbField) { + JsonbWriter writer; + JsonBinaryValue jsonb_value; + std::string test_data = R"({"a": ["1", "2"]})"; + THROW_IF_ERROR(jsonb_value.from_json_string(test_data.data(), test_data.size())); + Field jsonb_field_obj = JsonbField(jsonb_value.value(), jsonb_value.size()); + + // Test JsonbField using Field::dispatch pattern + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + jsonb_field_obj); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an object value + ASSERT_TRUE(doc->getValue()->isObject()); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_Array) { + JsonbWriter writer; + + // Create an array with mixed types + Array array_field; + array_field.push_back(123); + array_field.push_back("hello"); + array_field.push_back(456.789); + + Field array_obj = array_field; + + // Test Array using Field::dispatch pattern + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + array_obj); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an array value + ASSERT_TRUE(doc->getValue()->isArray()); + const ArrayVal& array = static_cast(*doc->getValue()); + ASSERT_EQ(array.numElem(), 3); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_NestedArray) { + JsonbWriter writer; + + // Create a nested array + Array inner_array; + inner_array.push_back(1); + inner_array.push_back(2); + + Array outer_array; + outer_array.push_back(inner_array); + outer_array.push_back("nested"); + + Field nested_array_obj = outer_array; + + // Test nested Array using Field::dispatch pattern + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + nested_array_obj); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an array value + ASSERT_TRUE(doc->getValue()->isArray()); + const ArrayVal& array = static_cast(*doc->getValue()); + ASSERT_EQ(array.numElem(), 2); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_LargeInt) { + JsonbWriter writer; + + // Test Int128 field using Field::dispatch pattern + Int128 test_value = 1234567890123456789; + Field largeint_field = test_value; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + largeint_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an int128 value + ASSERT_TRUE(doc->getValue()->isInt128()); + ASSERT_EQ(((const JsonbIntVal*)doc->getValue())->val(), test_value); +} + +TEST_F(ConvertFieldToTypeTest, FieldVisitorToJsonb_UInt128) { + JsonbWriter writer; + + // Test UInt128 field using Field::dispatch pattern + UInt128 test_value = 1234567890123456789; + Field uint128_field = test_value; + Field::dispatch([&writer](const auto& value) { FieldVisitorToJsonb()(value, &writer); }, + uint128_field); + + auto* output = writer.getOutput(); + ASSERT_NE(output, nullptr); + ASSERT_GT(output->getSize(), 0); + + // Verify the output is valid JSONB + JsonbDocument* doc = nullptr; + auto status = + JsonbDocument::checkAndCreateDocument(output->getBuffer(), output->getSize(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + + // Verify it's an int128 value + ASSERT_TRUE(doc->getValue()->isInt128()); + ASSERT_EQ(((const JsonbIntVal*)doc->getValue())->val(), static_cast(test_value)); +} + +// Test convert_field_to_type function with JSONB type (similar to convert_field_to_typeImpl) +TEST_F(ConvertFieldToTypeTest, ConvertFieldToType_ToJsonb) { + DataTypeJsonb jsonb_type; + + // Test converting Int64 to JSONB + { + Int64 test_value = 12345; + Field int_field = test_value; + Field result; + + convert_field_to_type(int_field, jsonb_type, &result); + + ASSERT_EQ(result.get_type(), Field::Types::JSONB); + ASSERT_FALSE(result.is_null()); + + const JsonbField& jsonb_result = result.get(); + ASSERT_NE(jsonb_result.get_value(), nullptr); + ASSERT_GT(jsonb_result.get_size(), 0); + + // Verify the JSONB content + JsonbDocument* doc = nullptr; + auto status = JsonbDocument::checkAndCreateDocument(jsonb_result.get_value(), + jsonb_result.get_size(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + ASSERT_TRUE(doc->getValue()->isInt64()); + ASSERT_EQ(((const JsonbIntVal*)doc->getValue())->val(), test_value); + } + + // Test converting String to JSONB + { + Field string_field = "hello world"; + Field result; + + convert_field_to_type(string_field, jsonb_type, &result); + + ASSERT_EQ(result.get_type(), Field::Types::JSONB); + ASSERT_FALSE(result.is_null()); + + const JsonbField& jsonb_result = result.get(); + ASSERT_NE(jsonb_result.get_value(), nullptr); + ASSERT_GT(jsonb_result.get_size(), 0); + + // Verify the JSONB content + JsonbDocument* doc = nullptr; + auto status = JsonbDocument::checkAndCreateDocument(jsonb_result.get_value(), + jsonb_result.get_size(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + ASSERT_TRUE(doc->getValue()->isString()); + const auto* string_val = static_cast(doc->getValue()); + std::string real_string(string_val->getBlob(), string_val->getBlobLen()); + ASSERT_EQ(real_string, string_field.get()); + } + + // Test converting Array to JSONB + { + Array array_field; + array_field.push_back(1); + array_field.push_back("test"); + array_field.push_back(3.14); + + Field array_obj = array_field; + Field result; + + convert_field_to_type(array_obj, jsonb_type, &result); + + ASSERT_EQ(result.get_type(), Field::Types::JSONB); + ASSERT_FALSE(result.is_null()); + + const JsonbField& jsonb_result = result.get(); + ASSERT_NE(jsonb_result.get_value(), nullptr); + ASSERT_GT(jsonb_result.get_size(), 0); + + // Verify the JSONB content + JsonbDocument* doc = nullptr; + auto status = JsonbDocument::checkAndCreateDocument(jsonb_result.get_value(), + jsonb_result.get_size(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + ASSERT_TRUE(doc->getValue()->isArray()); + const ArrayVal& array = static_cast(*doc->getValue()); + ASSERT_EQ(array.numElem(), 3); + } + + // Test converting JSONB to JSONB (should be no-op) + { + JsonbWriter test_writer; + test_writer.writeStartObject(); + test_writer.writeKey("key"); + test_writer.writeString("value"); + test_writer.writeEndObject(); + + auto* test_output = test_writer.getOutput(); + JsonbField original_jsonb(test_output->getBuffer(), test_output->getSize()); + Field jsonb_field = original_jsonb; + Field result; + + convert_field_to_type(jsonb_field, jsonb_type, &result); + + ASSERT_EQ(result.get_type(), Field::Types::JSONB); + ASSERT_FALSE(result.is_null()); + + const JsonbField& jsonb_result = result.get(); + ASSERT_NE(jsonb_result.get_value(), nullptr); + ASSERT_EQ(jsonb_result.get_size(), original_jsonb.get_size()); + ASSERT_EQ(memcmp(jsonb_result.get_value(), original_jsonb.get_value(), + original_jsonb.get_size()), + 0); + } +} + +// Test convert_field_to_type with nullable JSONB type +TEST_F(ConvertFieldToTypeTest, ConvertFieldToType_ToNullableJsonb) { + auto nullable_jsonb_type = + std::make_shared(std::make_shared()); + + // Test converting null field + { + Field null_field; + Field result; + + convert_field_to_type(null_field, *nullable_jsonb_type, &result); + + ASSERT_TRUE(result.is_null()); + } + + // Test converting non-null field + { + Field string_field = "test string"; + Field result; + + convert_field_to_type(string_field, *nullable_jsonb_type, &result); + + ASSERT_EQ(result.get_type(), Field::Types::JSONB); + ASSERT_FALSE(result.is_null()); + + const JsonbField& jsonb_result = result.get(); + ASSERT_NE(jsonb_result.get_value(), nullptr); + ASSERT_GT(jsonb_result.get_size(), 0); + + // Verify the JSONB content + JsonbDocument* doc = nullptr; + auto status = JsonbDocument::checkAndCreateDocument(jsonb_result.get_value(), + jsonb_result.get_size(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument: " << status.to_string(); + ASSERT_NE(doc, nullptr); + ASSERT_TRUE(doc->getValue()->isString()); + const auto* string_val = static_cast(doc->getValue()); + std::string real_string(string_val->getBlob(), string_val->getBlobLen()); + ASSERT_EQ(real_string, string_field.get()); + } +} + +// Test convert_field_to_type with array of JSONB +TEST_F(ConvertFieldToTypeTest, ConvertFieldToType_ArrayToJsonb) { + auto array_jsonb_type = std::make_shared(std::make_shared()); + + // Create an array with mixed types that will be converted to JSONB + Array array_field; + array_field.push_back(123); + array_field.push_back("hello"); + array_field.push_back(456.789); + + Field array_obj = array_field; + Field result; + + convert_field_to_type(array_obj, *array_jsonb_type, &result); + + ASSERT_EQ(result.get_type(), Field::Types::Array); + ASSERT_FALSE(result.is_null()); + + const Array& result_array = result.get(); + ASSERT_EQ(result_array.size(), 3); + + // Verify each element is converted to JSONB + for (size_t i = 0; i < result_array.size(); ++i) { + ASSERT_EQ(result_array[i].get_type(), Field::Types::JSONB); + ASSERT_FALSE(result_array[i].is_null()); + + const auto& jsonb_element = result_array[i].get(); + ASSERT_NE(jsonb_element.get_value(), nullptr); + ASSERT_GT(jsonb_element.get_size(), 0); + + // Verify the JSONB content + JsonbDocument* doc = nullptr; + auto status = JsonbDocument::checkAndCreateDocument(jsonb_element.get_value(), + jsonb_element.get_size(), &doc); + ASSERT_TRUE(status.ok()) << "Failed to create JsonbDocument for element " << i << ": " + << status.to_string(); + ASSERT_NE(doc, nullptr); + } +} + +// Test error cases +TEST_F(ConvertFieldToTypeTest, ConvertFieldToType_ErrorCases) { + DataTypeJsonb jsonb_type; + + // Test with unsupported types (should throw exception) + { + Field tuple_field = Tuple(); + + EXPECT_THROW( + { + Field result; + convert_field_to_type(tuple_field, jsonb_type, &result); + }, + doris::Exception); + } +} + +} // namespace doris::vectorized \ No newline at end of file diff --git a/be/test/vec/jsonb/json_parser_test.cpp b/be/test/vec/jsonb/json_parser_test.cpp new file mode 100644 index 00000000000000..924bd13197db7b --- /dev/null +++ b/be/test/vec/jsonb/json_parser_test.cpp @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/json/json_parser.h" + +#include + +#include + +#include "vec/core/field.h" +#include "vec/core/types.h" + +using doris::vectorized::JSONDataParser; +using doris::vectorized::SimdJSONParser; +using doris::vectorized::ParseConfig; + +TEST(JsonParserTest, ParseSimpleTypes) { + JSONDataParser parser; + ParseConfig config; + + // int + auto result = parser.parse("123", 3, config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + + // double + result = parser.parse("1.23", 4, config); + ASSERT_TRUE(result.has_value()); + + // bool + result = parser.parse("true", 4, config); + ASSERT_TRUE(result.has_value()); + + // null + result = parser.parse("null", 4, config); + ASSERT_TRUE(result.has_value()); + + // string + result = parser.parse("\"abc\"", 5, config); + ASSERT_TRUE(result.has_value()); +} + +TEST(JsonParserTest, ParseObjectAndArray) { + JSONDataParser parser; + ParseConfig config; + + // Object + auto result = parser.parse(R"({"a":1,"b":2})", 13, config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 2); + + // Array + result = parser.parse("[1,2,3]", 7, config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); +} + +TEST(JsonParserTest, ParseMultiLevelNestedArray) { + JSONDataParser parser; + ParseConfig config; + + auto result = parser.parse("[[1,2],[3,4]]", 13, config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + EXPECT_EQ(result->values[0].get_type(), doris::vectorized::Field::Types::Array); + + result = parser.parse("[[[1],[2]],[[3],[4]]]", 21, config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + EXPECT_EQ(result->values[0].get_type(), doris::vectorized::Field::Types::Array); + + result = parser.parse("[[1,2],[3],[4,5,6]]", 19, config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + + // Test complex nested structure + config.enable_flatten_nested = false; + std::string json1 = R"({"a":[[1,2],[3],[4,5,6]]})"; + // multi level nested array in object + result = parser.parse(json1.c_str(), json1.size(), config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + EXPECT_EQ(result->values[0].get_type(), doris::vectorized::Field::Types::Array); + + std::string json = R"({"nested": [{"a": [1,2,3]}]})"; + // result should be jsonbField + result = parser.parse(json.c_str(), json.size(), config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + EXPECT_EQ(result->values[0].get_type(), doris::vectorized::Field::Types::JSONB); + + // multi level nested array in nested array object + std::string json2 = R"({"a":[{"b":[[1,2,3]]}]})"; + result = parser.parse(json2.c_str(), json2.size(), config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + EXPECT_EQ(result->values[0].get_type(), doris::vectorized::Field::Types::JSONB); + + // test flatten nested + config.enable_flatten_nested = true; + EXPECT_ANY_THROW(parser.parse(json.c_str(), json.size(), config)); + // test flatten nested with multi level nested array + // no throw because it is not nested object array + result = parser.parse(json1.c_str(), json1.size(), config); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->values.size(), 1); + EXPECT_EQ(result->paths.size(), 1); + EXPECT_EQ(result->values[0].get_type(), doris::vectorized::Field::Types::Array); + + EXPECT_ANY_THROW(parser.parse(json2.c_str(), json2.size(), config)); +} + +TEST(JsonParserTest, ParseNestedAndFlatten) { + JSONDataParser parser; + ParseConfig config; + config.enable_flatten_nested = true; + + std::string json = R"({"a":[{"b":1},{"b":2}]})"; + auto result = parser.parse(json.c_str(), json.size(), config); + ASSERT_TRUE(result.has_value()); + EXPECT_GT(result->values.size(), 0); + + config.enable_flatten_nested = false; + std::string json2 = R"({"a":[{"b":1},{"b":2}]})"; + result = parser.parse(json2.c_str(), json2.size(), config); + ASSERT_TRUE(result.has_value()); +} + +TEST(JsonParserTest, ParseInvalidJson) { + JSONDataParser parser; + ParseConfig config; + + auto result = parser.parse("{a:1}", 5, config); + ASSERT_FALSE(result.has_value()); + + result = parser.parse("", 0, config); + ASSERT_FALSE(result.has_value()); +} + +TEST(JsonParserTest, ParseCornerCases) { + JSONDataParser parser; + ParseConfig config; + + auto result = parser.parse("{}", 2, config); + ASSERT_TRUE(result.has_value()); + + result = parser.parse("[]", 2, config); + ASSERT_TRUE(result.has_value()); + + result = parser.parse(R"({"a":"\n\t"})", 12, config); + ASSERT_TRUE(result.has_value()); +} diff --git a/regression-test/data/variant_p0/desc.out b/regression-test/data/variant_p0/desc.out index 1eff52e44842f3..71f804cc25c5bb 100644 --- a/regression-test/data/variant_p0/desc.out +++ b/regression-test/data/variant_p0/desc.out @@ -16,7 +16,7 @@ v.b json Yes false \N NONE v.c.c smallint Yes false \N NONE v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE -v.ddd.mxmxm json Yes false \N NONE +v.ddd.mxmxm array Yes false \N NONE v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE @@ -37,7 +37,7 @@ v.b json Yes false \N NONE v.c.c smallint Yes false \N NONE v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE -v.ddd.mxmxm json Yes false \N NONE +v.ddd.mxmxm array Yes false \N NONE v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE @@ -66,7 +66,7 @@ v.b json Yes false \N NONE v.c.c smallint Yes false \N NONE v.c.e double Yes false \N NONE v.ddd.aaa tinyint Yes false \N NONE -v.ddd.mxmxm json Yes false \N NONE +v.ddd.mxmxm array Yes false \N NONE v.oooo.xxxx.xxx tinyint Yes false \N NONE v.xxxx text Yes false \N NONE diff --git a/regression-test/data/variant_p0/nested2.out b/regression-test/data/variant_p0/nested2.out new file mode 100644 index 00000000000000..c7790a107de3da --- /dev/null +++ b/regression-test/data/variant_p0/nested2.out @@ -0,0 +1,337 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_desc_1 -- +k bigint Yes true \N +v variant Yes false \N NONE +v.nested.a array Yes false \N NONE +v.nested.b array Yes false \N NONE +v.nested.c array Yes false \N NONE + +-- !sql_0 -- +1 {"nested":[{"a":1,"c":1.1},{"b":"1"}]} + +-- !sql_1 -- +[1, null] + +-- !sql_2 -- +[null, "1"] + +-- !sql_3 -- +[1.1, null] + +-- !sql_4 -- +[{"a":1,"c":1.1}, {"b":"1"}] + +-- !sql_8 -- +[1, null] 2 + +-- !sql_9 -- +[null, 1] 2 + +-- !sql_10 -- +[1, null] 2 + +-- !sql_11 -- +["1", null] 2 + +-- !sql_12 -- +[null, "1"] 2 + +-- !sql_13 -- +["1.1", null] 2 + +-- !sql_14 -- +[1, null] 2 + +-- !sql_15 -- +[null, 1] 2 + +-- !sql_16 -- +[1.1, null] 2 + +-- !sql_desc_2 -- +k bigint Yes true \N +v variant Yes false \N NONE +v.nested.a array Yes false \N NONE +v.nested.b array Yes false \N NONE +v.nested.c array Yes false \N NONE + +-- !sql_0 -- +1 {"nested":[{"a":1.0,"c":1.1},{"b":"1"}]} +2 {"nested":[{"a":2.5,"b":123.1}]} + +-- !sql_1 -- +[1, null] +[2.5] + +-- !sql_2 -- +[null, ""1""] +["123.1"] + +-- !sql_3 -- +[1.1, null] +\N + +-- !sql_4 -- +[{"a":1.0,"c":1.1}, {"b":"1"}] +[{"a":2.5,"b":123.1}] + +-- !sql_8 -- +[1, null] 2 +[2] 1 + +-- !sql_9 -- +[null, 1] 2 +[123] 1 + +-- !sql_10 -- +[1, null] 2 +\N \N + +-- !sql_11 -- +["1", null] 2 +["2.5"] 1 + +-- !sql_12 -- +[null, "1"] 2 +["123.1"] 1 + +-- !sql_13 -- +["1.1", null] 2 +\N \N + +-- !sql_14 -- +[1, null] 2 +[2.5] 1 + +-- !sql_15 -- +[null, 1] 2 +[123.1] 1 + +-- !sql_16 -- +[1.1, null] 2 +\N \N + +-- !sql_0 -- +1 {"nested":[{"a":1.0,"c":1.1},{"b":"1"}]} +2 {"nested":[{"a":2.5,"b":123.1}]} + +-- !sql_1 -- +[1, null] +[2.5] + +-- !sql_2 -- +[null, ""1""] +["123.1"] + +-- !sql_3 -- +[1.1, null] +[null] + +-- !sql_4 -- +[{"a":1.0,"c":1.1}, {"b":"1"}] +[{"a":2.5,"b":123.1}] + +-- !sql_8 -- +[1, null] 2 +[2] 1 + +-- !sql_9 -- +[null, 1] 2 +[123] 1 + +-- !sql_10 -- +[1, null] 2 +[null] 1 + +-- !sql_11 -- +["1", null] 2 +["2.5"] 1 + +-- !sql_12 -- +[null, "1"] 2 +["123.1"] 1 + +-- !sql_13 -- +["1.1", null] 2 +[null] 1 + +-- !sql_14 -- +[1, null] 2 +[2.5] 1 + +-- !sql_15 -- +[null, 1] 2 +[123.1] 1 + +-- !sql_16 -- +[1.1, null] 2 +[null] 1 + +-- !sql_desc_4 -- +k bigint Yes true \N +v variant Yes false \N NONE +v.nested.a array Yes false \N NONE +v.nested.b array Yes false \N NONE +v.nested.c array Yes false \N NONE + +-- !sql_0 -- +1 {"nested":[{"a":1,"b":1.1},{"a":"1","b":"1","c":"1"}]} + +-- !sql_1 -- +["1", ""1""] + +-- !sql_2 -- +["1.1", ""1""] + +-- !sql_3 -- +[null, "1"] + +-- !sql_4 -- +[{"a":1,"b":1.1}, {"a":"1","b":"1","c":"1"}] + +-- !sql_8 -- +[1, 1] 2 + +-- !sql_9 -- +[1, 1] 2 + +-- !sql_10 -- +[null, 1] 2 + +-- !sql_11 -- +["1", "1"] 2 + +-- !sql_12 -- +["1.1", "1"] 2 + +-- !sql_13 -- +[null, "1"] 2 + +-- !sql_14 -- +[1, 1] 2 + +-- !sql_15 -- +[1.1, 1] 2 + +-- !sql_16 -- +[null, 1] 2 + +-- !sql_desc_5 -- +k bigint Yes true \N +v variant Yes false \N NONE +v.nested.a array Yes false \N NONE +v.nested.b array Yes false \N NONE +v.nested.c array Yes false \N NONE + +-- !sql_0 -- +1 {"nested":[{"a":1,"b":1.1},{"a":"1","b":"1","c":"1"}]} +2 {"nested":[{"a":1,"c":1.1}]} + +-- !sql_1 -- +["1", ""1""] +["1"] + +-- !sql_2 -- +["1.1", ""1""] +\N + +-- !sql_3 -- +[null, ""1""] +["1.1"] + +-- !sql_4 -- +[{"a":1,"b":1.1}, {"a":"1","b":"1","c":"1"}] +[{"a":1,"c":1.1}] + +-- !sql_8 -- +[1, 1] 2 +[1] 1 + +-- !sql_9 -- +[1, 1] 2 +\N \N + +-- !sql_10 -- +[null, 1] 2 +[1] 1 + +-- !sql_11 -- +["1", "1"] 2 +["1"] 1 + +-- !sql_12 -- +["1.1", "1"] 2 +\N \N + +-- !sql_13 -- +[null, "1"] 2 +["1.1"] 1 + +-- !sql_14 -- +[1, 1] 2 +[1] 1 + +-- !sql_15 -- +[1.1, 1] 2 +\N \N + +-- !sql_16 -- +[null, 1] 2 +[1.1] 1 + +-- !sql_0 -- +1 {"nested":[{"a":1,"b":1.1},{"a":"1","b":"1","c":"1"}]} +2 {"nested":[{"a":1,"c":1.1}]} + +-- !sql_1 -- +["1", ""1""] +["1"] + +-- !sql_2 -- +["1.1", ""1""] +[null] + +-- !sql_3 -- +[null, ""1""] +["1.1"] + +-- !sql_4 -- +[{"a":1,"b":1.1}, {"a":"1","b":"1","c":"1"}] +[{"a":1,"c":1.1}] + +-- !sql_8 -- +[1, 1] 2 +[1] 1 + +-- !sql_9 -- +[1, 1] 2 +[null] 1 + +-- !sql_10 -- +[null, 1] 2 +[1] 1 + +-- !sql_11 -- +["1", "1"] 2 +["1"] 1 + +-- !sql_12 -- +["1.1", "1"] 2 +[null] 1 + +-- !sql_13 -- +[null, "1"] 2 +["1.1"] 1 + +-- !sql_14 -- +[1, 1] 2 +[1] 1 + +-- !sql_15 -- +[1.1, 1] 2 +[null] 1 + +-- !sql_16 -- +[null, 1] 2 +[1.1] 1 + diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index 8ec837ff18bae2..8d87175d1d324c 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +import org.awaitility.Awaitility suite("regression_test_variant", "p0"){ diff --git a/regression-test/suites/variant_p0/nested2.groovy b/regression-test/suites/variant_p0/nested2.groovy new file mode 100644 index 00000000000000..8d48fcfce9b1c0 --- /dev/null +++ b/regression-test/suites/variant_p0/nested2.groovy @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// this test is used to test the type conflict of nested array +suite("variant_nested_type_conflict", "p0"){ + + try { + + def table_name = "var_nested_type_conflict" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """set describe_extend_variant_column = true""" + + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case + properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + """ + def sql_select_batch = { + qt_sql_0 """select * from ${table_name} order by k""" + + qt_sql_1 """select v['nested']['a'] from ${table_name} order by k""" + qt_sql_2 """select v['nested']['b'] from ${table_name} order by k""" + qt_sql_3 """select v['nested']['c'] from ${table_name} order by k""" + + qt_sql_4 """select v['nested'] from ${table_name} order by k""" + } + + def sql_test_cast_to_array = { + // test cast to array + qt_sql_8 """select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from ${table_name} order by k""" + qt_sql_9 """select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from ${table_name} order by k""" + qt_sql_10 """select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from ${table_name} order by k""" + + // test cast to array + qt_sql_11 """select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from ${table_name} order by k""" + qt_sql_12 """select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from ${table_name} order by k""" + qt_sql_13 """select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from ${table_name} order by k""" + + // test cast to array + qt_sql_14 """select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from ${table_name} order by k""" + qt_sql_15 """select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from ${table_name} order by k""" + qt_sql_16 """select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from ${table_name} order by k""" + + } + // insert Nested array in Nested array which is not supported + test { + sql """ + insert into ${table_name} values (1, '{"nested": [{"a": [1,2,3]}]}'); + """ + exception "Nesting of array in Nested array within variant subcolumns is currently not supported." + } + /// insert a array of object for a, b, c + // insert type conflict in multiple rows + sql """ + insert into ${table_name} values (1, '{"nested": [{"a": 1, "c": 1.1}, {"b": "1"}]}'); + """ + + // for cloud we should select first and then desc for syncing rowset to get latest schema + sql """ + select * from ${table_name} order by k limit 1; + """ + qt_sql_desc_1 """ + desc ${table_name}; + """ + // now select for a, b, c + sql_select_batch() + sql_test_cast_to_array() + /// insert a, b type changed to double + sql """ + insert into ${table_name} values (2, '{"nested": [{"a": 2.5, "b": 123.1}]}'); + """ + // for cloud we should select first and then desc for syncing rowset to get latest schema + sql """ + select * from ${table_name} order by k limit 1; + """ + qt_sql_desc_2 """ + desc ${table_name}; + """ + // now select for a, b, c + sql_select_batch() + sql_test_cast_to_array() + + // trigger and wait compaction + trigger_and_wait_compaction("${table_name}", "full") + + // now select for a, b, c + sql_select_batch() + sql_test_cast_to_array() + + sql """ truncate table ${table_name} """ + + + // insert type conflict in one row + sql """ + insert into ${table_name} values (1, '{"nested": [{"a": 1, "b": 1.1}, {"a": "1", "b": "1", "c": "1"}]}'); + """ + // for cloud we should select first and then desc for syncing rowset to get latest schema + sql """ + select * from ${table_name} order by k limit 1; + """ + qt_sql_desc_4 """ + desc ${table_name}; + """ + // now select for a, b, c + sql_select_batch() + sql_test_cast_to_array() + + // insert c type changed to double + sql """ + insert into ${table_name} values (2, '{"nested": [{"a": 1, "c": 1.1}]}'); + """ + // for cloud we should select first and then desc for syncing rowset to get latest schema + sql """ + select * from ${table_name} order by k limit 1; + """ + qt_sql_desc_5 """ + desc ${table_name}; + """ + // now select for a, b, c + sql_select_batch() + sql_test_cast_to_array() + + // trigger and wait compaction + trigger_and_wait_compaction("${table_name}", "full") + + // now select for a, b, c + sql_select_batch() + sql_test_cast_to_array() + + } finally { + } + +} From 80644430a22971959be4e1aa60ac6c6467bd57c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 14:12:57 +0800 Subject: [PATCH 252/572] branch-3.0: [fix](nereids)'VALUES IN' keyword should be optional in list partition definition #52986 (#53241) Cherry-picked from #52986 Co-authored-by: starocean999 --- .../src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +- .../partition_p0/list_partition/test_list_partition.groovy | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 38acd9f6e9fa57..4cc2eae2e1b85d 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1379,7 +1379,7 @@ stepPartitionDef ; inPartitionDef - : PARTITION (IF NOT EXISTS)? partitionName=identifier (VALUES IN ((LEFT_PAREN partitionValueLists+=partitionValueList + : PARTITION (IF NOT EXISTS)? partitionName=identifier ((VALUES IN)? ((LEFT_PAREN partitionValueLists+=partitionValueList (COMMA partitionValueLists+=partitionValueList)* RIGHT_PAREN) | constants=partitionValueList))? ; diff --git a/regression-test/suites/partition_p0/list_partition/test_list_partition.groovy b/regression-test/suites/partition_p0/list_partition/test_list_partition.groovy index a12879393e6a72..4ce19d7bb285b1 100644 --- a/regression-test/suites/partition_p0/list_partition/test_list_partition.groovy +++ b/regression-test/suites/partition_p0/list_partition/test_list_partition.groovy @@ -38,7 +38,7 @@ suite("test_list_partition", "p0") { AGGREGATE KEY(k1,k2,k3,k4,k5,k6,k10,k11,k12,k13,k14,k15,k7) PARTITION BY LIST(k1) ( PARTITION p1 VALUES IN ("1","2","3","4"), - PARTITION p2 VALUES IN ("5","6","7","8","9","10","11","12","13","14"), + PARTITION p2 ("5","6","7","8","9","10","11","12","13","14"), PARTITION p3 VALUES IN ("15") ) DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1") """ @@ -67,9 +67,9 @@ suite("test_list_partition", "p0") { AGGREGATE KEY(k1,k2,k3,k4,k5,k6,k10,k11,k7) PARTITION BY LIST(k1) ( PARTITION p1 VALUES IN ("-128","-127","-126","-125","-124","-123","-122","-121","-120","-119","-118","-117","-116","-115","-114","-113","-112","-111","-110","-109","-108","-107","-106","-105","-104","-103","-102","-101","-100","-99","-98","-97","-96","-95","-94","-93","-92","-91","-90","-89","-88","-87","-86","-85","-84","-83","-82","-81","-80","-79","-78","-77","-76","-75","-74","-73","-72","-71","-70","-69","-68","-67","-66","-65"), - PARTITION p2 VALUES IN ("-64","-63","-62","-61","-60","-59","-58","-57","-56","-55","-54","-53","-52","-51","-50","-49","-48","-47","-46","-45","-44","-43","-42","-41","-40","-39","-38","-37","-36","-35","-34","-33","-32","-31","-30","-29","-28","-27","-26","-25","-24","-23","-22","-21","-20","-19","-18","-17","-16","-15","-14","-13","-12","-11","-10","-9","-8","-7","-6","-5","-4","-3","-2","-1"), + PARTITION p2 ("-64","-63","-62","-61","-60","-59","-58","-57","-56","-55","-54","-53","-52","-51","-50","-49","-48","-47","-46","-45","-44","-43","-42","-41","-40","-39","-38","-37","-36","-35","-34","-33","-32","-31","-30","-29","-28","-27","-26","-25","-24","-23","-22","-21","-20","-19","-18","-17","-16","-15","-14","-13","-12","-11","-10","-9","-8","-7","-6","-5","-4","-3","-2","-1"), PARTITION p3 VALUES IN ("0","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31","32","33","34","35","36","37","38","39","40","41","42","43","44","45","46","47","48","49","50","51","52","53","54","55","56","57","58","59","60","61","62","63"), - PARTITION p4 VALUES IN ("64","65","66","67","68","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116","117","118","119","120","121","122","123","124","125","126","127") + PARTITION p4 ("64","65","66","67","68","69","70","71","72","73","74","75","76","77","78","79","80","81","82","83","84","85","86","87","88","89","90","91","92","93","94","95","96","97","98","99","100","101","102","103","104","105","106","107","108","109","110","111","112","113","114","115","116","117","118","119","120","121","122","123","124","125","126","127") ) DISTRIBUTED BY HASH(k1) BUCKETS 5 properties("replication_num" = "1") From 9d71ebae8f93d8891fd6c66cf7c0d4510a7447eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 14:16:49 +0800 Subject: [PATCH 253/572] branch-3.0: [fix](cloud) Fix roll-backed cumulative point of new tablet when doing schema change #53402 (#53446) Cherry-picked from #53402 Co-authored-by: Gavin Chou --- cloud/src/meta-service/meta_service.cpp | 3 ++- cloud/src/meta-service/meta_service_job.cpp | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 176882cb120394..87a6cf68e33915 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1816,8 +1816,9 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, code = MetaServiceCode::INVALID_ARGUMENT; ss << "no valid compaction_cnt or cumulative_point given. req_bc_cnt=" << req_bc_cnt << ", bc_cnt=" << bc_cnt << ", req_cc_cnt=" << req_cc_cnt << ", cc_cnt=" << cc_cnt - << ", req_cp=" << req_cp << ", cp=" << cp; + << ", req_cp=" << req_cp << ", cp=" << cp << " tablet_id=" << tablet_id; msg = ss.str(); + LOG(WARNING) << msg; return; } auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 000edc76a30395..6b60a0dd81c371 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -1204,7 +1204,7 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str //========================================================================== if (!schema_change.has_alter_version()) { code = MetaServiceCode::INVALID_ARGUMENT; - msg = "invalid alter_version"; + msg = "no alter_version for schema change job, tablet_id=" + std::to_string(tablet_id); return; } if (schema_change.alter_version() < 2) { @@ -1304,7 +1304,12 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str internal_get_tablet_stats(code, msg, txn.get(), instance_id, new_tablet_idx, *stats, detached_stats, config::snapshot_get_tablet_stats); // clang-format off - stats->set_cumulative_point(schema_change.output_cumulative_point()); + // ATTN: cumu point in job is from base tablet which may be fetched long time ago + // since the new tablet may have done cumu compactions with alter_version as initial cumu point + // current cumu point of new tablet may be larger than job.alter_version + // we need to keep the larger one in case of cumu point roll-back to + // break the basic assumptions of non-decreasing cumu point + stats->set_cumulative_point(std::max(schema_change.output_cumulative_point(), stats->cumulative_point())); stats->set_num_rows(stats->num_rows() + (schema_change.num_output_rows() - num_remove_rows)); stats->set_data_size(stats->data_size() + (schema_change.size_output_rowsets() - size_remove_rowsets)); stats->set_num_rowsets(stats->num_rowsets() + (schema_change.num_output_rowsets() - num_remove_rowsets)); From ba48ea5e4415b8ad171bcac9c935ecf44bf04688 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 14:17:27 +0800 Subject: [PATCH 254/572] branch-3.0: [fix](regression)Fix test_analyze_mv case. #53352 (#53475) Cherry-picked from #53352 Co-authored-by: James --- .../suites/statistics/test_analyze_mv.groovy | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy index aa8be109203bc1..508aa538a83639 100644 --- a/regression-test/suites/statistics/test_analyze_mv.groovy +++ b/regression-test/suites/statistics/test_analyze_mv.groovy @@ -680,20 +680,38 @@ suite("test_analyze_mv") { empty_test = sql """show column stats mvTestDup""" assertEquals(12, empty_test.size()) - for (int i = 0; i < 120; i++) { - result_row = sql """show index stats mvTestDup mv3""" - logger.info("mv3 stats: " + result_row) - if (result_row[0][4] == "0") { - break; + def result = sql """show frontends;""" + logger.info("show frontends result origin: " + result) + def host + def port + for (int i = 0; i < result.size(); i++) { + if (result[i][8] == "true") { + host = result[i][1] + port = result[i][4] } - Thread.sleep(5000) } - result_row = sql """show index stats mvTestDup mv3""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv3", result_row[0][1]) - assertEquals("0", result_row[0][3]) - assertEquals("0", result_row[0][4]) + def tokens = context.config.jdbcUrl.split('/') + def url=tokens[0] + "//" + host + ":" + port + logger.info("Master url is " + url) + connect(context.config.jdbcUser, context.config.jdbcPassword, url) { + sql """use test_analyze_mv""" + result = sql """show frontends;""" + logger.info("show frontends result master: " + result) + for (int i = 0; i < 120; i++) { + result_row = sql """show index stats mvTestDup mv3""" + logger.info("mv3 stats: " + result_row) + if (result_row[0][4] == "0") { + break; + } + Thread.sleep(5000) + } + result_row = sql """show index stats mvTestDup mv3""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mv3", result_row[0][1]) + assertEquals("0", result_row[0][3]) + assertEquals("0", result_row[0][4]) + } // ** Embedded test for skip auto analyze when table is empty again wait_row_count_reported("test_analyze_mv", "mvTestDup", 0, 4, "0") @@ -725,7 +743,7 @@ suite("test_analyze_mv") { sql """alter table mvTestDup index mv3 modify column mv_key1 set stats ('ndv'='5', 'num_nulls'='0', 'min_value'='0', 'max_value'='4', 'row_count'='5');""" sql """alter table mvTestDup index mv3 modify column `mva_SUM__CAST(``value1`` AS bigint)` set stats ('ndv'='10', 'num_nulls'='2', 'min_value'='1', 'max_value'='5', 'row_count'='11');""" - def result = sql """show column cached stats mvTestDup(key1)""" + result = sql """show column cached stats mvTestDup(key1)""" assertEquals(1, result.size()) assertEquals("key1", result[0][0]) assertEquals("mvTestDup", result[0][1]) From 362f65ac5436036bba5e379f896771dcafe25efa Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 18 Jul 2025 14:19:52 +0800 Subject: [PATCH 255/572] branch-3.0 [fix](nereids) fix ut case: StatsCalculatorTest (#53436) --- .../apache/doris/nereids/stats/StatsCalculatorTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java index cf91eacb51ca75..58bc51d88cc641 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java @@ -145,14 +145,14 @@ public void testFilter() { GroupExpression groupExpression = new GroupExpression(logicalFilter, ImmutableList.of(childGroup)); Group ownerGroup = new Group(null, groupExpression, null); StatsCalculator.estimate(groupExpression, null); - Assertions.assertEquals(49.945, ownerGroup.getStatistics().getRowCount(), 0.001); + Assertions.assertEquals(49.945, ownerGroup.getStatistics().getRowCount(), 0.1); LogicalFilter logicalFilterOr = new LogicalFilter<>(or, groupPlan); GroupExpression groupExpressionOr = new GroupExpression(logicalFilterOr, ImmutableList.of(childGroup)); Group ownerGroupOr = new Group(null, groupExpressionOr, null); StatsCalculator.estimate(groupExpressionOr, null); Assertions.assertEquals(1448.555, - ownerGroupOr.getStatistics().getRowCount(), 0.001); + ownerGroupOr.getStatistics().getRowCount(), 0.1); } // a, b are in (0,100) @@ -194,14 +194,14 @@ public void testFilterOutofRange() { Group ownerGroup = new Group(null, groupExpression, null); groupExpression.setOwnerGroup(ownerGroup); StatsCalculator.estimate(groupExpression, null); - Assertions.assertEquals(0, ownerGroup.getStatistics().getRowCount(), 0.001); + Assertions.assertEquals(0, ownerGroup.getStatistics().getRowCount(), 0.1); LogicalFilter logicalFilterOr = new LogicalFilter<>(or, groupPlan); GroupExpression groupExpressionOr = new GroupExpression(logicalFilterOr, ImmutableList.of(childGroup)); Group ownerGroupOr = new Group(null, groupExpressionOr, null); groupExpressionOr.setOwnerGroup(ownerGroupOr); StatsCalculator.estimate(groupExpressionOr, null); - Assertions.assertEquals(0, ownerGroupOr.getStatistics().getRowCount(), 0.001); + Assertions.assertEquals(0, ownerGroupOr.getStatistics().getRowCount(), 0.1); } // TODO: temporary disable this test, until we could get column stats // @Test From f2f9ecdc9aa4435090ba57b16746008d89b80218 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 14:20:36 +0800 Subject: [PATCH 256/572] branch-3.0: [fix](case) fix regression case test_cluster_management_auth #53336 (#53369) Cherry-picked from #53336 Co-authored-by: zhangdong --- .../test_cluster_management_auth.groovy | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/regression-test/suites/auth_call/test_cluster_management_auth.groovy b/regression-test/suites/auth_call/test_cluster_management_auth.groovy index f769f29c967abc..c3c1692c405109 100644 --- a/regression-test/suites/auth_call/test_cluster_management_auth.groovy +++ b/regression-test/suites/auth_call/test_cluster_management_auth.groovy @@ -76,10 +76,7 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") { // pipeline can't support delete node, it can affect other case if (is_exists_follower()) { connect(user, "${pwd}", context.config.jdbcUrl) { - test { - sql """show frontends""" - exception "denied" - } + sql """show frontends""" test { sql """ALTER SYSTEM add FOLLOWER '${follower_ip}:${follower_host}'""" exception "denied" @@ -93,10 +90,7 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") { if (is_exists_observer()) { connect(user, "${pwd}", context.config.jdbcUrl) { - test { - sql """show frontends""" - exception "denied" - } + sql """show frontends""" test { sql """ALTER SYSTEM add OBSERVER '${observer_ip}:${observer_host}'""" exception "denied" @@ -110,10 +104,8 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") { if (is_exists_backends()) { connect(user, "${pwd}", context.config.jdbcUrl) { - test { - sql """show backends""" - exception "denied" - } + sql """show backends""" + test { sql """ALTER SYSTEM add backend '${backend_ip}:${backend_host}'""" exception "denied" From cd8d3f87d5f77bcf6cbb02920cb11d6570da23e6 Mon Sep 17 00:00:00 2001 From: daidai Date: Fri, 18 Jul 2025 14:21:15 +0800 Subject: [PATCH 257/572] branch-3.0:[fix](function)fix month=0 of from_iso8601_date function. (#53050) (#53309) bp #53050 --- be/src/vec/runtime/vdatetime_value.h | 7 +- be/test/vec/function/function_time_test.cpp | 73 +++++++++++++++++++ .../test_from_iso8601_date.out | 52 +++++++++++++ .../test_from_iso8601_date.groovy | 63 ++++++++++++++++ 4 files changed, 194 insertions(+), 1 deletion(-) diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 12e961f5fa91bc..11dfb6e796bb0f 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -1211,13 +1211,18 @@ class DateV2Value { } date_v2_value_.year_ = val; } else if constexpr (unit == TimeUnit::MONTH) { - if (val > MAX_MONTH) [[unlikely]] { + DCHECK(date_v2_value_.year_ <= MAX_YEAR); + if (val > MAX_MONTH || val == 0) [[unlikely]] { return false; } date_v2_value_.month_ = val; } else if constexpr (unit == TimeUnit::DAY) { + DCHECK(date_v2_value_.year_ <= MAX_YEAR); DCHECK(date_v2_value_.month_ <= MAX_MONTH); DCHECK(date_v2_value_.month_ != 0); + if (val == 0) [[unlikely]] { + return false; + } if (val > S_DAYS_IN_MONTH[date_v2_value_.month_] && !(is_leap(date_v2_value_.year_) && date_v2_value_.month_ == 2 && val == 29)) { return false; diff --git a/be/test/vec/function/function_time_test.cpp b/be/test/vec/function/function_time_test.cpp index 1ea010e4afdbb3..3d71d70c4a77d4 100644 --- a/be/test/vec/function/function_time_test.cpp +++ b/be/test/vec/function/function_time_test.cpp @@ -1892,4 +1892,77 @@ TEST(VTimestampFunctionsTest, next_day_test) { data_set)); } } + +TEST(VTimestampFunctionsTest, from_iso8601_date) { + std::string func_name = "from_iso8601_date"; + InputTypeSet input_types = {TypeIndex::String}; + + DataSet data_set = { + {{std::string("2020-01-01")}, str_to_date_v2("2020-01-01", "%Y-%m-%d")}, + {{std::string("2020-01-01")}, str_to_date_v2("2020-01-01", "%Y-%m-%d")}, + {{std::string("-1")}, Null()}, + {{std::string("2025-07-11")}, str_to_date_v2("2025-07-11", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("2025-13-01")}, Null()}, + {{std::string("2020-W10")}, str_to_date_v2("2020-03-02", "%Y-%m-%d")}, + {{std::string("2025-W28")}, str_to_date_v2("2025-07-07", "%Y-%m-%d")}, + {{std::string("2025-W53")}, str_to_date_v2("2025-12-29", "%Y-%m-%d")}, + {{std::string("2025-W00")}, Null()}, + {{std::string("2020-123")}, str_to_date_v2("2020-05-02", "%Y-%m-%d")}, + {{std::string("2025-192")}, str_to_date_v2("2025-07-11", "%Y-%m-%d")}, + {{std::string("2024-366")}, str_to_date_v2("2024-12-31", "%Y-%m-%d")}, + {{std::string("2025-366")}, Null()}, + {{std::string("2025-000")}, str_to_date_v2("2024-12-31", "%Y-%m-%d")}, + {{std::string("2025/07/11")}, Null()}, + {{std::string("25-07-11")}, Null()}, + {{std::string("2025-7-11")}, Null()}, + {{std::string("invalid-date")}, Null()}, + {{std::string("2025-07-11T12:34:56")}, Null()}, + {{std::string("-1")}, Null()}, + {{std::string("9999-12-31")}, str_to_date_v2("9999-12-31", "%Y-%m-%d")}, + {{std::string("10000-01-01")}, Null()}, + {{std::string("0001-01-01")}, str_to_date_v2("0001-01-01", "%Y-%m-%d")}, + {{std::string("0000-12-31")}, str_to_date_v2("0000-12-31", "%Y-%m-%d")}, + {{std::string("-0001-01-01")}, Null()}, + {{std::string("2025-01-01")}, str_to_date_v2("2025-01-01", "%Y-%m-%d")}, + {{std::string("2025-12-31")}, str_to_date_v2("2025-12-31", "%Y-%m-%d")}, + {{std::string("2025-00-01")}, Null()}, + {{std::string("2025-13-01")}, Null()}, + {{std::string("2025--01-01")}, Null()}, + {{std::string("2025-01-31")}, str_to_date_v2("2025-01-31", "%Y-%m-%d")}, + {{std::string("2025-04-30")}, str_to_date_v2("2025-04-30", "%Y-%m-%d")}, + {{std::string("2025-02-28")}, str_to_date_v2("2025-02-28", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, + {{std::string("2025-01-32")}, Null()}, + {{std::string("2025-04-31")}, Null()}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("2025-02-30")}, Null()}, + {{std::string("2025-01-00")}, Null()}, + {{std::string("2025-01--01")}, Null()}, + {{std::string("2000-02-29")}, str_to_date_v2("2000-02-29", "%Y-%m-%d")}, + {{std::string("2024-02-29")}, str_to_date_v2("2024-02-29", "%Y-%m-%d")}, + {{std::string("1900-02-29")}, Null()}, + {{std::string("2100-02-29")}, Null()}, + {{std::string("2025-02-29")}, Null()}, + {{std::string("-2025-01-01")}, Null()}, + {{std::string("2025--07-01")}, Null()}, + {{std::string("2025-07--01")}, Null()}, + {{std::string("")}, Null()}, + {{std::string("2025")}, str_to_date_v2("2025-01-01", "%Y-%m-%d")}, + {{std::string("2025-07")}, str_to_date_v2("2025-07-01", "%Y-%m-%d")}, + {{std::string("99999-01-01")}, Null()}, + {{std::string("2025-123-01")}, Null()}, + {{std::string("2025-01-123")}, Null()}, + {{std::string("2025/01/01")}, Null()}, + {{std::string("2025.01.01")}, Null()}, + {{std::string("2025-01-01X")}, Null()}, + {{std::string("2025--01--01")}, Null()}, + {{std::string("abcd-01-01")}, Null()}, + {{std::string("2025-ab-01")}, Null()}, + {{std::string("2025-01-ab")}, Null()}, + }; + + static_cast(check_function(func_name, input_types, data_set)); +} } // namespace doris::vectorized diff --git a/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out b/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out index 43a4f0bd496c38..93b8bda36e1f92 100644 --- a/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out +++ b/regression-test/data/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.out @@ -242,3 +242,55 @@ \N \N +-- !test_99 -- +1 2023-01-01 +2 2023-12-31 +3 2020-02-29 +4 1970-01-01 +5 9999-12-31 +6 1000-01-01 +7 \N +8 \N +9 \N +10 \N +11 \N +12 2023-02-28 +13 2024-02-29 +14 \N +15 \N +16 \N +17 \N +18 \N +19 \N +20 2023-01-01 +21 \N +22 \N +23 \N +24 2023-01-02 +25 2023-01-01 +26 \N +27 \N +28 \N +29 \N +30 \N +31 \N +32 \N +33 \N +34 \N +35 \N +36 \N +37 \N +38 \N +39 \N +40 \N +41 \N +42 \N +43 \N +44 \N +45 \N +46 \N +47 \N +48 \N +49 \N +50 \N + diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy index 813c0ca49d94cf..79d508cae87da1 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_from_iso8601_date.groovy @@ -141,4 +141,67 @@ suite("test_from_iso8601_date") { qt_test_87 """ select from_iso8601_date(k1),from_iso8601_date(k2),from_iso8601_date(k3),from_iso8601_date(k11),from_iso8601_date(k22),from_iso8601_date(k33) from tb2 order by k0;""" qt_test_88 """ select from_iso8601_date(nullable(k1)),from_iso8601_date(k2),from_iso8601_date(k3),from_iso8601_date(nullable(k11)),from_iso8601_date(k22),from_iso8601_date(k33) from tb2 order by k0; """ qt_test_89 """ select from_iso8601_date(NULL) from tb2 order by k0; """ + + + sql """ + CREATE TABLE tb3 (id INT, date_str VARCHAR(255)) DISTRIBUTED BY HASH(id) BUCKETS 4 PROPERTIES ("replication_num" = "1"); """ + sql """ + INSERT INTO tb3 (id, date_str) VALUES + (1, '2023-01-01'), + (2, '2023-12-31'), + (3, '2020-02-29'), + (4, '1970-01-01'), + (5, '9999-12-31'), + (6, '1000-01-01'), + (7, '2023-13-01'), + (8, '2023-00-01'), + (9, '2023-01-00'), + (10, '2023-01-32'), + (11, '2023-02-29'), + (12, '2023-02-28'), + (13, '2024-02-29'), + (14, '2024-02-30'), + (15, '2023-01-01T12:00:00'), + (16, '2023-01-01 12:00:00'), + (17, '2023/01/01'), + (18, '01-01-2023'), + (19, '01/01/2023'), + (20, '20230101'), + (21, '2023-01-01Z'), + (22, '2023-01-01+08:00'), + (23, '2023-01-01-08:00'), + (24, '2023-W01-1'), + (25, '2023-001'), + (26, '2023-01-01T12:00:00.000Z'), + (27, '2023-01-01T12:00:00.000+08:00'), + (28, '2023-01-01T12:00:00.000-08:00'), + (29, '2023-01-01T12:00:00.123456Z'), + (30, '2023-01-01T12:00:00.123456+08:00'), + (31, '2023-01-01T12:00:00.123456-08:00'), + (32, '2023-01-01T24:00:00'), + (33, '2023-01-01T00:00:00.000000'), + (34, '2023-01-01T00:00:00.000001'), + (35, '2023-01-01T00:00:00.999999'), + (36, '2023-01-01T23:59:59.999999'), + (37, '2023-01-01T23:59:60'), + (38, '2023-01-01T23:59:59.9999999'), + (39, '2023-01-01T23:59:59.999999999'), + (40, '2023-01-01T23:59:59.999999999Z'), + (41, '2023-01-01T23:59:59.999999999+08:00'), + (42, '2023-01-01T23:59:59.999999999-08:00'), + (43, '2023-01-01T23:59:59.999999999999'), + (44, '2023-01-01T23:59:59.999999999999Z'), + (45, '2023-01-01T23:59:59.999999999999+08:00'), + (46, '2023-01-01T23:59:59.999999999999-08:00'), + (47, '2023-01-01T23:59:59.999999999999999'), + (48, '2023-01-01T23:59:59.999999999999999Z'), + (49, '2023-01-01T23:59:59.999999999999999+08:00'), + (50, '2023-01-01T23:59:59.999999999999999-08:00'); + """ + qt_test_99 """ SELECT id, from_iso8601_date(date_str) AS result FROM tb3 order by id; """ + + + + sql """ drop table tb2 """ + } \ No newline at end of file From b41ac481114fd01cde6f5456cd631cc88b73187a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 14:21:59 +0800 Subject: [PATCH 258/572] branch-3.0: [fix](regress) new thread should connect to cluster.jdbcUrl in docker regression suite #53234 (#53287) Cherry-picked from #53234 Co-authored-by: yujun --- .../org/apache/doris/regression/Config.groovy | 11 --- .../regression/action/WaitForAction.groovy | 8 ++- .../doris/regression/suite/Suite.groovy | 67 ++++++++++--------- .../regression/suite/SuiteCluster.groovy | 2 + .../regression/suite/SuiteContext.groovy | 27 +++++++- .../multi_cluster/test_auto_start.groovy | 6 +- .../test_fe_tablet_same_backend.groovy | 8 +-- .../multi_cluster/test_multi_follower.groovy | 6 +- .../multi_cluster/test_no_cluster_hits.groovy | 2 +- .../multi_cluster/test_rebalance.groovy | 12 ++-- .../test_warmup_rebalance.groovy | 2 +- .../node_mgr/test_cloud_decommission.groovy | 10 +-- .../node_mgr/test_sql_mode_node_mgr.groovy | 8 +-- .../test_retry_e-230_async_mtmv_job.groovy | 4 +- ..._clean_tablet_when_drop_force_table.groovy | 2 +- .../test_clean_tablet_when_rebalance.groovy | 4 +- .../suites/demo_p0/docker_action.groovy | 3 +- ...ablet_state_change_in_publish_phase.groovy | 2 +- ...l_multi_segments_re_calc_in_publish.groovy | 2 +- .../test_abort_txn_by_be.groovy | 2 +- .../test_abort_txn_by_fe.groovy | 2 +- .../suites/trash_p0/clean_trash.groovy | 2 +- 22 files changed, 108 insertions(+), 84 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy index da76c682c571fb..ceffa3529a21b2 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy @@ -1060,17 +1060,6 @@ class Config { return DriverManager.getConnection(jdbcUrl, 'root', '') } - Connection getConnectionByDbName(String dbName) { - String dbUrl = getConnectionUrlByDbName(dbName) - tryCreateDbIfNotExist(dbName) - log.info("connect to ${dbUrl}".toString()) - return DriverManager.getConnection(dbUrl, jdbcUser, jdbcPassword) - } - - String getConnectionUrlByDbName(String dbName) { - return buildUrlWithDb(jdbcUrl, dbName) - } - Connection getConnectionByArrowFlightSqlDbName(String dbName) { Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver") String arrowFlightSqlHost = otherConfigs.get("extArrowFlightSqlHost") diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy index 54ba7bb631ba57..f54d43758e10b1 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/WaitForAction.groovy @@ -62,8 +62,12 @@ class WaitForAction implements SuiteAction { if (forRollUp) { num = 8 } - Awaitility.await().atMost(time, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).and() - .pollInterval(100, TimeUnit.MILLISECONDS).await().until(() -> { + Awaitility + .with().pollInSameThread() + .await() + .atMost(time, TimeUnit.SECONDS) + .with().pollDelay(100, TimeUnit.MILLISECONDS).and() + .pollInterval(100, TimeUnit.MILLISECONDS).await().until({ log.info("sql is :\n${sql}") def (result, meta) = JdbcUtils.executeToList(context.getConnection(), sql) String res = result.get(0).get(num) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 85ec795cd0df24..e7d038a44eb48f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -23,8 +23,6 @@ import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.AmazonS3ClientBuilder -import static java.util.concurrent.TimeUnit.SECONDS - import com.google.common.base.Strings import com.google.common.collect.ImmutableList import com.google.common.collect.Maps @@ -278,13 +276,22 @@ class Suite implements GroovyInterceptable { return context.connect(user, password, connInfo.conn.getMetaData().getURL(), actionSupplier) } - public void dockerAwaitUntil(int atMostSeconds, int intervalSecond = 1, Closure actionSupplier) { - def connInfo = context.threadLocalConn.get() - Awaitility.await().atMost(atMostSeconds, SECONDS).pollInterval(intervalSecond, SECONDS).until( - { - connect(connInfo.username, connInfo.password, connInfo.conn.getMetaData().getURL(), actionSupplier) - } - ) + // delete 'dockerAwaitUntil', should call 'Awaitility.await()...' directly or use 'awaitUntil(..., f)' + // public void dockerAwaitUntil(int atMostSeconds, int intervalSecond = 1, Closure actionSupplier) { + // def connInfo = context.threadLocalConn.get() + // Awaitility.await().atMost(atMostSeconds, SECONDS).pollInterval(intervalSecond, SECONDS).until( + // { + // connect(connInfo.username, connInfo.password, connInfo.conn.getMetaData().getURL(), actionSupplier) + // } + // ) + // } + public void awaitUntil(int atMostSeconds, double intervalSecond = 1, Closure actionSupplier) { + Awaitility + .with().pollInSameThread() + .await() + .atMost(atMostSeconds, TimeUnit.SECONDS) + .pollInterval((int) (1000 * intervalSecond), TimeUnit.MILLISECONDS) + .until(actionSupplier) } // more explaination can see example file: demo_p0/docker_action.groovy @@ -319,6 +326,7 @@ class Suite implements GroovyInterceptable { private void dockerImpl(ClusterOptions options, boolean isCloud, Closure actionSupplier) throws Exception { logger.info("=== start run suite {} in {} mode. ===", name, (isCloud ? "cloud" : "not_cloud")) + def originConnection = context.threadLocalConn.get() try { cluster.destroy(true) cluster.init(options, isCloud) @@ -345,18 +353,18 @@ class Suite implements GroovyInterceptable { // wait be report Thread.sleep(5000) - def url = String.format( + def jdbcUrl = String.format( "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", fe.host, fe.queryPort) - def conn = DriverManager.getConnection(url, user, password) - def sql = "CREATE DATABASE IF NOT EXISTS " + context.dbName - logger.info("try create database if not exists {}", context.dbName) - JdbcUtils.executeToList(conn, sql) - - url = Config.buildUrlWithDb(url, context.dbName) - logger.info("connect to docker cluster: suite={}, url={}", name, url) - connect(user, password, url, actionSupplier) + cluster.jdbcUrl = jdbcUrl + context.threadLocalConn.remove() + actionSupplier.call() } finally { + if (originConnection == null) { + context.threadLocalConn.remove() + } else { + context.threadLocalConn.set(originConnection) + } if (!context.config.dockerEndNoKill) { cluster.destroy(context.config.dockerEndDeleteFiles) } @@ -650,7 +658,7 @@ class Suite implements GroovyInterceptable { } String getCurDbConnectUrl() { - return context.config.getConnectionUrlByDbName(getCurDbName()) + return Config.buildUrlWithDb(context.getJdbcUrl(), getCurDbName()) } long getDbId() { @@ -1893,8 +1901,7 @@ class Suite implements GroovyInterceptable { void waitAddFeFinished(String host, int port) { logger.info("waiting for ${host}:${port}") - Awaitility.await().atMost(60, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).and() - .pollInterval(100, TimeUnit.MILLISECONDS).await().until(() -> { + awaitUntil(60, 0.1) { def frontends = getFrontendIpEditlogPort() logger.info("frontends ${frontends}") boolean matched = false @@ -1906,12 +1913,12 @@ class Suite implements GroovyInterceptable { } } return matched; - }); + } } void waitDropFeFinished(String host, int port) { - Awaitility.await().atMost(60, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).and() - .pollInterval(100, TimeUnit.MILLISECONDS).await().until(() -> { + logger.info("waiting drop fe ${host}:${port}"); + awaitUntil(60, 0.1) { def frontends = getFrontendIpEditlogPort() boolean matched = false for (frontend: frontends) { @@ -1920,13 +1927,12 @@ class Suite implements GroovyInterceptable { } } return !matched; - }); + } } void waitAddBeFinished(String host, int port) { logger.info("waiting ${host}:${port} added"); - Awaitility.await().atMost(60, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).and() - .pollInterval(100, TimeUnit.MILLISECONDS).await().until(() -> { + awaitUntil(60, 0.1) { def ipList = [:] def portList = [:] getBackendIpHeartbeatPort(ipList, portList) @@ -1937,12 +1943,11 @@ class Suite implements GroovyInterceptable { } } return matched; - }); + } } void waitDropBeFinished(String host, int port) { - Awaitility.await().atMost(60, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).and() - .pollInterval(100, TimeUnit.MILLISECONDS).await().until(() -> { + awaitUntil(60, 0.1) { def ipList = [:] def portList = [:] getBackendIpHeartbeatPort(ipList, portList) @@ -1953,7 +1958,7 @@ class Suite implements GroovyInterceptable { } } return !matched; - }); + } } void waiteCreateTableFinished(String tableName) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index 9b948a3c3033da..7e2bc4d681cdc0 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -279,6 +279,8 @@ class SuiteCluster { static final Logger logger = LoggerFactory.getLogger(this.class) + // dockerImpl() will set jdbcUrl + String jdbcUrl = "" final String name final Config config private boolean running diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy index 076566651e5429..b303d1fbcd6952 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteContext.groovy @@ -145,7 +145,7 @@ class SuiteContext implements Closeable { def threadConnInfo = threadLocalConn.get() if (threadConnInfo == null) { threadConnInfo = new ConnectionInfo() - threadConnInfo.conn = config.getConnectionByDbName(dbName) + threadConnInfo.conn = getConnectionByDbName(dbName) threadConnInfo.username = config.jdbcUser threadConnInfo.password = config.jdbcPassword threadLocalConn.set(threadConnInfo) @@ -153,6 +153,31 @@ class SuiteContext implements Closeable { return threadConnInfo.conn } + Connection getConnectionByDbName(String dbName) { + def jdbcUrl = getJdbcUrl() + def jdbcConn = DriverManager.getConnection(jdbcUrl, config.jdbcUser, config.jdbcPassword) + try { + String sql = "CREATE DATABASE IF NOT EXISTS ${dbName}" + log.info("Try to create db, sql: ${sql}".toString()) + if (!config.dryRun) { + jdbcConn.withCloseable { conn -> JdbcUtils.executeToList(conn, sql) } + } + } catch (Throwable t) { + throw new IllegalStateException("Create database failed, jdbcUrl: ${jdbcUrl}", t) + } + def dbUrl = Config.buildUrlWithDb(jdbcUrl, dbName) + log.info("connect to ${dbUrl}".toString()) + return DriverManager.getConnection(dbUrl, config.jdbcUser, config.jdbcPassword) + } + + String getJdbcUrl() { + if (cluster.isRunning()) { + return cluster.jdbcUrl + } else { + return config.jdbcUrl + } + } + // like getConnection, but connect to FE master Connection getMasterConnection() { def threadConnInfo = threadLocalMasterConn.get() diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy index ceb8969b721580..a02481dcebfea0 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy @@ -126,7 +126,7 @@ suite('test_auto_start_in_cloud', 'multi_cluster, docker') { // cloud control set cluster status SUSPENDED set_cluster_status(uniqueId, cloudClusterId, "SUSPENDED", ms) - dockerAwaitUntil(5) { + awaitUntil(5) { tag = getCloudBeTagByName(clusterName) logger.info("tag = {}", tag) jsonObject = jsonSlurper.parseText(tag) @@ -154,7 +154,7 @@ suite('test_auto_start_in_cloud', 'multi_cluster, docker') { // cloud control future2 = thread { // check cluster "TO_RESUME" - dockerAwaitUntil(5) { + awaitUntil(5) { tag = getCloudBeTagByName(clusterName) logger.info("tag = {}", tag) jsonObject = jsonSlurper.parseText(tag) @@ -177,7 +177,7 @@ suite('test_auto_start_in_cloud', 'multi_cluster, docker') { // add 1 nodes, check it status NORMAL cluster.addBackend(1, null) - dockerAwaitUntil(5) { + awaitUntil(5) { result = sql """SHOW BACKENDS""" result.size() == 4 } diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_fe_tablet_same_backend.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_fe_tablet_same_backend.groovy index 973492bd4b52e3..5ecc610bf5af14 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_fe_tablet_same_backend.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_fe_tablet_same_backend.groovy @@ -98,7 +98,7 @@ suite('test_fe_tablet_same_backend', 'multi_cluster,docker') { } def checkAllTable = { isAllBeAliveOrDeadLong -> - dockerAwaitUntil(60) { + awaitUntil(60) { checkAllTableImpl(isAllBeAliveOrDeadLong, true) } checkAllTableImpl(isAllBeAliveOrDeadLong, false) @@ -128,7 +128,7 @@ suite('test_fe_tablet_same_backend', 'multi_cluster,docker') { checkAllTable(true) cluster.stopBackends(choseDeadBeIndex) - dockerAwaitUntil(60) { + awaitUntil(60) { def chosenBe = cluster.getBeByIndex(choseDeadBeIndex) !chosenBe.alive } @@ -144,13 +144,13 @@ suite('test_fe_tablet_same_backend', 'multi_cluster,docker') { def choseRestartFeIndex = cluster.getOneFollowerFe().index cluster.stopFrontends(choseRestartFeIndex) - dockerAwaitUntil(60) { + awaitUntil(60) { def chosenFe = cluster.getFeByIndex(choseRestartFeIndex) !chosenFe.alive } cluster.startFrontends(choseRestartFeIndex) - dockerAwaitUntil(60) { + awaitUntil(60) { def chosenFe = cluster.getFeByIndex(choseRestartFeIndex) chosenFe.alive } diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy index 792e2a98b248cb..1b7f33fdb1dfc1 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy @@ -77,7 +77,7 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { def toDropUniqueId // add new follower cluster.addFrontend(1, true) - dockerAwaitUntil(5) { + awaitUntil(5) { def ret = sql """SHOW FRONTENDS""" log.info("show frontends: {}", ret) ret.size() == 4 @@ -119,7 +119,7 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { drop_node(toDropUniqueId, toDropIP, 0, toDropPort, toDropType, feClusterName, feClusterId, ms) - dockerAwaitUntil(50) { + awaitUntil(50) { def ret = sql """SHOW FRONTENDS""" log.info("show frontends: {}", ret) ret.size() == 3 @@ -155,7 +155,7 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { drop_node(toDropUniqueId, toDropIP, 0, toDropPort, toDropType, feClusterName, feClusterId, ms) - dockerAwaitUntil(50) { + awaitUntil(50) { def ret = sql """SHOW FRONTENDS""" log.info("show frontends: {}", ret) ret.size() == 3 diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy index e1761a061ba754..193824a65408b9 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy @@ -145,7 +145,7 @@ suite('test_no_cluster_hits', 'multi_cluster, docker') { logger.info("ms addr={}, port={}", ms.host, ms.httpPort) drop_cluster(currentCluster.cluster, cloudClusterId, ms) - dockerAwaitUntil(5) { + awaitUntil(5) { result = sql_return_maparray """show clusters""" logger.info("show cluster2 : {}", result) result.size() == 0 diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy index c15157308c4903..83a8dc336debd7 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_rebalance.groovy @@ -76,13 +76,13 @@ suite('test_rebalance_in_cloud', 'multi_cluster,docker') { // add a be cluster.addBackend(1, null) - dockerAwaitUntil(30) { + awaitUntil(30) { def bes = sql """show backends""" log.info("bes: {}", bes) bes.size() == 2 } - dockerAwaitUntil(5) { + awaitUntil(5) { def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" log.info("replica distribution table100: {}", ret) ret.size() == 2 @@ -102,7 +102,7 @@ suite('test_rebalance_in_cloud', 'multi_cluster,docker') { } } - dockerAwaitUntil(5) { + awaitUntil(5) { def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table_p2 PARTITION(p1992)""" log.info("replica distribution table_p2: {}", ret) ret.size() == 2 @@ -181,13 +181,13 @@ suite('test_rebalance_in_cloud', 'multi_cluster,docker') { sql """admin set frontend config("enable_cloud_warm_up_for_rebalance"="true")""" // test rebalance thread still work - dockerAwaitUntil(30) { + awaitUntil(30) { def bes = sql """show backends""" log.info("bes: {}", bes) bes.size() == 3 } - dockerAwaitUntil(5) { + awaitUntil(5) { def ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" log.info("replica distribution table100: {}", ret) ret.size() == 3 @@ -205,7 +205,7 @@ suite('test_rebalance_in_cloud', 'multi_cluster,docker') { sleep(1 * 1000) } GetDebugPoint().disableDebugPointForAllFEs("CloudTabletRebalancer.checkInflghtWarmUpCacheAsync.beNull"); - dockerAwaitUntil(10) { + awaitUntil(10) { def ret = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" log.info("replica distribution table100: {}", ret) ret.any { row -> diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_warmup_rebalance.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_warmup_rebalance.groovy index b7de0c501618e8..aa35a70e121957 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_warmup_rebalance.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_warmup_rebalance.groovy @@ -98,7 +98,7 @@ suite('test_warmup_rebalance_in_cloud', 'multi_cluster, docker') { INSERT INTO table100 VALUES (1, 1, 100); """ - dockerAwaitUntil(5) { + awaitUntil(5) { ret = sql """ADMIN SHOW REPLICA DISTRIBUTION FROM table100""" log.info("replica distribution table100: {}", ret) ret.size() == 5 diff --git a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy index a8173108f0b951..dfe46054e24b76 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy @@ -25,7 +25,7 @@ suite("cloud_decommission", 'p0, docker') { def checkStatus = { ms, decommissionBeUniqueId, decommissionBe -> boolean found = false - dockerAwaitUntil(100) { + awaitUntil(100) { found = false def resp = get_cluster.call(decommissionBeUniqueId, ms) resp.each { cluster -> @@ -43,7 +43,7 @@ suite("cloud_decommission", 'p0, docker') { def dropAndCheckBe = { host, heartbeatPort -> sql """ ALTER SYSTEM DROPP BACKEND "${host}:${heartbeatPort}" """ - dockerAwaitUntil(100) { + awaitUntil(100) { def result = sql_return_maparray """ SHOW BACKENDS """ log.info("show backends result {}", result) def ret = result.find {it.Host == host && it.HeartbeatPort == heartbeatPort} @@ -60,7 +60,7 @@ suite("cloud_decommission", 'p0, docker') { def result = sql """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ assertEquals(result.size(), beNum) - dockerAwaitUntil(100) { + awaitUntil(100) { result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ if (beNum == 3) { result.every { Integer.valueOf((String) it.ReplicaNum) >= 15 && Integer.valueOf((String) it.ReplicaNum) <= 17 } @@ -95,7 +95,7 @@ suite("cloud_decommission", 'p0, docker') { d_node.call(firstDecommissionBeUniqueId, firstDecommissionBe.Host, firstDecommissionBe.HeartbeatPort, firstDecommissionBeClusterName, firstDecommissionBeCloudClusterId, ms) - dockerAwaitUntil(100) { + awaitUntil(100) { result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ result.any { Integer.valueOf((String) it.ReplicaNum) == 0 } } @@ -127,7 +127,7 @@ suite("cloud_decommission", 'p0, docker') { result = sql """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ assertEquals(result.size(), beNum - 1) - dockerAwaitUntil(100) { + awaitUntil(100) { result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ log.info("show replica result {}", result) def ret = result.findAll { Integer.valueOf((String) it.ReplicaNum) == 0 } diff --git a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy index b29cbef0f6855c..0d792950105133 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy @@ -329,7 +329,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') { logger.info("Dropping frontend index: {}, remove it from docker compose", dropFeInx) // Wait for the frontend to be fully dropped - dockerAwaitUntil(300) { + awaitUntil(300) { reconnectFe() def currentFrontends = sql_return_maparray("SHOW FRONTENDS") currentFrontends.size() == frontends.size() - 1 @@ -365,7 +365,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') { logger.info("Adding back frontend: {}", showFes) // Wait for the frontend to be fully added back - dockerAwaitUntil(300, 5) { + awaitUntil(300, 5) { def updatedFrontends = sql_return_maparray("SHOW FRONTENDS") updatedFrontends.size() == frontends.size() } @@ -404,7 +404,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') { reconnectFe() // Wait for the frontend to be fully dropped - dockerAwaitUntil(300, 5) { + awaitUntil(300, 5) { def updatedFrontends = sql_return_maparray("SHOW FRONTENDS") !updatedFrontends.any { it['Host'] == frontendToDrop.Host && it['EditLogPort'] == frontendToDrop.EditLogPort } } @@ -414,7 +414,7 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') { addList = cluster.addFrontend(1, true) logger.info("Up a new frontend, addList: {}", addList) - dockerAwaitUntil(300, 5) { + awaitUntil(300, 5) { def updatedFrontends = sql_return_maparray("SHOW FRONTENDS") updatedFrontends.size() == 3 } diff --git a/regression-test/suites/cloud_p0/query_retry/test_retry_e-230_async_mtmv_job.groovy b/regression-test/suites/cloud_p0/query_retry/test_retry_e-230_async_mtmv_job.groovy index 67bac61e8d909e..f7ddc494ffc450 100644 --- a/regression-test/suites/cloud_p0/query_retry/test_retry_e-230_async_mtmv_job.groovy +++ b/regression-test/suites/cloud_p0/query_retry/test_retry_e-230_async_mtmv_job.groovy @@ -86,7 +86,7 @@ suite("test_retry_e-230_async_mtmv_job", 'p0, docker') { """ def firstTaskId = getMvTaskId(0) def firstTask - dockerAwaitUntil(100) { + awaitUntil(100) { firstTask = getMvTask(firstTaskId) logger.info("firstTask = {}, Status = {}, bool = {}", firstTask, firstTask.Status, firstTask.Status[0] == "FAILED") firstTask.Status[0] as String == "FAILED" as String @@ -111,7 +111,7 @@ suite("test_retry_e-230_async_mtmv_job", 'p0, docker') { def futrue2 = thread { def secondTaskId = getMvTaskId(1) def secondTask - dockerAwaitUntil(100, 5) { + awaitUntil(100, 5) { secondTask = getMvTask(secondTaskId) logger.info("secondTask = {}", secondTask) secondTask.Status[0] == "SUCCESS" diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy index a65f59f85a1ee4..366ef5fa103459 100644 --- a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy +++ b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_drop_force_table.groovy @@ -127,7 +127,7 @@ suite('test_clean_tablet_when_drop_force_table', 'docker') { } def start = System.currentTimeMillis() / 1000 // tablet can't find in be - dockerAwaitUntil(500) { + awaitUntil(500) { def beTablets = getTabletAndBeHostFromBe(cluster.getAllBackends()).keySet() logger.info("before drop tablets {}, after tablets {}", beforeGetFromFe, beTablets) beforeGetFromFe.keySet().every { !getTabletAndBeHostFromBe(cluster.getAllBackends()).containsKey(it) } diff --git a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy index caac5b73cfdbba..9a91707731d53c 100644 --- a/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy +++ b/regression-test/suites/cloud_p0/tablets/test_clean_tablet_when_rebalance.groovy @@ -84,7 +84,7 @@ suite('test_clean_tablet_when_rebalance', 'docker') { } cluster.stopBackends(choseDeadBeIndex) - dockerAwaitUntil(50) { + awaitUntil(50) { def showTablets = sql_return_maparray("SHOW TABLETS FROM ${table}") def bes = showTablets .collect { it.BackendId } @@ -109,7 +109,7 @@ suite('test_clean_tablet_when_rebalance', 'docker') { def afterGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends()) logger.info("after stop one be, rehash fe tablets {}, be tablets {}", afterGetFromFe, afterGetFromBe) - dockerAwaitUntil(50) { + awaitUntil(50) { def showTablets = sql_return_maparray("SHOW TABLETS FROM ${table}") def bes = showTablets .collect { it.BackendId } diff --git a/regression-test/suites/demo_p0/docker_action.groovy b/regression-test/suites/demo_p0/docker_action.groovy index 7bea52989b59d2..7e111b4828595d 100644 --- a/regression-test/suites/demo_p0/docker_action.groovy +++ b/regression-test/suites/demo_p0/docker_action.groovy @@ -38,8 +38,7 @@ import org.apache.doris.regression.suite.ClusterOptions // NOTICE: // 1. Need add 'docker' to suite's group, and don't add 'nonConcurrent' to it; // 2. In docker closure: -// a. Don't use 'Awaitility.await()...until(f)', but use 'dockerAwaitUntil(..., f)'; -// b. Don't use java Thread, but use regress framework's ThreadAction(see example demo_p0/thread_action.groovy); +// a. remove function dockerAwaitUntil(...), should use 'Awaitility.await()...until(f)' directly or use 'awaitUntil(...)'; // 3. No need to use code ` if (isCloudMode()) { return } ` in docker suites, // instead should use `ClusterOptions.cloudMode = true/false` is enough. // Because when run docker suite without an external doris cluster, if suite use code `isCloudMode()`, it need specific -runMode=cloud/not_cloud. diff --git a/regression-test/suites/fault_injection_p0/cloud/test_tablet_state_change_in_publish_phase.groovy b/regression-test/suites/fault_injection_p0/cloud/test_tablet_state_change_in_publish_phase.groovy index 6b7102ed243d76..c3c245ce37ec77 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_tablet_state_change_in_publish_phase.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_tablet_state_change_in_publish_phase.groovy @@ -135,7 +135,7 @@ suite("test_tablet_state_change_in_publish_phase", "docker") { // let sc finish GetDebugPoint().disableDebugPointForAllBEs("CloudSchemaChangeJob.process_alter_tablet.sleep") - dockerAwaitUntil(30) { + awaitUntil(30) { def res = sql_return_maparray """ SHOW ALTER TABLE COLUMN WHERE TableName='${table1}' ORDER BY createtime DESC LIMIT 1 """ logger.info("alter status: ${res}") res[0].State as String == "FINISHED" diff --git a/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy b/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy index ff1d9531fb564d..c5fe06b19c367a 100644 --- a/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy +++ b/regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy @@ -155,7 +155,7 @@ suite("test_local_multi_segments_re_calc_in_publish", "docker") { Thread.sleep(1000) do_streamload_2pc_commit(txnId) - dockerAwaitUntil(30) { + awaitUntil(30) { def result = sql_return_maparray "show transaction from ${dbName} where id = ${txnId}" result[0].TransactionStatus as String == "VISIBLE" } diff --git a/regression-test/suites/schema_change_p0/test_abort_txn_by_be.groovy b/regression-test/suites/schema_change_p0/test_abort_txn_by_be.groovy index b1af2b1fcad76d..127465d66d85d0 100644 --- a/regression-test/suites/schema_change_p0/test_abort_txn_by_be.groovy +++ b/regression-test/suites/schema_change_p0/test_abort_txn_by_be.groovy @@ -98,7 +98,7 @@ suite('test_abort_txn_by_be', 'docker') { sleep 3000 } else { def dbId = getDbId() - dockerAwaitUntil(20, { + awaitUntil(20, { def txns = sql_return_maparray("show proc '/transactions/${dbId}/running'") txns.size() > 0 }) diff --git a/regression-test/suites/schema_change_p0/test_abort_txn_by_fe.groovy b/regression-test/suites/schema_change_p0/test_abort_txn_by_fe.groovy index d93e8a203e300e..91641ae483c64f 100644 --- a/regression-test/suites/schema_change_p0/test_abort_txn_by_fe.groovy +++ b/regression-test/suites/schema_change_p0/test_abort_txn_by_fe.groovy @@ -67,7 +67,7 @@ suite('test_abort_txn_by_fe', 'docker') { sleep 6000 } else { def dbId = getDbId() - dockerAwaitUntil(20, { + awaitUntil(20, { def txns = sql_return_maparray("show proc '/transactions/${dbId}/running'") txns.any { it.Label == loadLabel } }) diff --git a/regression-test/suites/trash_p0/clean_trash.groovy b/regression-test/suites/trash_p0/clean_trash.groovy index 525e8054ce5ca3..4c03f43b80700d 100644 --- a/regression-test/suites/trash_p0/clean_trash.groovy +++ b/regression-test/suites/trash_p0/clean_trash.groovy @@ -40,7 +40,7 @@ suite("test_clean_trash", "docker") { def checkFunc = { boolean trashZero -> def succ = false - dockerAwaitUntil(300) { + awaitUntil(300) { def bes = sql_return_maparray """show backends""" succ = bes.every { if (trashZero) { From 8a1f679fc8103f81d50b6748459b43722f60fea6 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 18 Jul 2025 14:26:57 +0800 Subject: [PATCH 259/572] [fix](warmup) fix show warm up tables (#53406) ### What problem does this PR solve? Related PR: #52291 #52514 Problem Summary: Fix show warm up jobs after merging periodic and event driven warmup feature. --- .../main/java/org/apache/doris/cloud/CacheHotspotManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index 6797ce45acb7de..417f1b8d233e55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -785,8 +785,8 @@ public long createJob(WarmUpClusterStmt stmt) throws AnalysisException { stmt.isForce()); } Map>> beToTabletIdBatches = splitBatch(beToWarmUpTablets); - warmUpJob = new CloudWarmUpJob(jobId, null, stmt.getDstClusterName(), - beToTabletIdBatches, JobType.TABLE); + warmUpJob = new CloudWarmUpJob(jobId, stmt.getDstClusterName(), + beToTabletIdBatches, JobType.TABLE, stmt.getTables(), stmt.isForce()); } else { CloudWarmUpJob.Builder builder = new CloudWarmUpJob.Builder() .setJobId(jobId) From f4f6e37f4937d5bea6c3117c1ae898c49eefe5ec Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 18 Jul 2025 15:36:46 +0800 Subject: [PATCH 260/572] [Feature](function) support function cot/sec/cosec (#52872) (#53274) pick from #52872 --- be/src/vec/functions/math.cpp | 27 ++++++++ be/test/vec/function/function_math_test.cpp | 31 +++++++++ .../doris/catalog/BuiltinScalarFunctions.java | 6 ++ .../executable/NumericArithmetic.java | 33 +++++++++ .../expressions/functions/scalar/Cosec.java | 68 +++++++++++++++++++ .../expressions/functions/scalar/Cot.java | 68 +++++++++++++++++++ .../expressions/functions/scalar/Sec.java | 68 +++++++++++++++++++ .../visitor/ScalarFunctionVisitor.java | 15 ++++ .../math_functions/test_triangle.out | 58 ++++++++++++++++ .../fold_constant_numeric_arithmatic.groovy | 33 +++++++++ .../math_functions/test_triangle.groovy | 50 ++++++++++++++ 11 files changed, 457 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cosec.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cot.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sec.java create mode 100644 regression-test/data/query_p0/sql_functions/math_functions/test_triangle.out create mode 100644 regression-test/suites/query_p0/sql_functions/math_functions/test_triangle.groovy diff --git a/be/src/vec/functions/math.cpp b/be/src/vec/functions/math.cpp index 2d9faaf19bc492..711a92ebb9df20 100644 --- a/be/src/vec/functions/math.cpp +++ b/be/src/vec/functions/math.cpp @@ -270,6 +270,30 @@ struct TanhName { }; using FunctionTanh = FunctionMathUnary>; +struct CotName { + static constexpr auto name = "cot"; +}; +double cot(double x) { + return 1.0 / std::tan(x); +} +using FunctionCot = FunctionMathUnary>; + +struct SecName { + static constexpr auto name = "sec"; +}; +double sec(double x) { + return 1.0 / std::cos(x); +} +using FunctionSec = FunctionMathUnary>; + +struct CosecName { + static constexpr auto name = "cosec"; +}; +double cosec(double x) { + return 1.0 / std::sin(x); +} +using FunctionCosec = FunctionMathUnary>; + template struct RadiansImpl { using ResultType = A; @@ -450,6 +474,9 @@ void register_function_math(SimpleFunctionFactory& factory) { factory.register_function(); factory.register_function(); factory.register_function(); + factory.register_function(); + factory.register_function(); + factory.register_function(); factory.register_function(); factory.register_alias("pow", "power"); factory.register_alias("pow", "dpow"); diff --git a/be/test/vec/function/function_math_test.cpp b/be/test/vec/function/function_math_test.cpp index 63a1b964a0c048..e4fec957f44299 100644 --- a/be/test/vec/function/function_math_test.cpp +++ b/be/test/vec/function/function_math_test.cpp @@ -119,6 +119,37 @@ TEST(MathFunctionTest, cbrt_test) { static_cast(check_function(func_name, input_types, data_set)); } +TEST(MathFunctionTest, cot_test) { + std::string func_name = "cot"; + + InputTypeSet input_types = {TypeIndex::Float64}; + + DataSet data_set = {{{1.0}, 0.6420926159343306}, {{M_PI / 4}, 1.0000000000000002}}; + + static_cast(check_function(func_name, input_types, data_set)); +} + +TEST(MathFunctionTest, sec_test) { + std::string func_name = "sec"; + + InputTypeSet input_types = {TypeIndex::Float64}; + + DataSet data_set = {{{1.0}, 1.8508157176809255}, {{1000.0}, 1.7781600385912715}}; + + static_cast(check_function(func_name, input_types, data_set)); +} + +TEST(MathFunctionTest, cosec_test) { + std::string func_name = "cosec"; + + InputTypeSet input_types = {TypeIndex::Float64}; + + DataSet data_set = { + {{1.0}, 1.1883951057781212}, {{2.0}, 1.0997501702946164}, {{1000.0}, 1.20936599707935}}; + + static_cast(check_function(func_name, input_types, data_set)); +} + TEST(MathFunctionTest, tan_test) { std::string func_name = "tan"; //tan(x) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java index 7d9314cb74fd75..46ac4856f00426 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinScalarFunctions.java @@ -124,8 +124,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTo; import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosec; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh; import org.apache.doris.nereids.trees.expressions.functions.scalar.CosineDistance; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cot; import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual; import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; @@ -365,6 +367,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Rpad; import org.apache.doris.nereids.trees.expressions.functions.scalar.Rtrim; import org.apache.doris.nereids.trees.expressions.functions.scalar.RtrimIn; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Sec; import org.apache.doris.nereids.trees.expressions.functions.scalar.SecToTime; import org.apache.doris.nereids.trees.expressions.functions.scalar.Second; import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondCeil; @@ -601,7 +604,9 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(ConvertTo.class, "convert_to"), scalar(ConvertTz.class, "convert_tz"), scalar(Cos.class, "cos"), + scalar(Cosec.class, "cosec"), scalar(Cosh.class, "cosh"), + scalar(Cot.class, "cot"), scalar(CosineDistance.class, "cosine_distance"), scalar(CountEqual.class, "countequal"), scalar(CreateMap.class, "map"), @@ -856,6 +861,7 @@ public class BuiltinScalarFunctions implements FunctionHelper { scalar(Rpad.class, "rpad"), scalar(Rtrim.class, "rtrim"), scalar(RtrimIn.class, "rtrim_in"), + scalar(Sec.class, "sec"), scalar(Second.class, "second"), scalar(SecondCeil.class, "second_ceil"), scalar(SecondFloor.class, "second_floor"), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java index 1368d8adcf7049..abf6363c03597a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java @@ -915,6 +915,39 @@ public static Expression tan(DoubleLiteral first) { return checkOutputBoundary(new DoubleLiteral(Math.tan(first.getValue()))); } + /** + * cot + */ + @ExecFunction(name = "cot") + public static Expression cot(DoubleLiteral first) { + if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { + return new NullLiteral(DoubleType.INSTANCE); + } + return checkOutputBoundary(new DoubleLiteral(1.0 / Math.tan(first.getValue()))); + } + + /** + * cot + */ + @ExecFunction(name = "sec") + public static Expression sec(DoubleLiteral first) { + if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { + return new NullLiteral(DoubleType.INSTANCE); + } + return checkOutputBoundary(new DoubleLiteral(1.0 / Math.cos(first.getValue()))); + } + + /** + * cosec + */ + @ExecFunction(name = "cosec") + public static Expression cosec(DoubleLiteral first) { + if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { + return new NullLiteral(DoubleType.INSTANCE); + } + return checkOutputBoundary(new DoubleLiteral(1.0 / Math.sin(first.getValue()))); + } + /** * asin */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cosec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cosec.java new file mode 100644 index 00000000000000..26919889a3a02f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cosec.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DoubleType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/**S + * ScalarFunction 'cosec'. This class is generated by GenerateFunction. + */ +public class Cosec extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) + ); + + /** + * constructor with 1 argument. + */ + public Cosec(Expression arg) { + super("cosec", arg); + } + + /** + * withChildren. + */ + @Override + public Cosec withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new Cosec(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitCosec(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cot.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cot.java new file mode 100644 index 00000000000000..842b035b6dfa87 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Cot.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DoubleType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + * ScalarFunction 'cot'. This class is generated by GenerateFunction. + */ +public class Cot extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) + ); + + /** + * constructor with 1 argument. + */ + public Cot(Expression arg) { + super("cot", arg); + } + + /** + * withChildren. + */ + @Override + public Cot withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new Cot(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitCot(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sec.java new file mode 100644 index 00000000000000..dc6a188cd77116 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Sec.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.functions.scalar; + +import org.apache.doris.catalog.FunctionSignature; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.types.DoubleType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/**S + * ScalarFunction 'sec'. This class is generated by GenerateFunction. + */ +public class Sec extends ScalarFunction + implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullable { + + public static final List SIGNATURES = ImmutableList.of( + FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE) + ); + + /** + * constructor with 1 argument. + */ + public Sec(Expression arg) { + super("sec", arg); + } + + /** + * withChildren. + */ + @Override + public Sec withChildren(List children) { + Preconditions.checkArgument(children.size() == 1); + return new Sec(children.get(0)); + } + + @Override + public List getSignatures() { + return SIGNATURES; + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitSec(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java index f85ce2fbed3b80..16a6bf61a9d2e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ScalarFunctionVisitor.java @@ -131,8 +131,10 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTo; import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosec; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh; import org.apache.doris.nereids.trees.expressions.functions.scalar.CosineDistance; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cot; import org.apache.doris.nereids.trees.expressions.functions.scalar.CountEqual; import org.apache.doris.nereids.trees.expressions.functions.scalar.Crc32; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; @@ -365,6 +367,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Rtrim; import org.apache.doris.nereids.trees.expressions.functions.scalar.RtrimIn; import org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunction; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Sec; import org.apache.doris.nereids.trees.expressions.functions.scalar.Second; import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondCeil; import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondFloor; @@ -2073,6 +2076,18 @@ default R visitTanh(Tanh tanh, C context) { return visitScalarFunction(tanh, context); } + default R visitCot(Cot cot, C context) { + return visitScalarFunction(cot, context); + } + + default R visitSec(Sec sec, C context) { + return visitScalarFunction(sec, context); + } + + default R visitCosec(Cosec cosec, C context) { + return visitScalarFunction(cosec, context); + } + default R visitTimeDiff(TimeDiff timeDiff, C context) { return visitScalarFunction(timeDiff, context); } diff --git a/regression-test/data/query_p0/sql_functions/math_functions/test_triangle.out b/regression-test/data/query_p0/sql_functions/math_functions/test_triangle.out new file mode 100644 index 00000000000000..7fa4c9b3be9dd8 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/math_functions/test_triangle.out @@ -0,0 +1,58 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test -- +1 1.0000000000000002 +2 6.123233995736766E-17 +3 -8.165619676597685E15 +4 inf +5 0.6420926159343306 +6 -0.6420926159343306 +7 \N +8 1.830487721712452 +9 -1.830487721712452 +10 1.5423510453569202 +11 -1.5423510453569202 +12 9999999.999999966 +13 9999999.999999966 +14 -2.1573538153608958 +15 2.1573538153608958 +16 8.165619676597685E15 +17 -6.123233995736766E-17 + +-- !test -- +1 1.414213562373095 +2 1.633123935319537E16 +3 -1.0 +4 1.0 +5 1.8508157176809255 +6 1.8508157176809255 +7 \N +8 1.139493927324549 +9 1.139493927324549 +10 -1.1917935066878957 +11 -1.1917935066878957 +12 1.000000000000005 +13 1.000000000000005 +14 -1.102207252910033 +15 -1.102207252910033 +16 -1.0 +17 1.633123935319537E16 + +-- !test -- +1 1.4142135623730951 +2 1.0 +3 8.165619676597685E15 +4 inf +5 1.1883951057781212 +6 -1.1883951057781212 +7 \N +8 2.085829642933488 +9 -2.085829642933488 +10 -1.8381639608896658 +11 1.8381639608896658 +12 1.0000000000000017E7 +13 1.0000000000000017E7 +14 2.3778510223839118 +15 -2.3778510223839118 +16 -8.165619676597685E15 +17 -1.0 + diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy index 7aa09d8a18666c..97a70ac3c0c23a 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy @@ -474,6 +474,39 @@ suite("fold_constant_numeric_arithmatic") { testFoldConst("SELECT TANH(-0.5), TANH(0.5), TANH(10), TANH(-10)") testFoldConst("SELECT TANH(-20), TANH(20), TANH(1E-7), TANH(-1E-7)") +//Cot function cases + testFoldConst("SELECT COT(PI() / 4)") + testFoldConst("SELECT COT(PI())") + testFoldConst("SELECT COT(PI() / 2)") + // testFoldConst("SELECT COT(0)") need rethink inf behavior + testFoldConst("SELECT COT(1)") + testFoldConst("SELECT COT(-1)") + testFoldConst("SELECT COT(NULL)") + testFoldConst("SELECT COT(-0.5), COT(0.5), COT(10), COT(-10)") + testFoldConst("SELECT COT(-20), COT(20), COT(1E-7), COT(-1E-7)") + +//Sec function cases + testFoldConst("SELECT SEC(PI() / 4)") + testFoldConst("SELECT SEC(PI())") + // testFoldConst("SELECT SEC(PI() / 2)") need rethink inf behavior + testFoldConst("SELECT SEC(0)") + testFoldConst("SELECT SEC(1)") + testFoldConst("SELECT SEC(-1)") + testFoldConst("SELECT SEC(NULL)") + testFoldConst("SELECT SEC(-0.5), SEC(0.5), SEC(10), SEC(-10)") + testFoldConst("SELECT SEC(-20), SEC(20), SEC(1E-7), SEC(-1E-7)") + +//Cosec function cases + testFoldConst("SELECT COSEC(PI() / 4)") + // testFoldConst("SELECT COSEC(PI())") need rethink inf behavior + testFoldConst("SELECT COSEC(PI() / 2)") + // testFoldConst("SELECT COSEC(0)") need rethink inf behavior + testFoldConst("SELECT COSEC(1)") + testFoldConst("SELECT COSEC(-1)") + testFoldConst("SELECT COSEC(NULL)") + testFoldConst("SELECT COSEC(-0.5), COSEC(0.5), COSEC(10), COSEC(-10)") + testFoldConst("SELECT COSEC(-20), COSEC(20), COSEC(1E-7), COSEC(-1E-7)") + //Truncate function cases testFoldConst("SELECT TRUNCATE(123.456, 2) AS truncate_case_1") //truncate(123.456, 2) = 123.45 testFoldConst("SELECT TRUNCATE(-123.456, 1) AS truncate_case_2") //truncate(-123.456, 1) = -123.4 diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_triangle.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_triangle.groovy new file mode 100644 index 00000000000000..53e2b478ebd039 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_triangle.groovy @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_triangle") { + sql """ drop table if exists test_triangle; """ + sql """ create table test_triangle( + k1 int, + v1 double + ) distributed by hash (k1) buckets 1 + properties ("replication_num"="1"); + """ + sql """ insert into test_triangle values + (1,PI() / 4), + (2,PI() / 2), + (3,PI()), + (4,0), + (5,1), + (6,-1), + (7,NULL), + (8,0.5), + (9,-0.5), + (10,10), + (11,-10), + (12,1E-7), + (13,1E-7), + (14,1E7), + (15,-1E7), + (16,-PI()), + (17,-PI()/2) + """ + + qt_test "select k1,COT(v1) from test_triangle order by k1;" + qt_test "select k1,SEC(v1) from test_triangle order by k1;" + qt_test "select k1,COSEC(v1) from test_triangle order by k1;" +} + From 2ea0e0f30451e9c9a9e5ca72ba3ae23f72b88a5d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 18 Jul 2025 15:55:55 +0800 Subject: [PATCH 261/572] [opt](nereids) use one phase aggregate for group_concat with order #53066 (#53207) cherry pick from #53066 --- .../translator/PhysicalPlanTranslator.java | 10 ++------ .../translator/PlanTranslatorContext.java | 13 ----------- .../implementation/AggregateStrategies.java | 18 ++++++++++++--- .../functions/agg/AggregateFunction.java | 4 ++++ .../functions/agg/GroupConcat.java | 23 +++++++++++++++++++ .../org/apache/doris/qe/SessionVariable.java | 13 +++++++++++ .../nereids_syntax_p0/group_concat.groovy | 16 +++++++++++++ 7 files changed, 73 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 2c7e1988c48e14..7ffe57168c244a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -302,8 +302,7 @@ public PlanFragment visitPhysicalDistribute(PhysicalDistribute d List> distributeExprLists = getDistributeExprs(child); // TODO: why need set streaming here? should remove this. if (inputFragment.getPlanRoot() instanceof AggregationNode - && child instanceof PhysicalHashAggregate - && context.getFirstAggregateInFragment(inputFragment) == child) { + && child instanceof PhysicalHashAggregate) { PhysicalHashAggregate hashAggregate = (PhysicalHashAggregate) child; if (hashAggregate.getAggPhase() == AggPhase.LOCAL && hashAggregate.getAggMode() == AggMode.INPUT_TO_BUFFER @@ -1027,17 +1026,12 @@ public PlanFragment visitPhysicalHashAggregate( default: throw new RuntimeException("Unsupported agg phase: " + aggregate.getAggPhase()); } - // TODO: use to set useStreamingAgg, we should remove it by set it in Nereids - PhysicalHashAggregate firstAggregateInFragment = context.getFirstAggregateInFragment(inputPlanFragment); - if (firstAggregateInFragment == null) { - context.setFirstAggregateInFragment(inputPlanFragment, aggregate); - } // in pipeline engine, we use parallel scan by default, but it broke the rule of data distribution // so, if we do final phase or merge without exchange. // we need turn of parallel scan to ensure to get correct result. PlanNode leftMostNode = inputPlanFragment.getPlanRoot(); - while (leftMostNode.getChildren().size() != 0 && !(leftMostNode instanceof ExchangeNode)) { + while (!leftMostNode.getChildren().isEmpty() && !(leftMostNode instanceof ExchangeNode)) { leftMostNode = leftMostNode.getChild(0); } // TODO: nereids forbid all parallel scan under aggregate temporary, because nereids could generate diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index 64a015dd5d2184..227dca12a5e6cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -37,7 +37,6 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.planner.CTEScanNode; import org.apache.doris.planner.PlanFragment; @@ -54,7 +53,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -96,9 +94,6 @@ public class PlanTranslatorContext { private final IdGenerator nodeIdGenerator = PlanNodeId.createGenerator(); - private final IdentityHashMap firstAggInFragment - = new IdentityHashMap<>(); - private final Map bufferedSlotRefForWindow = Maps.newHashMap(); private TupleDescriptor bufferedTupleForWindow = null; @@ -257,14 +252,6 @@ public List getScanNodes() { return scanNodes; } - public PhysicalHashAggregate getFirstAggregateInFragment(PlanFragment planFragment) { - return firstAggInFragment.get(planFragment); - } - - public void setFirstAggregateInFragment(PlanFragment planFragment, PhysicalHashAggregate aggregate) { - firstAggInFragment.put(planFragment, aggregate); - } - public Map getBufferedSlotRefForWindow() { return bufferedSlotRefForWindow; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index 0539a08d3d794b..ace11cd3eb3bed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -919,7 +919,16 @@ private boolean enablePushDownStringMinMax() { private List> onePhaseAggregateWithoutDistinct( LogicalAggregate logicalAgg, ConnectContext connectContext) { RequireProperties requireGather = RequireProperties.of(PhysicalProperties.GATHER); - AggregateParam inputToResultParam = AggregateParam.LOCAL_RESULT; + boolean canBeBanned = true; + for (AggregateFunction aggregateFunction : logicalAgg.getAggregateFunctions()) { + if (aggregateFunction.forceSkipRegulator(AggregatePhase.ONE)) { + canBeBanned = false; + break; + } + } + AggregateParam inputToResultParam = new AggregateParam( + AggregateParam.LOCAL_RESULT.aggPhase, AggregateParam.LOCAL_RESULT.aggMode, canBeBanned + ); List newOutput = ExpressionUtils.rewriteDownShortCircuit( logicalAgg.getOutputExpressions(), outputChild -> { if (outputChild instanceof AggregateFunction) { @@ -935,8 +944,11 @@ private List> onePhaseAggregateWithoutDistinct( if (logicalAgg.getGroupByExpressions().isEmpty()) { // TODO: usually bad, disable it until we could do better cost computation. - // return ImmutableList.of(gatherLocalAgg); - return ImmutableList.of(); + if (!canBeBanned) { + return ImmutableList.of(gatherLocalAgg); + } else { + return ImmutableList.of(); + } } else { RequireProperties requireHash = RequireProperties.of( PhysicalProperties.createHash(logicalAgg.getGroupByExpressions(), ShuffleType.REQUIRE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java index 7aa5da02ff375e..ce8ec70e765745 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/AggregateFunction.java @@ -66,6 +66,10 @@ public AggregateFunction withChildren(List children) { return withDistinctAndChildren(distinct, children); } + public boolean forceSkipRegulator(AggregatePhase aggregatePhase) { + return false; + } + public abstract AggregateFunction withDistinctAndChildren(boolean distinct, List children); /** getIntermediateTypes */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java index 2505329b2fe901..2157deeeb4d872 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java @@ -27,6 +27,7 @@ import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -133,6 +134,28 @@ public List getSignatures() { return SIGNATURES; } + @Override + public boolean supportAggregatePhase(AggregatePhase aggregatePhase) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getSessionVariable().useOnePhaseAggForGroupConcatWithOrder + && children.stream().anyMatch(OrderExpression.class::isInstance) + && aggregatePhase != AggregatePhase.ONE) { + return false; + } + return true; + } + + @Override + public boolean forceSkipRegulator(AggregatePhase aggregatePhase) { + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getSessionVariable().useOnePhaseAggForGroupConcatWithOrder + && children.stream().anyMatch(OrderExpression.class::isInstance) + && aggregatePhase == AggregatePhase.ONE) { + return true; + } + return false; + } + public MultiDistinctGroupConcat convertToMultiDistinct() { Preconditions.checkArgument(distinct, "can't convert to multi_distinct_group_concat because there is no distinct args"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 3a7ee7c8f2e2f0..6400ae19a7b582 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -410,6 +410,8 @@ public class SessionVariable implements Serializable, Writable { public static final String ENABLE_UNICODE_NAME_SUPPORT = "enable_unicode_name_support"; public static final String GROUP_CONCAT_MAX_LEN = "group_concat_max_len"; + public static final String USE_ONE_PHASE_AGG_FOR_GROUP_CONCAT_WITH_ORDER + = "use_one_phase_agg_for_group_concat_with_order"; public static final String ENABLE_TWO_PHASE_READ_OPT = "enable_two_phase_read_opt"; public static final String TOPN_OPT_LIMIT_THRESHOLD = "topn_opt_limit_threshold"; @@ -1630,6 +1632,17 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = GROUP_CONCAT_MAX_LEN) public long groupConcatMaxLen = 2147483646; + @VariableMgr.VarAttr( + name = USE_ONE_PHASE_AGG_FOR_GROUP_CONCAT_WITH_ORDER, + needForward = true, + fuzzy = true, + description = { + "允许使用一阶段聚合来执行带有order的group_concat函数", + "Enable to use one stage aggregation to execute the group_concat function with order" + } + ) + public boolean useOnePhaseAggForGroupConcatWithOrder = false; + // Whether enable two phase read optimization // 1. read related rowids along with necessary column data // 2. spawn fetch RPC to other nodes to get related data by sorted rowids diff --git a/regression-test/suites/nereids_syntax_p0/group_concat.groovy b/regression-test/suites/nereids_syntax_p0/group_concat.groovy index b46091616ba2cc..ab50cb74e9699d 100644 --- a/regression-test/suites/nereids_syntax_p0/group_concat.groovy +++ b/regression-test/suites/nereids_syntax_p0/group_concat.groovy @@ -90,5 +90,21 @@ suite("group_concat") { LEFT OUTER JOIN test_group_concat_distinct_tbl3 tbl3 ON tbl3.tbl3_id2 = tbl2.tbl2_id2 GROUP BY tbl1.tbl1_id1 """ + + sql "set use_one_phase_agg_for_group_concat_with_order=true" + + explain { + sql "select group_concat(tbl3_name, ',' order by tbl3_id2) FROM test_group_concat_distinct_tbl3" + check { explainStr -> + assertFalse(explainStr.contains("partial_group_concat")) + } + } + + explain { + sql "select group_concat(tbl3_name, ',' order by tbl3_id2) FROM test_group_concat_distinct_tbl3 group by tbl3_name" + check { explainStr -> + assertFalse(explainStr.contains("partial_group_concat")) + } + } }() } From c557c4457fdbb6fcf497b71ca8ab73ce3eb0a4a5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Jul 2025 18:57:14 +0800 Subject: [PATCH 262/572] branch-3.0: [fix](case) do not define global var in groovy #53502 (#53551) Cherry-picked from #53502 Co-authored-by: Yongqiang YANG --- .../fault_injection_p0/test_disable_move_memtable.groovy | 6 +++--- .../suites/load_p0/stream_load/test_json_load.groovy | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy b/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy index 5bca47cb71c1b1..1513d1ec75d60b 100644 --- a/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy +++ b/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy @@ -248,9 +248,9 @@ suite("test_disable_move_memtable", "nonConcurrent") { try { GetDebugPoint().enableDebugPointForAllBEs(injection) if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load_with_properties.json" def test_load_label = UUID.randomUUID().toString().replaceAll("-", "") load_from_hdfs_norm.call(tableName, test_load_label, hdfs_csv_file_path, "json", diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index cbecb831cfcad7..a3d6eb6932de43 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -654,9 +654,9 @@ suite("test_json_load", "p0,nonConcurrent") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName =getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName =getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_file_path = uploadToHdfs "load_p0/stream_load/simple_object_json.json" def format = "json" From 3f04357905e156c1ff885688ca38db0220dddfce Mon Sep 17 00:00:00 2001 From: Luwei Date: Fri, 18 Jul 2025 20:41:02 +0800 Subject: [PATCH 263/572] [enhance](compaction) limit time series table max version using maximum of current backend (#53244) (#53562) pick master #53244 --- be/src/olap/base_tablet.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 56ffc2d100e628..6738a9c41c2c33 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -1718,7 +1719,8 @@ void TabletReadSource::fill_delete_predicates() { int32_t BaseTablet::max_version_config() { int32_t max_version = tablet_meta()->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY - ? config::time_series_max_tablet_version_num + ? std::max(config::time_series_max_tablet_version_num, + config::max_tablet_version_num) : config::max_tablet_version_num; return max_version; } From c90842bd433fc503d41634bf8e962a3b0825101c Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 18 Jul 2025 20:55:42 +0800 Subject: [PATCH 264/572] [Bug](datev2) fix wrong result of predicate about cast(cast(datetimev2 as datev2) as datetimev2) #53485 (#53513) pick from #53485 --- be/src/pipeline/exec/scan_operator.cpp | 3 +- regression-test/data/cast_p0/cast_ignore.out | 6 ++++ .../suites/cast_p0/cast_ignore.groovy | 35 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/cast_p0/cast_ignore.out create mode 100644 regression-test/suites/cast_p0/cast_ignore.groovy diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 686fe6d2960966..94d6b6d4db78be 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -473,7 +473,8 @@ bool ScanLocalState::_is_predicate_acting_on_slot( // the type of predicate not match the slot's type return false; } - } else if (child_contains_slot->type().is_datetime_type() && + } else if ((child_contains_slot->type().is_datetime_type() || + child_contains_slot->type().is_datetime_v2_type()) && child_contains_slot->node_type() == doris::TExprNodeType::CAST_EXPR) { // Expr `CAST(CAST(datetime_col AS DATE) AS DATETIME) = datetime_literal` should not be // push down. diff --git a/regression-test/data/cast_p0/cast_ignore.out b/regression-test/data/cast_p0/cast_ignore.out new file mode 100644 index 00000000000000..ef9a4ecb109c7a --- /dev/null +++ b/regression-test/data/cast_p0/cast_ignore.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test -- +1 2023-10-01 2023-10-01T01:00 + +-- !test -- + diff --git a/regression-test/suites/cast_p0/cast_ignore.groovy b/regression-test/suites/cast_p0/cast_ignore.groovy new file mode 100644 index 00000000000000..261b44c1164514 --- /dev/null +++ b/regression-test/suites/cast_p0/cast_ignore.groovy @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("cast_ignore") { + sql "drop table if exists tdate" + sql """ + create table tdate( + k1 int, + kdate date, + kdatetime datetime +) distributed by hash (k1) buckets 1 +properties ("replication_num"="1"); + """ + sql """ +insert into tdate values(1,'2023-10-01','2023-10-01 01:00:00'), +(2,'2023-10-02','2023-10-02 01:00:00'), +(3,'2023-10-03','2023-10-03 01:00:00'); +""" + qt_test "select k1,kdate,kdatetime from tdate where cast(cast(kdatetime as date) as datetime)='2023-10-01';" + qt_test "select k1,kdate,kdatetime from tdate where kdatetime='2023-10-01';" +} From e7c41278cf946f3f347a4010c54e4a5ed1226fd8 Mon Sep 17 00:00:00 2001 From: Mryange Date: Sat, 19 Jul 2025 10:23:52 +0800 Subject: [PATCH 265/572] [branch-3.0](core)Fix format_round will be core when entering multiple lines (#53515) ### What problem does this PR solve? https://github.com/apache/doris/pull/53314 --- be/src/vec/functions/function_string.h | 34 ++++++++++++------- .../math_functions/test_format_round.out | 6 ++++ .../math_functions/test_format_round.groovy | 13 +++---- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index b837a2220ce77f..09122aaf51c4e2 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -1807,11 +1807,15 @@ class FunctionStringFormatRound : public IFunction { bool is_const; std::tie(argument_column_2, is_const) = unpack_if_const(block.get_by_position(arguments[1]).column); + auto* result_column = assert_cast(res_column.get()); - auto result_column = assert_cast(res_column.get()); - - RETURN_IF_ERROR(Impl::execute(context, result_column, argument_column, argument_column_2, - input_rows_count)); + if (is_const) { + RETURN_IF_ERROR(Impl::template execute(context, result_column, argument_column, + argument_column_2, input_rows_count)); + } else { + RETURN_IF_ERROR(Impl::template execute(context, result_column, argument_column, + argument_column_2, input_rows_count)); + } block.replace_by_position(result, std::move(res_column)); return Status::OK(); @@ -3180,8 +3184,8 @@ StringRef do_format_round(FunctionContext* context, UInt32 scale, T int_value, T } // Note string value must be valid decimal string which contains two digits after the decimal point -static StringRef do_format_round(FunctionContext* context, const string& value, - Int32 decimal_places) { +static inline StringRef do_format_round(FunctionContext* context, const string& value, + Int32 decimal_places) { bool is_positive = (value[0] != '-'); int32_t result_len = value.size() + @@ -3361,6 +3365,7 @@ struct FormatRoundDoubleImpl { return {std::make_shared(), std::make_shared()}; } + template static Status execute(FunctionContext* context, ColumnString* result_column, const ColumnPtr col_ptr, ColumnPtr decimal_places_col_ptr, size_t input_rows_count) { @@ -3369,7 +3374,7 @@ struct FormatRoundDoubleImpl { const auto* data_column = assert_cast(col_ptr.get()); // when scale is above 38, we will go here for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", decimal_places); @@ -3390,6 +3395,7 @@ struct FormatRoundInt64Impl { return {std::make_shared(), std::make_shared()}; } + template static Status execute(FunctionContext* context, ColumnString* result_column, const ColumnPtr col_ptr, ColumnPtr decimal_places_col_ptr, size_t input_rows_count) { @@ -3397,7 +3403,7 @@ struct FormatRoundInt64Impl { const auto& arg_column_data_2 = assert_cast(decimal_places_col_ptr.get())->get_data(); for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", decimal_places); @@ -3417,6 +3423,7 @@ struct FormatRoundInt128Impl { return {std::make_shared(), std::make_shared()}; } + template static Status execute(FunctionContext* context, ColumnString* result_column, const ColumnPtr col_ptr, ColumnPtr decimal_places_col_ptr, size_t input_rows_count) { @@ -3427,7 +3434,7 @@ struct FormatRoundInt128Impl { // get "170,141,183,460,469,231,731,687,303,715,884,105,727.00" in doris, // see https://github.com/apache/doris/blob/788abf2d7c3c7c2d57487a9608e889e7662d5fb2/be/src/vec/data_types/data_type_number_base.cpp#L124 for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", decimal_places); @@ -3448,13 +3455,14 @@ struct FormatRoundDecimalImpl { std::make_shared()}; } + template static Status execute(FunctionContext* context, ColumnString* result_column, ColumnPtr col_ptr, ColumnPtr decimal_places_col_ptr, size_t input_rows_count) { const auto& arg_column_data_2 = assert_cast(decimal_places_col_ptr.get())->get_data(); if (auto* decimalv2_column = check_and_get_column>(*col_ptr)) { for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", @@ -3475,7 +3483,7 @@ struct FormatRoundDecimalImpl { check_and_get_column>(*col_ptr)) { const UInt32 scale = decimal32_column->get_scale(); for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", @@ -3494,7 +3502,7 @@ struct FormatRoundDecimalImpl { check_and_get_column>(*col_ptr)) { const UInt32 scale = decimal64_column->get_scale(); for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", @@ -3513,7 +3521,7 @@ struct FormatRoundDecimalImpl { check_and_get_column>(*col_ptr)) { const UInt32 scale = decimal128_column->get_scale(); for (size_t i = 0; i < input_rows_count; i++) { - int32_t decimal_places = arg_column_data_2[i]; + int32_t decimal_places = arg_column_data_2[index_check_const(i)]; if (decimal_places < 0) { return Status::InvalidArgument( "The second argument is {}, it can not be less than 0.", diff --git a/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out b/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out index 42ce5b1ff2833c..aa9ccc5aa30ce8 100644 --- a/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out +++ b/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out @@ -25,19 +25,25 @@ -- !select_default -- 1 123 123456 123455677788 123456.1234567 123456.123456700 +2 123 12313 94720913 434.1234567 34.123456700 -- !format_round_8 -- 123.000000 +123.000000 -- !format_round_9 -- +12,313.000000 123,456.000000 -- !format_round_10 -- 123,455,677,788.000000 +94,720,913.000000 -- !format_round_12 -- 123,456.123457 +434.123457 -- !format_round_13 -- 123,456.123457 +34.123457 diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy index dfd4a057a36984..1af614f6aac207 100644 --- a/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy +++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy @@ -46,15 +46,16 @@ suite("test_format_round", "p0") { """ sql """ INSERT INTO test_format_round VALUES - (1, 123, 123456, 123455677788, 123456.1234567, 123456.1234567); + (1, 123, 123456, 123455677788, 123456.1234567, 123456.1234567), + (2, 123, 12313, 094720913, 434.1234567, 34.1234567); """ qt_select_default """ SELECT * FROM test_format_round t ORDER BY user_id; """ - order_qt_format_round_8 """ select format_round(int_col, 6) from test_format_round""" - order_qt_format_round_9 """ select format_round(bigint_col, 6) from test_format_round""" - order_qt_format_round_10 """ select format_round(largeint_col, 6) from test_format_round""" - order_qt_format_round_12 """ select format_round(double_col, 6) from test_format_round""" - order_qt_format_round_13 """ select format_round(decimal_col, 6) from test_format_round""" + order_qt_format_round_8 """ select format_round(int_col, 6) from test_format_round order by user_id""" + order_qt_format_round_9 """ select format_round(bigint_col, 6) from test_format_round order by user_id""" + order_qt_format_round_10 """ select format_round(largeint_col, 6) from test_format_round order by user_id""" + order_qt_format_round_12 """ select format_round(double_col, 6) from test_format_round order by user_id""" + order_qt_format_round_13 """ select format_round(decimal_col, 6) from test_format_round order by user_id""" test { sql """select format_round(1234567.8910, -1) """ From 4595c4fdf8a928e69649295fdbfa20598be94b05 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sun, 20 Jul 2025 06:52:57 -0700 Subject: [PATCH 266/572] branch-3.0: [fix](test) fix some regression test cases (#53588) (#53589) bp #53588 And fix some test cases in branch-3.0 --- .../iceberg/IcebergTransaction.java | 11 ++++-- .../hive/test_information_schema_external.out | 4 +- .../test_information_schema_external.groovy | 37 +++++++------------ .../iceberg/test_s3tables_write_insert.groovy | 17 --------- 4 files changed, 23 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index c7d7212335a224..f691f14ebb65a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -84,7 +84,6 @@ public void beginInsert(SimpleTableInfo tableInfo) throws UserException { } catch (Exception e) { throw new UserException("Failed to begin insert for iceberg table " + tableInfo, e); } - } public void finishInsert(SimpleTableInfo tableInfo, Optional insertCtx) { @@ -133,8 +132,14 @@ private void updateManifestAfterInsert(TUpdateMode updateMode) { @Override public void commit() throws UserException { - // commit the iceberg transaction - transaction.commitTransaction(); + try { + // commit the iceberg transaction + ops.getPreExecutionAuthenticator().execute(() -> { + transaction.commitTransaction(); + }); + } catch (Exception e) { + throw new UserException("Failed to commit iceberg transaction for table " + tableInfo, e); + } } @Override diff --git a/regression-test/data/external_table_p0/hive/test_information_schema_external.out b/regression-test/data/external_table_p0/hive/test_information_schema_external.out index 199b9e0b12305e..6142e37c2b8905 100644 --- a/regression-test/data/external_table_p0/hive/test_information_schema_external.out +++ b/regression-test/data/external_table_p0/hive/test_information_schema_external.out @@ -127,7 +127,7 @@ test_information_schema_external_hive2 tpch1_parquet orders BASE TABLE hms 0 \N test_information_schema_external_hive2 tpch1_parquet partsupp BASE TABLE hms 0 \N -- !views_1 -- -\N info_schema_ext_db test_view NONE NO root@% DEFINER utf8 \N +internal info_schema_ext_db test_view SELECT `internal`.`info_schema_ext_db`.`ab`.`id` AS `a` FROM `internal`.`info_schema_ext_db`.`ab` NONE NO root@% DEFINER utf8 \N -- !views_2 -- @@ -259,7 +259,7 @@ test_information_schema_external_hive3 tpch1_parquet orders BASE TABLE hms 0 \N test_information_schema_external_hive3 tpch1_parquet partsupp BASE TABLE hms 0 \N -- !views_1 -- -\N info_schema_ext_db test_view NONE NO root@% DEFINER utf8 \N +internal info_schema_ext_db test_view SELECT `internal`.`info_schema_ext_db`.`ab`.`id` AS `a` FROM `internal`.`info_schema_ext_db`.`ab` NONE NO root@% DEFINER utf8 \N -- !views_2 -- diff --git a/regression-test/suites/external_table_p0/hive/test_information_schema_external.groovy b/regression-test/suites/external_table_p0/hive/test_information_schema_external.groovy index 71cd62f8d6e453..04b155826b6fdb 100644 --- a/regression-test/suites/external_table_p0/hive/test_information_schema_external.groovy +++ b/regression-test/suites/external_table_p0/hive/test_information_schema_external.groovy @@ -37,14 +37,17 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte );""" def db_name = "info_schema_ext_db" + sql """drop database if exists ${db_name} force""" + sql """drop database if exists ${db_name}_1 force""" + sql """drop database if exists ${db_name}_2 force""" //schemata order_qt_schemata_1 """ select * from ${catalog_name}.information_schema.schemata where CATALOG_NAME = "${catalog_name}" and SCHEMA_NAME = "default"; """ - sql """ create database if not exists ${db_name}_1; """ - sql """ create database if not exists ${db_name}_2; """ + sql """ create database ${db_name}_1; """ + sql """ create database ${db_name}_2; """ order_qt_schemata_2 """ select * from internal.information_schema.schemata where CATALOG_NAME = "internal" and SCHEMA_NAME = "${db_name}_1"; @@ -53,7 +56,7 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte select * from internal.information_schema.schemata where CATALOG_NAME = "internal" and SCHEMA_NAME = "${db_name}_2"; """ - sql """ drop database if exists ${db_name}_1 """ + sql """ drop database ${db_name}_1 """ order_qt_schemata_4 """ select * from internal.information_schema.schemata where CATALOG_NAME = "internal" and SCHEMA_NAME = "${db_name}_1"; @@ -62,7 +65,7 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte select * from internal.information_schema.schemata where CATALOG_NAME = "internal" and SCHEMA_NAME = "${db_name}_2"; """ - sql """ drop database if exists ${db_name}_2 """ + sql """ drop database ${db_name}_2 """ order_qt_schemata_6 """ select * from internal.information_schema.schemata where CATALOG_NAME = "internal" and SCHEMA_NAME = "${db_name}_1"; @@ -80,12 +83,8 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte where CATALOG_NAME = "internal" and SCHEMA_NAME = "infomation_schema_ext"; """ - sql """ drop database if exists ${db_name}_1 """ - sql """ drop database if exists ${db_name}_2 """ - //columns - sql """ create database if not exists ${db_name}; """ - sql """ drop table if exists ${db_name}.abcd """ + sql """ create database ${db_name}; """ sql """ CREATE TABLE ${db_name}.abcd ( `id` int(11) not null , @@ -99,7 +98,6 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte select * from internal.information_schema.columns where TABLE_CATALOG = "internal" and TABLE_SCHEMA = "${db_name}"; """ - sql """ drop table if exists ${db_name} """ order_qt_columns_2 """ select * from internal.information_schema.columns where TABLE_CATALOG = "internal" and TABLE_SCHEMA = "${db_name}"; @@ -124,8 +122,8 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte select * from ${catalog_name}.information_schema.columns where TABLE_CATALOG = "${catalog_name}" and TABLE_SCHEMA = "tpch1_parquet" and TABLE_NAME = "partsupp"; """ - sql """ drop table if exists ${db_name}.abcd """ - sql """ drop database if exists ${db_name}; """ + sql """ drop table ${db_name}.abcd """ + sql """ drop database ${db_name}; """ //metadata_name_ids order_qt_ids_1 """ @@ -192,10 +190,9 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte """ //views - sql """ create database if not exists ${db_name}; """ - sql """ drop table if exists ${db_name}.ab """ + sql """ create database ${db_name}; """ sql """ - CREATE TABLE ${db_name}.ab ( + CREATE TABLE ${db_name}.ab ( `id` int(11) not null , `name` string ) @@ -203,9 +200,8 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES("replication_num" = "1"); """ - sql """ drop VIEW IF EXISTS ${db_name}.test_view """ sql """ - CREATE VIEW IF NOT EXISTS ${db_name}.test_view (a) + CREATE VIEW ${db_name}.test_view (a) AS SELECT id as a FROM ${db_name}.ab """ @@ -218,13 +214,6 @@ suite("test_information_schema_external", "p0,external,hive,external_docker,exte select * from internal.information_schema.views where TABLE_SCHEMA = "${db_name}" and TABLE_NAME = "test_view"; """ - sql """ drop VIEW IF EXISTS ${db_name}.test_view """ - sql """ drop table if exists ${db_name}.ab """ - sql """ drop database if exists ${db_name}; """ - - - - sql """drop catalog if exists ${catalog_name}""" } finally { } } diff --git a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy index 670b3e1120610c..05c7eaeed473e1 100644 --- a/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy +++ b/regression-test/suites/external_table_p2/iceberg/test_s3tables_write_insert.groovy @@ -617,23 +617,6 @@ suite("test_s3tables_write_insert", "p2,external,iceberg,external_remote,externa order_qt_q03 """ select * from ${all_types_partition_table}; """ - // just test - sql """ - SELECT - CASE - WHEN file_size_in_bytes BETWEEN 0 AND 8 * 1024 * 1024 THEN '0-8M' - WHEN file_size_in_bytes BETWEEN 8 * 1024 * 1024 + 1 AND 32 * 1024 * 1024 THEN '8-32M' - WHEN file_size_in_bytes BETWEEN 2 * 1024 * 1024 + 1 AND 128 * 1024 * 1024 THEN '32-128M' - WHEN file_size_in_bytes BETWEEN 128 * 1024 * 1024 + 1 AND 512 * 1024 * 1024 THEN '128-512M' - WHEN file_size_in_bytes > 512 * 1024 * 1024 THEN '> 512M' - ELSE 'Unknown' - END AS SizeRange, - COUNT(*) AS FileNum - FROM ${all_types_partition_table}\$data_files - GROUP BY - SizeRange; - """ - sql """ DROP TABLE ${all_types_partition_table}; """ } From c3608e083f85164dd7f7e6d84174af2b0150f03e Mon Sep 17 00:00:00 2001 From: morrySnow Date: Sun, 20 Jul 2025 21:59:04 +0800 Subject: [PATCH 267/572] branch-3.0: [opt](group_concat) allow args be types other than string #52805 (#53531) cherry picked from #52805 --- .../functions/agg/GroupConcat.java | 44 +++++------ .../agg/MultiDistinctGroupConcat.java | 74 ++++++++++--------- .../group_concat/test_group_concat.out | 6 +- .../group_concat/test_group_concat.groovy | 16 +++- 4 files changed, 80 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java index 2157deeeb4d872..d9a717f8156b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupConcat.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -40,10 +39,18 @@ public class GroupConcat extends NullableAggregateFunction implements ExplicitlyCastableSignature { - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), + private static final List ONE_ARG = ImmutableList.of( + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT) + ); + private static final List ONE_ARG_WITH_ORDER_BY = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) - .varArgs(VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX), + .varArgs(VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX) + ); + private static final List TWO_ARGS = ImmutableList.of( + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) + ); + private static final List TWO_ARGS_WITH_ORDER_BY = ImmutableList.of( FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) .varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX) ); @@ -94,23 +101,6 @@ public List getDistinctArguments() { } } - @Override - public void checkLegalityBeforeTypeCoercion() { - DataType typeOrArg0 = getArgumentType(0); - if (!typeOrArg0.isStringLikeType() && !typeOrArg0.isNullType()) { - throw new AnalysisException( - "group_concat requires first parameter to be of type STRING: " + this.toSql()); - } - - if (nonOrderArguments == 2) { - DataType typeOrArg1 = getArgumentType(1); - if (!typeOrArg1.isStringLikeType() && !typeOrArg1.isNullType()) { - throw new AnalysisException( - "group_concat requires second parameter to be of type STRING: " + this.toSql()); - } - } - } - @Override public GroupConcat withAlwaysNullable(boolean alwaysNullable) { return new GroupConcat(distinct, alwaysNullable, children); @@ -131,7 +121,17 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public List getSignatures() { - return SIGNATURES; + if (nonOrderArguments == 2) { + if (arity() >= 3) { + return TWO_ARGS_WITH_ORDER_BY; + } + return TWO_ARGS; + } else { + if (arity() >= 2) { + return ONE_ARG_WITH_ORDER_BY; + } + return ONE_ARG; + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java index 30bce48d67c7ee..8c9665fee40d1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/MultiDistinctGroupConcat.java @@ -23,8 +23,6 @@ import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; -import org.apache.doris.nereids.types.CharType; -import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.AnyDataType; import org.apache.doris.nereids.util.ExpressionUtils; @@ -37,26 +35,24 @@ /** MultiDistinctGroupConcat */ public class MultiDistinctGroupConcat extends NullableAggregateFunction implements ExplicitlyCastableSignature, MultiDistinction { - public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT), - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT, - AnyDataType.INSTANCE_WITHOUT_INDEX), - FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).varArgs(VarcharType.SYSTEM_DEFAULT, - VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX), - - FunctionSignature.ret(StringType.INSTANCE).args(StringType.INSTANCE), - FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE, - AnyDataType.INSTANCE_WITHOUT_INDEX), - FunctionSignature.ret(StringType.INSTANCE).varArgs(StringType.INSTANCE, - StringType.INSTANCE, AnyDataType.INSTANCE_WITHOUT_INDEX), - - FunctionSignature.ret(CharType.SYSTEM_DEFAULT).args(CharType.SYSTEM_DEFAULT), - FunctionSignature.ret(CharType.SYSTEM_DEFAULT).varArgs(CharType.SYSTEM_DEFAULT, - AnyDataType.INSTANCE_WITHOUT_INDEX), - FunctionSignature.ret(CharType.SYSTEM_DEFAULT).varArgs(CharType.SYSTEM_DEFAULT, - CharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX)); + private static final List ONE_ARG = ImmutableList.of( + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT) + ); + private static final List ONE_ARG_WITH_ORDER_BY = ImmutableList.of( + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .varArgs(VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX) + ); + private static final List TWO_ARGS = ImmutableList.of( + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT) + ); + private static final List TWO_ARGS_WITH_ORDER_BY = ImmutableList.of( + FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT) + .varArgs(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, AnyDataType.INSTANCE_WITHOUT_INDEX) + ); private final boolean mustUseMultiDistinctAgg; + private final int nonOrderArguments; /** * constructor with 1 argument with other arguments. @@ -79,8 +75,8 @@ private MultiDistinctGroupConcat(boolean alwaysNullable, Expression arg, private MultiDistinctGroupConcat(boolean mustUseMultiDistinctAgg, boolean alwaysNullable, List args) { super("multi_distinct_group_concat", false, alwaysNullable, args); - checkArguments(children); this.mustUseMultiDistinctAgg = mustUseMultiDistinctAgg; + this.nonOrderArguments = findOrderExprIndex(children); } @Override @@ -99,7 +95,6 @@ public MultiDistinctGroupConcat withAlwaysNullable(boolean alwaysNullable) { */ @Override public MultiDistinctGroupConcat withDistinctAndChildren(boolean distinct, List children) { - checkArguments(children); return new MultiDistinctGroupConcat(mustUseMultiDistinctAgg, alwaysNullable, children); } @@ -110,10 +105,30 @@ public R accept(ExpressionVisitor visitor, C context) { @Override public List getSignatures() { - return SIGNATURES; + if (nonOrderArguments == 2) { + if (arity() >= 3) { + return TWO_ARGS_WITH_ORDER_BY; + } + return TWO_ARGS; + } else { + if (arity() >= 2) { + return ONE_ARG_WITH_ORDER_BY; + } + return ONE_ARG; + } + } + + @Override + public boolean mustUseMultiDistinctAgg() { + return mustUseMultiDistinctAgg || children.stream().anyMatch(OrderExpression.class::isInstance); } - private void checkArguments(List children) { + @Override + public Expression withMustUseMultiDistinctAgg(boolean mustUseMultiDistinctAgg) { + return new MultiDistinctGroupConcat(mustUseMultiDistinctAgg, alwaysNullable, children); + } + + private int findOrderExprIndex(List children) { Preconditions.checkArgument(children().size() >= 1, "children's size should >= 1"); boolean foundOrderExpr = false; int firstOrderExrIndex = 0; @@ -133,15 +148,6 @@ private void checkArguments(List children) { throw new AnalysisException( "multi_distinct_group_concat requires one or two parameters: " + children); } - } - - @Override - public boolean mustUseMultiDistinctAgg() { - return mustUseMultiDistinctAgg || children.stream().anyMatch(OrderExpression.class::isInstance); - } - - @Override - public Expression withMustUseMultiDistinctAgg(boolean mustUseMultiDistinctAgg) { - return new MultiDistinctGroupConcat(mustUseMultiDistinctAgg, alwaysNullable, children); + return firstOrderExrIndex; } } diff --git a/regression-test/data/query_p0/group_concat/test_group_concat.out b/regression-test/data/query_p0/group_concat/test_group_concat.out index 3065713cf55c49..29afb6c950d88d 100644 --- a/regression-test/data/query_p0/group_concat/test_group_concat.out +++ b/regression-test/data/query_p0/group_concat/test_group_concat.out @@ -39,6 +39,10 @@ false 1 2 1 2 +-- !select_12 -- +1 2 +1 2 + -- !select_13 -- 1 2 1 2 @@ -68,7 +72,7 @@ false 2 23,222,22,211,21 -- !select_group_concat_order_by1 -- -1,11,2,21,21,211,22,222,23,3 3,23,222,22,211,21,21,2,11,1 +1,11,2,21,21,211,22,222,23,3 3223222222222112212212221121 -- !select_group_concat_order_by2 -- 1,11,2,21,21,211,22,222,23,3 3,23,222,22,211,21,21,2,11,1 diff --git a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy index ee9bab29e8e166..a60514c719f687 100644 --- a/regression-test/suites/query_p0/group_concat/test_group_concat.groovy +++ b/regression-test/suites/query_p0/group_concat/test_group_concat.groovy @@ -72,7 +72,16 @@ suite("test_group_concat", "query,p0,arrow_flight_sql") { qt_select_12 """ select - group_concat( distinct b1, '?'), group_concat( distinct b3, '?') + group_concat( distinct b1, 123), group_concat( distinct b3, '?') + from + table_group_concat + group by + b2; + """ + + qt_select_12 """ + select + multi_distinct_group_concat(b1, 123), multi_distinct_group_concat(b3, '?') from table_group_concat group by @@ -109,7 +118,7 @@ suite("test_group_concat", "query,p0,arrow_flight_sql") { select * from table_group_concat order by b1, b2, b3; """ qt_select_group_concat_order_by_desc1 """ - SELECT b1, group_concat(cast(abs(b2) as varchar) order by abs(b2) desc) FROM table_group_concat group by b1 order by b1 + SELECT b1, group_concat(abs(b2) order by abs(b2) desc) FROM table_group_concat group by b1 order by b1 """ qt_select_group_concat_order_by_desc2 """ @@ -119,12 +128,13 @@ suite("test_group_concat", "query,p0,arrow_flight_sql") { SELECT b1, group_concat(cast(abs(b3) as varchar) order by abs(b2) desc, b3 desc) FROM table_group_concat group by b1 order by b1 """ qt_select_group_concat_order_by1 """ - select group_concat(b3,',' order by b3 asc),group_concat(b3,',' order by b3 desc) from table_group_concat; + select group_concat(b3,',' order by b3 asc),group_concat(b3,'2' order by b3 desc) from table_group_concat; """ sql """create view if not exists test_view as select group_concat(b3,',' order by b3 asc),group_concat(b3,',' order by b3 desc) from table_group_concat;""" qt_select_group_concat_order_by2 """ select * from test_view; """ + sql """drop view if exists test_view""" } From da37cabc903cfae22602978f0b4534c5659de4fc Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Sun, 20 Jul 2025 21:59:46 +0800 Subject: [PATCH 268/572] [Cherry-Pick](branch-3.0) Pick "[Enhancement](Log) Missing rowset clone should not print stack log (#53193)" (#53527) Pick #53193 --- be/src/olap/cumulative_compaction.cpp | 2 +- be/src/olap/storage_engine.cpp | 5 +++-- be/src/olap/task/engine_publish_version_task.cpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 2a2fdb51eb2f4e..8db397aa577ccd 100644 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -206,7 +206,7 @@ Status CumulativeCompaction::pick_rowsets_to_compact() { Status st = _engine.submit_clone_task(tablet(), missing_versions.back().first); if (!st) { LOG_WARNING("cumulative compaction failed to submit missing rowset clone task.") - .tag("st", st.to_string()) + .tag("st", st.msg()) .tag("tablet_id", _tablet->tablet_id()) .tag("version", missing_versions.back().first) .tag("replica_id", tablet()->replica_id()) diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index df8c6631e20036..bcb9dd8453feb8 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -1472,6 +1472,7 @@ bool StorageEngine::get_peers_replica_backends(int64_t tablet_id, std::vector backends; if (!get_peers_replica_backends(tablet->tablet_id(), &backends)) { - LOG(WARNING) << tablet->tablet_id() << " tablet doesn't have peer replica backends"; - return Status::InternalError(""); + return Status::Error( + "get_peers_replica_backends failed."); } TAgentTaskRequest task; TCloneReq req; diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp index 6c37e55da757e4..13e7dcd97aab4a 100644 --- a/be/src/olap/task/engine_publish_version_task.cpp +++ b/be/src/olap/task/engine_publish_version_task.cpp @@ -228,7 +228,7 @@ Status EnginePublishVersionTask::execute() { if (!st) { LOG_WARNING( "mow publish failed to submit missing rowset clone task.") - .tag("st", st.to_string()) + .tag("st", st.msg()) .tag("tablet_id", tablet->tablet_id()) .tag("version", version.first - 1) .tag("replica_id", tablet->replica_id()) From 8518d9701eddbed83c51e955b100cecefec2ea2b Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 20 Jul 2025 22:01:55 +0800 Subject: [PATCH 269/572] branch-3.0:[fix](jni)add ExceptionCheck for safe jni call. (#51913) (#53342) bp #51913 --- be/src/util/jni-util.cpp | 65 ++++++++++++++----- be/src/util/jni-util.h | 6 +- .../aggregate_function_java_udaf.h | 12 ++-- .../table/trino_connector_jni_reader.cpp | 3 + be/src/vec/exec/jni_connector.cpp | 43 ++++++++---- be/src/vec/exec/jni_connector.h | 20 +++--- be/src/vec/exec/vjdbc_connector.cpp | 48 +++++++++----- be/src/vec/exec/vjdbc_connector.h | 6 +- .../table_function/udf_table_function.cpp | 12 ++-- .../exprs/table_function/udf_table_function.h | 2 + be/src/vec/functions/function_java_udf.cpp | 14 ++-- be/src/vec/functions/function_java_udf.h | 2 + .../pipeline/external/conf/be.conf | 4 +- regression-test/pipeline/p0/conf/be.conf | 4 +- 14 files changed, 165 insertions(+), 76 deletions(-) diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index c2510336129fed..21cfbf36b1cef5 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -311,66 +311,83 @@ Status JniUtil::GetJniExceptionMsg(JNIEnv* env, bool log_stack, const string& pr return Status::RuntimeError("{}{}", prefix, msg_str_guard.get()); } -jobject JniUtil::convert_to_java_map(JNIEnv* env, const std::map& map) { - //TODO: ADD EXCEPTION CHECK. +Status JniUtil::convert_to_java_map(JNIEnv* env, const std::map& map, + jobject* hashmap_object) { jclass hashmap_class = env->FindClass("java/util/HashMap"); + RETURN_ERROR_IF_EXC(env); jmethodID hashmap_constructor = env->GetMethodID(hashmap_class, "", "(I)V"); - jobject hashmap_object = env->NewObject(hashmap_class, hashmap_constructor, map.size()); + RETURN_ERROR_IF_EXC(env); + jobject hashmap_local_object = env->NewObject(hashmap_class, hashmap_constructor, map.size()); + RETURN_ERROR_IF_EXC(env); jmethodID hashmap_put = env->GetMethodID( hashmap_class, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;"); + RETURN_ERROR_IF_EXC(env); for (const auto& it : map) { jstring key = env->NewStringUTF(it.first.c_str()); jstring value = env->NewStringUTF(it.second.c_str()); - env->CallObjectMethod(hashmap_object, hashmap_put, key, value); + env->CallObjectMethod(hashmap_local_object, hashmap_put, key, value); + RETURN_ERROR_IF_EXC(env); env->DeleteLocalRef(key); env->DeleteLocalRef(value); } env->DeleteLocalRef(hashmap_class); - return hashmap_object; + RETURN_IF_ERROR(LocalToGlobalRef(env, hashmap_local_object, hashmap_object)); + return Status::OK(); } -std::map JniUtil::convert_to_cpp_map(JNIEnv* env, jobject map) { - std::map resultMap; - +Status JniUtil::convert_to_cpp_map(JNIEnv* env, jobject map, + std::map* resultMap) { // Get the class and method ID of the java.util.Map interface jclass mapClass = env->FindClass("java/util/Map"); + RETURN_ERROR_IF_EXC(env); jmethodID entrySetMethod = env->GetMethodID(mapClass, "entrySet", "()Ljava/util/Set;"); // Get the class and method ID of the java.util.Set interface jclass setClass = env->FindClass("java/util/Set"); + RETURN_ERROR_IF_EXC(env); jmethodID iteratorSetMethod = env->GetMethodID(setClass, "iterator", "()Ljava/util/Iterator;"); // Get the class and method ID of the java.util.Iterator interface jclass iteratorClass = env->FindClass("java/util/Iterator"); + RETURN_ERROR_IF_EXC(env); jmethodID hasNextMethod = env->GetMethodID(iteratorClass, "hasNext", "()Z"); jmethodID nextMethod = env->GetMethodID(iteratorClass, "next", "()Ljava/lang/Object;"); // Get the class and method ID of the java.util.Map.Entry interface jclass entryClass = env->FindClass("java/util/Map$Entry"); + RETURN_ERROR_IF_EXC(env); jmethodID getKeyMethod = env->GetMethodID(entryClass, "getKey", "()Ljava/lang/Object;"); jmethodID getValueMethod = env->GetMethodID(entryClass, "getValue", "()Ljava/lang/Object;"); // Call the entrySet method to get the set of key-value pairs jobject entrySet = env->CallObjectMethod(map, entrySetMethod); + RETURN_ERROR_IF_EXC(env); // Call the iterator method on the set to iterate over the key-value pairs jobject iteratorSet = env->CallObjectMethod(entrySet, iteratorSetMethod); + RETURN_ERROR_IF_EXC(env); // Iterate over the key-value pairs while (env->CallBooleanMethod(iteratorSet, hasNextMethod)) { + RETURN_ERROR_IF_EXC(env); + // Get the current entry jobject entry = env->CallObjectMethod(iteratorSet, nextMethod); + RETURN_ERROR_IF_EXC(env); // Get the key and value from the entry jobject javaKey = env->CallObjectMethod(entry, getKeyMethod); + RETURN_ERROR_IF_EXC(env); + jobject javaValue = env->CallObjectMethod(entry, getValueMethod); + RETURN_ERROR_IF_EXC(env); // Convert the key and value to C++ strings const char* key = env->GetStringUTFChars(static_cast(javaKey), nullptr); const char* value = env->GetStringUTFChars(static_cast(javaValue), nullptr); // Store the key-value pair in the map - resultMap[key] = value; + (*resultMap)[key] = value; // Release the string references env->ReleaseStringUTFChars(static_cast(javaKey), key); @@ -390,7 +407,7 @@ std::map JniUtil::convert_to_cpp_map(JNIEnv* env, jobj env->DeleteLocalRef(iteratorClass); env->DeleteLocalRef(entryClass); - return resultMap; + return Status::OK(); } Status JniUtil::GetGlobalClassRef(JNIEnv* env, const char* class_str, jclass* class_ref) { @@ -439,13 +456,12 @@ Status JniUtil::init_jni_scanner_loader(JNIEnv* env) { env->GetMethodID(jni_scanner_loader_cls, "loadAllScannerJars", "()V"); RETURN_ERROR_IF_EXC(env); - jni_scanner_loader_obj_ = + jobject jni_scanner_loader_local_obj = env->NewObject(jni_scanner_loader_cls, jni_scanner_loader_constructor); + jni_scanner_loader_obj_ = env->NewGlobalRef(jni_scanner_loader_local_obj); + RETURN_ERROR_IF_EXC(env); + env->DeleteLocalRef(jni_scanner_loader_local_obj); RETURN_ERROR_IF_EXC(env); - if (jni_scanner_loader_obj_ == NULL) { - if (env->ExceptionOccurred()) env->ExceptionDescribe(); - return Status::InternalError("Failed to create ScannerLoader object."); - } env->CallVoidMethod(jni_scanner_loader_obj_, load_jni_scanner); RETURN_ERROR_IF_EXC(env); @@ -464,8 +480,12 @@ Status JniUtil::init_jni_scanner_loader(JNIEnv* env) { Status JniUtil::clean_udf_class_load_cache(const std::string& function_signature) { JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); + jstring function_signature_jstr = env->NewStringUTF(function_signature.c_str()); + RETURN_ERROR_IF_EXC(env); env->CallVoidMethod(jni_scanner_loader_obj_, _clean_udf_cache_method_id, - env->NewStringUTF(function_signature.c_str())); + function_signature_jstr); + RETURN_ERROR_IF_EXC(env); + env->DeleteLocalRef(function_signature_jstr); RETURN_ERROR_IF_EXC(env); return Status::OK(); } @@ -473,11 +493,20 @@ Status JniUtil::clean_udf_class_load_cache(const std::string& function_signature Status JniUtil::get_jni_scanner_class(JNIEnv* env, const char* classname, jclass* jni_scanner_class) { // Get JNI scanner class by class name; - jobject loaded_class_obj = env->CallObjectMethod( - jni_scanner_loader_obj_, jni_scanner_loader_method_, env->NewStringUTF(classname)); + jstring class_name_str = env->NewStringUTF(classname); RETURN_ERROR_IF_EXC(env); + + jobject loaded_class_obj = env->CallObjectMethod(jni_scanner_loader_obj_, + jni_scanner_loader_method_, class_name_str); + RETURN_ERROR_IF_EXC(env); + *jni_scanner_class = reinterpret_cast(env->NewGlobalRef(loaded_class_obj)); RETURN_ERROR_IF_EXC(env); + + env->DeleteLocalRef(loaded_class_obj); + RETURN_ERROR_IF_EXC(env); + env->DeleteLocalRef(class_name_str); + RETURN_ERROR_IF_EXC(env); return Status::OK(); } diff --git a/be/src/util/jni-util.h b/be/src/util/jni-util.h index df332951afebb8..1a2e751b69ed7b 100644 --- a/be/src/util/jni-util.h +++ b/be/src/util/jni-util.h @@ -101,8 +101,10 @@ class JniUtil { return INITIAL_RESERVED_BUFFER_SIZE << n; } static Status get_jni_scanner_class(JNIEnv* env, const char* classname, jclass* loaded_class); - static jobject convert_to_java_map(JNIEnv* env, const std::map& map); - static std::map convert_to_cpp_map(JNIEnv* env, jobject map); + static Status convert_to_java_map(JNIEnv* env, const std::map& map, + jobject* hashmap_object); + static Status convert_to_cpp_map(JNIEnv* env, jobject map, + std::map* resultMap); static size_t get_max_jni_heap_memory_size(); static Status clean_udf_class_load_cache(const std::string& function_signature); diff --git a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h index 366b242927aae1..a215a3a7f8c48f 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h +++ b/be/src/vec/aggregate_functions/aggregate_function_java_udaf.h @@ -132,11 +132,13 @@ struct AggregateJavaUdafData { {"meta_address", std::to_string((long)input_table.get())}, {"required_fields", input_table_schema.first}, {"columns_types", input_table_schema.second}}; - jobject input_map = JniUtil::convert_to_java_map(env, input_params); + jobject input_map = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, input_params, &input_map)); // invoke add batch env->CallObjectMethod(executor_obj, executor_add_batch_id, is_single_place, row_num_start, row_num_end, places_address, place_offset, input_map); - env->DeleteLocalRef(input_map); + RETURN_ERROR_IF_EXC(env); + env->DeleteGlobalRef(input_map); return JniUtil::GetJniExceptionMsg(env); } @@ -200,10 +202,12 @@ struct AggregateJavaUdafData { std::map output_params = {{"is_nullable", output_nullable}, {"required_fields", output_table_schema.first}, {"columns_types", output_table_schema.second}}; - jobject output_map = JniUtil::convert_to_java_map(env, output_params); + jobject output_map = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, output_params, &output_map)); long output_address = env->CallLongMethod(executor_obj, executor_get_value_id, place, output_map); - env->DeleteLocalRef(output_map); + RETURN_ERROR_IF_EXC(env); + env->DeleteGlobalRef(output_map); RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); return JniConnector::fill_block(&output_block, {0}, output_address); } diff --git a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp index 3a7b28b91a4421..49ad7e4f600ba0 100644 --- a/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp +++ b/be/src/vec/exec/format/table/trino_connector_jni_reader.cpp @@ -121,9 +121,12 @@ Status TrinoConnectorJniReader::_set_spi_plugins_dir() { // call: setPluginsDir(String pluginsDir) jstring trino_connector_plugin_path = env->NewStringUTF(doris::config::trino_connector_plugin_dir.c_str()); + RETURN_ERROR_IF_EXC(env); env->CallStaticVoidMethod(plugin_loader_cls, set_plugins_dir_method, trino_connector_plugin_path); RETURN_ERROR_IF_EXC(env); + env->DeleteLocalRef(trino_connector_plugin_path); + RETURN_ERROR_IF_EXC(env); return Status::OK(); } diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index 9940821fcfca6f..e7ec3fd721ee40 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -141,27 +141,36 @@ Status JniConnector::get_next_block(Block* block, size_t* read_rows, bool* eof) Status JniConnector::get_table_schema(std::string& table_schema_str) { JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); - // Call org.apache.doris.jni.JniScanner#getTableSchema - // return the TableSchema information + jstring jstr = (jstring)env->CallObjectMethod(_jni_scanner_obj, _jni_scanner_get_table_schema); RETURN_ERROR_IF_EXC(env); - table_schema_str = env->GetStringUTFChars(jstr, nullptr); + + const char* cstr = env->GetStringUTFChars(jstr, nullptr); RETURN_ERROR_IF_EXC(env); + + if (cstr == nullptr) { + return Status::RuntimeError("GetStringUTFChars returned null"); + } + + table_schema_str = std::string(cstr); // copy to std::string + env->ReleaseStringUTFChars(jstr, cstr); + env->DeleteLocalRef(jstr); return Status::OK(); } -std::map JniConnector::get_statistics(JNIEnv* env) { +Status JniConnector::get_statistics(JNIEnv* env, std::map* result) { + result->clear(); jobject metrics = env->CallObjectMethod(_jni_scanner_obj, _jni_scanner_get_statistics); jthrowable exc = (env)->ExceptionOccurred(); if (exc != nullptr) { LOG(WARNING) << "get_statistics has error: " << JniUtil::GetJniExceptionMsg(env).to_string(); env->DeleteLocalRef(metrics); - return std::map {}; + return Status::OK(); } - std::map result = JniUtil::convert_to_cpp_map(env, metrics); + RETURN_IF_ERROR(JniUtil::convert_to_cpp_map(env, metrics, result)); env->DeleteLocalRef(metrics); - return result; + return Status::OK(); } Status JniConnector::close() { @@ -172,8 +181,11 @@ Status JniConnector::close() { // _fill_block may be failed and returned, we should release table in close. // org.apache.doris.common.jni.JniScanner#releaseTable is idempotent env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_release_table); + RETURN_ERROR_IF_EXC(env); env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_close); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(_jni_scanner_obj); + RETURN_ERROR_IF_EXC(env); } if (_jni_scanner_cls != nullptr) { // _jni_scanner_cls may be null if init connector failed @@ -193,7 +205,7 @@ Status JniConnector::close() { Status JniConnector::_init_jni_scanner(JNIEnv* env, int batch_size) { RETURN_IF_ERROR( JniUtil::get_jni_scanner_class(env, _connector_class.c_str(), &_jni_scanner_cls)); - if (_jni_scanner_cls == nullptr) { + if (_jni_scanner_cls == nullptr) [[unlikely]] { if (env->ExceptionOccurred()) { env->ExceptionDescribe(); } @@ -206,14 +218,15 @@ Status JniConnector::_init_jni_scanner(JNIEnv* env, int batch_size) { RETURN_ERROR_IF_EXC(env); // prepare constructor parameters - jobject hashmap_object = JniUtil::convert_to_java_map(env, _scanner_params); + jobject hashmap_object; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, _scanner_params, &hashmap_object)); jobject jni_scanner_obj = env->NewObject(_jni_scanner_cls, scanner_constructor, batch_size, hashmap_object); RETURN_ERROR_IF_EXC(env); // prepare constructor parameters - env->DeleteLocalRef(hashmap_object); + env->DeleteGlobalRef(hashmap_object); RETURN_ERROR_IF_EXC(env); _jni_scanner_open = env->GetMethodID(_jni_scanner_cls, "open", "()V"); @@ -226,6 +239,7 @@ Status JniConnector::_init_jni_scanner(JNIEnv* env, int batch_size) { _jni_scanner_release_table = env->GetMethodID(_jni_scanner_cls, "releaseTable", "()V"); _jni_scanner_get_statistics = env->GetMethodID(_jni_scanner_cls, "getStatistics", "()Ljava/util/Map;"); + RETURN_ERROR_IF_EXC(env); RETURN_IF_ERROR(JniUtil::LocalToGlobalRef(env, jni_scanner_obj, &_jni_scanner_obj)); env->DeleteLocalRef(jni_scanner_obj); RETURN_ERROR_IF_EXC(env); @@ -748,7 +762,14 @@ void JniConnector::_collect_profile_before_close() { return; } // update scanner metrics - for (const auto& metric : get_statistics(env)) { + std::map statistics_result; + st = get_statistics(env, &statistics_result); + if (!st) { + LOG(WARNING) << "failed to get_statistics when collect profile: " << st; + return; + } + + for (const auto& metric : statistics_result) { std::vector type_and_name = split(metric.first, ":"); if (type_and_name.size() != 2) { LOG(WARNING) << "Name of JNI Scanner metric should be pattern like " diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h index df4c85e7614b0a..3ddf75ee6a51fd 100644 --- a/be/src/vec/exec/jni_connector.h +++ b/be/src/vec/exec/jni_connector.h @@ -241,7 +241,7 @@ class JniConnector : public ProfileCollector { /** * Get performance metrics from java scanner */ - std::map get_statistics(JNIEnv* env); + Status get_statistics(JNIEnv* env, std::map* result); /** * Call java side function JniScanner.getTableSchema. @@ -296,15 +296,15 @@ class JniConnector : public ProfileCollector { bool _closed = false; bool _scanner_opened = false; - jclass _jni_scanner_cls; - jobject _jni_scanner_obj; - jmethodID _jni_scanner_open; - jmethodID _jni_scanner_get_next_batch; - jmethodID _jni_scanner_get_table_schema; - jmethodID _jni_scanner_close; - jmethodID _jni_scanner_release_column; - jmethodID _jni_scanner_release_table; - jmethodID _jni_scanner_get_statistics; + jclass _jni_scanner_cls = nullptr; + jobject _jni_scanner_obj = nullptr; + jmethodID _jni_scanner_open = nullptr; + jmethodID _jni_scanner_get_next_batch = nullptr; + jmethodID _jni_scanner_get_table_schema = nullptr; + jmethodID _jni_scanner_close = nullptr; + jmethodID _jni_scanner_release_column = nullptr; + jmethodID _jni_scanner_release_table = nullptr; + jmethodID _jni_scanner_get_statistics = nullptr; TableMetaAddress _table_meta; diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index f3f4b4d8a1c6fe..a8d8d6fd210405 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -77,10 +77,13 @@ Status JdbcConnector::close(Status /*unused*/) { JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); env->CallNonvirtualVoidMethod(_executor_obj, _executor_clazz, _executor_close_id); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(_executor_factory_clazz); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(_executor_clazz); RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); env->DeleteGlobalRef(_executor_obj); + RETURN_ERROR_IF_EXC(env); return Status::OK(); } @@ -100,7 +103,8 @@ Status JdbcConnector::open(RuntimeState* state, bool read) { GetStaticMethodID(_executor_factory_clazz, "getExecutorClass", "(Lorg/apache/doris/thrift/TOdbcTableType;)Ljava/lang/String;")); - jobject jtable_type = _get_java_table_type(env, _conn_param.table_type); + jobject jtable_type = nullptr; + RETURN_IF_ERROR(_get_java_table_type(env, _conn_param.table_type, &jtable_type)); JNI_CALL_METHOD_CHECK_EXCEPTION_DELETE_REF( jobject, executor_name, env, @@ -110,8 +114,11 @@ Status JdbcConnector::open(RuntimeState* state, bool read) { const char* executor_name_str = env->GetStringUTFChars((jstring)executor_name, nullptr); RETURN_IF_ERROR(JniUtil::get_jni_scanner_class(env, executor_name_str, &_executor_clazz)); - env->DeleteLocalRef(jtable_type); + + env->DeleteGlobalRef(jtable_type); + RETURN_ERROR_IF_EXC(env); env->ReleaseStringUTFChars((jstring)executor_name, executor_name_str); + RETURN_ERROR_IF_EXC(env); #undef GET_BASIC_JAVA_CLAZZ RETURN_IF_ERROR(_register_func_id(env)); @@ -236,6 +243,7 @@ Status JdbcConnector::get_next(bool* eos, Block* block, int batch_size) { SCOPED_RAW_TIMER(&_jdbc_statistic._has_next_timer); // Timer for hasNext check has_next = env->CallNonvirtualBooleanMethod(_executor_obj, _executor_clazz, _executor_has_next_id); + RETURN_ERROR_IF_EXC(env); } // _has_next_timer stops here if (has_next != JNI_TRUE) { @@ -248,10 +256,10 @@ Status JdbcConnector::get_next(bool* eos, Block* block, int batch_size) { auto column_size = _tuple_desc->slots().size(); auto slots = _tuple_desc->slots(); - jobject map; + jobject map = nullptr; { SCOPED_RAW_TIMER(&_jdbc_statistic._prepare_params_timer); // Timer for preparing params - map = _get_reader_params(block, env, column_size); + RETURN_IF_ERROR(_get_reader_params(block, env, column_size, &map)); } // _prepare_params_timer stops here long address = 0; @@ -264,7 +272,8 @@ Status JdbcConnector::get_next(bool* eos, Block* block, int batch_size) { } // _get_block_address_timer stops here RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); - env->DeleteLocalRef(map); + env->DeleteGlobalRef(map); + RETURN_ERROR_IF_EXC(env); std::vector all_columns; for (size_t i = 0; i < column_size; ++i) { @@ -315,10 +324,12 @@ Status JdbcConnector::exec_stmt_write(Block* block, const VExprContextSPtrs& out std::map write_params = {{"meta_address", std::to_string(meta_address)}, {"required_fields", table_schema.first}, {"columns_types", table_schema.second}}; - jobject hashmap_object = JniUtil::convert_to_java_map(env, write_params); + jobject hashmap_object = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, write_params, &hashmap_object)); + env->CallNonvirtualIntMethod(_executor_obj, _executor_clazz, _executor_stmt_write_id, hashmap_object); - env->DeleteLocalRef(hashmap_object); + env->DeleteGlobalRef(hashmap_object); RETURN_ERROR_IF_EXC(env); *num_rows_sent = block->rows(); return Status::OK(); @@ -396,7 +407,8 @@ Status JdbcConnector::_register_func_id(JNIEnv* env) { return Status::OK(); } -jobject JdbcConnector::_get_reader_params(Block* block, JNIEnv* env, size_t column_size) { +Status JdbcConnector::_get_reader_params(Block* block, JNIEnv* env, size_t column_size, + jobject* ans) { std::ostringstream columns_nullable; std::ostringstream columns_replace_string; std::ostringstream required_fields; @@ -447,7 +459,7 @@ jobject JdbcConnector::_get_reader_params(Block* block, JNIEnv* env, size_t colu {"replace_string", columns_replace_string.str()}, {"required_fields", required_fields.str()}, {"columns_types", columns_types.str()}}; - return JniUtil::convert_to_java_map(env, reader_params); + return JniUtil::convert_to_java_map(env, reader_params, ans); } Status JdbcConnector::_cast_string_to_special(Block* block, JNIEnv* env, size_t column_size) { @@ -605,13 +617,17 @@ Status JdbcConnector::_cast_string_to_json(const SlotDescriptor* slot_desc, Bloc return Status::OK(); } -jobject JdbcConnector::_get_java_table_type(JNIEnv* env, TOdbcTableType::type tableType) { - jclass enumClass = env->FindClass("org/apache/doris/thrift/TOdbcTableType"); - jmethodID findByValueMethod = env->GetStaticMethodID( - enumClass, "findByValue", "(I)Lorg/apache/doris/thrift/TOdbcTableType;"); - jobject javaEnumObj = - env->CallStaticObjectMethod(enumClass, findByValueMethod, static_cast(tableType)); - return javaEnumObj; +Status JdbcConnector::_get_java_table_type(JNIEnv* env, TOdbcTableType::type table_type, + jobject* java_enum_obj) { + jclass enum_class = env->FindClass("org/apache/doris/thrift/TOdbcTableType"); + jmethodID find_by_value_method = env->GetStaticMethodID( + enum_class, "findByValue", "(I)Lorg/apache/doris/thrift/TOdbcTableType;"); + jobject java_enum_local_obj = env->CallStaticObjectMethod(enum_class, find_by_value_method, + static_cast(table_type)); + RETURN_ERROR_IF_EXC(env); + RETURN_IF_ERROR(JniUtil::LocalToGlobalRef(env, java_enum_local_obj, java_enum_obj)); + env->DeleteLocalRef(java_enum_local_obj); + return Status::OK(); } std::string JdbcConnector::_get_real_url(const std::string& url) { diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h index a09d390dc7c925..3505d830a363bc 100644 --- a/be/src/vec/exec/vjdbc_connector.h +++ b/be/src/vec/exec/vjdbc_connector.h @@ -127,7 +127,7 @@ class JdbcConnector : public TableConnector { private: Status _register_func_id(JNIEnv* env); - jobject _get_reader_params(Block* block, JNIEnv* env, size_t column_size); + Status _get_reader_params(Block* block, JNIEnv* env, size_t column_size, jobject* ans); Status _cast_string_to_special(Block* block, JNIEnv* env, size_t column_size); Status _cast_string_to_hll(const SlotDescriptor* slot_desc, Block* block, int column_index, @@ -136,7 +136,9 @@ class JdbcConnector : public TableConnector { int rows); Status _cast_string_to_json(const SlotDescriptor* slot_desc, Block* block, int column_index, int rows); - jobject _get_java_table_type(JNIEnv* env, TOdbcTableType::type tableType); + + Status _get_java_table_type(JNIEnv* env, TOdbcTableType::type table_type, + jobject* java_enum_obj); std::string _get_real_url(const std::string& url); diff --git a/be/src/vec/exprs/table_function/udf_table_function.cpp b/be/src/vec/exprs/table_function/udf_table_function.cpp index bb794737a3d38a..0a51cd6a54dec1 100644 --- a/be/src/vec/exprs/table_function/udf_table_function.cpp +++ b/be/src/vec/exprs/table_function/udf_table_function.cpp @@ -115,7 +115,8 @@ Status UDFTableFunction::process_init(Block* block, RuntimeState* state) { {"required_fields", input_table_schema.first}, {"columns_types", input_table_schema.second}}; - jobject input_map = JniUtil::convert_to_java_map(env, input_params); + jobject input_map = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, input_params, &input_map)); _array_result_column = _return_type->create_column(); _result_column_idx = block->columns(); block->insert({_array_result_column, _return_type, "res"}); @@ -125,13 +126,16 @@ Status UDFTableFunction::process_init(Block* block, RuntimeState* state) { {"required_fields", output_table_schema.first}, {"columns_types", output_table_schema.second}}; - jobject output_map = JniUtil::convert_to_java_map(env, output_params); + jobject output_map = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, output_params, &output_map)); DCHECK(_jni_ctx != nullptr); DCHECK(_jni_ctx->executor != nullptr); long output_address = env->CallLongMethod(_jni_ctx->executor, _jni_ctx->executor_evaluate_id, input_map, output_map); - env->DeleteLocalRef(input_map); - env->DeleteLocalRef(output_map); + RETURN_ERROR_IF_EXC(env); + env->DeleteGlobalRef(input_map); + RETURN_ERROR_IF_EXC(env); + env->DeleteGlobalRef(output_map); RETURN_ERROR_IF_EXC(env); RETURN_IF_ERROR(JniConnector::fill_block(block, {_result_column_idx}, output_address)); block->erase(_result_column_idx); diff --git a/be/src/vec/exprs/table_function/udf_table_function.h b/be/src/vec/exprs/table_function/udf_table_function.h index b09371984675df..88427d7a257798 100644 --- a/be/src/vec/exprs/table_function/udf_table_function.h +++ b/be/src/vec/exprs/table_function/udf_table_function.h @@ -77,7 +77,9 @@ class UDFTableFunction final : public TableFunction { return status; } env->CallNonvirtualVoidMethodA(executor, executor_cl, executor_close_id, nullptr); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(executor); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(executor_cl); RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); is_closed = true; diff --git a/be/src/vec/functions/function_java_udf.cpp b/be/src/vec/functions/function_java_udf.cpp index fd55cdaddb2de8..5136081cee64ca 100644 --- a/be/src/vec/functions/function_java_udf.cpp +++ b/be/src/vec/functions/function_java_udf.cpp @@ -105,19 +105,23 @@ Status JavaFunctionCall::execute_impl(FunctionContext* context, Block& block, {"meta_address", std::to_string((long)input_table.get())}, {"required_fields", input_table_schema.first}, {"columns_types", input_table_schema.second}}; - jobject input_map = JniUtil::convert_to_java_map(env, input_params); + jobject input_map = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, input_params, &input_map)); auto output_table_schema = JniConnector::parse_table_schema(&block, {result}, true); std::string output_nullable = block.get_by_position(result).type->is_nullable() ? "true" : "false"; std::map output_params = {{"is_nullable", output_nullable}, {"required_fields", output_table_schema.first}, {"columns_types", output_table_schema.second}}; - jobject output_map = JniUtil::convert_to_java_map(env, output_params); + jobject output_map = nullptr; + RETURN_IF_ERROR(JniUtil::convert_to_java_map(env, output_params, &output_map)); long output_address = env->CallLongMethod(jni_ctx->executor, jni_ctx->executor_evaluate_id, input_map, output_map); - env->DeleteLocalRef(input_map); - env->DeleteLocalRef(output_map); - RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); + RETURN_ERROR_IF_EXC(env); + env->DeleteGlobalRef(input_map); + RETURN_ERROR_IF_EXC(env); + env->DeleteGlobalRef(output_map); + RETURN_ERROR_IF_EXC(env); return JniConnector::fill_block(&block, {result}, output_address); } diff --git a/be/src/vec/functions/function_java_udf.h b/be/src/vec/functions/function_java_udf.h index e35fc67881acb5..4c358c71bc66ec 100644 --- a/be/src/vec/functions/function_java_udf.h +++ b/be/src/vec/functions/function_java_udf.h @@ -144,7 +144,9 @@ class JavaFunctionCall : public IFunctionBase { return status; } env->CallNonvirtualVoidMethodA(executor, executor_cl, executor_close_id, nullptr); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(executor); + RETURN_ERROR_IF_EXC(env); env->DeleteGlobalRef(executor_cl); RETURN_IF_ERROR(JniUtil::GetJniExceptionMsg(env)); is_closed = true; diff --git a/regression-test/pipeline/external/conf/be.conf b/regression-test/pipeline/external/conf/be.conf index 61ba46218ecb4e..4e38d398899eb0 100644 --- a/regression-test/pipeline/external/conf/be.conf +++ b/regression-test/pipeline/external/conf/be.conf @@ -20,10 +20,10 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` PPROF_TMPDIR="$DORIS_HOME/log/" # For jdk 8 -JAVA_OPTS="-Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" +JAVA_OPTS="-Xcheck:jni -Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # For jdk 17, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DORIS_HOME/log/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" +JAVA_OPTS_FOR_JDK_17="-Xcheck:jni -Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DORIS_HOME/log/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # Set your own JAVA_HOME # JAVA_HOME=/path/to/jdk/ diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf index 9b0bc0f8e7184b..277a3e8e94d8cb 100644 --- a/regression-test/pipeline/p0/conf/be.conf +++ b/regression-test/pipeline/p0/conf/be.conf @@ -20,10 +20,10 @@ CUR_DATE=`date +%Y%m%d-%H%M%S` PPROF_TMPDIR="$DORIS_HOME/log/" # For jdk 8 -JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" +JAVA_OPTS="-Xcheck:jni -Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # For jdk 17, this JAVA_OPTS will be used as default JVM options -JAVA_OPTS_FOR_JDK_17="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DORIS_HOME/log/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" +JAVA_OPTS_FOR_JDK_17="-Xcheck:jni -Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc*:$DORIS_HOME/log/be.gc.log.$CUR_DATE:time,uptime:filecount=10,filesize=50M -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.management/sun.management=ALL-UNNAMED -Dcom.mysql.cj.disableAbandonedConnectionCleanup=true" # Set your own JAVA_HOME # JAVA_HOME=/path/to/jdk/ From 21dc357c4bcbb78b3450983de24854e17f3ba3df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 20 Jul 2025 22:02:44 +0800 Subject: [PATCH 270/572] branch-3.0: [fix](ci) cloud_p0 gen default.profraw #53333 (#53391) Cherry-picked from #53333 Co-authored-by: Dongyang Li --- regression-test/pipeline/common/doris-utils.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh index d1b6e71835a812..44f075e14e1855 100644 --- a/regression-test/pipeline/common/doris-utils.sh +++ b/regression-test/pipeline/common/doris-utils.sh @@ -68,6 +68,7 @@ function start_doris_ms() { if [[ ${i} -ge 5 ]]; then echo -e "INFO: doris meta-service started,\n$("${DORIS_HOME}"/ms/lib/doris_cloud --version)" fi + cd - || return 1 } function start_doris_recycler() { @@ -87,6 +88,7 @@ function start_doris_recycler() { if [[ ${i} -ge 5 ]]; then echo -e "INFO: doris recycler started,\n$("${DORIS_HOME}"/ms/lib/doris_cloud --version)" fi + cd - || return 1 } function install_java() { @@ -150,6 +152,7 @@ function start_doris_be() { JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')" export JAVA_HOME fi + cd "${DORIS_HOME}"/be || return 1 ASAN_SYMBOLIZER_PATH="$(command -v llvm-symbolizer)" if [[ -z "${ASAN_SYMBOLIZER_PATH}" ]]; then ASAN_SYMBOLIZER_PATH='/var/local/ldb-toolchain/bin/llvm-symbolizer'; fi export ASAN_SYMBOLIZER_PATH @@ -159,7 +162,7 @@ function start_doris_be() { ulimit -n 200000 && ulimit -c unlimited && swapoff -a && - "${DORIS_HOME}"/be/bin/start_be.sh --daemon + ./bin/start_be.sh --daemon sleep 2 local i=1 @@ -171,8 +174,9 @@ function start_doris_be() { fi done if [[ ${i} -ge 5 ]]; then - echo "INFO: doris be started, be version: $("${DORIS_HOME}"/be/lib/doris_be --version)" + echo "INFO: doris be started, be version: $("${DORIS_HOME}"/be/bin/start_be.sh --version)" fi + cd - || return 1 } function add_doris_be_to_fe() { From 93beba425a790558dfeab059e8fbdbd26b94b552 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 21 Jul 2025 04:53:33 +0800 Subject: [PATCH 271/572] branch-3.0:[echcement](hive)support read hive table that change timestamp column to bigint. (#52954) (#53307) bp #52954 --- .../vec/exec/format/column_type_convert.cpp | 29 +++ be/src/vec/exec/format/column_type_convert.h | 58 +++++ .../format/parquet/parquet_column_convert.cpp | 5 +- be/test/vec/exec/column_type_convert_test.cpp | 202 ++++++++++++++++++ 4 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 be/test/vec/exec/column_type_convert_test.cpp diff --git a/be/src/vec/exec/format/column_type_convert.cpp b/be/src/vec/exec/format/column_type_convert.cpp index a2c226c91d6799..90efabcd847b04 100644 --- a/be/src/vec/exec/format/column_type_convert.cpp +++ b/be/src/vec/exec/format/column_type_convert.cpp @@ -61,6 +61,13 @@ namespace doris::vectorized::converter { M(TYPE_DATETIMEV2) \ M(TYPE_DATEV2) +#define FOR_LOGICAL_INTEGER_TYPES(M) \ + M(TYPE_TINYINT) \ + M(TYPE_SMALLINT) \ + M(TYPE_INT) \ + M(TYPE_BIGINT) \ + M(TYPE_LARGEINT) + static bool _is_numeric_type(PrimitiveType type) { switch (type) { case TYPE_BOOLEAN: @@ -242,6 +249,23 @@ static std::unique_ptr _numeric_to_decimal_converter( } } +static std::unique_ptr _datetime_to_numeric_converter( + const TypeDescriptor& src_type, const DataTypePtr& dst_type) { + PrimitiveType dst_primitive_type = + remove_nullable(dst_type)->get_type_as_type_descriptor().type; + switch (dst_primitive_type) { +#define DISPATCH(DST_TYPE) \ + case DST_TYPE: { \ + return std::make_unique>(); \ + } + FOR_LOGICAL_INTEGER_TYPES(DISPATCH) +#undef DISPATCH + default: { + return std::make_unique(src_type, dst_type); + } + }; +} + static std::unique_ptr _decimal_to_numeric_converter( const TypeDescriptor& src_type, const DataTypePtr& dst_type) { PrimitiveType src_primitive_type = src_type.type; @@ -316,6 +340,11 @@ std::unique_ptr ColumnTypeConverter::get_converter( return std::make_unique>(); } + // datetime to bigint (ms) + if (src_primitive_type == TYPE_DATETIMEV2 && _is_numeric_type(dst_primitive_type)) { + return _datetime_to_numeric_converter(src_type, dst_type); + } + // numeric to decimal if (_is_numeric_type(src_primitive_type) && _is_decimal_type(dst_primitive_type)) { return _numeric_to_decimal_converter(src_type, dst_type); diff --git a/be/src/vec/exec/format/column_type_convert.h b/be/src/vec/exec/format/column_type_convert.h index d4a8186549ab1d..0825a5f0186892 100644 --- a/be/src/vec/exec/format/column_type_convert.h +++ b/be/src/vec/exec/format/column_type_convert.h @@ -17,6 +17,8 @@ #pragma once +#include + #include "gutil/strings/numbers.h" #include "vec/columns/column_string.h" #include "vec/core/types.h" @@ -412,6 +414,62 @@ class CastStringConverter : public ColumnTypeConverter { } }; +template +class DateTimeToNumericConverter : public ColumnTypeConverter { +public: + Status convert(ColumnPtr& src_col, MutableColumnPtr& dst_col) override { + using SrcColumnType = typename PrimitiveTypeTraits::ColumnType; + using DstColumnType = typename PrimitiveTypeTraits::ColumnType; + using SrcCppType = typename PrimitiveTypeTraits::CppType; + using DstCppType = typename PrimitiveTypeTraits::CppType; + + ColumnPtr from_col = remove_nullable(src_col); + MutableColumnPtr to_col = remove_nullable(dst_col->get_ptr())->assume_mutable(); + + NullMap* null_map = nullptr; + if (dst_col->is_nullable()) { + null_map = &reinterpret_cast(dst_col.get()) + ->get_null_map_data(); + } + + size_t rows = from_col->size(); + auto& src_data = static_cast(from_col.get())->get_data(); + size_t start_idx = to_col->size(); + to_col->resize(start_idx + rows); + auto& data = static_cast(*to_col.get()).get_data(); + + for (int i = 0; i < rows; ++i) { + const SrcCppType& src_value = src_data[i]; + auto& dst_value = reinterpret_cast(data[start_idx + i]); + + int64_t ts_s = 0; + if (!src_value.unix_timestamp(&ts_s, cctz::utc_time_zone())) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } + auto micro = src_value.microsecond(); + int64_t ts_ms = ts_s * 1000 + micro / 1000; + if constexpr (DstPrimitiveType != TYPE_LARGEINT && DstPrimitiveType != TYPE_BIGINT) { + if ((Int64)std::numeric_limits::min() > ts_ms || + ts_ms > (Int64)std::numeric_limits::max()) { + if (null_map == nullptr) { + return Status::InternalError("Failed to cast value '{}' to {} column", + src_data[i], dst_col->get_name()); + } else { + (*null_map)[start_idx + i] = 1; + } + } + } + dst_value = static_cast(ts_ms); + } + return Status::OK(); + } +}; + // only support date & datetime v2 template class TimeV2Converter : public ColumnTypeConverter { diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp index 0a5ef2913dd940..a80a26390dbf57 100644 --- a/be/src/vec/exec/format/parquet/parquet_column_convert.cpp +++ b/be/src/vec/exec/format/parquet/parquet_column_convert.cpp @@ -286,9 +286,8 @@ std::unique_ptr PhysicalToLogicalConverter::get_conv convert_params->reset_time_scale_if_missing(9); physical_converter.reset(new Int96toTimestamp()); } else if (src_physical_type == tparquet::Type::INT64) { - convert_params->reset_time_scale_if_missing( - remove_nullable(dst_logical_type)->get_scale()); - physical_converter.reset(new Int64ToTimestamp()); + convert_params->reset_time_scale_if_missing(src_logical_type.scale); + physical_converter = std::make_unique(); } else { physical_converter.reset(new UnsupportedConverter(src_physical_type, src_logical_type)); } diff --git a/be/test/vec/exec/column_type_convert_test.cpp b/be/test/vec/exec/column_type_convert_test.cpp new file mode 100644 index 00000000000000..f883b00ce299aa --- /dev/null +++ b/be/test/vec/exec/column_type_convert_test.cpp @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exec/format/column_type_convert.h" + +#include + +#include + +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_nullable.h" +#include "vec/columns/column_string.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_date.h" +#include "vec/data_types/data_type_date_time.h" +#include "vec/data_types/data_type_decimal.h" +#include "vec/data_types/data_type_factory.hpp" +#include "vec/data_types/data_type_nullable.h" +#include "vec/data_types/data_type_number.h" +#include "vec/data_types/data_type_string.h" + +namespace doris::vectorized { + +class ColumnTypeConverterTest : public testing::Test { +public: + ColumnTypeConverterTest() = default; + virtual ~ColumnTypeConverterTest() = default; +}; + +TEST_F(ColumnTypeConverterTest, TestDateTimeV2ToNumericConversions) { + using namespace doris::vectorized; + auto make_datetimev2_col = + [](const std::vector>& datetimes) { + auto col = ColumnDateTimeV2::create(); + for (const auto& [y, m, d, h, min, s, micro] : datetimes) { + DateV2Value v; + v.unchecked_set_time(y, m, d, h, min, s, micro); + col->get_data().push_back(*reinterpret_cast(&v)); + } + return col; + }; + + auto parse_datetimev2_str = [](const std::string& datetime_str) { + UInt64 x = 0; + ReadBuffer buf((char*)datetime_str.data(), datetime_str.size()); + bool ok = read_datetime_v2_text_impl(x, buf, 6); + CHECK(ok) << "parse_datetimev2_str failed for: " << datetime_str; + return x; + }; + + // 1. DATETIMEV2 -> BIGINT + { + TypeDescriptor src_type(TYPE_DATETIMEV2); + auto dst_type = std::make_shared(); + auto converter = converter::ColumnTypeConverter::get_converter(src_type, dst_type); + + ASSERT_TRUE(converter->support()); + + // 2024-01-01 00:00:00.123456 + auto src_col = make_datetimev2_col({{2024, 1, 1, 0, 0, 0, 123456}}); + auto dst_col = dst_type->create_column(); + auto mutable_dst = dst_col->assume_mutable(); + + Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); + ASSERT_TRUE(st.ok()); + + auto& dst_data = static_cast(*mutable_dst).get_data(); + ASSERT_EQ(1, dst_data.size()); + EXPECT_EQ(1704067200123, dst_data[0]); + } + + // 2. DATETIMEV2 -> INT + { + TypeDescriptor src_type(TYPE_DATETIMEV2); + auto dst_type = std::make_shared(); + auto nullable_dst_type = std::make_shared(dst_type); + auto converter = converter::ColumnTypeConverter::get_converter(src_type, nullable_dst_type); + + ASSERT_TRUE(converter->support()); + + // 1970-01-01 00:00:00.000000 + // 3000-01-01 00:00:00.000000 + auto src_col = make_datetimev2_col({{1970, 1, 1, 0, 0, 0, 0}, {3000, 1, 1, 0, 0, 0, 0}}); + auto dst_col = nullable_dst_type->create_column(); + auto mutable_dst = dst_col->assume_mutable(); + auto& nullable_col = static_cast(*mutable_dst); + auto& null_map = nullable_col.get_null_map_data(); + null_map.resize_fill(src_col->size(), 0); + + Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); + ASSERT_TRUE(st.ok()); + auto& nested_col = static_cast(nullable_col.get_nested_column()); + auto& dst_data = nested_col.get_data(); + + ASSERT_EQ(2, nested_col.size()); + EXPECT_EQ(0, null_map[0]); + ASSERT_EQ(0, dst_data[0]); + EXPECT_EQ(1, null_map[1]); + } + + // 3. DATETIMEV2 -> INT, non-nullable + { + TypeDescriptor src_type(TYPE_DATETIMEV2); + auto dst_type = std::make_shared(); + auto converter = converter::ColumnTypeConverter::get_converter(src_type, dst_type); + + ASSERT_TRUE(converter->support()); + + // 3000-01-01 00:00:00.000000（会溢出int32） + auto src_col = make_datetimev2_col({{3000, 1, 1, 0, 0, 0, 0}}); + auto dst_col = dst_type->create_column(); + auto mutable_dst = dst_col->assume_mutable(); + + Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); + ASSERT_FALSE(st.ok()); + } + + { + TypeDescriptor src_type(TYPE_DATETIMEV2); + auto dst_type = std::make_shared(); + auto nullable_dst_type = std::make_shared(dst_type); + auto converter = converter::ColumnTypeConverter::get_converter(src_type, nullable_dst_type); + + ASSERT_TRUE(converter->support()); + + auto src_col = ColumnDateTimeV2::create(); + src_col->get_data().push_back(parse_datetimev2_str("2024-01-01 12:34:56.123456")); + src_col->get_data().push_back(parse_datetimev2_str("1970-01-01 00:00:00.000000")); + src_col->get_data().push_back(parse_datetimev2_str("3000-01-01 00:00:00.000000")); + src_col->get_data().push_back(parse_datetimev2_str("1900-01-01 00:00:00.000000")); + src_col->get_data().push_back(parse_datetimev2_str("1999-12-31 23:59:59.999999")); + src_col->get_data().push_back(parse_datetimev2_str("2000-01-01 00:00:00.000000")); + src_col->get_data().push_back(parse_datetimev2_str("2025-07-08 16:00:00.123456")); + src_col->get_data().push_back(parse_datetimev2_str("2100-01-01 00:00:00.000000")); + src_col->get_data().push_back(parse_datetimev2_str("9999-12-31 23:59:59.999999")); + src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 12:00:00.000001")); + src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 13:00:00.000002")); + src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 14:00:00.000004")); + src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 12:00:00")); + src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 13:00:00")); + src_col->get_data().push_back(parse_datetimev2_str("2022-05-01 14:00:00")); + + auto dst_col = nullable_dst_type->create_column(); + auto mutable_dst = dst_col->assume_mutable(); + auto& nullable_col = static_cast(*mutable_dst); + auto& null_map = nullable_col.get_null_map_data(); + null_map.resize_fill(src_col->size(), 0); + + Status st = converter->convert(reinterpret_cast(src_col), mutable_dst); + ASSERT_TRUE(st.ok()); + + ASSERT_EQ(15, null_map.size()); + EXPECT_EQ(0, null_map[0]); + EXPECT_EQ(0, null_map[1]); + EXPECT_EQ(0, null_map[2]); + EXPECT_EQ(0, null_map[3]); + EXPECT_EQ(0, null_map[4]); + EXPECT_EQ(0, null_map[5]); + EXPECT_EQ(0, null_map[6]); + EXPECT_EQ(0, null_map[7]); + EXPECT_EQ(0, null_map[8]); + EXPECT_EQ(0, null_map[9]); + EXPECT_EQ(0, null_map[10]); + EXPECT_EQ(0, null_map[11]); + EXPECT_EQ(0, null_map[12]); + EXPECT_EQ(0, null_map[13]); + EXPECT_EQ(0, null_map[14]); + + auto& dst_data = static_cast(nullable_col.get_nested_column()).get_data(); + ASSERT_EQ(15, dst_data.size()); + EXPECT_EQ(1704112496123L, dst_data[0]); + EXPECT_EQ(0L, dst_data[1]); + EXPECT_EQ(32503680000000L, dst_data[2]); + EXPECT_EQ(-2208988800000L, dst_data[3]); + EXPECT_EQ(946684799999L, dst_data[4]); + EXPECT_EQ(946684800000L, dst_data[5]); + EXPECT_EQ(1751990400123, dst_data[6]); + EXPECT_EQ(4102444800000L, dst_data[7]); + EXPECT_EQ(253402300799999, dst_data[8]); + EXPECT_EQ(1651406400000, dst_data[9]); + EXPECT_EQ(1651410000000, dst_data[10]); + EXPECT_EQ(1651413600000, dst_data[11]); + EXPECT_EQ(1651406400000, dst_data[12]); + EXPECT_EQ(1651410000000, dst_data[13]); + EXPECT_EQ(1651413600000, dst_data[14]); + } +} +} // namespace doris::vectorized \ No newline at end of file From 67bf3ee8bbaefacf25ab98a8698ee246d4b8c960 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 10:17:31 +0800 Subject: [PATCH 272/572] branch-3.0: [enhance](load) delete temporary files and disk detection #49962 (#53599) Cherry-picked from #49962 Co-authored-by: kkop <2449402815@qq.com> Co-authored-by: Xin Liao --- be/src/http/action/stream_load.cpp | 12 ++- be/src/http/action/stream_load.h | 2 +- be/src/runtime/load_path_mgr.cpp | 68 +++++++++++-- be/src/runtime/load_path_mgr.h | 9 +- .../runtime/stream_load/stream_load_context.h | 2 + be/test/runtime/stream_load_parquet_test.cpp | 99 +++++++++++++++++++ 6 files changed, 179 insertions(+), 13 deletions(-) create mode 100644 be/test/runtime/stream_load_parquet_test.cpp diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp index e3abd5a8a5d807..1f40fed8c175d4 100644 --- a/be/src/http/action/stream_load.cpp +++ b/be/src/http/action/stream_load.cpp @@ -153,6 +153,9 @@ void StreamLoadAction::handle(HttpRequest* req) { // update statistics streaming_load_requests_total->increment(1); streaming_load_duration_ms->increment(ctx->load_cost_millis); + if (!ctx->data_saved_path.empty()) { + _exec_env->load_path_mgr()->clean_tmp_files(ctx->data_saved_path); + } } Status StreamLoadAction::_handle(std::shared_ptr ctx) { @@ -432,13 +435,14 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req, ctx->pipe = pipe; RETURN_IF_ERROR(_exec_env->new_load_stream_mgr()->put(ctx->id, ctx)); } else { - RETURN_IF_ERROR(_data_saved_path(http_req, &request.path)); + RETURN_IF_ERROR(_data_saved_path(http_req, &request.path, ctx->body_bytes)); auto file_sink = std::make_shared(request.path); RETURN_IF_ERROR(file_sink->open()); request.__isset.path = true; request.fileType = TFileType::FILE_LOCAL; request.__set_file_size(ctx->body_bytes); ctx->body_sink = file_sink; + ctx->data_saved_path = request.path; } if (!http_req->header(HTTP_COLUMNS).empty()) { request.__set_columns(http_req->header(HTTP_COLUMNS)); @@ -716,9 +720,11 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req, return _exec_env->stream_load_executor()->execute_plan_fragment(ctx); } -Status StreamLoadAction::_data_saved_path(HttpRequest* req, std::string* file_path) { +Status StreamLoadAction::_data_saved_path(HttpRequest* req, std::string* file_path, + int64_t file_bytes) { std::string prefix; - RETURN_IF_ERROR(_exec_env->load_path_mgr()->allocate_dir(req->param(HTTP_DB_KEY), "", &prefix)); + RETURN_IF_ERROR(_exec_env->load_path_mgr()->allocate_dir(req->param(HTTP_DB_KEY), "", &prefix, + file_bytes)); timeval tv; gettimeofday(&tv, nullptr); struct tm tm; diff --git a/be/src/http/action/stream_load.h b/be/src/http/action/stream_load.h index d1de89c9397018..bf359317f94ba1 100644 --- a/be/src/http/action/stream_load.h +++ b/be/src/http/action/stream_load.h @@ -47,7 +47,7 @@ class StreamLoadAction : public HttpHandler { private: Status _on_header(HttpRequest* http_req, std::shared_ptr ctx); Status _handle(std::shared_ptr ctx); - Status _data_saved_path(HttpRequest* req, std::string* file_path); + Status _data_saved_path(HttpRequest* req, std::string* file_path, int64_t file_bytes); Status _process_put(HttpRequest* http_req, std::shared_ptr ctx); void _save_stream_load_record(std::shared_ptr ctx, const std::string& str); Status _handle_group_commit(HttpRequest* http_req, std::shared_ptr ctx); diff --git a/be/src/runtime/load_path_mgr.cpp b/be/src/runtime/load_path_mgr.cpp index f1899aadb28281..5b88b45f722ced 100644 --- a/be/src/runtime/load_path_mgr.cpp +++ b/be/src/runtime/load_path_mgr.cpp @@ -86,7 +86,7 @@ Status LoadPathMgr::init() { } Status LoadPathMgr::allocate_dir(const std::string& db, const std::string& label, - std::string* prefix) { + std::string* prefix, int64_t file_bytes) { Status status = _init_once.call([this] { for (auto& store_path : _exec_env->store_paths()) { _path_vec.push_back(store_path.path + "/" + MINI_PREFIX); @@ -96,24 +96,52 @@ Status LoadPathMgr::allocate_dir(const std::string& db, const std::string& label std::string path; auto size = _path_vec.size(); auto retry = size; + auto exceed_capacity_path_num = 0; + size_t disk_capacity_bytes = 0; + size_t available_bytes = 0; while (retry--) { - { - // add SHARD_PREFIX for compatible purpose - std::lock_guard l(_lock); - std::string shard = SHARD_PREFIX + std::to_string(_next_shard++ % MAX_SHARD_NUM); - path = _path_vec[_idx] + "/" + db + "/" + shard + "/" + label; - _idx = (_idx + 1) % size; + // Call get_space_info to get disk space information. + // If the call fails or the disk space is insufficient, + // increment the count of paths exceeding capacity and proceed to the next loop iteration. + std::string base_path = _path_vec[_idx].substr(0, _path_vec[_idx].find("/" + MINI_PREFIX)); + if (!io::global_local_filesystem() + ->get_space_info(base_path, &disk_capacity_bytes, &available_bytes) + .ok() || + !check_disk_space(disk_capacity_bytes, available_bytes, file_bytes)) { + ++exceed_capacity_path_num; + continue; } + // add SHARD_PREFIX for compatible purpose + std::lock_guard l(_lock); + std::string shard = SHARD_PREFIX + std::to_string(_next_shard++ % MAX_SHARD_NUM); + path = _path_vec[_idx] + "/" + db + "/" + shard + "/" + label; + _idx = (_idx + 1) % size; status = io::global_local_filesystem()->create_directory(path); if (LIKELY(status.ok())) { *prefix = path; return Status::OK(); } } - + if (exceed_capacity_path_num == size) { + return Status::Error("exceed capacity limit."); + } return status; } +bool LoadPathMgr::check_disk_space(size_t disk_capacity_bytes, size_t available_bytes, + int64_t file_bytes) { + bool is_available = false; + int64_t remaining_bytes = available_bytes - file_bytes; + double used_ratio = 1.0 - static_cast(remaining_bytes) / disk_capacity_bytes; + is_available = !(used_ratio >= config::storage_flood_stage_usage_percent / 100.0 && + remaining_bytes <= config::storage_flood_stage_left_capacity_bytes); + if (!is_available) { + LOG(WARNING) << "Exceed capacity limit. disk_capacity: " << disk_capacity_bytes + << ", available: " << available_bytes << ", file_bytes: " << file_bytes; + } + return is_available; +} + bool LoadPathMgr::is_too_old(time_t cur_time, const std::string& label_dir, int64_t reserve_hours) { struct stat dir_stat; if (stat(label_dir.c_str(), &dir_stat)) { @@ -178,6 +206,30 @@ void LoadPathMgr::process_path(time_t now, const std::string& path, int64_t rese } } +void LoadPathMgr::clean_files_in_path_vec(const std::string& path) { + // Check if the path contains "/"+MINI_PREFIX. If not, return directly. + if (path.find("/" + MINI_PREFIX) == std::string::npos) { + return; + } + + bool exists = false; + // Check if the path exists + Status status = io::global_local_filesystem()->exists(path, &exists); + if (!status.ok()) { + LOG(WARNING) << "Failed to check if path exists: " << path << ", error: " << status; + return; + } + if (exists) { + // If the path exists, delete the file or directory corresponding to that path + status = io::global_local_filesystem()->delete_directory_or_file(path); + if (status.ok()) { + LOG(INFO) << "Delete path success: " << path; + } else { + LOG(WARNING) << "Delete path failed: " << path << ", error: " << status; + } + } +} + void LoadPathMgr::clean_one_path(const std::string& path) { bool exists = true; std::vector dbs; diff --git a/be/src/runtime/load_path_mgr.h b/be/src/runtime/load_path_mgr.h index e07971709591d1..3a111d55a545b3 100644 --- a/be/src/runtime/load_path_mgr.h +++ b/be/src/runtime/load_path_mgr.h @@ -45,7 +45,10 @@ class LoadPathMgr { Status init(); void stop(); - Status allocate_dir(const std::string& db, const std::string& label, std::string* prefix); + Status allocate_dir(const std::string& db, const std::string& label, std::string* prefix, + int64_t file_bytes); + + bool check_disk_space(size_t disk_capacity_bytes, size_t available_bytes, int64_t file_bytes); void get_load_data_path(std::vector* data_paths); @@ -54,6 +57,8 @@ class LoadPathMgr { std::string get_load_error_absolute_path(const std::string& file_path); const std::string& get_load_error_file_dir() const { return _error_log_dir; } + void clean_tmp_files(const std::string& file_path) { clean_files_in_path_vec(file_path); } + private: bool is_too_old(time_t cur_time, const std::string& label_dir, int64_t reserve_hours); void clean_one_path(const std::string& path); @@ -61,6 +66,8 @@ class LoadPathMgr { void clean(); void process_path(time_t now, const std::string& path, int64_t reserve_hours); + void clean_files_in_path_vec(const std::string& path); + ExecEnv* _exec_env = nullptr; std::mutex _lock; std::vector _path_vec; diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h index 93f76fad4e613c..ff9f0b8be60ad4 100644 --- a/be/src/runtime/stream_load/stream_load_context.h +++ b/be/src/runtime/stream_load/stream_load_context.h @@ -105,6 +105,8 @@ class StreamLoadContext { } } + std::string data_saved_path; + std::string to_json() const; std::string prepare_stream_load_record(const std::string& stream_load_record); diff --git a/be/test/runtime/stream_load_parquet_test.cpp b/be/test/runtime/stream_load_parquet_test.cpp new file mode 100644 index 00000000000000..76fab7782867f6 --- /dev/null +++ b/be/test/runtime/stream_load_parquet_test.cpp @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gtest/gtest.h" +#include "olap/storage_engine.h" +#include "runtime/exec_env.h" +#include "runtime/load_path_mgr.h" +namespace doris { + +class LoadPathMgrTest : public testing::Test { +protected: + void SetUp() override { + _exec_env = ExecEnv::GetInstance(); + + _load_path_mgr = std::make_unique(_exec_env); + // create tmp file + _test_dir = "/tmp/test_clean_file"; + _test_dir1 = "/tmp/test_clean_file/mini_download"; + _test_dir2 = "/tmp/test_clean_file1/mini_download/test.parquet"; + + auto result = io::global_local_filesystem()->delete_directory_or_file(_test_dir1); + result = io::global_local_filesystem()->create_directory(_test_dir1); + EXPECT_TRUE(result.ok()); + + result = io::global_local_filesystem()->delete_directory_or_file(_test_dir2); + result = io::global_local_filesystem()->create_directory(_test_dir2); + EXPECT_TRUE(result.ok()); + + const_cast&>(_exec_env->store_paths()).emplace_back(_test_dir, 1024); + } + + void TearDown() override { + const_cast&>(_exec_env->store_paths()).clear(); + _load_path_mgr->stop(); + _exec_env->destroy(); + } + + ExecEnv* _exec_env; + std::unique_ptr _load_path_mgr; + std::string _test_dir; + std::string _test_dir1; + std::string _test_dir2; +}; + +TEST_F(LoadPathMgrTest, CheckDiskSpaceTest) { + // Check disk space + bool is_available = false; + size_t disk_capacity_bytes = 10; + size_t available_bytes = 9; + int64_t file_bytes = 1; + is_available = + _load_path_mgr->check_disk_space(disk_capacity_bytes, available_bytes, file_bytes); + ASSERT_TRUE(is_available); + + // Check disk space + is_available = false; + disk_capacity_bytes = 10; + available_bytes = 2; + file_bytes = 1; + is_available = + _load_path_mgr->check_disk_space(disk_capacity_bytes, available_bytes, file_bytes); + ASSERT_FALSE(is_available); + + std::string prefix; + Status status = _load_path_mgr->allocate_dir("tmp", "test_label1", &prefix, 1); + EXPECT_TRUE(status.ok()); + std::cout << "NormalAllocation: " << prefix.size() << std::endl; + EXPECT_FALSE(prefix.empty()); + + prefix.clear(); + status = _load_path_mgr->allocate_dir("tmp", "test_label2", &prefix, 999999999999999999); + EXPECT_TRUE(!status.ok()); + std::cout << "UnNormalAllocation: " << prefix.size() << std::endl; + EXPECT_TRUE(prefix.empty()); + + std::cout << "clean_tmp_files" << std::endl; + bool exists = false; + status = io::global_local_filesystem()->exists(_test_dir2, &exists); + EXPECT_TRUE(exists); + _load_path_mgr->clean_tmp_files(_test_dir2); + status = io::global_local_filesystem()->exists(_test_dir2, &exists); + EXPECT_FALSE(exists); +} + +} // namespace doris \ No newline at end of file From a68908691f382361744f4ea908dd76fca5eb1f4c Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Mon, 21 Jul 2025 11:14:49 +0800 Subject: [PATCH 273/572] [test](p2) allow protocol prefix in S3 endpoint format in test_broker_load (#53525) (#53530) cherry-pick #53525 --- .../broker_load/test_broker_load.groovy | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy index 01e1560895205d..f7fffce41afb70 100644 --- a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy +++ b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy @@ -310,37 +310,37 @@ suite("test_broker_load_p2", "p2") { "unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000" // case 30 ] - def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 0 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 1 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 2 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 3 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 4 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 5 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 6 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 7 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 8 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 9 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 10 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 11 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 12 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 13 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 14 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 15 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 16 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 17 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 18 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 19 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 20 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 21 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 22 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 23 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 24 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 25 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 26 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 27 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 28 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 29 - "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0" // case 30 + def task_info = ["${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 0 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 1 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 2 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 3 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 4 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 5 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 6 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 7 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 8 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 9 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 10 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 11 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 12 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 13 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 14 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 15 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 16 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 17 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 18 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 19 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 20 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 21 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 22 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 23 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 24 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 25 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 26 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 27 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 28 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0", // case 29 + "${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0" // case 30 ] def error_msg = ["", // case 0 @@ -451,7 +451,7 @@ suite("test_broker_load_p2", "p2") { logger.info("Load status: " + logStr + ", label: $label") if (result[0][2].equals("FINISHED")) { logger.info("Load FINISHED " + label) - assertTrue(result[0][6].contains(task_info[i])) + assertTrue(result[0][6].contains(task_info[i]), "expected: " + task_info[i] + ", actual: " + result[0][6] + ", label: $label") def load_counters = etl_info[i].split('; '); for (String counter : load_counters) { assertTrue(result[0][5].contains(counter), "expected: " + counter + ", actual: " + result[0][5] + ", label: $label") @@ -460,7 +460,7 @@ suite("test_broker_load_p2", "p2") { } if (result[0][2].equals("CANCELLED")) { logger.info("Load result: " + result[0]) - assertTrue(result[0][6].contains(task_info[i])) + assertTrue(result[0][6].contains(task_info[i]), "expected: " + task_info[i] + ", actual: " + result[0][6] + ", label: $label") assertTrue(result[0][7].contains(error_msg[i]), "expected: " + error_msg[i] + ", actual: " + result[0][7] + ", label: $label") break; } From ba592ed1229aa92962eb896681d85aafc27da63f Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 21 Jul 2025 12:21:43 +0800 Subject: [PATCH 274/572] [Chore](thirdparty) sync thirdparty from master (#53608) sync thirdparty from master --- thirdparty/CHANGELOG.md | 32 ++++- thirdparty/build-thirdparty.sh | 93 +++++++++++-- thirdparty/download-thirdparty.sh | 79 ++++++++++- .../patches/aws-sdk-cpp-1.11.119-cmake.patch | 11 ++ .../azure-sdk-for-cpp-azure-core_1.16.0.patch | 24 ++++ thirdparty/patches/brpc-uuid-string.patch | 123 ++++++++++++++++++ thirdparty/patches/faiss-1.10.0.patch | 66 ++++++++++ thirdparty/patches/flatbuffers-2.0.0.patch | 11 ++ thirdparty/patches/grpc-1.54.3.patch | 11 ++ thirdparty/patches/libdivide-5.0.patch | 11 ++ thirdparty/patches/libuuid-1.0.3.patch | 19 +++ .../thrift-0.16-fix_mutex_include.patch | 9 ++ thirdparty/vars.sh | 49 ++++--- 13 files changed, 507 insertions(+), 31 deletions(-) create mode 100644 thirdparty/patches/aws-sdk-cpp-1.11.119-cmake.patch create mode 100644 thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch create mode 100644 thirdparty/patches/brpc-uuid-string.patch create mode 100644 thirdparty/patches/faiss-1.10.0.patch create mode 100644 thirdparty/patches/flatbuffers-2.0.0.patch create mode 100644 thirdparty/patches/grpc-1.54.3.patch create mode 100644 thirdparty/patches/libdivide-5.0.patch create mode 100644 thirdparty/patches/libuuid-1.0.3.patch create mode 100644 thirdparty/patches/thrift-0.16-fix_mutex_include.patch diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md index 350e914572d539..528c29a13d274e 100644 --- a/thirdparty/CHANGELOG.md +++ b/thirdparty/CHANGELOG.md @@ -2,6 +2,14 @@ This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/doris, and the tag is `build-env-${version}` +## 20250715 + +- Modified: azure-core 1.10.3 -> 1.16.0 + +## 20250617 + +- Modified: simdjson 3.0.1 -> 3.11.6 + ## 20250416 - Mofified: patch re2 to set `ignore_replace_escape` option @@ -10,18 +18,38 @@ This file contains version of the third-party dependency libraries in the build- - Modified: jindofs 6.3.4 -> 6.8.2 -## 20250402 +## 20250405 -- Added: icu 75-1, develop ICU tokenizer based on ICU library. +- Modified: libdivide 5.0 -> 5.2 + +## 20250325 + +- Added: jindofs 6.3.4 with jindofs ubuntu jar ## 20250324 - Added: pugixml 1.15, a C++ XML processing library. +## 20250319 + +- Added: jindofs 6.3.4 + +## 20250318 + +- Modified: hadoop-libs 3.3.6.5 -> 3.3.6.6 + ## 20250225 - Modified: hadoop-libs 3.3.6.4 -> 3.3.6.5 +## 20250123 + +- Added: icu 75-1, develop ICU tokenizer based on ICU library. + +## 20241031 + +- Modified: hadoop-libs 3.3.6.3 -> 3.3.6.4 + ## 20240815 - Modified: arrow 15.0.2 -> 17.0.0 diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index bb3cc427ab3574..6084026fd00304 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -165,6 +165,12 @@ if [[ "${CC}" == *gcc ]]; then warning_array_parameter='-Wno-array-parameter' warning_narrowing='-Wno-narrowing' warning_dangling_reference='-Wno-dangling-reference' + + gcc_major_version=$("${CC}" -dumpversion | cut -d. -f1) + if [[ "${gcc_major_version}" -ge 15 ]]; then + warning_deprecated_literal_operator='-Wno-deprecated-literal-operator' + fi + boost_toolset='gcc' elif [[ "${CC}" == *clang ]]; then warning_uninitialized='-Wno-uninitialized' @@ -758,10 +764,10 @@ build_mysql() { fi if [[ "${KERNEL}" != 'Darwin' ]]; then - cflags='-static -pthread -lrt' + cflags='-static -pthread -lrt -std=gnu89' cxxflags='-static -pthread -lrt' else - cflags='-pthread' + cflags='-pthread -std=gnu89' cxxflags='-pthread' fi @@ -875,7 +881,7 @@ build_cyrus_sasl() { check_if_source_exist "${CYRUS_SASL_SOURCE}" cd "${TP_SOURCE_DIR}/${CYRUS_SASL_SOURCE}" - CFLAGS="-fPIC -Wno-implicit-function-declaration" \ + CFLAGS="-fPIC -std=gnu89 -Wno-implicit-function-declaration" \ CPPFLAGS="-I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ LIBS="-lcrypto" \ @@ -919,7 +925,7 @@ build_odbc() { cd "${TP_SOURCE_DIR}/${ODBC_SOURCE}" - CFLAGS="-I${TP_INCLUDE_DIR} -Wno-int-conversion -Wno-implicit-function-declaration" \ + CFLAGS="-I${TP_INCLUDE_DIR} -Wno-int-conversion -std=gnu89 -Wno-implicit-function-declaration" \ LDFLAGS="-L${TP_LIB_DIR}" \ ./configure --prefix="${TP_INSTALL_DIR}" --with-included-ltdl --enable-static=yes --enable-shared=no @@ -1052,13 +1058,13 @@ build_arrow() { -DBoost_USE_STATIC_RUNTIME=ON \ -DARROW_GFLAGS_USE_SHARED=OFF \ -Dgflags_ROOT="${TP_INSTALL_DIR}" \ - -DGLOG_ROOT="${TP_INSTALL_DIR}" \ - -DRE2_ROOT="${TP_INSTALL_DIR}" \ + -Dglog_ROOT="${TP_INSTALL_DIR}" \ + -Dre2_ROOT="${TP_INSTALL_DIR}" \ -DZLIB_SOURCE=SYSTEM \ -DZLIB_LIBRARY="${TP_INSTALL_DIR}/lib/libz.a" -DZLIB_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ -DRapidJSON_SOURCE=SYSTEM \ -DRapidJSON_ROOT="${TP_INSTALL_DIR}" \ - -DORC_ROOT="${TP_INSTALL_DIR}" \ + -Dorc_ROOT="${TP_INSTALL_DIR}" \ -Dxsimd_SOURCE=BUNDLED \ -DBrotli_SOURCE=BUNDLED \ -DARROW_LZ4_USE_SHARED=OFF \ @@ -1069,7 +1075,7 @@ build_arrow() { -Dzstd_SOURCE=SYSTEM \ -DSnappy_LIB="${TP_INSTALL_DIR}/lib/libsnappy.a" -DSnappy_INCLUDE_DIR="${TP_INSTALL_DIR}/include" \ -DSnappy_SOURCE=SYSTEM \ - -DBOOST_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \ + -DBoost_ROOT="${TP_INSTALL_DIR}" --no-warn-unused-cli \ -DARROW_JEMALLOC=OFF -DARROW_MIMALLOC=OFF \ -DJEMALLOC_HOME="${TP_INSTALL_DIR}" \ -DARROW_THRIFT_USE_SHARED=OFF \ @@ -1352,7 +1358,7 @@ build_aws_sdk() { -DCMAKE_PREFIX_PATH="${TP_INSTALL_DIR}" -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \ -DCURL_LIBRARY_RELEASE="${TP_INSTALL_DIR}/lib/libcurl.a" -DZLIB_LIBRARY_RELEASE="${TP_INSTALL_DIR}/lib/libz.a" \ -DBUILD_ONLY="core;s3;s3-crt;transfer;identity-management;sts" \ - -DCMAKE_CXX_FLAGS="-Wno-nonnull -Wno-deprecated-declarations ${warning_dangling_reference}" -DCPP_STANDARD=17 + -DCMAKE_CXX_FLAGS="-Wno-nonnull ${warning_deprecated_literal_operator} -Wno-deprecated-declarations ${warning_dangling_reference}" -DCPP_STANDARD=17 cd "${BUILD_DIR}" @@ -1471,7 +1477,7 @@ build_krb5() { with_crypto_impl='--with-crypto-impl=openssl' fi - CFLAGS="-fcommon -fPIC -I${TP_INSTALL_DIR}/include" LDFLAGS="-L${TP_INSTALL_DIR}/lib" \ + CFLAGS="-fcommon -fPIC -I${TP_INSTALL_DIR}/include -std=gnu89" LDFLAGS="-L${TP_INSTALL_DIR}/lib" \ ../configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static \ --without-keyutils ${with_crypto_impl:+${with_crypto_impl}} @@ -1531,6 +1537,9 @@ build_jemalloc_doris() { WITH_LG_PAGE='' fi + # It is not easy to remove `with-jemalloc-prefix`, which may affect the compatibility between third-party and old version codes. + # Also, will building failed on Mac, it said can't find mallctl symbol. because jemalloc's default prefix on macOS is "je_", not "". + # Maybe can use alias instead of overwrite. CFLAGS="${cflags}" ../configure --prefix="${TP_INSTALL_DIR}" --with-install-suffix="_doris" "${WITH_LG_PAGE}" \ --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared @@ -1744,7 +1753,7 @@ build_jsoncpp() { rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" cd "${BUILD_DIR}" - "${CMAKE_CMD}" -G "${GENERATOR}" -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DJSONCPP_WITH_TESTS=OFF -DBUILD_STATIC_LIBS=ON -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install } @@ -1814,7 +1823,7 @@ build_azure() { AZURE_PORTS="vcpkg-custom-ports" AZURE_MANIFEST_DIR="." - "${CMAKE_CMD}" -G "${GENERATOR}" -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. + "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_CXX_FLAGS="-Wno-maybe-uninitialized" -DDISABLE_RUST_IN_BUILD=ON -DVCPKG_MANIFEST_MODE=ON -DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}" -DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}" -DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release .. "${BUILD_SYSTEM}" -j "${PARALLEL}" "${BUILD_SYSTEM}" install fi @@ -1882,6 +1891,66 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } +build_openblas() { + check_if_source_exist "${OPENBLAS_SOURCE}" + cd "${TP_SOURCE_DIR}/${OPENBLAS_SOURCE}" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + OPENBLAS_CMAKE_OPTIONS=( + "-DCMAKE_PREFIX_PATH=${TP_INSTALL_DIR}" + "-DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}" + "-DCMAKE_BUILD_TYPE=Release" + "-DBUILD_WITHOUT_LAPACK=OFF" + "-DNO_SHARED=TRUE" + "-DNO_AVX512=TRUE" + "-DC_LAPACK=TRUE" + "-DUSE_OPENMP=TRUE" + "-DBUILD_STATIC_LIBS=ON" + "-DNOFORTRAN=TRUE" + "-DBUILD_TESTING=OFF" + "-DBUILD_RELAPACK=ON" + "-DBUILD_BENCHMARKS=OFF" + ) + + echo "Building openblas at $(pwd) with cmake parameters: ${OPENBLAS_CMAKE_OPTIONS[*]}" + + "${CMAKE_CMD}" -G "${GENERATOR}" "${OPENBLAS_CMAKE_OPTIONS[@]}" .. + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install +} + +build_faiss() { + check_if_source_exist "${FAISS_SOURCE}" + echo "Building faiss ${FAISS_SOURCE}" + cd "${TP_SOURCE_DIR}" + # if faiss dir not exists, create a symlink to faiss source dir + # this symlink is necessary since faiss source code must be compiled in a directory named faiss. + if [[ ! -d "${TP_SOURCE_DIR}/faiss" ]]; then + ln -s "${FAISS_SOURCE}" faiss + fi + cd "${TP_SOURCE_DIR}/faiss" + + rm -rf "${BUILD_DIR}" + mkdir -p "${BUILD_DIR}" + cd "${BUILD_DIR}" + + FAISS_CMAKE_OPTIONS=( + "-DDORIS_THIRD_LIB_INSTALL_DIR=${TP_INSTALL_DIR}" + "-DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}" + "-DCMAKE_BUILD_TYPE=Release" + "-DFAISS_ENABLE_GPU=OFF" + "-DFAISS_ENABLE_PYTHON=OFF" + ) + + echo "Building faiss at $(pwd) with cmake parameters: ${FAISS_CMAKE_OPTIONS[*]}" + + "${CMAKE_CMD}" -G "${GENERATOR}" "${FAISS_CMAKE_OPTIONS[@]}" .. + "${BUILD_SYSTEM}" -j "${PARALLEL}" + "${BUILD_SYSTEM}" install +} + if [[ "${#packages[@]}" -eq 0 ]]; then packages=( jindofs diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 5ca4eb44080005..54bcf2148d289f 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -425,6 +425,7 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AWS_SDK " ]]; then bash ./prefetch_crt_dependency.sh fi patch -p1 <"${TP_PATCH_DIR}/aws-sdk-cpp-1.11.119.patch" + patch -p1 <"${TP_PATCH_DIR}/aws-sdk-cpp-1.11.119-cmake.patch" else bash ./prefetch_crt_dependency.sh fi @@ -436,7 +437,7 @@ fi # patch simdjson, change simdjson::dom::element_type::BOOL to BOOLEAN to avoid conflict with odbc macro BOOL if [[ " ${TP_ARCHIVES[*]} " =~ " SIMDJSON " ]]; then - if [[ "${SIMDJSON_SOURCE}" = "simdjson-3.0.1" ]]; then + if [[ "${SIMDJSON_SOURCE}" = "simdjson-3.11.6" ]]; then cd "${TP_SOURCE_DIR}/${SIMDJSON_SOURCE}" if [[ ! -f "${PATCHED_MARK}" ]]; then patch -p1 <"${TP_PATCH_DIR}/simdjson-3.0.1.patch" @@ -489,6 +490,58 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " BASE64 " ]]; then echo "Finished patching ${BASE64_SOURCE}" fi +# patch libuuid +if [[ " ${TP_ARCHIVES[*]} " =~ " LIBUUID " ]]; then + if [[ "${LIBUUID_SOURCE}" = "libuuid-1.0.3" ]]; then + cd "${TP_SOURCE_DIR}/${LIBUUID_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/libuuid-1.0.3.patch" + touch "${PATCHED_MARK}" + fi + cd - + fi + echo "Finished patching ${LIBUUID_SOURCE}" +fi + +# patch libdivide +if [[ " ${TP_ARCHIVES[*]} " =~ " LIBDIVIDE " ]]; then + if [[ "${LIBDIVIDE_SOURCE}" = "libdivide-5.0" ]]; then + cd "${TP_SOURCE_DIR}/${LIBDIVIDE_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/libdivide-5.0.patch" + touch "${PATCHED_MARK}" + fi + cd - + fi + echo "Finished patching ${LIBDIVIDE_SOURCE}" +fi + +# patch grpc +if [[ " ${TP_ARCHIVES[*]} " =~ " GRPC " ]]; then + if [[ "${GRPC_SOURCE}" = "grpc-1.54.3" ]]; then + cd "${TP_SOURCE_DIR}/${GRPC_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/grpc-1.54.3.patch" + touch "${PATCHED_MARK}" + fi + cd - + fi + echo "Finished patching ${GRPC_SOURCE}" +fi + +# patch flatbuffer +if [[ " ${TP_ARCHIVES[*]} " =~ " FLATBUFFERS " ]]; then + if [[ "${FLATBUFFERS_SOURCE}" = "flatbuffers-2.0.0" ]]; then + cd "${TP_SOURCE_DIR}/${FLATBUFFERS_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/flatbuffers-2.0.0.patch" + touch "${PATCHED_MARK}" + fi + cd - + fi + echo "Finished patching ${FLATBUFFERS_SOURCE}" +fi + # patch krb if [[ " ${TP_ARCHIVES[*]} " =~ " KRB5 " ]]; then if [[ "${KRB5_SOURCE}" = "krb5-1.19" ]]; then @@ -537,6 +590,19 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " THRIFT " ]]; then echo "Finished patching ${THRIFT_SOURCE}" fi +# patch faiss cmake so that we can use openblas +if [[ " ${TP_ARCHIVES[*]} " =~ " FAISS " ]]; then + if [[ "${FAISS_SOURCE}" = "faiss-1.10.0" ]]; then + cd "${TP_SOURCE_DIR}/${FAISS_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p2 <"${TP_PATCH_DIR}/faiss-1.10.0.patch" + touch "${PATCHED_MARK}" + fi + cd - + fi + echo "Finished patching ${FAISS_SOURCE}" +fi + # patch re2 if [[ " ${TP_ARCHIVES[*]} " =~ " RE2 " ]]; then if [[ "${RE2_SOURCE}" == 're2-2021-02-02' ]]; then @@ -553,4 +619,15 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " RE2 " ]]; then echo "Finished patching ${RE2_SOURCE}" fi +# patch azure +if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then + cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}" + if [[ ! -f "${PATCHED_MARK}" ]]; then + patch -p1 <"${TP_PATCH_DIR}/azure-sdk-for-cpp-azure-core_1.16.0.patch" + touch "${PATCHED_MARK}" + fi + cd - + echo "Finished patching ${GRPC_SOURCE}" +fi + # vim: ts=4 sw=4 ts=4 tw=100: diff --git a/thirdparty/patches/aws-sdk-cpp-1.11.119-cmake.patch b/thirdparty/patches/aws-sdk-cpp-1.11.119-cmake.patch new file mode 100644 index 00000000000000..0f9e38be507969 --- /dev/null +++ b/thirdparty/patches/aws-sdk-cpp-1.11.119-cmake.patch @@ -0,0 +1,11 @@ +--- a/crt/aws-crt-cpp/crt/aws-c-cal/CMakeLists.txt 2025-01-06 21:23:51.057566544 +0800 ++++ b/crt/aws-crt-cpp/crt/aws-c-cal/CMakeLists.txt 2025-01-06 16:50:56.480567259 +0800 +@@ -8,7 +8,7 @@ + cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags + endif() + +-set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) ++# set(CMAKE_FIND_PACKAGE_PREFER_CONFIG TRUE) + + option(BYO_CRYPTO "Set this if you want to provide your own cryptography implementation. This will cause the defaults to not be compiled." OFF) + option(USE_OPENSSL "Set this if you want to use your system's OpenSSL 1.0.2/1.1.1 compatible libcrypto" OFF) diff --git a/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch b/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch new file mode 100644 index 00000000000000..c0e8def8f267ec --- /dev/null +++ b/thirdparty/patches/azure-sdk-for-cpp-azure-core_1.16.0.patch @@ -0,0 +1,24 @@ +From 0c18bd91955f6ad48582c01901ffadd4ad5a149c Mon Sep 17 00:00:00 2001 +From: BiteTheDDDDt +Date: Tue, 15 Jul 2025 12:57:25 +0800 +Subject: [PATCH] resolve missing uint8_t define + +--- + .../azure-security-attestation/src/private/crypto/inc/crypto.hpp | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp +index 9a98f204..8f338436 100644 +--- a/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp ++++ b/sdk/attestation/azure-security-attestation/src/private/crypto/inc/crypto.hpp +@@ -2,6 +2,7 @@ + // Licensed under the MIT License. + + #pragma once ++#include + #include + #include + #include +-- +2.43.5 + diff --git a/thirdparty/patches/brpc-uuid-string.patch b/thirdparty/patches/brpc-uuid-string.patch new file mode 100644 index 00000000000000..99606aefce3bc3 --- /dev/null +++ b/thirdparty/patches/brpc-uuid-string.patch @@ -0,0 +1,123 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -110,6 +110,34 @@ set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") + endif() + set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} -Wno-deprecated-declarations -Wno-inconsistent-missing-override") ++ ++ # Check if xcrun exists and get SDK version if it does ++ execute_process( ++ COMMAND which xcrun ++ RESULT_VARIABLE XCRUN_RESULT ++ OUTPUT_QUIET ++ ERROR_QUIET ++ ) ++ ++ if(XCRUN_RESULT EQUAL 0) ++ # xcrun exists, detect SDK version ++ execute_process( ++ COMMAND xcrun --sdk macosx --show-sdk-version ++ OUTPUT_VARIABLE MACOSX_SDK_VERSION ++ OUTPUT_STRIP_TRAILING_WHITESPACE ++ ) ++ message(STATUS "Detected macOS SDK version: ${MACOSX_SDK_VERSION}") ++ ++ if(MACOSX_SDK_VERSION VERSION_LESS 10.14) ++ message(STATUS "macOS SDK version < 10.14; not applying _DARWIN_C_SOURCE or uuid fix.") ++ else() ++ message(STATUS "macOS SDK version >= 10.14; applying Darwin-specific uuid fix.") ++ add_definitions(-D_DARWIN_C_SOURCE) ++ add_definitions(-DUSE_DARWIN_UUID_FIX) ++ endif() ++ else() ++ message(STATUS "xcrun not found; skipping Darwin-specific SDK checks.") ++ endif() + endif() + + set(CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} ${DEFINE_CLOCK_GETTIME} -DBRPC_WITH_GLOG=${WITH_GLOG_VAL} -DBRPC_WITH_RDMA=${WITH_RDMA_VAL} -DGFLAGS_NS=${GFLAGS_NS}") + +diff --git a/src/brpc/macos_uuid_fix.h b/src/brpc/macos_uuid_fix.h +new file mode 100644 +index 00000000..623a6454 +--- /dev/null ++++ b/src/brpc/macos_uuid_fix.h +@@ -0,0 +1,9 @@ ++#ifdef __APPLE__ ++#ifdef USE_DARWIN_UUID_FIX ++#include ++#ifndef _UUID_STRING_T ++#define _UUID_STRING_T ++typedef char uuid_string_t[37]; ++#endif ++#endif ++#endif + +diff --git a/src/butil/mac/bundle_locations.mm b/src/butil/mac/bundle_locations.mm +--- a/src/butil/mac/bundle_locations.mm ++++ b/src/butil/mac/bundle_locations.mm +@@ -1,7 +1,8 @@ + // Copyright (c) 2012 The Chromium Authors. All rights reserved. + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file. + ++#include "brpc/macos_uuid_fix.h" + #include "butil/mac/bundle_locations.h" + + #include "butil/logging.h" + +diff --git a/src/butil/mac/foundation_util.h b/src/butil/mac/foundation_util.h +--- a/src/butil/mac/foundation_util.h ++++ b/src/butil/mac/foundation_util.h +@@ -2,6 +2,7 @@ + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file. + ++#include "brpc/macos_uuid_fix.h" + #ifndef BUTIL_MAC_FOUNDATION_UTIL_H_ + #define BUTIL_MAC_FOUNDATION_UTIL_H_ + + +diff --git a/src/butil/mac/foundation_util.mm b/src/butil/mac/foundation_util.mm +--- a/src/butil/mac/foundation_util.mm ++++ b/src/butil/mac/foundation_util.mm +@@ -2,6 +2,7 @@ + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file. + ++#include "brpc/macos_uuid_fix.h" + #include "butil/mac/foundation_util.h" + + #include + +diff --git a/src/butil/threading/platform_thread_mac.mm b/src/butil/threading/platform_thread_mac.mm +--- a/src/butil/threading/platform_thread_mac.mm ++++ b/src/butil/threading/platform_thread_mac.mm +@@ -3,3 +3,4 @@ + // found in the LICENSE file. + ++#include "brpc/macos_uuid_fix.h" + #include "butil/threading/platform_thread.h" + +diff --git a/src/butil/file_util_mac.mm b/src/butil/file_util_mac.mm +--- a/src/butil/file_util_mac.mm ++++ b/src/butil/file_util_mac.mm +@@ -2,6 +2,7 @@ + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file. + ++#include "brpc/macos_uuid_fix.h" + #include "butil/file_util.h" + + #import + +diff --git a/src/butil/strings/sys_string_conversions_mac.mm b/src/butil/strings/sys_string_conversions_mac.mm +--- a/src/butil/strings/sys_string_conversions_mac.mm ++++ b/src/butil/strings/sys_string_conversions_mac.mm +@@ -2,5 +2,6 @@ + // Use of this source code is governed by a BSD-style license that can be + // found in the LICENSE file. + ++#include "brpc/macos_uuid_fix.h" + #include "butil/mac/foundation_util.h" + #include "butil/mac/scoped_cftyperef.h" + diff --git a/thirdparty/patches/faiss-1.10.0.patch b/thirdparty/patches/faiss-1.10.0.patch new file mode 100644 index 00000000000000..8279f4d71a1174 --- /dev/null +++ b/thirdparty/patches/faiss-1.10.0.patch @@ -0,0 +1,66 @@ +--- src/faiss-1.10.0/faiss/CMakeLists.txt 2025-02-01 05:52:00.000000000 +0800 ++++ src/faiss-1.10.0/faiss/CMakeLists.txt.new 2025-03-28 19:45:37.513624103 +0800 +@@ -381,19 +381,51 @@ + target_link_libraries(faiss_avx512 PRIVATE ${MKL_LIBRARIES}) + target_link_libraries(faiss_avx512_spr PRIVATE ${MKL_LIBRARIES}) + else() +- find_package(BLAS REQUIRED) +- target_link_libraries(faiss PRIVATE ${BLAS_LIBRARIES}) +- target_link_libraries(faiss_avx2 PRIVATE ${BLAS_LIBRARIES}) +- target_link_libraries(faiss_avx512 PRIVATE ${BLAS_LIBRARIES}) +- target_link_libraries(faiss_avx512_spr PRIVATE ${BLAS_LIBRARIES}) +- target_link_libraries(faiss_sve PRIVATE ${BLAS_LIBRARIES}) ++ # If not found through find_package, look in the DORIS_THIRD_LIB_INSTALL_DIR ++ if(DEFINED DORIS_THIRD_LIB_INSTALL_DIR) ++ set(OpenBLAS_ROOT ${DORIS_THIRD_LIB_INSTALL_DIR}) ++ ++ # Check if libopenblas exists in DORIS_THIRD_LIB_INSTALL_DIR ++ if(EXISTS "${DORIS_THIRD_LIB_INSTALL_DIR}/lib/libopenblas.a") ++ set(OpenBLAS_LIB "${DORIS_THIRD_LIB_INSTALL_DIR}/lib/libopenblas.a") ++ endif() ++ # Terminate if OpenBLAS_LIB is not found ++ if(NOT OpenBLAS_LIB) ++ message(WARNING "OpenBLAS not found in DORIS_THIRD_LIB_INSTALL_DIR: ${DORIS_THIRD_LIB_INSTALL_DIR}") ++ endif() + +- find_package(LAPACK REQUIRED) +- target_link_libraries(faiss PRIVATE ${LAPACK_LIBRARIES}) +- target_link_libraries(faiss_avx2 PRIVATE ${LAPACK_LIBRARIES}) +- target_link_libraries(faiss_avx512 PRIVATE ${LAPACK_LIBRARIES}) +- target_link_libraries(faiss_avx512_spr PRIVATE ${LAPACK_LIBRARIES}) +- target_link_libraries(faiss_sve PRIVATE ${LAPACK_LIBRARIES}) ++ if(OpenBLAS_LIB) ++ set(OpenBLAS_LIBRARIES ${OpenBLAS_LIB}) ++ set(OpenBLAS_FOUND TRUE) ++ message(STATUS "Found OpenBLAS in DORIS_THIRD_LIB_INSTALL_DIR: ${OpenBLAS_LIB}") ++ endif() ++ else() ++ message(WARNING "DORIS_THIRD_LIB_INSTALL_DIR is not defined. Please set it to the directory where OpenBLAS is installed.") ++ endif() ++ ++ if(OpenBLAS_FOUND) ++ message(STATUS "Using OpenBLAS: ${OpenBLAS_LIBRARIES}") ++ target_link_libraries(faiss PRIVATE ${OpenBLAS_LIBRARIES}) ++ target_link_libraries(faiss_avx2 PRIVATE ${OpenBLAS_LIBRARIES}) ++ target_link_libraries(faiss_avx512 PRIVATE ${OpenBLAS_LIBRARIES}) ++ target_link_libraries(faiss_avx512_spr PRIVATE ${OpenBLAS_LIBRARIES}) ++ target_link_libraries(faiss_sve PRIVATE ${OpenBLAS_LIBRARIES}) ++ else() ++ # Fall back to separate BLAS and LAPACK if OpenBLAS is not found ++ find_package(BLAS REQUIRED) ++ target_link_libraries(faiss PRIVATE ${BLAS_LIBRARIES}) ++ target_link_libraries(faiss_avx2 PRIVATE ${BLAS_LIBRARIES}) ++ target_link_libraries(faiss_avx512 PRIVATE ${BLAS_LIBRARIES}) ++ target_link_libraries(faiss_avx512_spr PRIVATE ${BLAS_LIBRARIES}) ++ target_link_libraries(faiss_sve PRIVATE ${BLAS_LIBRARIES}) ++ ++ find_package(LAPACK REQUIRED) ++ target_link_libraries(faiss PRIVATE ${LAPACK_LIBRARIES}) ++ target_link_libraries(faiss_avx2 PRIVATE ${LAPACK_LIBRARIES}) ++ target_link_libraries(faiss_avx512 PRIVATE ${LAPACK_LIBRARIES}) ++ target_link_libraries(faiss_avx512_spr PRIVATE ${LAPACK_LIBRARIES}) ++ target_link_libraries(faiss_sve PRIVATE ${LAPACK_LIBRARIES}) ++ endif() + endif() + + install(TARGETS faiss diff --git a/thirdparty/patches/flatbuffers-2.0.0.patch b/thirdparty/patches/flatbuffers-2.0.0.patch new file mode 100644 index 00000000000000..096f9cb7ad4dcc --- /dev/null +++ b/thirdparty/patches/flatbuffers-2.0.0.patch @@ -0,0 +1,11 @@ +--- a/include/flatbuffers/stl_emulation.h 2021-05-11 02:45:16.000000000 +0800 ++++ b/include/flatbuffers/stl_emulation.h 2025-01-06 22:43:47.538175771 +0800 +@@ -625,7 +625,7 @@ + private: + // This is a naive implementation with 'count_' member even if (Extent != dynamic_extent). + pointer const data_; +- const size_type count_; ++ size_type count_; + }; + + #if !defined(FLATBUFFERS_SPAN_MINIMAL) diff --git a/thirdparty/patches/grpc-1.54.3.patch b/thirdparty/patches/grpc-1.54.3.patch new file mode 100644 index 00000000000000..1f211eac3f7669 --- /dev/null +++ b/thirdparty/patches/grpc-1.54.3.patch @@ -0,0 +1,11 @@ +--- a/src/core/lib/promise/detail/basic_seq.h 2025-01-06 22:41:37.857651534 +0800 ++++ b/src/core/lib/promise/detail/basic_seq.h 2025-01-06 22:41:42.308703853 +0800 +@@ -471,7 +471,7 @@ + cur_ = next; + state_.~State(); + Construct(&state_, +- Traits::template CallSeqFactory(f_, *cur_, std::move(arg))); ++ Traits::CallSeqFactory(f_, *cur_, std::move(arg))); + return PollNonEmpty(); + }); + } diff --git a/thirdparty/patches/libdivide-5.0.patch b/thirdparty/patches/libdivide-5.0.patch new file mode 100644 index 00000000000000..e5d8b952d7570e --- /dev/null +++ b/thirdparty/patches/libdivide-5.0.patch @@ -0,0 +1,11 @@ +--- a/libdivide.h 2025-01-06 22:27:33.308725176 +0800 ++++ b/libdivide.h 2025-01-06 22:27:41.517821664 +0800 +@@ -3017,7 +3017,7 @@ + T recover() const { return div.recover(); } + + bool operator==(const divider &other) const { +- return div.denom.magic == other.denom.magic && div.denom.more == other.denom.more; ++ return div.denom.magic == other.div.denom.magic && div.denom.more == other.div.denom.more; + } + + bool operator!=(const divider &other) const { return !(*this == other); } diff --git a/thirdparty/patches/libuuid-1.0.3.patch b/thirdparty/patches/libuuid-1.0.3.patch new file mode 100644 index 00000000000000..30a2cb8f6b4919 --- /dev/null +++ b/thirdparty/patches/libuuid-1.0.3.patch @@ -0,0 +1,19 @@ +--- a/gen_uuid.c 2025-01-06 21:27:05.241857362 +0800 ++++ b/gen_uuid.c 2025-01-06 17:30:03.201419206 +0800 +@@ -38,6 +38,8 @@ + */ + #define _SVID_SOURCE + ++#include "c.h" ++ + #ifdef _WIN32 + #define _WIN32_WINNT 0x0500 + #include +@@ -91,7 +93,6 @@ + #include "uuidP.h" + #include "uuidd.h" + #include "randutils.h" +-#include "c.h" + + #ifdef HAVE_TLS + #define THREAD_LOCAL static __thread diff --git a/thirdparty/patches/thrift-0.16-fix_mutex_include.patch b/thirdparty/patches/thrift-0.16-fix_mutex_include.patch new file mode 100644 index 00000000000000..ca4315fde3a701 --- /dev/null +++ b/thirdparty/patches/thrift-0.16-fix_mutex_include.patch @@ -0,0 +1,9 @@ +--- a/lib/cpp/src/thrift/concurrency/Mutex.h ++++ b/lib/cpp/src/thrift/concurrency/Mutex.h +@@ -19,4 +19,5 @@ + #ifndef _THRIFT_CONCURRENCY_MUTEX_H_ + #define _THRIFT_CONCURRENCY_MUTEX_H_ 1 + ++#include + #include + #include diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index d694968d86dcbb..bc11d26d6b1999 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -253,6 +253,8 @@ GRPC_SOURCE=grpc-1.54.3 GRPC_MD5SUM="af00a2edeae0f02bb25917cc3473b7de" # arrow +# Arrow 19.0.1 will MacOS compile error and decimal type error when convert to Parquet. +# https://github.com/apache/doris/pull/51217 ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-17.0.0.tar.gz" ARROW_NAME="apache-arrow-17.0.0.tar.gz" ARROW_SOURCE="arrow-apache-arrow-17.0.0" @@ -394,10 +396,10 @@ HDFS3_SOURCE="doris-thirdparty-libhdfs3-v2.3.9" HDFS3_MD5SUM="b3eaa03e5b184521e5ad5bf6cabea97e" #libdivide -LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/5.0.tar.gz" -LIBDIVIDE_NAME="libdivide-5.0.tar.gz" -LIBDIVIDE_SOURCE="libdivide-5.0" -LIBDIVIDE_MD5SUM="7fd16b0bb4ab6812b2e2fdc7bfb81641" +LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/refs/tags/v5.2.0.tar.gz" +LIBDIVIDE_NAME="libdivide-5.2.0.tar.gz" +LIBDIVIDE_SOURCE="libdivide-5.2.0" +LIBDIVIDE_MD5SUM="4ba77777192c295d6de2b86d88f3239a" #pdqsort PDQSORT_DOWNLOAD="https://raw.githubusercontent.com/orlp/pdqsort/b1ef26a55cdb60d236a5cb199c4234c704f46726/pdqsort.h" @@ -419,10 +421,10 @@ XSIMD_SOURCE=xsimd-13.0.0 XSIMD_MD5SUM="c661deb91836e82d3070f81032014fe6" # simdjson -SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.0.1.tar.gz" -SIMDJSON_NAME=simdjson-3.0.1.tar.gz -SIMDJSON_SOURCE=simdjson-3.0.1 -SIMDJSON_MD5SUM="993576b47249f2bade2bfb2552b2896a" +SIMDJSON_DOWNLOAD="https://github.com/simdjson/simdjson/archive/refs/tags/v3.11.6.tar.gz" +SIMDJSON_NAME=simdjson-3.11.6.tar.gz +SIMDJSON_SOURCE=simdjson-3.11.6 +SIMDJSON_MD5SUM="e7d9c814a4fdd6e47119ce5cf4240f4e" # nlohmann_json NLOHMANN_JSON_DOWNLOAD="https://github.com/nlohmann/json/archive/refs/tags/v3.10.1.tar.gz" @@ -461,10 +463,10 @@ FAST_FLOAT_SOURCE=fast_float-3.9.0 FAST_FLOAT_MD5SUM="5656b0d8b150a3b157cfb092d214f6ea" # libhdfs -HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.6.5-for-doris.tar.gz" -HADOOP_LIBS_NAME="hadoop-3.3.6.5-for-doris.tar.gz" -HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.5-for-doris" -HADOOP_LIBS_MD5SUM="c13f90ceb5ef9460b7d736c49d9b79fd" +HADOOP_LIBS_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/hadoop-3.3.6.6-for-doris.tar.gz" +HADOOP_LIBS_NAME="hadoop-3.3.6.6-for-doris.tar.gz" +HADOOP_LIBS_SOURCE="doris-thirdparty-hadoop-3.3.6.6-for-doris" +HADOOP_LIBS_MD5SUM="13b66d5f2abffd1740e692b65df5962e" # AvxToNeon AVX2NEON_DOWNLOAD="https://github.com/kunpengcompute/AvxToNeon/archive/refs/tags/v1.0.0.tar.gz" @@ -509,10 +511,10 @@ BASE64_SOURCE="base64-0.5.2" BASE64_MD5SUM="49e5a6c98bd0192aedd16c16eec39974" # azure blob -AZURE_DOWNLOAD="https://github.com/Azure/azure-sdk-for-cpp/archive/azure-core_1.10.3.tar.gz" -AZURE_NAME="azure-core_1.10.3.tar.gz" -AZURE_SOURCE="azure-sdk-for-cpp-azure-core_1.10.3" -AZURE_MD5SUM="aa470cfdba93dd69a6f3112a6958c13c" +AZURE_DOWNLOAD="https://github.com/Azure/azure-sdk-for-cpp/archive/azure-core_1.16.0.tar.gz" +AZURE_NAME="azure-core_1.16.0.tar.gz" +AZURE_SOURCE="azure-sdk-for-cpp-azure-core_1.16.0" +AZURE_MD5SUM="0fb23eab695dc24b94456bf461e6af3b" # libdragonbox for faster double/float to string DRAGONBOX_DOWNLOAD="https://github.com/jk-jeon/dragonbox/archive/refs/tags/1.1.3.tar.gz" @@ -538,6 +540,19 @@ PUGIXML_NAME=pugixml-1.15.tar.gz PUGIXML_SOURCE=pugixml-1.15 PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895" +# openblas +OPENBLAS_DOWNLOAD="https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.29/OpenBLAS-0.3.29.tar.gz" +OPENBLAS_NAME="OpenBLAS-0.3.29.tar.gz" +OPENBLAS_SOURCE="OpenBLAS-0.3.29" +OPENBLAS_MD5SUM="853a0c5c0747c5943e7ef4bbb793162d" + +# faiss +FAISS_DOWNLOAD="https://github.com/facebookresearch/faiss/archive/refs/tags/v1.10.0.tar.gz" +FAISS_NAME="faiss-1.10.0.tar.gz" +FAISS_SOURCE="faiss-1.10.0" +FAISS_MD5SUM="f31edf2492808b27cc963d0ab316a205" + + # all thirdparties which need to be downloaded is set in array TP_ARCHIVES export TP_ARCHIVES=( 'LIBEVENT' @@ -618,6 +633,8 @@ export TP_ARCHIVES=( 'ICU' 'JINDOFS' 'PUGIXML' + 'OPENBLAS' + 'FAISS' ) if [[ "$(uname -s)" == 'Darwin' ]]; then From 6e07b9289e184a6cebf4817e37ef77e507bbbabf Mon Sep 17 00:00:00 2001 From: hui lai Date: Mon, 21 Jul 2025 17:34:29 +0800 Subject: [PATCH 275/572] branch-3.0: [fix](csv reader) fix data loss when concurrency read using multi char line delimiter (#53374) (#53634) pick (#53374) Multiple concurrent split file locations will be determined in plan phase, if the split point happens to be in the middle of the multi char line delimiter: - The previous concurrent will read the complete row1 and read a little more to read the line delimiter. - The latter concurrency will start reading from half of the multi char line delimiter, and row2 is the first line of this concurrency, but the first line in the middle range is always discarded, so row2 will be lost. ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/exec/format/csv/csv_reader.cpp | 8 +- .../ddl/test_multi_char_line_delimiter.sql | 76 ++++++++++++++++++ .../test_multi_char_line_delimiter.groovy | 77 +++++++++++++++++++ 3 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql create mode 100644 regression-test/suites/load_p2/broker_load/test_multi_char_line_delimiter.groovy diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp index 4c5adf6f4b15d7..3e3ee25284fe17 100644 --- a/be/src/vec/exec/format/csv/csv_reader.cpp +++ b/be/src/vec/exec/format/csv/csv_reader.cpp @@ -300,8 +300,12 @@ Status CsvReader::init_reader(bool is_load) { _file_compress_type != TFileCompressType::PLAIN)) { return Status::InternalError("For now we do not support split compressed file"); } - start_offset -= 1; - _size += 1; + // pre-read to promise first line skipped always read + int64_t pre_read_len = std::min( + static_cast(_params.file_attributes.text_params.line_delimiter.size()), + start_offset); + start_offset -= pre_read_len; + _size += pre_read_len; // not first range will always skip one line _skip_lines = 1; } diff --git a/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql b/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql new file mode 100644 index 00000000000000..c0be1514f90a62 --- /dev/null +++ b/regression-test/suites/load_p2/broker_load/ddl/test_multi_char_line_delimiter.sql @@ -0,0 +1,76 @@ +CREATE TABLE `test_multi_char_line_delimiter` ( + `col1` bigint NULL, + `col2` bigint NULL, + `col3` varchar(765) NULL, + `col4` varchar(765) NULL, + `col5` bigint NULL, + `col6` bigint NULL, + `col7` int NULL, + `col8` int NULL, + `col9` tinyint NULL, + `col10` bigint NULL, + `col11` datetime NULL, + `col12` bigint NULL, + `col13` bigint NULL, + `col14` int NULL, + `col15` bigint MIN NULL, + `col16` decimal(19,4) SUM NULL, + `col17` decimal(19,4) SUM NULL, + `col18` decimal(19,4) SUM NULL, + `col19` decimal(19,4) SUM NULL, + `col20` decimal(19,4) SUM NULL, + `col21` decimal(19,4) SUM NULL, + `col22` decimal(19,4) SUM NULL, + `col23` decimal(19,4) SUM NULL, + `col24` decimal(19,4) SUM NULL, + `col25` decimal(19,4) SUM NULL, + `col26` decimal(19,4) SUM NULL, + `col27` decimal(19,4) SUM NULL, + `col28` decimal(19,4) SUM NULL, + `col29` decimal(19,4) SUM NULL, + `col30` decimal(19,4) SUM NULL, + `col31` decimal(19,4) SUM NULL, + `col32` decimal(19,4) SUM NULL, + `col33` decimal(19,4) SUM NULL DEFAULT "0", + `col34` decimal(19,4) SUM NULL DEFAULT "0", + `col35` decimal(19,4) SUM NULL DEFAULT "0", + `col36` decimal(19,4) SUM NULL DEFAULT "0", + `col37` decimal(19,4) SUM NULL DEFAULT "0", + `col38` decimal(19,4) SUM NULL DEFAULT "0", + `col39` decimal(19,4) SUM NULL DEFAULT "0", + `col40` decimal(19,4) SUM NULL DEFAULT "0", + `col41` decimal(19,4) SUM NULL DEFAULT "0", + `col42` decimal(19,4) SUM NULL DEFAULT "0", + `col43` decimal(19,4) SUM NULL DEFAULT "0", + `col44` decimal(19,4) SUM NULL DEFAULT "0", + `col45` decimal(19,4) SUM NULL DEFAULT "0", + `col46` decimal(19,4) SUM NULL DEFAULT "0", + `col47` decimal(19,4) SUM NULL DEFAULT "0", + `col48` decimal(19,4) SUM NULL DEFAULT "0", + `col49` decimal(19,4) SUM NULL DEFAULT "0", + `col50` decimal(19,4) SUM NULL DEFAULT "0", + `col51` decimal(19,4) SUM NULL, + `col52` datetime MIN NULL, + `col53` bigint MIN NULL, + `col54` datetime MAX NULL, + `col55` bigint MAX NULL, + `col56` tinyint MIN NULL, + `col57` bitmap BITMAP_UNION NOT NULL DEFAULT BITMAP_EMPTY +) ENGINE=OLAP +AGGREGATE KEY(`col1`, `col2`, `col3`, `col4`, `col5`, `col6`, `col7`, `col8`, `col9`, `col10`, `col11`, `col12`, `col13`, `col14`) +PARTITION BY RANGE(`col12`, `col11`) +(PARTITION p_default VALUES [("0", '1900-01-01 00:00:00'), ("99999", '2030-01-01 00:00:00'))) +DISTRIBUTED BY HASH(`col8`) BUCKETS 1 +PROPERTIES ( +"file_cache_ttl_seconds" = "0", +"is_being_synced" = "false", +"storage_medium" = "hdd", +"storage_format" = "V2", +"inverted_index_storage_format" = "V2", +"light_schema_change" = "true", +"disable_auto_compaction" = "false", +"enable_single_replica_compaction" = "false", +"group_commit_interval_ms" = "10000", +"group_commit_data_bytes" = "134217728", +"replication_allocation" = "tag.location.default: 1" +); \ No newline at end of file diff --git a/regression-test/suites/load_p2/broker_load/test_multi_char_line_delimiter.groovy b/regression-test/suites/load_p2/broker_load/test_multi_char_line_delimiter.groovy new file mode 100644 index 00000000000000..184b3d5cf07da8 --- /dev/null +++ b/regression-test/suites/load_p2/broker_load/test_multi_char_line_delimiter.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_multi_char_line_delimiter", "p2") { + def s3BucketName = getS3BucketName() + def s3Endpoint = getS3Endpoint() + def s3Region = getS3Region() + def ak = getS3AK() + def sk = getS3SK() + def tableName = "test_multi_char_line_delimiter" + def label = "test_multi_char_line_delimiter" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql new File("""${context.file.parent}/ddl/${tableName}.sql""").text + sql """ + LOAD LABEL ${label} + ( + DATA INFILE("s3://${s3BucketName}/regression/load/data/test_multi_char_line_delimiter*.csv") + INTO TABLE ${tableName} + COLUMNS TERMINATED BY "\tcolumn_separator" + LINES TERMINATED BY "\nline_delimiter" + FORMAT AS CSV + (`col1`,`col2`,`col3`,`col4`,`col5`,`col6`,`col7`,`col8`,`col9`,`col10`,`col11`,`col12`,`col13`,`col14`,`col15`,`col16`,`col17`,`col18`,`col19`,`col20`,`col21`,`col22`,`col23`,`col24`,`col25`,`col26`,`col27`,`col28`,`col29`,`col30`,`col31`,`col32`,`col33`,`col34`,`col35`,`col36`,`col37`,`col38`,`col39`,`col40`,`col41`,`col42`,`col43`,`col44`,`col45`,`col46`,`col47`,`col48`,`col49`,`col50`,`col51`,`col52`,`col53`,`col54`,`col55`,`col56`,`col57`) + SET(`col1`=`col1`,`col2`=`col2`,`col3`=`col3`,`col4`=`col4`,`col5`=`col5`,`col6`=`col6`,`col7`=`col7`,`col8`=`col8`,`col9`=`col9`,`col10`=`col10`,`col11`=`col11`,`col12`=`col12`,`col13`=`col13`,`col14`=`col14`,`col15`=`col15`,`col16`=`col16`,`col17`=`col17`,`col18`=`col18`,`col19`=`col19`,`col20`=`col20`,`col21`=`col21`,`col22`=`col22`,`col23`=`col23`,`col24`=`col24`,`col25`=`col25`,`col26`=`col26`,`col27`=`col27`,`col28`=`col28`,`col29`=`col29`,`col30`=`col30`,`col31`=`col31`,`col32`=`col32`,`col33`=`col33`,`col34`=`col34`,`col35`=`col35`,`col36`=`col36`,`col37`=`col37`,`col38`=`col38`,`col39`=`col39`,`col40`=`col40`,`col41`=`col41`,`col42`=`col42`,`col43`=`col43`,`col44`=`col44`,`col45`=`col45`,`col46`=`col46`,`col47`=`col47`,`col48`=`col48`,`col49`=`col49`,`col50`=`col50`,`col51`=`col51`,`col52`=`col52`,`col53`=`col53`,`col54`=`col54`,`col55`=`col55`,`col56`=`col56`,col57=bitmap_from_string(col57)) + ) + WITH S3 + ( + "s3.region" = "${s3Region}", + "s3.endpoint" = "${s3Endpoint}", + "s3.access_key" = "${ak}", + "s3.secret_key" = "${sk}" + ) + PROPERTIES + ( + "timeout" = "3600", + "load_parallelism" = "4" + ); + """ + + def max_try_milli_secs = 600000 + while (max_try_milli_secs > 0) { + def String[][] result = sql """ show load where label="$label"; """ + logger.info("Load status: " + result[0][2] + ", label: $label") + if (result[0][2].equals("FINISHED")) { + logger.info("Load FINISHED " + label) + break; + } + if (result[0][2].equals("CANCELLED")) { + assertTrue(false, "load failed: $result") + break; + } + Thread.sleep(1000) + max_try_milli_secs -= 1000 + if(max_try_milli_secs <= 0) { + assertTrue(1 == 2, "load Timeout: $label") + } + } + + def result = sql """ select count(*) from ${tableName}; """ + logger.info("result: ${result[0][0]}") + assertTrue(result[0][0] == 2060625, "load result is not correct") + sql """ DROP TABLE IF EXISTS ${tableName} """ +} \ No newline at end of file From cc045b3347efbd1dfa3b13b6b6151a7464b61401 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Mon, 21 Jul 2025 17:40:32 +0800 Subject: [PATCH 276/572] [fix & opt](orc) ORC-1525: Fix bad read in RleDecoderV2::readByte and Decompress zlib by libdeflate. (#53610) Cherry-pick #51775 --- be/src/apache-orc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/apache-orc b/be/src/apache-orc index 0182042e141250..8c0b7f877684c4 160000 --- a/be/src/apache-orc +++ b/be/src/apache-orc @@ -1 +1 @@ -Subproject commit 0182042e141250802b1a6c1d7a5317b0055c776b +Subproject commit 8c0b7f877684c4d2b5745ab271aab900f85c5cee From e79e6b168244bd814ef729d8f16745253008ed56 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Mon, 21 Jul 2025 04:28:54 -0700 Subject: [PATCH 277/572] branch-3.0: [fix](catalog) fix deadlock of catalog and database(#53626) (#53628) bp #53626 --- .../doris/datasource/ExternalDatabase.java | 86 ++++++++++--------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 9b726e6b1de257..c122c8c568f82b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -155,50 +155,56 @@ public boolean isInitialized() { return initialized; } - public final synchronized void makeSureInitialized() { - if (isInitializing) { - return; - } - isInitializing = true; - try { - extCatalog.makeSureInitialized(); - if (!initialized) { - if (extCatalog.getUseMetaCache().get()) { - if (metaCache == null) { - metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( - name, - OptionalLong.of(86400L), - OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_meta_object_cache_num, - ignored -> listTableNames(), - localTableName -> Optional.ofNullable( - buildTableForInit(null, localTableName, - Util.genIdByName(extCatalog.getName(), name, localTableName), - extCatalog, - this, true)), - (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); - } - setLastUpdateTime(System.currentTimeMillis()); - } else { - if (!Env.getCurrentEnv().isMaster()) { - // Forward to master and wait the journal to replay. - int waitTimeOut = ConnectContext.get() == null ? 300 : ConnectContext.get().getExecTimeout(); - MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); - try { - remoteExecutor.forward(extCatalog.getId(), id); - } catch (Exception e) { - Util.logAndThrowRuntimeException(LOG, - String.format("failed to forward init external db %s operation to master", name), - e); + public final void makeSureInitialized() { + // Must call this method before any operation on the database to avoid deadlock of synchronized block + extCatalog.makeSureInitialized(); + synchronized (this) { + if (isInitializing) { + return; + } + isInitializing = true; + try { + if (!initialized) { + if (extCatalog.getUseMetaCache().get()) { + if (metaCache == null) { + metaCache = Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache( + name, + OptionalLong.of(86400L), + OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), + Config.max_meta_object_cache_num, + ignored -> listTableNames(), + localTableName -> Optional.ofNullable( + buildTableForInit(null, localTableName, + Util.genIdByName(extCatalog.getName(), name, localTableName), + extCatalog, + this, true)), + (key, value, cause) + -> value.ifPresent(ExternalTable::unsetObjectCreated)); + } + setLastUpdateTime(System.currentTimeMillis()); + } else { + if (!Env.getCurrentEnv().isMaster()) { + // Forward to master and wait the journal to replay. + int waitTimeOut = ConnectContext.get() == null ? 300 + : ConnectContext.get().getExecTimeout(); + MasterCatalogExecutor remoteExecutor = new MasterCatalogExecutor(waitTimeOut * 1000); + try { + remoteExecutor.forward(extCatalog.getId(), id); + } catch (Exception e) { + Util.logAndThrowRuntimeException(LOG, + String.format("failed to forward init external db %s operation to master", + name), + e); + } + return; } - return; + init(); } - init(); + initialized = true; } - initialized = true; + } finally { + isInitializing = false; } - } finally { - isInitializing = false; } } From 237adc2659b56b113af8e70c69410d194f51f9c4 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Mon, 21 Jul 2025 19:58:45 +0800 Subject: [PATCH 278/572] [fix](warmup) avoid calling recycle_cache after rebalance (#53339) (#53523) backport #53339 --- be/src/cloud/cloud_tablet.cpp | 74 +++++++++++++------------- be/src/cloud/cloud_tablet.h | 9 +++- be/src/cloud/cloud_warm_up_manager.cpp | 26 ++++----- be/src/cloud/cloud_warm_up_manager.h | 5 +- 4 files changed, 57 insertions(+), 57 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index e4c62c592f0821..f96f2a9deb7291 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -491,7 +491,11 @@ uint64_t CloudTablet::delete_expired_stale_rowsets() { _reconstruct_version_tracker_if_necessary(); } _tablet_meta->delete_bitmap()->remove_stale_delete_bitmap_from_queue(version_to_delete); - recycle_cached_data(expired_rowsets); + auto recycled_rowsets = recycle_cached_data(expired_rowsets); + if (!recycled_rowsets.empty()) { + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(tablet_id(), recycled_rowsets); + } if (config::enable_mow_verbose_log) { LOG_INFO("finish delete_expired_stale_rowset for tablet={}", tablet_id()); } @@ -535,33 +539,31 @@ void CloudTablet::remove_unused_rowsets() { } } - std::vector rowset_ids; - std::vector num_segments; - std::vector> index_file_names; + { + std::vector recycled_rowsets; + + for (auto& rs : removed_rowsets) { + auto index_names = rs->get_index_file_names(); + recycled_rowsets.emplace_back(rs->rowset_id(), rs->num_segments(), index_names); + int64_t segment_size_sum = 0; + for (int32_t i = 0; i < rs->num_segments(); i++) { + segment_size_sum += rs->rowset_meta()->segment_file_size(i); + } + g_file_cache_recycle_cached_data_segment_num << rs->num_segments(); + g_file_cache_recycle_cached_data_segment_size << segment_size_sum; + g_file_cache_recycle_cached_data_index_num << index_names.size(); + } - for (auto& rs : removed_rowsets) { - rowset_ids.push_back(rs->rowset_id()); - num_segments.push_back(rs->num_segments()); - auto index_names = rs->get_index_file_names(); - index_file_names.push_back(index_names); - int64_t segment_size_sum = 0; - for (int32_t i = 0; i < rs->num_segments(); i++) { - segment_size_sum += rs->rowset_meta()->segment_file_size(i); + if (recycled_rowsets.size() > 0) { + auto& manager = + ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(tablet_id(), recycled_rowsets); } - g_file_cache_recycle_cached_data_segment_num << rs->num_segments(); - g_file_cache_recycle_cached_data_segment_size << segment_size_sum; - g_file_cache_recycle_cached_data_index_num << index_names.size(); } - if (removed_rowsets.size() > 0) { - auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); - manager.recycle_cache(tablet_id(), rowset_ids, num_segments, index_file_names); - - LOG(INFO) << "tablet_id=" << tablet_id() - << ", unused_rowset size=" << _unused_rowsets.size() - << ", removed_rowsets_num=" << removed_rowsets.size() - << ", cost(us)=" << watch.get_elapse_time_us(); - } + LOG(INFO) << "tablet_id=" << tablet_id() << ", unused_rowset size=" << _unused_rowsets.size() + << ", removed_rowsets_num=" << removed_rowsets.size() + << ", cost(us)=" << watch.get_elapse_time_us(); } void CloudTablet::update_base_size(const Rowset& rs) { @@ -572,14 +574,17 @@ void CloudTablet::update_base_size(const Rowset& rs) { } void CloudTablet::clear_cache() { - CloudTablet::recycle_cached_data(get_snapshot_rowset(true)); + auto recycled_rowsets = CloudTablet::recycle_cached_data(get_snapshot_rowset(true)); + if (!recycled_rowsets.empty()) { + auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); + manager.recycle_cache(tablet_id(), recycled_rowsets); + } _engine.tablet_mgr().erase_tablet(tablet_id()); } -void CloudTablet::recycle_cached_data(const std::vector& rowsets) { - std::vector rowset_ids; - std::vector num_segments; - std::vector> index_file_names; +std::vector CloudTablet::recycle_cached_data( + const std::vector& rowsets) { + std::vector recycled_rowsets; for (const auto& rs : rowsets) { // rowsets and tablet._rs_version_map each hold a rowset shared_ptr, so at this point, the reference count of the shared_ptr is at least 2. if (rs.use_count() > 2) { @@ -588,10 +593,9 @@ void CloudTablet::recycle_cached_data(const std::vector& rowset continue; } rs->clear_cache(); - rowset_ids.push_back(rs->rowset_id()); - num_segments.push_back(rs->num_segments()); auto index_names = rs->get_index_file_names(); - index_file_names.push_back(index_names); + recycled_rowsets.emplace_back(rs->rowset_id(), rs->num_segments(), index_names); + int64_t segment_size_sum = 0; for (int32_t i = 0; i < rs->num_segments(); i++) { segment_size_sum += rs->rowset_meta()->segment_file_size(i); @@ -600,11 +604,7 @@ void CloudTablet::recycle_cached_data(const std::vector& rowset g_file_cache_recycle_cached_data_segment_size << segment_size_sum; g_file_cache_recycle_cached_data_index_num << index_names.size(); } - if (!rowsets.empty()) { - auto& manager = ExecEnv::GetInstance()->storage_engine().to_cloud().cloud_warm_up_manager(); - manager.recycle_cache(rowsets.front()->rowset_meta()->tablet_id(), rowset_ids, num_segments, - index_file_names); - } + return recycled_rowsets; } void CloudTablet::reset_approximate_stats(int64_t num_rowsets, int64_t num_segments, diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index df573314938339..d1ea7dec379684 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -41,6 +41,12 @@ struct SyncRowsetStats { int64_t tablet_meta_cache_miss {0}; }; +struct RecycledRowsets { + RowsetId rowset_id; + int64_t num_segments; + std::vector index_file_names; +}; + class CloudTablet final : public BaseTablet { public: CloudTablet(CloudStorageEngine& engine, TabletMetaSharedPtr tablet_meta); @@ -267,7 +273,8 @@ class CloudTablet final : public BaseTablet { void add_unused_rowsets(const std::vector& rowsets); void remove_unused_rowsets(); - static void recycle_cached_data(const std::vector& rowsets); + static std::vector recycle_cached_data( + const std::vector& rowsets); private: // FIXME(plat1ko): No need to record base size if rowsets are ordered by version diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index f270ee5ff78460..f9a256d2b97e4a 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -583,36 +583,30 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta) { } } -void CloudWarmUpManager::recycle_cache( - int64_t tablet_id, const std::vector& rowset_ids, - const std::vector& num_segments, - const std::vector>& index_file_names) { - LOG(INFO) << "recycle_cache: tablet_id=" << tablet_id << ", num_rowsets=" << rowset_ids.size(); +void CloudWarmUpManager::recycle_cache(int64_t tablet_id, + const std::vector& rowsets) { + LOG(INFO) << "recycle_cache: tablet_id=" << tablet_id << ", num_rowsets=" << rowsets.size(); auto replicas = get_replica_info(tablet_id); if (replicas.empty()) { return; } - if (rowset_ids.size() != num_segments.size()) { - LOG(WARNING) << "recycle_cache: rowset_ids size mismatch with num_segments"; - return; - } PRecycleCacheRequest request; - for (int i = 0; i < rowset_ids.size(); i++) { + for (const auto& rowset : rowsets) { RecycleCacheMeta* meta = request.add_cache_metas(); meta->set_tablet_id(tablet_id); - meta->set_rowset_id(rowset_ids[i].to_string()); - meta->set_num_segments(num_segments[i]); - for (const auto& name : index_file_names[i]) { + meta->set_rowset_id(rowset.rowset_id.to_string()); + meta->set_num_segments(rowset.num_segments); + for (const auto& name : rowset.index_file_names) { meta->add_index_file_names(name); } - g_file_cache_recycle_cache_requested_segment_num << num_segments[i]; - g_file_cache_recycle_cache_requested_index_num << index_file_names[i].size(); + g_file_cache_recycle_cache_requested_segment_num << rowset.num_segments; + g_file_cache_recycle_cache_requested_index_num << rowset.index_file_names.size(); } + auto dns_cache = ExecEnv::GetInstance()->dns_cache(); for (auto& replica : replicas) { // send sync request std::string host = replica.host; - auto dns_cache = ExecEnv::GetInstance()->dns_cache(); if (dns_cache == nullptr) { LOG(WARNING) << "DNS cache is not initialized, skipping hostname resolve"; } else if (!is_valid_ip(replica.host)) { diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 13ba906a4e5844..6feef0e9d421ea 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -27,6 +27,7 @@ #include #include "cloud/cloud_storage_engine.h" +#include "cloud/cloud_tablet.h" #include "common/status.h" #include "gen_cpp/BackendService.h" @@ -73,9 +74,7 @@ class CloudWarmUpManager { void warm_up_rowset(RowsetMeta& rs_meta); - void recycle_cache(int64_t tablet_id, const std::vector& rowset_ids, - const std::vector& num_segments, - const std::vector>& index_file_names); + void recycle_cache(int64_t tablet_id, const std::vector& rowsets); private: void handle_jobs(); From 6ae399f3f04ec039b7a4577037f24f0e81e9ca11 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:08:41 +0800 Subject: [PATCH 279/572] branch-3.0: [chore](job) display sequence column in show routine load #53441 (#53503) Cherry-picked from #53441 Co-authored-by: hui lai --- .../load/routineload/RoutineLoadJob.java | 10 ++++++-- .../test_show_routine_load.groovy | 24 +++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 5ea984ebfac23f..30ef3f54f1aec2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -1839,8 +1839,10 @@ private String getTaskStatistic() { } } + // jobPropertiesJsonString contains both load properties and job properties defined in CreateRoutineLoadStmt public String jobPropertiesToJsonString() { Map jobProperties = Maps.newHashMap(); + // load properties defined in CreateRoutineLoadStmt jobProperties.put("partitions", partitions == null ? STAR_STRING : Joiner.on(",").join(partitions.getPartitionNames())); jobProperties.put("columnToColumnExpr", columnDescs == null @@ -1855,6 +1857,12 @@ public String jobPropertiesToJsonString() { jobProperties.put(LoadStmt.KEY_IN_PARAM_LINE_DELIMITER, lineDelimiter == null ? "\n" : lineDelimiter.toString()); } + jobProperties.put(LoadStmt.KEY_IN_PARAM_DELETE_CONDITION, + deleteCondition == null ? STAR_STRING : deleteCondition.toSqlWithoutTbl()); + jobProperties.put(LoadStmt.KEY_IN_PARAM_SEQUENCE_COL, + sequenceCol == null ? STAR_STRING : sequenceCol); + + // job properties defined in CreateRoutineLoadStmt jobProperties.put(CreateRoutineLoadStmt.PARTIAL_COLUMNS, String.valueOf(isPartialUpdate)); jobProperties.put(CreateRoutineLoadStmt.MAX_ERROR_NUMBER_PROPERTY, String.valueOf(maxErrorNum)); jobProperties.put(CreateRoutineLoadStmt.MAX_BATCH_INTERVAL_SEC_PROPERTY, String.valueOf(maxBatchIntervalS)); @@ -1866,8 +1874,6 @@ public String jobPropertiesToJsonString() { String.valueOf(desireTaskConcurrentNum)); jobProperties.put(LoadStmt.EXEC_MEM_LIMIT, String.valueOf(execMemLimit)); jobProperties.put(LoadStmt.KEY_IN_PARAM_MERGE_TYPE, mergeType.toString()); - jobProperties.put(LoadStmt.KEY_IN_PARAM_DELETE_CONDITION, - deleteCondition == null ? STAR_STRING : deleteCondition.toSqlWithoutTbl()); jobProperties.putAll(this.jobProperties); Gson gson = new GsonBuilder().disableHtmlEscaping().create(); return gson.toJson(jobProperties); diff --git a/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy b/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy index 6075dc20dbe67f..d6b31db11f9d27 100644 --- a/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy @@ -46,9 +46,7 @@ suite("test_show_routine_load","p0") { producer.send(record) } } - } - if (enabled != null && enabled.equalsIgnoreCase("true")) { def tableName = "test_show_routine_load" sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ @@ -66,6 +64,7 @@ suite("test_show_routine_load","p0") { PROPERTIES ("replication_allocation" = "tag.location.default: 1"); """ + // test show routine load command try { sql """ CREATE ROUTINE LOAD testShow ON ${tableName} @@ -146,5 +145,26 @@ suite("test_show_routine_load","p0") { sql "stop routine load for testShow" sql "stop routine load for testShow1" } + + // test show routine load properties + try { + sql """ + CREATE ROUTINE LOAD testShow ON ${tableName} + COLUMNS TERMINATED BY ",", + ORDER BY k1 + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${kafkaCsvTpoics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + def res = sql "show routine load for testShow" + def json = parseJson(res[0][11]) + log.info("routine load job properties: ${res[0][11].toString()}".toString()) + assertEquals("k1", json.sequence_col.toString()) + } finally { + sql "stop routine load for testShow" + } } } From 6065b3a4693b300501b46c7117897231c0c103f4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:09:35 +0800 Subject: [PATCH 280/572] branch-3.0: [regression](load) add a case for using the '$.' symbol in routine load #53442 (#53601) Cherry-picked from #53442 Co-authored-by: lw112 <131352377+felixwluo@users.noreply.github.com> --- ...test_routine_load_jsonpath_dollar_job.json | 3 + .../test_routine_load_jsonpath_dollar.groovy | 159 ++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 regression-test/suites/load_p0/routine_load/data/test_routine_load_jsonpath_dollar_job.json create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy diff --git a/regression-test/suites/load_p0/routine_load/data/test_routine_load_jsonpath_dollar_job.json b/regression-test/suites/load_p0/routine_load/data/test_routine_load_jsonpath_dollar_job.json new file mode 100644 index 00000000000000..86a8a9b2bcf3ca --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/data/test_routine_load_jsonpath_dollar_job.json @@ -0,0 +1,3 @@ +{"time": 1752600673, "id": 1, "name": "test1", "extra": "field1"} +{"time": 1752600674, "id": 2, "name": "test2", "extra": "field2"} +{"time": 1752600675, "id": 3, "name": "test3", "extra": "field3"} \ No newline at end of file diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy new file mode 100644 index 00000000000000..a4ade95f27d4a2 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord + +suite("test_routine_load_jsonpath_dollar", "p0") { + def tableName = "test_routine_load_jsonpath_dollar" + def jobName = "test_routine_load_jsonpath_dollar_job" + + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + // Send test data to Kafka + def props = new Properties() + props.put("bootstrap.servers", "${externalEnvIp}:${kafka_port}".toString()) + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") + def producer = new KafkaProducer<>(props) + + def kafkaJson = new File("""${context.file.parent}/data/${jobName}.json""").text + def lines = kafkaJson.readLines() + lines.each { line -> + logger.info("=====${line}========") + def record = new ProducerRecord<>(jobName, null, line) + producer.send(record) + } + producer.close() + + try { + sql """ + DROP TABLE IF EXISTS ${tableName} + """ + + sql """ + CREATE TABLE ${tableName} ( + time DATETIME, + id INT, + name VARCHAR(50), + content TEXT + ) + UNIQUE KEY(time, id) + DISTRIBUTED BY HASH(time, id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + // Create routine load job with $. in jsonpaths + sql """ + CREATE ROUTINE LOAD ${jobName} ON ${tableName} + COLUMNS(ot,time=from_unixtime(`ot`), id, name, content), + PRECEDING FILTER ((`ot` > 0) AND (`id` != '')) + PROPERTIES + ( + "format" = "json", + "jsonpaths" = '[\"\$.time\", \"\$.id\", \"\$.name\", \"\$.\"]', + "max_batch_interval" = "5", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200", + "max_error_number" = "0", + "strip_outer_array" = "false", + "strict_mode" = "false" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${externalEnvIp}:${kafka_port}", + "kafka_topic" = "${jobName}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + + sql "sync" + + // Wait for routine load to be in RUNNING state + def count = 0 + while (true) { + sleep(1000) + def res = sql "show routine load for ${jobName}" + def state = res[0][8].toString() + log.info("routine load state: ${state}") + if (state == "RUNNING") { + break + } + if (count >= 60) { + log.error("routine load failed to start after 60 seconds") + assertEquals("RUNNING", state) + break + } + count++ + } + + // Wait for data to be loaded + count = 0 + while (true) { + def res = sql "select count(*) from ${tableName}" + def state = sql "show routine load for ${jobName}" + log.info("routine load state: ${state[0][8].toString()}") + log.info("routine load statistic: ${state[0][14].toString()}") + if (res[0][0] > 0) { + break + } + if (count >= 60) { + log.error("routine load can not load data for long time") + break + } + sleep(5000) + count++ + } + + sql "sync" + def result = sql "select * from ${tableName} order by time, id" + log.info("Loaded data: ${result}") + + def rowCount = sql "select count(*) from ${tableName}" + assertTrue(rowCount[0][0] > 0, "No data was loaded") + + def contentCheck = sql "select content from ${tableName} where id = 1" + assertTrue(contentCheck.size() > 0, "No data found for id = 1") + def jsonContent = contentCheck[0][0].toString() + assertTrue(jsonContent.contains("test1"), "Content should contain the full JSON with 'test1'") + assertTrue(jsonContent.contains("field1"), "Content should contain the full JSON with 'field1'") + assertTrue(jsonContent.contains("time"), "Content should contain the full JSON with 'time' field") + + def specificData = sql "select date_format(time, '%Y-%m-%dT%H:%i:%s'), id, name from ${tableName} where id = 1" + assertEquals("2025-07-16T01:31:13", specificData[0][0]) + assertEquals(1, specificData[0][1]) + assertEquals("test1", specificData[0][2]) + } finally { + try { + sql "stop routine load for ${jobName}" + } catch (Exception e) { + log.info("Stop routine load failed: ${e.getMessage()}") + } + + try { + sql "DROP TABLE IF EXISTS ${tableName}" + } catch (Exception e) { + log.info("Drop table failed: ${e.getMessage()}") + } + } + } +} \ No newline at end of file From a9d6def4142b864ebd465dd448c9f527d094afbb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:11:03 +0800 Subject: [PATCH 281/572] branch-3.0: [fix](docker case) Fix cluster id check causes cloud be startup abnormality #53444 (#53593) Cherry-picked from #53444 Co-authored-by: deardeng --- be/src/cloud/cloud_storage_engine.cpp | 14 ++-- docker/runtime/doris-compose/command.py | 4 +- .../regression/suite/SuiteCluster.groovy | 8 +-- .../multi_cluster/test_auto_start.groovy | 2 +- .../cloud_p0/node_mgr/test_ms_api.groovy | 66 +++++++++---------- .../node_mgr/test_rename_compute_group.groovy | 6 +- .../node_mgr/test_sql_mode_node_mgr.groovy | 44 +++++++++++-- 7 files changed, 87 insertions(+), 57 deletions(-) diff --git a/be/src/cloud/cloud_storage_engine.cpp b/be/src/cloud/cloud_storage_engine.cpp index 6cac3c7f8d063a..90b3ca0bfa6e8d 100644 --- a/be/src/cloud/cloud_storage_engine.cpp +++ b/be/src/cloud/cloud_storage_engine.cpp @@ -1175,7 +1175,7 @@ Status CloudStorageEngine::_check_all_root_path_cluster_id() { return Status::OK(); } else { // If no cluster id file exists, use the configured cluster id - RETURN_IF_ERROR(set_cluster_id(_effective_cluster_id)); + return set_cluster_id(_effective_cluster_id); } } if (cluster_ids.size() > 1) { @@ -1184,12 +1184,12 @@ Status CloudStorageEngine::_check_all_root_path_cluster_id() { "different cluster ids: {}", fmt::join(cluster_ids, ", ")); } - if (_effective_cluster_id != -1 && *cluster_ids.begin() != _effective_cluster_id) { - RETURN_NOT_OK_STATUS_WITH_WARN( - Status::Corruption("multiple cluster ids is not equal. config::cluster_id={}, " - "storage path cluster_id={}", - _effective_cluster_id, *cluster_ids.begin()), - "cluster id not equal"); + if (_effective_cluster_id != -1 && !cluster_ids.empty() && + *cluster_ids.begin() != _effective_cluster_id) { + return Status::Corruption( + "multiple cluster ids is not equal. config::cluster_id={}, " + "storage path cluster_id={}", + _effective_cluster_id, *cluster_ids.begin()); } return Status::OK(); } diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py index 1194b3d1f152d4..57f302dcc45a8c 100644 --- a/docker/runtime/doris-compose/command.py +++ b/docker/runtime/doris-compose/command.py @@ -483,14 +483,14 @@ def add_parser(self, args_parsers): if self._support_boolean_action(): parser.add_argument( "--be-cluster-id", - default=True, + default=False, action=self._get_parser_bool_action(False), help="Do not set BE cluster ID in conf. Default is False.") else: parser.add_argument( "--no-be-cluster-id", dest='be_cluster_id', - default=True, + default=False, action=self._get_parser_bool_action(False), help="Do not set BE cluster ID in conf. Default is False.") diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index 7e2bc4d681cdc0..61758f9c5ddeb9 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -128,7 +128,7 @@ class ServerNode { static void fromCompose(ServerNode node, ListHeader header, int index, List fields) { node.index = index node.host = (String) fields.get(header.indexOf('IP')) - node.httpPort = (Integer) fields.get(header.indexOf('http_port')) + node.httpPort = (int) toLongOrDefault(fields.get(header.indexOf('http_port')), -1) node.alive = fields.get(header.indexOf('alive')) == 'true' node.path = (String) fields.get(header.indexOf('path')) } @@ -179,8 +179,8 @@ class Frontend extends ServerNode { static Frontend fromCompose(ListHeader header, int index, List fields) { Frontend fe = new Frontend() ServerNode.fromCompose(fe, header, index, fields) - fe.queryPort = (Integer) fields.get(header.indexOf('query_port')) - fe.editLogPort = (Integer) fields.get(header.indexOf('edit_log_port')) + fe.queryPort = (int) toLongOrDefault(fields.get(header.indexOf('query_port')), -1) + fe.editLogPort = (int) toLongOrDefault(fields.get(header.indexOf('edit_log_port')), -1) fe.isMaster = fields.get(header.indexOf('is_master')) == 'true' return fe } @@ -208,7 +208,7 @@ class Backend extends ServerNode { static Backend fromCompose(ListHeader header, int index, List fields) { Backend be = new Backend() ServerNode.fromCompose(be, header, index, fields) - be.heartbeatPort = (Integer) fields.get(header.indexOf('heartbeat_port')) + be.heartbeatPort = (int) toLongOrDefault(fields.get(header.indexOf('heartbeat_port')), -1) be.backendId = toLongOrDefault(fields.get(header.indexOf('backend_id')), -1L) be.tabletNum = (int) toLongOrDefault(fields.get(header.indexOf('tablet_num')), 0L) diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy index a02481dcebfea0..ef49d7ee2176a5 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_auto_start.groovy @@ -152,7 +152,7 @@ suite('test_auto_start_in_cloud', 'multi_cluster, docker') { // insert // cloud control - future2 = thread { + def future2 = thread { // check cluster "TO_RESUME" awaitUntil(5) { tag = getCloudBeTagByName(clusterName) diff --git a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy index b094b5354af1d6..9cdc56e561a85e 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy @@ -429,7 +429,7 @@ suite('test_ms_api', 'p0, docker') { obj:[ak:"test-ak2", sk:"test-sk2", bucket:"test-bucket", prefix: "test-prefix", endpoint: "test-endpoint", region:"test-region", provider:"COS"]] jsonOutput = new JsonOutput() - addObjInfoBody = jsonOutput.toJson(add_obj_info_api_body) + def addObjInfoBody = jsonOutput.toJson(add_obj_info_api_body) add_obj_info_api.call(msHttpPort, addObjInfoBody) { @@ -625,7 +625,7 @@ suite('test_ms_api', 'p0, docker') { def clusterName2 = "cluster_name2" def clusterId2 = "cluster_id2" def nodeList1 = [node1] - clusterMap1 = [cluster_name: "${clusterName2}", cluster_id:"${clusterId2}", type:"COMPUTE", nodes:nodeList1] + def clusterMap1 = [cluster_name: "${clusterName2}", cluster_id:"${clusterId2}", type:"COMPUTE", nodes:nodeList1] instance = [instance_id: "${instance_id}", cluster: clusterMap1] jsonOutput = new JsonOutput() def addNewComputeGroupBody = jsonOutput.toJson(instance) @@ -1091,9 +1091,9 @@ suite('test_ms_api', 'p0, docker') { def compute_ip1 = "182.0.0.1" def heartbeatPort = 9050 def nodeMap = [cloud_unique_id: "${cloudUniqueId}", ip: "${compute_ip1}", heartbeat_port: "${heartbeatPort}"] - nodeList = [nodeMap] - clusterMap = [cluster_name: "${clusterName}", cluster_id:"${clusterId}", type:"COMPUTE", nodes:nodeList] - instance = [instance_id: "${instance_id}", cluster: clusterMap] + def nodeList = [nodeMap] + def clusterMap = [cluster_name: "${clusterName}", cluster_id:"${clusterId}", type:"COMPUTE", nodes:nodeList] + def instance = [instance_id: "${instance_id}", cluster: clusterMap] def addComputeGroupBody = jsonOutput.toJson(instance) add_cluster_api.call(msHttpPort, addComputeGroupBody) { respCode, body -> @@ -1104,18 +1104,18 @@ suite('test_ms_api', 'p0, docker') { // 1. Test that a node cannot be repeatedly added to multiple clusters // 1.1 compute node - node1 = [cloud_unique_id: "${cloudUniqueId}", ip : "${compute_ip1}", heartbeat_port: 9050] - add_nodes = [node1] + def node1 = [cloud_unique_id: "${cloudUniqueId}", ip : "${compute_ip1}", heartbeat_port: 9050] + def add_nodes = [node1] def otherClusterName = "compute_name_1_other" def otherClusterId = "compute_id_1_other" - add_nodes_cluster = [cluster_name: "${otherClusterName}", cluster_id: "${otherClusterId}", type: "COMPUTE", nodes: add_nodes] + def add_nodes_cluster = [cluster_name: "${otherClusterName}", cluster_id: "${otherClusterId}", type: "COMPUTE", nodes: add_nodes] def addNodeToOtherCluster = [instance_id: "${instance_id}", cluster: add_nodes_cluster] jsonOutput = new JsonOutput() - addNodeToOtherClusterbody = jsonOutput.toJson(addNodeToOtherCluster) + def addNodeToOtherClusterbody = jsonOutput.toJson(addNodeToOtherCluster) add_cluster_api.call(msHttpPort, addNodeToOtherClusterbody) { respCode, body -> log.info("add node to other compute group http cli result: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("ALREADY_EXISTED")) assertTrue(json.msg.contains("compute node endpoint has been added")) } @@ -1143,18 +1143,18 @@ suite('test_ms_api', 'p0, docker') { assertTrue(json.code.equalsIgnoreCase("OK")) } - node_fe_other = [cloud_unique_id: "${cloudUniqueId}", ip : "${ip3}", edit_log_port: 8050, node_type:"FE_FOLLOWER"] + def node_fe_other = [cloud_unique_id: "${cloudUniqueId}", ip : "${ip3}", edit_log_port: 8050, node_type:"FE_FOLLOWER"] add_nodes = [node_fe_other] otherClusterName = "RESERVED_CLUSTER_ID_FOR_SQL_SERVER_OTHER" otherClusterId = "RESERVED_CLUSTER_ID_FOR_SQL_SERVER_OTHER" add_nodes_cluster = [cluster_name: "${otherClusterName}", cluster_id: "${otherClusterId}", type:"SQL", nodes: add_nodes] def addNodeToOtherClusterFE = [instance_id: "${instance_id}", cluster: add_nodes_cluster] jsonOutput = new JsonOutput() - addNodeToOtherFEClusterbody = jsonOutput.toJson(addNodeToOtherClusterFE) + def addNodeToOtherFEClusterbody = jsonOutput.toJson(addNodeToOtherClusterFE) add_cluster_api.call(msHttpPort, addNodeToOtherFEClusterbody) { respCode, body -> log.info("add node to other compute group http cli result: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("ALREADY_EXISTED")) assertTrue(json.msg.contains("sql node endpoint has been added")) } @@ -1174,7 +1174,7 @@ suite('test_ms_api', 'p0, docker') { add_cluster_api.call(msHttpPort, addNodesClusterFailedBody) { respCode, body -> log.info("add two observer fe failed test http cli result: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("INVALID_ARGUMENT")) assertTrue(json.msg.contains("cluster is SQL type, but not set master and follower node, master count=0 follower count=0 so sql cluster can't get a Master node")) } @@ -1198,7 +1198,7 @@ suite('test_ms_api', 'p0, docker') { add_node_api.call(msHttpPort, addSomeFENodesFailed) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) // failed, due to two master node // if force_change_to_multi_follower_mode == false, check type not changed, FE_MASTER log.info("add some fe failed nodes http cli result: ${body} ${respCode} ${json}".toString()) @@ -1208,7 +1208,7 @@ suite('test_ms_api', 'p0, docker') { add_node_api.call(msHttpPort, addSomeFENodesSucc) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("add some fe nodes http cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("OK")) } @@ -1226,7 +1226,7 @@ suite('test_ms_api', 'p0, docker') { drop_node_api.call(msHttpPort, dropAllFeNodesFailedJson) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("drop all fe nodes failed http cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("INTERNAL_ERROR")) assertTrue(json.msg.contains("instance invalid, cant modify, plz check")) @@ -1235,7 +1235,7 @@ suite('test_ms_api', 'p0, docker') { get_instance_api.call(msHttpPort, instance_id) { respCode, body -> log.info("add Master-observer mode get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result def FECluster = result.clusters.find { @@ -1314,7 +1314,7 @@ suite('test_ms_api', 'p0, docker') { def feNodeMap2 = [cloud_unique_id: "${cloudUniqueId}", ip: "${ip2}", edit_log_port: "${edit_log_port}", node_type:"FE_OBSERVER"] def feNodeList = [feNodeMap1, feNodeMap2] def feClusterMap = [cluster_name: "${feClusterName}", cluster_id:"${feClusterId}", type:"SQL", nodes:feNodeList] - instance = [instance_id: "${instance_id}", cluster: feClusterMap] + def instance = [instance_id: "${instance_id}", cluster: feClusterMap] jsonOutput = new JsonOutput() def addSqlGroupBody = jsonOutput.toJson(instance) @@ -1327,7 +1327,7 @@ suite('test_ms_api', 'p0, docker') { get_instance_api.call(msHttpPort, instance_id) { respCode, body -> log.info("add Master-observer mode get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result def FECluster = result.clusters.find { @@ -1354,7 +1354,7 @@ suite('test_ms_api', 'p0, docker') { get_cluster_api.call(msHttpPort, getClusterByNameBody) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("get FE cluster http cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result @@ -1369,7 +1369,7 @@ suite('test_ms_api', 'p0, docker') { get_instance_api.call(msHttpPort, instance_id) { respCode, body -> log.info("after get cluster get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result def FECluster = result.clusters.find { @@ -1394,7 +1394,7 @@ suite('test_ms_api', 'p0, docker') { drop_node_api.call(msHttpPort, delFeObserverNodesBody) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("drop fe observer node http cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("INTERNAL_ERROR")) assertTrue(json.msg.contains("drop fe node not in safe time, try later")) @@ -1411,14 +1411,14 @@ suite('test_ms_api', 'p0, docker') { drop_cluster_api.call(msHttpPort, dropFeClusterBody) { respCode, body -> log.info("drop fe cluster http cli result: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("NOT_FOUND")) assertTrue(json.msg.contains("drop fe cluster not in safe time, try later")) } get_cluster_api.call(msHttpPort, getClusterByNameBody) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("get FE cluster after drop observer http cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result @@ -1442,14 +1442,14 @@ suite('test_ms_api', 'p0, docker') { // after inject, drop fe node, drop fe cluster all succ drop_node_api.call(msHttpPort, delFeObserverNodesBody) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("after inject drop fe observer nodeshttp cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("OK")) } get_cluster_api.call(msHttpPort, getClusterByNameBody) { respCode, body -> - json = parseJson(body) + def json = parseJson(body) log.info("get FE cluster after drop observer http cli result: ${body} ${respCode} ${json}".toString()) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result @@ -1462,14 +1462,14 @@ suite('test_ms_api', 'p0, docker') { drop_cluster_api.call(msHttpPort, dropFeClusterBody) { respCode, body -> log.info("drop fe cluster http cli result: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) } get_instance_api.call(msHttpPort, instance_id) { respCode, body -> log.info("after get cluster get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) def result = json.result def FECluster = result.clusters.find { @@ -1482,8 +1482,8 @@ suite('test_ms_api', 'p0, docker') { def compute_ip1 = "182.0.0.1" def heartbeatPort = 9050 def nodeMap = [cloud_unique_id: "${cloudUniqueId}", ip: "${compute_ip1}", heartbeat_port: "${heartbeatPort}"] - nodeList = [nodeMap] - clusterMap = [cluster_name: "${clusterName}", cluster_id:"${clusterId}", type:"COMPUTE", nodes:nodeList] + def nodeList = [nodeMap] + def clusterMap = [cluster_name: "${clusterName}", cluster_id:"${clusterId}", type:"COMPUTE", nodes:nodeList] instance = [instance_id: "${instance_id}", cluster: clusterMap] def addComputeGroupBody = jsonOutput.toJson(instance) add_cluster_api.call(msHttpPort, addComputeGroupBody) { @@ -1496,7 +1496,7 @@ suite('test_ms_api', 'p0, docker') { get_instance_api.call(msHttpPort, instance_id) { respCode, body -> log.info("after get cluster get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) } @@ -1512,7 +1512,7 @@ suite('test_ms_api', 'p0, docker') { respCode, body -> log.info("drop compute group http cli result: ${body} ${respCode}".toString()) assertEquals(404, respCode) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("NOT_FOUND")) } } diff --git a/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy b/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy index fd97b15eb51e0a..a59ab605447054 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_rename_compute_group.groovy @@ -159,7 +159,7 @@ suite('test_rename_compute_group', 'docker, p0') { // tag = {"cloud_unique_id" : "compute_node_4", "compute_group_status" : "NORMAL", "private_endpoint" : "", "compute_group_name" : "newcluster1", "location" : "default", "public_endpoint" : "", "compute_group_id" : "newcluster1_id"} def toDropIP = cluster.getBeByIndex(4).host - toDropUniqueId = findToDropUniqueId.call(cloudClusterId, toDropIP, ms) + def toDropUniqueId = findToDropUniqueId.call(cloudClusterId, toDropIP, ms) drop_node(toDropUniqueId, toDropIP, 9050, 0, "", clusterName, cloudClusterId, ms) // check have empty compute group @@ -168,7 +168,7 @@ suite('test_rename_compute_group', 'docker, p0') { get_instance_api(msHttpPort, "default_instance_id") { respCode, body -> log.info("before drop node get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) def clusters = json.result.clusters assertTrue(clusters.any { cluster -> @@ -197,7 +197,7 @@ suite('test_rename_compute_group', 'docker, p0') { get_instance_api(msHttpPort, "default_instance_id") { respCode, body -> log.info("after drop node get instance resp: ${body} ${respCode}".toString()) - json = parseJson(body) + def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("OK")) def clusters = json.result.clusters assertTrue(clusters.any { cluster -> diff --git a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy index 0d792950105133..daf83add6d26b1 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_sql_mode_node_mgr.groovy @@ -28,6 +28,8 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') { new ClusterOptions(), new ClusterOptions(), new ClusterOptions(), + new ClusterOptions(), + new ClusterOptions(), ] for (options in clusterOptions) { @@ -46,21 +48,49 @@ suite('test_sql_mode_node_mgr', 'multi_cluster,docker,p1') { "heartbeat_interval_second=1",] } + // Private deployment + // fe cluster id（docker compose 生成的），be cluster id 为空，ms endpoint 是配置的 clusterOptions[0].sqlModeNodeMgr = true; - clusterOptions[0].beClusterId = true; + clusterOptions[0].beClusterId = false; clusterOptions[0].beMetaServiceEndpoint = true; + // fe cluster id（docker compose 生成的），be cluster id 为空，ms endpoint 下发的 clusterOptions[1].sqlModeNodeMgr = true; clusterOptions[1].beClusterId = false; clusterOptions[1].beMetaServiceEndpoint = false; - clusterOptions[2].sqlModeNodeMgr = false; + // fe cluster id（docker compose 生成的），be cluster id （docker compose 生成的），ms endpoint 下发的 + clusterOptions[2].sqlModeNodeMgr = true; clusterOptions[2].beClusterId = true; - clusterOptions[2].beMetaServiceEndpoint = true; - - clusterOptions[3].sqlModeNodeMgr = false; - clusterOptions[3].beClusterId = false; - clusterOptions[3].beMetaServiceEndpoint = false; + clusterOptions[2].beMetaServiceEndpoint = false; + + // fe cluster id（docker compose 生成的），be cluster id （docker compose 生成的），ms endpoint 是配置的 + clusterOptions[3].sqlModeNodeMgr = true; + clusterOptions[3].beClusterId = true; + clusterOptions[3].beMetaServiceEndpoint = true; + + // saas + // fe cluster id（随机生成），be cluster id 是空，ms endpoint 是配置的 + clusterOptions[4].sqlModeNodeMgr = false; + clusterOptions[4].beClusterId = false; + clusterOptions[4].beMetaServiceEndpoint = true; + + // fe cluster id（随机生成）， be cluster id 是空，ms endpoint 用的是fe 下发的 + clusterOptions[5].sqlModeNodeMgr = false; + clusterOptions[5].beClusterId = false; + clusterOptions[5].beMetaServiceEndpoint = false; + + /* + fe cluster id（随机生成） 不等于 be cluster id（docker compose 配置1234567） + clusterOptions[].sqlModeNodeMgr = false; + clusterOptions[].beClusterId = true; + clusterOptions[].beMetaServiceEndpoint = true; + + fe cluster id（随机生成） 不等于 be cluster id（docker compose 配置1234567） + clusterOptions[].sqlModeNodeMgr = false; + clusterOptions[].beClusterId = true; + clusterOptions[].beMetaServiceEndpoint = false; + */ def inject_to_ms_api = { msHttpPort, key, value, check_func -> httpTest { From 0c9cda7e87acaf7bcbd1efc750bfda3d02a9e73a Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 21 Jul 2025 20:12:05 +0800 Subject: [PATCH 282/572] branch-3.0: [fix](case)Fix regression case: cse.groovy (#53434) --- regression-test/suites/tpch_sf0.1_p1/sql/cse.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/tpch_sf0.1_p1/sql/cse.groovy b/regression-test/suites/tpch_sf0.1_p1/sql/cse.groovy index 0aed62a3717a2d..74e928fe5fa1cb 100644 --- a/regression-test/suites/tpch_sf0.1_p1/sql/cse.groovy +++ b/regression-test/suites/tpch_sf0.1_p1/sql/cse.groovy @@ -22,6 +22,7 @@ suite('cse') { sql "set enable_nereids_planner=true;" sql "set enable_fallback_to_original_planner=false;" + sql "set enable_two_phase_read_opt=false;" def q1 = """select s_suppkey,n_regionkey,(s_suppkey + n_regionkey) + 1 as x, (s_suppkey + n_regionkey) + 2 as y from supplier join nation on s_nationkey=n_nationkey order by s_suppkey , n_regionkey limit 10 ; From 484807ab68875124aa996fa12b10085ac3b76d99 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Mon, 21 Jul 2025 20:12:44 +0800 Subject: [PATCH 283/572] branch-3.0: [fix](sql_select_limit) sql_select_limit should not affect DML #53379 (#53528) cherry picked from #53379 --- .../rules/rewrite/AddDefaultLimit.java | 10 +++++ .../org/apache/doris/qe/SessionVariable.java | 2 +- .../session_variable/test_default_limit.out | 12 ++++++ .../test_default_limit.groovy | 38 +++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/nereids_p0/session_variable/test_default_limit.out diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java index 7887836475c821..8885092051beea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AddDefaultLimit.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalTableSink; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.qe.ConnectContext; @@ -36,6 +37,10 @@ public class AddDefaultLimit extends DefaultPlanRewriter imple @Override public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (jobContext.getCascadesContext().getConnectContext() == null + || !jobContext.getCascadesContext().getConnectContext().getState().isQuery()) { + return plan; + } return plan.accept(this, jobContext.getCascadesContext().getStatementContext()); } @@ -52,6 +57,11 @@ public Plan visit(Plan plan, StatementContext context) { return plan; } + @Override + public Plan visitLogicalTableSink(LogicalTableSink logicalTableSink, StatementContext context) { + return logicalTableSink; + } + // should add limit under anchor to keep optimize opportunity @Override public Plan visitLogicalCTEAnchor(LogicalCTEAnchor cteAnchor, diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 6400ae19a7b582..806f23bc8d90f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -903,7 +903,7 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = SQL_AUTO_IS_NULL) public boolean sqlAutoIsNull = false; - @VariableMgr.VarAttr(name = SQL_SELECT_LIMIT) + @VariableMgr.VarAttr(name = SQL_SELECT_LIMIT, needForward = true) private long sqlSelectLimit = Long.MAX_VALUE; // this is used to make c3p0 library happy diff --git a/regression-test/data/nereids_p0/session_variable/test_default_limit.out b/regression-test/data/nereids_p0/session_variable/test_default_limit.out new file mode 100644 index 00000000000000..c44140a25c3621 --- /dev/null +++ b/regression-test/data/nereids_p0/session_variable/test_default_limit.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !baseall_should_delete_2_lines -- +1 1 1 +2 2 2 + +-- !unique_should_delete_2_lines_and_update_1_line -- +3 4 3 + +-- !bigtable_should_insert_2_lines -- +1 1 1 +2 2 2 + diff --git a/regression-test/suites/nereids_p0/session_variable/test_default_limit.groovy b/regression-test/suites/nereids_p0/session_variable/test_default_limit.groovy index 2854d87b8e359a..214f4a304cbc96 100644 --- a/regression-test/suites/nereids_p0/session_variable/test_default_limit.groovy +++ b/regression-test/suites/nereids_p0/session_variable/test_default_limit.groovy @@ -281,4 +281,42 @@ suite('test_default_limit') { ''' assertEquals(res.size(), 8) } + + // test dml + sql 'set default_order_by_limit = -1' + sql 'set sql_select_limit = 1' + + sql """truncate table baseall""" + sql """truncate table bigtable""" + sql """drop table if exists unique_table""" + sql """create table unique_table ( + k0 int, + k1 int, + k2 int + ) + unique key (k0) + distributed by hash(k0) buckets 16 + properties( + 'replication_num'='1' + ) + """ + sql """insert into baseall values(1, 1, 1), (2, 2, 2),(3, 3, 3), (4, 4, 4)""" + sql """insert into unique_table values(1, 1, 1), (2, 2, 2),(3, 3, 3)""" + sql "sync" + // should execute successful + sql "delete from baseall where k0 in (3, 4)" + sql "sync" + // should insert 2 lines + sql "insert into bigtable select * from baseall" + sql "sync" + // should update 2 lines + sql "update unique_table set k1 = 4 where k1 in (2, 3, 4)" + sql "sync" + // should delete 2 lines + sql "delete from unique_table where k0 = 1 or k0 = 2" + sql "sync" + sql 'set sql_select_limit = -1' + qt_baseall_should_delete_2_lines "select * from baseall order by k0" + qt_unique_should_delete_2_lines_and_update_1_line "select * from unique_table order by k0" + qt_bigtable_should_insert_2_lines "select * from bigtable order by k0" } \ No newline at end of file From f3e353f6934ca74123eea5ca52fa7edd3151c8af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:14:54 +0800 Subject: [PATCH 284/572] branch-3.0: [fix](test) Fix ttl partition compensate mv not be chosen stable by cbo #53330 (#53469) Cherry-picked from #53330 Co-authored-by: seawinde --- .../unioin_rewrite_grace_big.out | 2 ++ .../unioin_rewrite_grace_big.groovy | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out index 1e718b4e6127d3..489e7d4706b78c 100644 --- a/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out +++ b/regression-test/data/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.out @@ -64,10 +64,12 @@ -- !query_16_0_before -- 2023-10-18 2023-10-18 2 3 436.80 2023-10-19 2023-10-19 2 3 398.00 +2023-10-19 2023-12-19 2 3 398.00 -- !query_16_0_after -- 2023-10-18 2023-10-18 2 3 436.80 2023-10-19 2023-10-19 2 3 398.00 +2023-10-19 2023-12-19 2 3 398.00 -- !query_17_0_before -- 2023-09-17 2023-09-17 \N 2 3 \N diff --git a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy index 55394292d0b75c..5f6c7fcf62f37d 100644 --- a/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/union_rewrite_grace_big/unioin_rewrite_grace_big.groovy @@ -323,7 +323,12 @@ suite("union_rewrite_grace_big") { """ sql """ insert into lineitem_static values - (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'); + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'), + (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'), + (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'), + (3, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'), + (4, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '${today_str}', '${today_str}', '${today_str}', 'a', 'b', 'yyyyyyyyy'); """ multi_sql """ @@ -336,7 +341,7 @@ suite("union_rewrite_grace_big") { select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem_static - left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + left join orders on l_orderkey = o_orderkey group by l_shipdate, o_orderdate, @@ -346,7 +351,7 @@ suite("union_rewrite_grace_big") { def query_ttl_all_partition_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem_static - left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + left join orders on l_orderkey = o_orderkey group by l_shipdate, o_orderdate, @@ -356,7 +361,7 @@ suite("union_rewrite_grace_big") { def query_ttl_partition_sql = """ select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total from lineitem_static - left join orders on l_orderkey = o_orderkey and l_shipdate = o_orderdate + left join orders on l_orderkey = o_orderkey where (l_shipdate>= '2023-10-18' and l_shipdate <= '2023-10-19') group by l_shipdate, From 366e25930da1ffd9c289ad290971e8c74e50ea41 Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 21 Jul 2025 20:58:39 +0800 Subject: [PATCH 285/572] [Chore](workflow) change workflow's thirdparty prebuild to 3.0 (#53639) change workflow's thirdparty prebuild to 3.0 --- .github/workflows/be-ut-mac.yml | 10 ++-------- .github/workflows/build-extension.yml | 9 +-------- .github/workflows/build-thirdparty.yml | 6 +++--- .github/workflows/code-checks.yml | 2 +- 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/.github/workflows/be-ut-mac.yml b/.github/workflows/be-ut-mac.yml index 4f3db038dbab83..c1b516d0e74add 100644 --- a/.github/workflows/be-ut-mac.yml +++ b/.github/workflows/be-ut-mac.yml @@ -85,14 +85,8 @@ jobs: brew install "${cellars[@]}" || true pushd thirdparty - branch="${{ github.base_ref }}" - if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' ]]; then - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz \ - -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz - else - curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz" \ - -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz - fi + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation-3.0/doris-thirdparty-prebuilt-darwin-x86_64.tar.xz \ + -o doris-thirdparty-prebuilt-darwin-x86_64.tar.xz tar -xvf doris-thirdparty-prebuilt-darwin-x86_64.tar.xz popd diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml index 0c53abb1da3bfd..62b9725d0d0a6b 100644 --- a/.github/workflows/build-extension.yml +++ b/.github/workflows/build-extension.yml @@ -66,14 +66,7 @@ jobs: - name: Setup thrift run: | pushd thirdparty - branch="${{ github.base_ref }}" - if [[ -z "${branch}" ]] || [[ "${branch}" == 'master' || "${branch}" == 'branch-3.0' || "${branch}" == 'branch-2.1' ]]; then - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \ - -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz - else - curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-${branch/branch-/}/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" \ - -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz - fi + curl -L "https://github.com/apache/doris-thirdparty/releases/download/automation-3.0/doris-thirdparty-prebuilt-linux-x86_64.tar.xz" -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz popd export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}" diff --git a/.github/workflows/build-thirdparty.yml b/.github/workflows/build-thirdparty.yml index 2185237973ac58..e6cbd464c8a562 100644 --- a/.github/workflows/build-thirdparty.yml +++ b/.github/workflows/build-thirdparty.yml @@ -72,7 +72,7 @@ jobs: - name: Download run: | cd thirdparty - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation-3.0/doris-thirdparty-source.tgz \ -o doris-thirdparty-source.tgz tar -zxvf doris-thirdparty-source.tgz @@ -138,7 +138,7 @@ jobs: - name: Download run: | cd thirdparty - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation-3.0/doris-thirdparty-source.tgz \ -o doris-thirdparty-source.tgz tar -zxvf doris-thirdparty-source.tgz @@ -196,7 +196,7 @@ jobs: - name: Download run: | cd thirdparty - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-source.tgz \ + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation-3.0/doris-thirdparty-source.tgz \ -o doris-thirdparty-source.tgz tar -zxvf doris-thirdparty-source.tgz diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 43a4f74d2a67ac..9c8c62263fa5d5 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -99,7 +99,7 @@ jobs: sudo DEBIAN_FRONTEND=noninteractive apt install --yes tzdata byacc pushd thirdparty - curl -L https://github.com/apache/doris-thirdparty/releases/download/automation/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \ + curl -L https://github.com/apache/doris-thirdparty/releases/download/automation-3.0/doris-thirdparty-prebuilt-linux-x86_64.tar.xz \ -o doris-thirdparty-prebuilt-linux-x86_64.tar.xz tar -xvf doris-thirdparty-prebuilt-linux-x86_64.tar.xz popd From 2b2677c2b526f67466576713315720a408eaf49f Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 22 Jul 2025 14:34:22 +0800 Subject: [PATCH 286/572] [Enhancement](multi-catalog) Add PredicateFilterTime, DictFilterRewriteTime, LazyReadFilteredRows profile metrics in parquet orc profiles. (#53609) ### What problem does this PR solve? Problem Summary: ### Release note Cherry-pick #51248 --- be/src/vec/exec/format/orc/vorc_reader.cpp | 135 ++++++++------- be/src/vec/exec/format/orc/vorc_reader.h | 10 +- .../format/parquet/vparquet_group_reader.cpp | 159 ++++++++++-------- .../format/parquet/vparquet_group_reader.h | 2 + .../exec/format/parquet/vparquet_reader.cpp | 4 + .../vec/exec/format/parquet/vparquet_reader.h | 2 + 6 files changed, 175 insertions(+), 137 deletions(-) diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index 2e7047aecf9188..acd979678fb4fc 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -200,7 +200,9 @@ void OrcReader::_collect_profile_before_close() { COUNTER_UPDATE(_orc_profile.set_fill_column_time, _statistics.set_fill_column_time); COUNTER_UPDATE(_orc_profile.decode_value_time, _statistics.decode_value_time); COUNTER_UPDATE(_orc_profile.decode_null_map_time, _statistics.decode_null_map_time); - COUNTER_UPDATE(_orc_profile.filter_block_time, _statistics.filter_block_time); + COUNTER_UPDATE(_orc_profile.predicate_filter_time, _statistics.predicate_filter_time); + COUNTER_UPDATE(_orc_profile.dict_filter_rewrite_time, _statistics.dict_filter_rewrite_time); + COUNTER_UPDATE(_orc_profile.lazy_read_filtered_rows, _statistics.lazy_read_filtered_rows); if (_file_input_stream != nullptr) { _file_input_stream->collect_profile_before_close(); @@ -234,8 +236,16 @@ void OrcReader::_init_profile() { ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeValueTime", orc_profile, 1); _orc_profile.decode_null_map_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DecodeNullMapTime", orc_profile, 1); - _orc_profile.filter_block_time = - ADD_CHILD_TIMER_WITH_LEVEL(_profile, "FilterBlockTime", orc_profile, 1); + _orc_profile.predicate_filter_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PredicateFilterTime", orc_profile, 1); + _orc_profile.dict_filter_rewrite_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DictFilterRewriteTime", orc_profile, 1); + _orc_profile.lazy_read_filtered_rows = + ADD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByLazyRead", TUnit::UNIT, 1); + _orc_profile.selected_row_group_count = + ADD_COUNTER_WITH_LEVEL(_profile, "SelectedRowGroupCount", TUnit::UNIT, 1); + _orc_profile.evaluated_row_group_count = + ADD_COUNTER_WITH_LEVEL(_profile, "EvaluatedRowGroupCount", TUnit::UNIT, 1); } } @@ -1714,15 +1724,18 @@ Status OrcReader::get_next_block_impl(Block* block, size_t* read_rows, bool* eof *read_rows = 0; return Status::OK(); } - _execute_filter_position_delete_rowids(*_filter); { - SCOPED_RAW_TIMER(&_statistics.decode_null_map_time); - RETURN_IF_CATCH_EXCEPTION( - Block::filter_block_internal(block, columns_to_filter, *_filter)); + SCOPED_RAW_TIMER(&_statistics.predicate_filter_time); + _execute_filter_position_delete_rowids(*_filter); + { + SCOPED_RAW_TIMER(&_statistics.decode_null_map_time); + RETURN_IF_CATCH_EXCEPTION( + Block::filter_block_internal(block, columns_to_filter, *_filter)); + } + Block::erase_useless_column(block, column_to_keep); + RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); + *read_rows = block->rows(); } - Block::erase_useless_column(block, column_to_keep); - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); - *read_rows = block->rows(); } else { uint64_t rr; SCOPED_RAW_TIMER(&_statistics.column_read_time); @@ -1799,63 +1812,60 @@ Status OrcReader::get_next_block_impl(Block* block, size_t* read_rows, bool* eof return Status::OK(); } - _build_delete_row_filter(block, _batch->numElements); - - std::vector columns_to_filter; - int column_to_keep = block->columns(); - columns_to_filter.resize(column_to_keep); - for (uint32_t i = 0; i < column_to_keep; ++i) { - columns_to_filter[i] = i; - } - if (!_lazy_read_ctx.conjuncts.empty()) { - VExprContextSPtrs filter_conjuncts; - filter_conjuncts.insert(filter_conjuncts.end(), _filter_conjuncts.begin(), - _filter_conjuncts.end()); - for (auto& conjunct : _dict_filter_conjuncts) { - filter_conjuncts.emplace_back(conjunct); - } - for (auto& conjunct : _non_dict_filter_conjuncts) { - filter_conjuncts.emplace_back(conjunct); - } - std::vector filters; - if (_delete_rows_filter_ptr) { - filters.push_back(_delete_rows_filter_ptr.get()); - } - IColumn::Filter result_filter(block->rows(), 1); - bool can_filter_all = false; - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( - filter_conjuncts, &filters, block, &result_filter, &can_filter_all)); - if (can_filter_all) { - for (auto& col : columns_to_filter) { - std::move(*block->get_by_position(col).column).assume_mutable()->clear(); + { + SCOPED_RAW_TIMER(&_statistics.predicate_filter_time); + _build_delete_row_filter(block, _batch->numElements); + + std::vector columns_to_filter; + int column_to_keep = block->columns(); + columns_to_filter.resize(column_to_keep); + for (uint32_t i = 0; i < column_to_keep; ++i) { + columns_to_filter[i] = i; + } + if (!_lazy_read_ctx.conjuncts.empty()) { + VExprContextSPtrs filter_conjuncts; + filter_conjuncts.insert(filter_conjuncts.end(), _filter_conjuncts.begin(), + _filter_conjuncts.end()); + for (auto& conjunct : _dict_filter_conjuncts) { + filter_conjuncts.emplace_back(conjunct); } - Block::erase_useless_column(block, column_to_keep); - return _convert_dict_cols_to_string_cols(block, &batch_vec); - } - _execute_filter_position_delete_rowids(result_filter); - { - SCOPED_RAW_TIMER(&_statistics.filter_block_time); + for (auto& conjunct : _non_dict_filter_conjuncts) { + filter_conjuncts.emplace_back(conjunct); + } + std::vector filters; + if (_delete_rows_filter_ptr) { + filters.push_back(_delete_rows_filter_ptr.get()); + } + IColumn::Filter result_filter(block->rows(), 1); + bool can_filter_all = false; + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( + filter_conjuncts, &filters, block, &result_filter, &can_filter_all)); + if (can_filter_all) { + for (auto& col : columns_to_filter) { + std::move(*block->get_by_position(col).column).assume_mutable()->clear(); + } + Block::erase_useless_column(block, column_to_keep); + return _convert_dict_cols_to_string_cols(block, &batch_vec); + } + _execute_filter_position_delete_rowids(result_filter); RETURN_IF_CATCH_EXCEPTION( Block::filter_block_internal(block, columns_to_filter, result_filter)); - } - Block::erase_useless_column(block, column_to_keep); - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); - } else { - if (_delete_rows_filter_ptr) { - _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr); - SCOPED_RAW_TIMER(&_statistics.filter_block_time); - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(block, columns_to_filter, - (*_delete_rows_filter_ptr))); + Block::erase_useless_column(block, column_to_keep); } else { - std::unique_ptr filter(new IColumn::Filter(block->rows(), 1)); - _execute_filter_position_delete_rowids(*filter); - SCOPED_RAW_TIMER(&_statistics.filter_block_time); - RETURN_IF_CATCH_EXCEPTION( - Block::filter_block_internal(block, columns_to_filter, (*filter))); + if (_delete_rows_filter_ptr) { + _execute_filter_position_delete_rowids(*_delete_rows_filter_ptr); + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal( + block, columns_to_filter, (*_delete_rows_filter_ptr))); + } else { + std::unique_ptr filter(new IColumn::Filter(block->rows(), 1)); + _execute_filter_position_delete_rowids(*filter); + RETURN_IF_CATCH_EXCEPTION( + Block::filter_block_internal(block, columns_to_filter, (*filter))); + } + Block::erase_useless_column(block, column_to_keep); } - Block::erase_useless_column(block, column_to_keep); - RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); } + RETURN_IF_ERROR(_convert_dict_cols_to_string_cols(block, &batch_vec)); *read_rows = block->rows(); } return Status::OK(); @@ -1899,6 +1909,7 @@ void OrcReader::_build_delete_row_filter(const Block* block, size_t rows) { } Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t size, void* arg) { + SCOPED_RAW_TIMER(&_statistics.predicate_filter_time); Block* block = (Block*)arg; size_t origin_column_num = block->columns(); @@ -1999,6 +2010,7 @@ Status OrcReader::filter(orc::ColumnVectorBatch& data, uint16_t* sel, uint16_t s sel[new_size] = i; new_size += result_filter_data[i] ? 1 : 0; } + _statistics.lazy_read_filtered_rows += static_cast(size - new_size); data.numElements = new_size; return Status::OK(); } @@ -2072,6 +2084,7 @@ bool OrcReader::_can_filter_by_dict(int slot_id) { Status OrcReader::on_string_dicts_loaded( std::unordered_map& file_column_name_to_dict_map, bool* is_stripe_filtered) { + SCOPED_RAW_TIMER(&_statistics.dict_filter_rewrite_time); *is_stripe_filtered = false; for (auto it = _dict_filter_cols.begin(); it != _dict_filter_cols.end();) { std::string& dict_filter_col_name = it->first; diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h index 866d22ff56cc20..551c000dca3b9c 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.h +++ b/be/src/vec/exec/format/orc/vorc_reader.h @@ -128,7 +128,9 @@ class OrcReader : public GenericReader { int64_t set_fill_column_time = 0; int64_t decode_value_time = 0; int64_t decode_null_map_time = 0; - int64_t filter_block_time = 0; + int64_t predicate_filter_time = 0; + int64_t dict_filter_rewrite_time = 0; + int64_t lazy_read_filtered_rows = 0; }; OrcReader(RuntimeProfile* profile, RuntimeState* state, const TFileScanRangeParams& params, @@ -226,7 +228,11 @@ class OrcReader : public GenericReader { RuntimeProfile::Counter* set_fill_column_time = nullptr; RuntimeProfile::Counter* decode_value_time = nullptr; RuntimeProfile::Counter* decode_null_map_time = nullptr; - RuntimeProfile::Counter* filter_block_time = nullptr; + RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; + RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; + RuntimeProfile::Counter* selected_row_group_count = nullptr; + RuntimeProfile::Counter* evaluated_row_group_count = nullptr; }; class ORCFilterImpl : public orc::ORCFilter { diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 00beb59e7dcb64..573190bfda435a 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -319,45 +319,46 @@ Status RowGroupReader::next_batch(Block* block, size_t batch_size, size_t* read_ *read_rows = block->rows(); return Status::OK(); } + { + SCOPED_RAW_TIMER(&_predicate_filter_time); + RETURN_IF_ERROR(_build_pos_delete_filter(*read_rows)); - RETURN_IF_ERROR(_build_pos_delete_filter(*read_rows)); - - std::vector columns_to_filter; - int column_to_keep = block->columns(); - columns_to_filter.resize(column_to_keep); - for (uint32_t i = 0; i < column_to_keep; ++i) { - columns_to_filter[i] = i; - } - if (!_lazy_read_ctx.conjuncts.empty()) { - std::vector filters; - if (_position_delete_ctx.has_filter) { - filters.push_back(_pos_delete_filter_ptr.get()); + std::vector columns_to_filter; + int column_to_keep = block->columns(); + columns_to_filter.resize(column_to_keep); + for (uint32_t i = 0; i < column_to_keep; ++i) { + columns_to_filter[i] = i; } - IColumn::Filter result_filter(block->rows(), 1); - bool can_filter_all = false; + if (!_lazy_read_ctx.conjuncts.empty()) { + std::vector filters; + if (_position_delete_ctx.has_filter) { + filters.push_back(_pos_delete_filter_ptr.get()); + } + IColumn::Filter result_filter(block->rows(), 1); + bool can_filter_all = false; - { - SCOPED_RAW_TIMER(&_predicate_filter_time); - RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( - _filter_conjuncts, &filters, block, &result_filter, &can_filter_all)); - } + { + RETURN_IF_ERROR_OR_CATCH_EXCEPTION(VExprContext::execute_conjuncts( + _filter_conjuncts, &filters, block, &result_filter, &can_filter_all)); + } - if (can_filter_all) { - for (auto& col : columns_to_filter) { - std::move(*block->get_by_position(col).column).assume_mutable()->clear(); + if (can_filter_all) { + for (auto& col : columns_to_filter) { + std::move(*block->get_by_position(col).column).assume_mutable()->clear(); + } + Block::erase_useless_column(block, column_to_keep); + _convert_dict_cols_to_string_cols(block); + return Status::OK(); } + + RETURN_IF_CATCH_EXCEPTION( + Block::filter_block_internal(block, columns_to_filter, result_filter)); Block::erase_useless_column(block, column_to_keep); - _convert_dict_cols_to_string_cols(block); - return Status::OK(); + } else { + RETURN_IF_CATCH_EXCEPTION( + RETURN_IF_ERROR(_filter_block(block, column_to_keep, columns_to_filter))); } - - RETURN_IF_CATCH_EXCEPTION( - Block::filter_block_internal(block, columns_to_filter, result_filter)); - Block::erase_useless_column(block, column_to_keep); _convert_dict_cols_to_string_cols(block); - } else { - RETURN_IF_CATCH_EXCEPTION( - RETURN_IF_ERROR(_filter_block(block, column_to_keep, columns_to_filter))); } *read_rows = block->rows(); return Status::OK(); @@ -457,50 +458,57 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re RETURN_IF_ERROR(_build_pos_delete_filter(pre_read_rows)); - // generate filter vector - if (_lazy_read_ctx.resize_first_column) { - // VExprContext.execute has an optimization, the filtering is executed when block->rows() > 0 - // The following process may be tricky and time-consuming, but we have no other way. - block->get_by_position(0).column->assume_mutable()->resize(pre_read_rows); - } - result_filter.assign(pre_read_rows, static_cast(1)); bool can_filter_all = false; - std::vector filters; - if (_position_delete_ctx.has_filter) { - filters.push_back(_pos_delete_filter_ptr.get()); - } - - VExprContextSPtrs filter_contexts; - for (auto& conjunct : _filter_conjuncts) { - filter_contexts.emplace_back(conjunct); - } - { SCOPED_RAW_TIMER(&_predicate_filter_time); - RETURN_IF_ERROR(VExprContext::execute_conjuncts(filter_contexts, &filters, block, - &result_filter, &can_filter_all)); - } - if (_lazy_read_ctx.resize_first_column) { - // We have to clean the first column to insert right data. - block->get_by_position(0).column->assume_mutable()->clear(); + // generate filter vector + if (_lazy_read_ctx.resize_first_column) { + // VExprContext.execute has an optimization, the filtering is executed when block->rows() > 0 + // The following process may be tricky and time-consuming, but we have no other way. + block->get_by_position(0).column->assume_mutable()->resize(pre_read_rows); + } + result_filter.assign(pre_read_rows, static_cast(1)); + std::vector filters; + if (_position_delete_ctx.has_filter) { + filters.push_back(_pos_delete_filter_ptr.get()); + } + + VExprContextSPtrs filter_contexts; + for (auto& conjunct : _filter_conjuncts) { + filter_contexts.emplace_back(conjunct); + } + + { + SCOPED_RAW_TIMER(&_predicate_filter_time); + RETURN_IF_ERROR(VExprContext::execute_conjuncts(filter_contexts, &filters, block, + &result_filter, &can_filter_all)); + } + + if (_lazy_read_ctx.resize_first_column) { + // We have to clean the first column to insert right data. + block->get_by_position(0).column->assume_mutable()->clear(); + } } const uint8_t* __restrict filter_map_data = result_filter.data(); filter_map_ptr.reset(new FilterMap()); RETURN_IF_ERROR(filter_map_ptr->init(filter_map_data, pre_read_rows, can_filter_all)); if (filter_map_ptr->filter_all()) { - for (auto& col : _lazy_read_ctx.predicate_columns.first) { - // clean block to read predicate columns - block->get_by_name(col).column->assume_mutable()->clear(); - } - for (auto& col : _lazy_read_ctx.predicate_partition_columns) { - block->get_by_name(col.first).column->assume_mutable()->clear(); - } - for (auto& col : _lazy_read_ctx.predicate_missing_columns) { - block->get_by_name(col.first).column->assume_mutable()->clear(); + { + SCOPED_RAW_TIMER(&_predicate_filter_time); + for (auto& col : _lazy_read_ctx.predicate_columns.first) { + // clean block to read predicate columns + block->get_by_name(col).column->assume_mutable()->clear(); + } + for (auto& col : _lazy_read_ctx.predicate_partition_columns) { + block->get_by_name(col.first).column->assume_mutable()->clear(); + } + for (auto& col : _lazy_read_ctx.predicate_missing_columns) { + block->get_by_name(col.first).column->assume_mutable()->clear(); + } + Block::erase_useless_column(block, origin_column_num); } - Block::erase_useless_column(block, origin_column_num); if (!pre_eof) { // If continuous batches are skipped, we can cache them to skip a whole page @@ -554,18 +562,21 @@ Status RowGroupReader::_do_lazy_read(Block* block, size_t batch_size, size_t* re // we set pre_read_rows as batch_size for lazy read columns, so pre_eof != lazy_eof // filter data in predicate columns, and remove filter column - if (filter_map.has_filter()) { - if (block->columns() == origin_column_num) { - // the whole row group has been filtered by _lazy_read_ctx.vconjunct_ctx, and batch_eof is - // generated from next batch, so the filter column is removed ahead. - DCHECK_EQ(block->rows(), 0); + { + SCOPED_RAW_TIMER(&_predicate_filter_time); + if (filter_map.has_filter()) { + if (block->columns() == origin_column_num) { + // the whole row group has been filtered by _lazy_read_ctx.vconjunct_ctx, and batch_eof is + // generated from next batch, so the filter column is removed ahead. + DCHECK_EQ(block->rows(), 0); + } else { + RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal( + block, _lazy_read_ctx.all_predicate_col_ids, result_filter)); + Block::erase_useless_column(block, origin_column_num); + } } else { - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal( - block, _lazy_read_ctx.all_predicate_col_ids, result_filter)); Block::erase_useless_column(block, origin_column_num); } - } else { - Block::erase_useless_column(block, origin_column_num); } _convert_dict_cols_to_string_cols(block); @@ -782,6 +793,7 @@ Status RowGroupReader::_filter_block(Block* block, int column_to_keep, } Status RowGroupReader::_rewrite_dict_predicates() { + SCOPED_RAW_TIMER(&_dict_filter_rewrite_time); for (auto it = _dict_filter_cols.begin(); it != _dict_filter_cols.end();) { std::string& dict_filter_col_name = it->first; int slot_id = it->second; @@ -847,7 +859,6 @@ Status RowGroupReader::_rewrite_dict_predicates() { IColumn::Filter result_filter(temp_block.rows(), 1); bool can_filter_all; { - SCOPED_RAW_TIMER(&_predicate_filter_time); RETURN_IF_ERROR(VExprContext::execute_conjuncts(ctxs, nullptr, &temp_block, &result_filter, &can_filter_all)); } diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.h b/be/src/vec/exec/format/parquet/vparquet_group_reader.h index 8106241014ba63..758d376396a14f 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.h @@ -158,6 +158,7 @@ class RowGroupReader : public ProfileCollector { Status next_batch(Block* block, size_t batch_size, size_t* read_rows, bool* batch_eof); int64_t lazy_read_filtered_rows() const { return _lazy_read_filtered_rows; } int64_t predicate_filter_time() const { return _predicate_filter_time; } + int64_t dict_filter_rewrite_time() const { return _dict_filter_rewrite_time; } ParquetColumnReader::Statistics statistics(); void set_remaining_rows(int64_t rows) { _remaining_rows = rows; } @@ -213,6 +214,7 @@ class RowGroupReader : public ProfileCollector { const LazyReadContext& _lazy_read_ctx; int64_t _lazy_read_filtered_rows = 0; int64_t _predicate_filter_time = 0; + int64_t _dict_filter_rewrite_time = 0; // If continuous batches are skipped, we can cache them to skip a whole page size_t _cached_filtered_rows = 0; std::unique_ptr _pos_delete_filter_ptr; diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp index 7e0012486d63b5..28e42bd5e95ac3 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp @@ -183,6 +183,8 @@ void ParquetReader::_init_profile() { _profile, "ParsePageHeaderNum", TUnit::UNIT, parquet_profile, 1); _parquet_profile.predicate_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "PredicateFilterTime", parquet_profile, 1); + _parquet_profile.dict_filter_rewrite_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "DictFilterRewriteTime", parquet_profile, 1); } } @@ -589,6 +591,7 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof) _column_statistics.merge(column_st); _statistics.lazy_read_filtered_rows += _current_group_reader->lazy_read_filtered_rows(); _statistics.predicate_filter_time += _current_group_reader->predicate_filter_time(); + _statistics.dict_filter_rewrite_time += _current_group_reader->dict_filter_rewrite_time(); if (_read_row_groups.size() == 0) { *eof = true; } else { @@ -1033,6 +1036,7 @@ void ParquetReader::_collect_profile() { COUNTER_UPDATE(_parquet_profile.parse_page_header_num, _column_statistics.parse_page_header_num); COUNTER_UPDATE(_parquet_profile.predicate_filter_time, _statistics.predicate_filter_time); + COUNTER_UPDATE(_parquet_profile.dict_filter_rewrite_time, _statistics.dict_filter_rewrite_time); COUNTER_UPDATE(_parquet_profile.file_read_time, _column_statistics.read_time); COUNTER_UPDATE(_parquet_profile.file_read_calls, _column_statistics.read_calls); COUNTER_UPDATE(_parquet_profile.file_meta_read_calls, _column_statistics.meta_read_calls); diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.h b/be/src/vec/exec/format/parquet/vparquet_reader.h index 0d62bf9f3cd7b5..e24071093b635c 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_reader.h @@ -92,6 +92,7 @@ class ParquetReader : public GenericReader { int64_t read_page_index_time = 0; int64_t parse_page_index_time = 0; int64_t predicate_filter_time = 0; + int64_t dict_filter_rewrite_time = 0; }; ParquetReader(RuntimeProfile* profile, const TFileScanRangeParams& params, @@ -190,6 +191,7 @@ class ParquetReader : public GenericReader { RuntimeProfile::Counter* skip_page_header_num = nullptr; RuntimeProfile::Counter* parse_page_header_num = nullptr; RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; }; Status _open_file(); From a629f41d5aca9cf5dc5f056266903ee92671d057 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Tue, 22 Jul 2025 19:44:39 +0800 Subject: [PATCH 287/572] [Bug](topn) variant column read in topn may coredump (#52585) ### What problem does this PR solve? cherry pick https://github.com/apache/doris/pull/52573 Problem Summary: --- be/src/common/config.cpp | 2 ++ be/src/exec/rowid_fetcher.cpp | 10 +++++++--- be/src/olap/rowset/segment_v2/segment.cpp | 4 ++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 46e68829558b46..0c83fc76e1318c 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1966,6 +1966,8 @@ Status set_fuzzy_configs() { // if have set enable_fuzzy_mode=true in be.conf, will fuzzy those field and values fuzzy_field_and_value["disable_storage_page_cache"] = ((distribution(*generator) % 2) == 0) ? "true" : "false"; + fuzzy_field_and_value["disable_segment_cache"] = + ((distribution(*generator) % 2) == 0) ? "true" : "false"; fuzzy_field_and_value["enable_system_metrics"] = ((distribution(*generator) % 2) == 0) ? "true" : "false"; fuzzy_field_and_value["enable_set_in_bitmap_value"] = diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index b26ad63d71b34b..d6c47201dadf0a 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -442,9 +442,13 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequest& request, iterator_map[iterator_key].segment = segment; } segment = iterator_item.segment; - RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema, desc.slots()[x], - row_id, column, stats, - iterator_item.iterator)); + try { + RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema, desc.slots()[x], + row_id, column, stats, + iterator_item.iterator)); + } catch (const Exception& e) { + return Status::Error(e.code(), "Row id fetch failed because {}", e.what()); + } } } // serialize block if not empty diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp index 86884d7adcec47..e29ec8908147fb 100644 --- a/be/src/olap/rowset/segment_v2/segment.cpp +++ b/be/src/olap/rowset/segment_v2/segment.cpp @@ -1125,6 +1125,10 @@ Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescripto vectorized::PathInDataPtr path = std::make_shared( schema.column_by_uid(slot->col_unique_id()).name_lower_case(), slot->column_paths()); + + // here need create column readers to make sure column reader is created before seek_and_read_by_rowid + // if segment cache miss, column reader will be created to make sure the variant column result not coredump + RETURN_IF_ERROR(_create_column_readers_once(&stats)); auto storage_type = get_data_type_of(ColumnIdentifier {.unique_id = slot->col_unique_id(), .path = path, .is_nullable = slot->is_nullable()}, From 0ea385d160bd1b5ca8e7584df298b533d08f29a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 19:48:01 +0800 Subject: [PATCH 288/572] branch-3.0: [fix](test) Enable `enable_inverted_index_v1_for_variant` for check before quit test #53607 (#53664) Cherry-picked from #53607 Co-authored-by: Uniqueyou --- .../suites/check_before_quit/check_before_quit.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy index 9e624fa1ead88d..bd79bc11bb30d0 100644 --- a/regression-test/suites/check_before_quit/check_before_quit.groovy +++ b/regression-test/suites/check_before_quit/check_before_quit.groovy @@ -235,6 +235,9 @@ suite("check_before_quit", "nonConcurrent,p0") { def failureList = [] sql "set enable_decimal256 = true;" + sql """ + ADMIN SET ALL FRONTENDS CONFIG ('enable_inverted_index_v1_for_variant' = 'true'); + """ for (int i = 0; i < num; i++) { def db = allDataBases[i][0] From f46ac481f575679148b30a09360585fa42ad7cc4 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Wed, 23 Jul 2025 09:35:33 +0800 Subject: [PATCH 289/572] branch-3.0: [test](inverted index) make test_show_index_data regression case more stable #53705 (#53726) cherry pick from #53705 --- .../show_data/test_show_index_data.groovy | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy b/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy index 6881c6fe01c010..9f7de0dc6c762e 100644 --- a/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy +++ b/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy @@ -135,7 +135,7 @@ suite("test_show_index_data", "p1") { // trigger compactions for all tablets in ${tableName} for (def tablet in tablets) { String tablet_id = tablet.TabletId - def backend_id = tablet.BackendId + String backend_id = tablet.BackendId def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -148,7 +148,7 @@ suite("test_show_index_data", "p1") { Awaitility.await().atMost(30, TimeUnit.MINUTES).untilAsserted(() -> { Thread.sleep(30000) String tablet_id = tablet.TabletId - def backend_id = tablet.BackendId + String backend_id = tablet.BackendId def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -188,9 +188,19 @@ suite("test_show_index_data", "p1") { Thread.sleep(10000) def result = sql """ show data all;""" logger.info("show data all; result is: ${result}") - def currentLocalIndexSize = convert_size.call(result[0][4]) - def currentSegmentIndexSize = convert_size.call(result[0][3]) + // Find the row for our table + def tableRow = null + for (def row in result) { + if (row[0] == show_table_name) { + tableRow = row + break + } + } + assertNotNull(tableRow, "Could not find table ${show_table_name} in show data result") + + def currentLocalIndexSize = convert_size.call(tableRow[4]) + def currentSegmentIndexSize = convert_size.call(tableRow[3]) if (expect_idx == FileSizeChange.LARGER) { assertTrue(currentLocalIndexSize > localIndexSize) } else if (expect_idx == FileSizeChange.SMALLER) { @@ -199,13 +209,13 @@ suite("test_show_index_data", "p1") { assertTrue(check_size_equal(currentLocalIndexSize, localIndexSize)) } - if (expect_data == FileSizeChange.LARGER) { + /*if (expect_data == FileSizeChange.LARGER) { assertTrue(currentSegmentIndexSize > localSegmentSize) } else if (expect_data == FileSizeChange.SMALLER) { assertTrue(currentSegmentIndexSize < localSegmentSize) } else { assertTrue(check_size_equal(currentSegmentIndexSize, localSegmentSize)) - } + }*/ assertTrue(currentLocalIndexSize != 0) assertTrue(currentSegmentIndexSize != 0) localIndexSize = currentLocalIndexSize @@ -370,6 +380,12 @@ suite("test_show_index_data", "p1") { executor.shutdown() executor.awaitTermination(30, TimeUnit.MINUTES) + Awaitility.await().atMost(30, TimeUnit.MINUTES).untilAsserted(() -> { + sql """ SYNC """ + def count = sql "select count(*) from ${show_table_name}" + assertTrue(count[0][0] > 0) + }) + // 2. check show data check_show_data.call(FileSizeChange.LARGER, FileSizeChange.LARGER) From 4130b8e22e6d38995f778a7b7b23b0aa02075f6c Mon Sep 17 00:00:00 2001 From: airborne12 Date: Wed, 23 Jul 2025 09:45:35 +0800 Subject: [PATCH 290/572] branch-3.0: [fix](inverted index) set ENABLE_ADD_INDEX_FOR_NEW_DATA session var need forward flag to true #53690 (#53728) cherry pick from #53690 --- .../src/main/java/org/apache/doris/qe/SessionVariable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 806f23bc8d90f1..4dcb828dc84483 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2511,7 +2511,7 @@ public boolean isEnableSortSpill() { return enableSortSpill; } - @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true, description = { + @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, needForward = true, description = { "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引", "Whether to enable add index mode that only affects new data, " + "when enabled new indexes only affect subsequently written data, " From 5e8d460f0e779743349743b26de10e51441079d9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:49:42 +0800 Subject: [PATCH 291/572] branch-3.0: [fix](load) stream load should not overwrite existing error status in filtered rows check #53393 (#53695) Cherry-picked from #53393 Co-authored-by: Kaijie Chen --- be/src/runtime/stream_load/stream_load_executor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/runtime/stream_load/stream_load_executor.cpp b/be/src/runtime/stream_load/stream_load_executor.cpp index 1c1fa992f50d8b..65b428e4b4fbc1 100644 --- a/be/src/runtime/stream_load/stream_load_executor.cpp +++ b/be/src/runtime/stream_load/stream_load_executor.cpp @@ -86,7 +86,7 @@ Status StreamLoadExecutor::execute_plan_fragment(std::shared_ptrloaded_bytes = state->num_bytes_load_total(); int64_t num_selected_rows = ctx->number_total_rows - ctx->number_unselected_rows; ctx->error_url = to_load_error_http_path(state->get_error_log_file_path()); - if (!ctx->group_commit && num_selected_rows > 0 && + if (status->ok() && !ctx->group_commit && num_selected_rows > 0 && (double)ctx->number_filtered_rows / num_selected_rows > ctx->max_filter_ratio) { // NOTE: Do not modify the error message here, for historical reasons, // some users may rely on this error message. From 71562efcde997cf747ff26d00c46bf13fb48b7c0 Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 23 Jul 2025 09:50:37 +0800 Subject: [PATCH 292/572] branch-3.0: [fix](cloud) Fix modify cluster network not work in follower introduced by #52294 #53568 (#53704) cherry pick from #53568 --- .../cloud/catalog/CloudClusterChecker.java | 8 +- .../multi_cluster/test_change_node_net.groovy | 103 ++++++++++-------- 2 files changed, 62 insertions(+), 49 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java index ca81b165cb96ff..a87582e1d835de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudClusterChecker.java @@ -209,7 +209,8 @@ private void updateIfComputeNodeEndpointChanged(ClusterPB remoteClusterPb, Backe boolean netChanged = false; String remotePublicEndpoint = remoteClusterPb.getPublicEndpoint(); String localPublicEndpoint = be.getTagMap().get(Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT); - if (!localPublicEndpoint.equals(remotePublicEndpoint)) { + if ((localPublicEndpoint == null && !Strings.isNullOrEmpty(remotePublicEndpoint)) + || (localPublicEndpoint != null && !localPublicEndpoint.equals(remotePublicEndpoint))) { LOG.info("be {} has changed public_endpoint from {} to {}", be, localPublicEndpoint, remotePublicEndpoint); be.getTagMap().put(Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, remotePublicEndpoint); @@ -218,7 +219,8 @@ private void updateIfComputeNodeEndpointChanged(ClusterPB remoteClusterPb, Backe String remotePrivateEndpoint = remoteClusterPb.getPrivateEndpoint(); String localPrivateEndpoint = be.getTagMap().get(Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT); - if (!localPrivateEndpoint.equals(remotePrivateEndpoint)) { + if (localPrivateEndpoint == null && !Strings.isNullOrEmpty(remotePrivateEndpoint) + || (localPrivateEndpoint != null && !localPrivateEndpoint.equals(remotePrivateEndpoint))) { LOG.info("be {} has changed private_endpoint from {} to {}", be, localPrivateEndpoint, remotePrivateEndpoint); be.getTagMap().put(Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, remotePrivateEndpoint); @@ -226,7 +228,7 @@ private void updateIfComputeNodeEndpointChanged(ClusterPB remoteClusterPb, Backe } if (netChanged) { // edit log - Env.getCurrentEnv().getEditLog().logBackendStateChange(be); + Env.getCurrentEnv().getEditLog().logModifyBackend(be); } } diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy index 3cc6991fa207de..991e73cd9c0f66 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_change_node_net.groovy @@ -20,11 +20,19 @@ import groovy.json.JsonSlurper import groovy.json.JsonOutput suite('test_change_node_net', 'multi_cluster,docker') { - def options = new ClusterOptions() - options.feConfigs += [ - 'cloud_cluster_check_interval_second=5', + def clusterOptions = [ + new ClusterOptions(), + new ClusterOptions(), ] - options.cloudMode = true + for (options in clusterOptions) { + options.feConfigs += [ + 'cloud_cluster_check_interval_second=5', + ] + options.feNum = 3 + options.cloudMode = true + } + clusterOptions[0].connectToFollower = true + clusterOptions[1].connectToFollower = false def token = "greedisgood9999" def update_cluster_endpoint_api = { msHttpPort, request_body, check_func -> @@ -37,6 +45,8 @@ suite('test_change_node_net', 'multi_cluster,docker') { } def showClusterBackends = { clusterName -> + // The new optimizer has a bug, all queries are forwarded to the master. Including show backend + sql """set forward_to_master=false""" def bes = sql_return_maparray "show backends" def clusterBes = bes.findAll { be -> be.Tag.contains(clusterName) } def backendMap = clusterBes.collectEntries { be -> @@ -45,58 +55,59 @@ suite('test_change_node_net', 'multi_cluster,docker') { logger.info("Collected BackendId and Tag map: {}", backendMap) backendMap } + for (options in clusterOptions) { + docker(options) { + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + logger.info("ms addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) - docker(options) { - def ms = cluster.getAllMetaservices().get(0) - def msHttpPort = ms.host + ":" + ms.httpPort - logger.info("ms addr={}, port={}, ms endpoint={}", ms.host, ms.httpPort, msHttpPort) - - def clusterName = "newcluster1" - // 添加一个新的cluster add_new_cluster - cluster.addBackend(3, clusterName) - - def result = sql """show clusters""" - logger.info("show cluster1 : {}", result) + def clusterName = "newcluster1" + // 添加一个新的cluster add_new_cluster + cluster.addBackend(3, clusterName) + + def result = sql """show clusters""" + logger.info("show cluster1 : {}", result) - def beforeBackendMap = showClusterBackends.call(clusterName) + def beforeBackendMap = showClusterBackends.call(clusterName) - def tag = beforeBackendMap.entrySet().iterator().next().Value - assertNotNull(tag) - def jsonSlurper = new JsonSlurper() - def jsonObject = jsonSlurper.parseText(tag) - def cloudUniqueId = jsonObject.cloud_unique_id - def clusterId = jsonObject.compute_group_id - def before_public_endpoint = jsonObject.public_endpoint - def after_private_endpoint = jsonObject.private_endpoint + def tag = beforeBackendMap.entrySet().iterator().next().Value + assertNotNull(tag) + def jsonSlurper = new JsonSlurper() + def jsonObject = jsonSlurper.parseText(tag) + def cloudUniqueId = jsonObject.cloud_unique_id + def clusterId = jsonObject.compute_group_id + def before_public_endpoint = jsonObject.public_endpoint + def after_private_endpoint = jsonObject.private_endpoint - def changeCluster = [cluster_id: "${clusterId}", public_endpoint: "test_public_endpoint", private_endpoint: "test_private_endpoint"] - def updateClusterEndpointBody = [cloud_unique_id: "${cloudUniqueId}", cluster: changeCluster] - def jsonOutput = new JsonOutput() - def updateClusterEndpointJson = jsonOutput.toJson(updateClusterEndpointBody) + def changeCluster = [cluster_id: "${clusterId}", public_endpoint: "test_public_endpoint", private_endpoint: "test_private_endpoint"] + def updateClusterEndpointBody = [cloud_unique_id: "${cloudUniqueId}", cluster: changeCluster] + def jsonOutput = new JsonOutput() + def updateClusterEndpointJson = jsonOutput.toJson(updateClusterEndpointBody) - update_cluster_endpoint_api.call(msHttpPort, updateClusterEndpointJson) { - respCode, body -> - def json = parseJson(body) - log.info("update cluster endpoint result: ${body} ${respCode} ${json}".toString()) - } + update_cluster_endpoint_api.call(msHttpPort, updateClusterEndpointJson) { + respCode, body -> + def json = parseJson(body) + log.info("update cluster endpoint result: ${body} ${respCode} ${json}".toString()) + } - def futrue = thread { - // check 15s - for (def i = 0; i < 15; i++) { - def afterBackendMap = showClusterBackends.call(clusterName) - if (i > 5) { - // cloud_cluster_check_interval_second = 5 - afterBackendMap.each { key, value -> - assert value.contains("test_public_endpoint") : "Value for key ${key} does not contain 'test_public_endpoint'" - assert value.contains("test_private_endpoint") : "Value for key ${key} does not contain 'test_private_endpoint'" + def futrue = thread { + // check 15s + for (def i = 0; i < 15; i++) { + def afterBackendMap = showClusterBackends.call(clusterName) + if (i > 5) { + // cloud_cluster_check_interval_second = 5 + afterBackendMap.each { key, value -> + assert value.contains("test_public_endpoint") : "Value for key ${key} does not contain 'test_public_endpoint'" + assert value.contains("test_private_endpoint") : "Value for key ${key} does not contain 'test_private_endpoint'" + } } + // check beid not changed + assertEquals(afterBackendMap.keySet(), beforeBackendMap.keySet()) + sleep(1 * 1000) } - // check beid not changed - assertEquals(afterBackendMap.keySet(), beforeBackendMap.keySet()) - sleep(1 * 1000) } + futrue.get() } - futrue.get() } } From 0bc9f88229bfba5266bae6cefbeb9a7c97502ace Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:51:27 +0800 Subject: [PATCH 293/572] branch-3.0: [fix](warmup) prevent NPE when upgrading from older versions #53555 (#53666) Cherry-picked from #53555 Co-authored-by: Kaijie Chen --- .../src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java index 0bd655893a7a7c..aacec3f3f997f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java @@ -393,7 +393,7 @@ public List getJobInfo() { info.add(Long.toString(maxBatchSize)); info.add(TimeUtils.longToTimeStringWithms(finishedTimeMs)); info.add(errMsg); - info.add(tables.stream() + info.add(tables == null ? "" : tables.stream() .map(t -> StringUtils.isEmpty(t.getRight()) ? t.getLeft() + "." + t.getMiddle() : t.getLeft() + "." + t.getMiddle() + "." + t.getRight()) From 7844a651071e30a3408a9086ed7fe1b597529029 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:52:06 +0800 Subject: [PATCH 294/572] branch-3.0: [chore](load) optimize show create load error message #53694 (#53730) Cherry-picked from #53694 Co-authored-by: hui lai --- .../apache/doris/load/loadv2/LoadManager.java | 5 ++- .../broker_load/test_show_create_load.groovy | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/load_p0/broker_load/test_show_create_load.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java index f661513cd66558..89f7df1fcf8e56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java @@ -553,7 +553,10 @@ public List> getCreateLoadStmt(long dbId, String label) throw throw new DdlException("Label does not exist: " + label); } } else { - throw new DdlException("Database does not exist"); + // If dbId is not found in dbIdToLabelToLoadJobs, + // it means the database has no label records, + // so throw a "Label does not exist" error. + throw new DdlException("Label does not exist: " + label); } return result; } finally { diff --git a/regression-test/suites/load_p0/broker_load/test_show_create_load.groovy b/regression-test/suites/load_p0/broker_load/test_show_create_load.groovy new file mode 100644 index 00000000000000..8881e35ef8e75c --- /dev/null +++ b/regression-test/suites/load_p0/broker_load/test_show_create_load.groovy @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_create_load", "load_p0") { + // test unknown db + try { + sql "show create load for unknown_db.test_label" + } catch (Exception e) { + logger.info("result: ${e.message}") + assertTrue(e.message.contains(" Unknown database 'unknown_db'")) + } + + // test unknown label + try { + sql "show create load for ${context.dbName}.unknown_label" + } catch (Exception e) { + logger.info("result: ${e.message}") + assertTrue(e.message.contains("Label does not exist: unknown_label")) + } +} \ No newline at end of file From 4aabc77fa07ef64c9598e7c5033798c1247b358d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:52:48 +0800 Subject: [PATCH 295/572] branch-3.0: [Fix](case) Fix `test_table_properties`/`test_primary_key_partial_update_complex_type` #53701 (#53716) Cherry-picked from #53701 Co-authored-by: bobhan1 --- .../suites/query_p0/system/test_table_properties.groovy | 1 + .../partial_update/test_partial_update_complex_type.groovy | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/query_p0/system/test_table_properties.groovy b/regression-test/suites/query_p0/system/test_table_properties.groovy index 7465497ae76d08..766bcf86fe391d 100644 --- a/regression-test/suites/query_p0/system/test_table_properties.groovy +++ b/regression-test/suites/query_p0/system/test_table_properties.groovy @@ -36,6 +36,7 @@ suite("test_table_properties") { UNIQUE KEY(`user_id`, `username`) DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 PROPERTIES ( + "enable_mow_light_delete" = "false", "replication_allocation" = "tag.location.default: 1" ); """ diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy index 502c4284ebf75a..2ae6e741385e82 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_complex_type.groovy @@ -131,7 +131,8 @@ suite("test_primary_key_partial_update_complex_type", "p0") { `c_map` MAP not null) UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "true", - "store_row_column" = "${use_row_store}","disable_auto_compaction" = "true"); """ + "store_row_column" = "${use_row_store}","disable_auto_compaction" = "true", + "enable_mow_light_delete" = "false"); """ sql """insert into ${tableName2} values(2, "doris2", '{"jsonk2": 333, "jsonk4": 444}', [300, 400], {3, 4}, {'a': 2})""" sql """insert into ${tableName2} values(1, "doris1", '{"jsonk1": 123, "jsonk2": 456}', [100, 200], {1, 2}, {'b': 3})""" From 3f7daf01b219e56f822da6c2efd139cb67340db4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 09:53:29 +0800 Subject: [PATCH 296/572] branch-3.0: [test](inverted index) make test more stable for test_write_inverted_index_exception_fault_injection #53686 (#53715) Cherry-picked from #53686 Co-authored-by: airborne12 --- .../test_write_inverted_index_exception_fault_injection.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy index 52ce8f87a7443a..2c421c2ab42de1 100644 --- a/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_write_inverted_index_exception_fault_injection.groovy @@ -91,6 +91,7 @@ suite("test_write_inverted_index_exception_fault_injection", "nonConcurrent") { sql """ INSERT INTO ${tableName} VALUES (2, "bason", 11, [79, 85, 97], ["singing", "dancing"], "bason is good at singing", ["bason is very clever", "bason is very healthy"]); """ sql """ INSERT INTO ${tableName} VALUES (3, "andy", 10, [89, 80, 98], ["football", "basketball"], "andy is good at sports", ["andy has a good heart", "andy is so nice"]); """ sql """ INSERT INTO ${tableName} VALUES (3, "bason", 11, [79, 85, 97], ["singing", "dancing"], "bason is good at singing", ["bason is very clever", "bason is very healthy"]); """ + sql """ SYNC """ } def check_count = { String tableName, int count -> From 3fdefdaf5c3cf321e612d71249124a9487c9f9c0 Mon Sep 17 00:00:00 2001 From: deardeng <565620795@qq.com> Date: Wed, 23 Jul 2025 09:55:52 +0800 Subject: [PATCH 297/572] [fix](docker case) Fix some docker case (#53404) (#53717) ### What problem does this PR solve? pick from https://github.com/apache/doris/pull/53404/files --- cloud/src/common/config.h | 1 + cloud/src/resource-manager/resource_manager.cpp | 6 ++++-- docker/runtime/doris-compose/cluster.py | 2 +- .../cloud_p0/multi_cluster/test_multi_follower.groovy | 5 ++++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 6fd933d3a152a6..b767d696034322 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -320,4 +320,5 @@ CONF_mString(ca_cert_file_paths, "/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;" "/etc/ssl/ca-bundle.pem"); +CONF_Bool(enable_check_fe_drop_in_safe_time, "true"); } // namespace doris::cloud::config diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp index 0e4133d0198e45..b6f184929ac4f4 100644 --- a/cloud/src/resource-manager/resource_manager.cpp +++ b/cloud/src/resource-manager/resource_manager.cpp @@ -687,7 +687,8 @@ std::pair ResourceManager::drop_cluster( if (i.type() == ClusterPB::SQL) { for (auto& fe_node : i.nodes()) { // check drop fe cluster - if (!is_sql_node_exceeded_safe_drop_time(fe_node)) { + if (config::enable_check_fe_drop_in_safe_time && + !is_sql_node_exceeded_safe_drop_time(fe_node)) { ss << "drop fe cluster not in safe time, try later, cluster=" << i.DebugString(); msg = ss.str(); @@ -1350,7 +1351,8 @@ std::string ResourceManager::modify_nodes(const std::string& instance_id, } // check drop fe node - if (ClusterPB::SQL == c.type() && !is_sql_node_exceeded_safe_drop_time(copy_node)) { + if (ClusterPB::SQL == c.type() && config::enable_check_fe_drop_in_safe_time && + !is_sql_node_exceeded_safe_drop_time(copy_node)) { s << "drop fe node not in safe time, try later, node=" << copy_node.DebugString(); err = s.str(); LOG(WARNING) << err; diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py index 00fe9ddc3866c1..8d4d65097e9a2f 100644 --- a/docker/runtime/doris-compose/cluster.py +++ b/docker/runtime/doris-compose/cluster.py @@ -547,7 +547,7 @@ def docker_env(self): if self.cluster.is_cloud: envs["CLOUD_UNIQUE_ID"] = self.cloud_unique_id() if self.meta["is_cloud_follower"]: - envs["is_fe_follower"] = 1 + envs["IS_FE_FOLLOWER"] = 1 envs["MY_QUERY_PORT"] = self.meta["ports"]["query_port"] envs["MY_EDITLOG_PORT"] = self.meta["ports"]["edit_log_port"] return envs diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy index 1b7f33fdb1dfc1..d524eb08bd1f67 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_multi_follower.groovy @@ -30,6 +30,9 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { 'sys_log_verbose_modules=org', 'heartbeat_interval_second=1' ] + options.msConfigs += [ + 'enable_check_fe_drop_in_safe_time=false' + ] options.setFeNum(3) options.setBeNum(1) options.cloudMode = true @@ -99,7 +102,7 @@ suite('test_multi_followr_in_cloud', 'multi_cluster, docker') { logger.info("ms addr={}, port={}", ms.host, ms.httpPort) // drop a follwer def findToDropUniqueId = { clusterId, hostIP, metaServices -> - ret = get_instance(metaServices) + def ret = get_instance(metaServices) def toDropCluster = ret.clusters.find { it.cluster_id.contains(clusterId) } From d0957aec321e882a27a56bec63313befbcbbe83e Mon Sep 17 00:00:00 2001 From: starocean999 Date: Wed, 23 Jul 2025 09:57:25 +0800 Subject: [PATCH 298/572] [enhancement](nereids)remove subqueryExpr member from LogicalApply (#53183) ### What problem does this PR solve? pick from master https://github.com/apache/doris/pull/50378 --- .../rules/analysis/SubqueryToApply.java | 18 +- .../rules/rewrite/ExistsApplyToJoin.java | 5 +- .../nereids/rules/rewrite/InApplyToJoin.java | 21 +- .../rules/rewrite/PullUpCteAnchor.java | 24 +- .../rewrite/PullUpProjectUnderApply.java | 3 +- .../rules/rewrite/ScalarApplyToJoin.java | 2 +- .../UnCorrelatedApplyAggregateFilter.java | 3 +- .../rewrite/UnCorrelatedApplyFilter.java | 3 +- .../UnCorrelatedApplyProjectFilter.java | 3 +- .../trees/copier/LogicalPlanDeepCopier.java | 10 +- .../trees/plans/logical/LogicalApply.java | 104 ++- .../rules/rewrite/ExistsApplyToJoinTest.java | 17 +- .../nereids_p0/test_cte_subquery_many.out | 4 + .../nereids_p0/test_cte_subquery_many.groovy | 696 ++++++++++++++++++ 14 files changed, 827 insertions(+), 86 deletions(-) create mode 100644 regression-test/data/nereids_p0/test_cte_subquery_many.out create mode 100644 regression-test/suites/nereids_p0/test_cte_subquery_many.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java index cacd3d9cd6ce02..27e6f446682da9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; @@ -471,9 +472,24 @@ private Pair> addApply(SubqueryExpr subquery, needRuntimeAssertCount = true; } } + LogicalApply.SubQueryType subQueryType; + boolean isNot = false; + Optional compareExpr = Optional.empty(); + if (subquery instanceof InSubquery) { + subQueryType = LogicalApply.SubQueryType.IN_SUBQUERY; + isNot = ((InSubquery) subquery).isNot(); + compareExpr = Optional.of(((InSubquery) subquery).getCompareExpr()); + } else if (subquery instanceof Exists) { + subQueryType = LogicalApply.SubQueryType.EXITS_SUBQUERY; + isNot = ((Exists) subquery).isNot(); + } else if (subquery instanceof ScalarSubquery) { + subQueryType = LogicalApply.SubQueryType.SCALAR_SUBQUERY; + } else { + throw new AnalysisException(String.format("Unsupported subquery : %s", subquery.toString())); + } LogicalApply newApply = new LogicalApply( subquery.getCorrelateSlots(), - subquery, Optional.empty(), + subQueryType, isNot, compareExpr, subquery.getTypeCoercionExpr(), Optional.empty(), markJoinSlot, needAddScalarSubqueryOutputToProjects, isProject, isMarkJoinSlotNotNull, childPlan, subquery.getQueryPlan()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java index 930e4c467dfe1a..683833248560e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoin.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; @@ -94,7 +93,7 @@ public Rule build() { private Plan correlatedToJoin(LogicalApply apply) { Optional correlationFilter = apply.getCorrelationFilter(); - if (((Exists) apply.getSubqueryExpr()).isNot()) { + if (apply.isNot()) { return new LogicalJoin<>(JoinType.LEFT_ANTI_JOIN, ExpressionUtils.EMPTY_CONDITION, correlationFilter.map(ExpressionUtils::extractConjunction).orElse(ExpressionUtils.EMPTY_CONDITION), new DistributeHint(DistributeType.NONE), @@ -110,7 +109,7 @@ private Plan correlatedToJoin(LogicalApply apply) { } private Plan unCorrelatedToJoin(LogicalApply unapply) { - if (((Exists) unapply.getSubqueryExpr()).isNot()) { + if (unapply.isNot()) { return unCorrelatedNotExist(unapply); } else { return unCorrelatedExist(unapply); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java index 67276e92bb31d2..a004b7fc569e95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/InApplyToJoin.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.IsNull; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Not; @@ -36,7 +35,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -82,9 +80,9 @@ select t1.k1 from bigtable t1 left semi join (select bitmap_union(k2) x from bit List outputExpressions = Lists.newArrayList(alias); LogicalAggregate agg = new LogicalAggregate(groupExpressions, outputExpressions, apply.right()); - Expression compareExpr = ((InSubquery) apply.getSubqueryExpr()).getCompareExpr(); + Expression compareExpr = apply.getCompareExpr().get(); Expression expr = new BitmapContains(agg.getOutput().get(0), compareExpr); - if (((InSubquery) apply.getSubqueryExpr()).isNot()) { + if (apply.isNot()) { expr = new Not(expr); } return new LogicalJoin<>(JoinType.LEFT_SEMI_JOIN, Lists.newArrayList(), @@ -95,19 +93,18 @@ select t1.k1 from bigtable t1 left semi join (select bitmap_union(k2) x from bit } //in-predicate to equal - InSubquery inSubquery = ((InSubquery) apply.getSubqueryExpr()); Expression predicate; - Expression left = inSubquery.getCompareExpr(); + Expression left = apply.getCompareExpr().get(); // TODO: trick here, because when deep copy logical plan the apply right child // is not same with query plan in subquery expr, since the scan node copy twice - Expression right = inSubquery.getSubqueryOutput((LogicalPlan) apply.right()); + Expression right = apply.getSubqueryOutput(); if (apply.isMarkJoin()) { List joinConjuncts = apply.getCorrelationFilter().isPresent() ? ExpressionUtils.extractConjunction(apply.getCorrelationFilter().get()) : Lists.newArrayList(); predicate = new EqualTo(left, right); List markConjuncts = Lists.newArrayList(predicate); - if (!predicate.nullable() || (apply.isMarkJoinSlotNotNull() && !inSubquery.isNot())) { + if (!predicate.nullable() || (apply.isMarkJoinSlotNotNull() && !apply.isNot())) { // we can merge mark conjuncts with hash conjuncts in 2 scenarios // 1. the mark join predicate is not nullable, so no null value would be produced // 2. semi join with non-nullable mark slot. @@ -117,7 +114,7 @@ select t1.k1 from bigtable t1 left semi join (select bitmap_union(k2) x from bit markConjuncts.clear(); } return new LogicalJoin<>( - inSubquery.isNot() ? JoinType.LEFT_ANTI_JOIN : JoinType.LEFT_SEMI_JOIN, + apply.isNot() ? JoinType.LEFT_ANTI_JOIN : JoinType.LEFT_SEMI_JOIN, Lists.newArrayList(), joinConjuncts, markConjuncts, new DistributeHint(DistributeType.NONE), apply.getMarkJoinSlotReference(), apply.children(), null); @@ -127,7 +124,7 @@ select t1.k1 from bigtable t1 left semi join (select bitmap_union(k2) x from bit // so we need check both correlated slot and correlation filter exists // before creating LogicalJoin node if (apply.isCorrelated() && apply.getCorrelationFilter().isPresent()) { - if (inSubquery.isNot()) { + if (apply.isNot()) { predicate = ExpressionUtils.and(ExpressionUtils.or(new EqualTo(left, right), new IsNull(left), new IsNull(right)), apply.getCorrelationFilter().get()); @@ -140,7 +137,7 @@ select t1.k1 from bigtable t1 left semi join (select bitmap_union(k2) x from bit } List conjuncts = ExpressionUtils.extractConjunction(predicate); - if (inSubquery.isNot()) { + if (apply.isNot()) { return new LogicalJoin<>( predicate.nullable() && !apply.isCorrelated() ? JoinType.NULL_AWARE_LEFT_ANTI_JOIN @@ -159,6 +156,6 @@ select t1.k1 from bigtable t1 left semi join (select bitmap_union(k2) x from bit private boolean needBitmapUnion(LogicalApply apply) { return apply.right().getOutput().get(0).getDataType().isBitmapType() - && !((InSubquery) apply.getSubqueryExpr()).getCompareExpr().getDataType().isBitmapType(); + && !apply.getCompareExpr().get().getDataType().isBitmapType(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java index 4bfb36db8134c1..09b7098b9339c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpCteAnchor.java @@ -18,12 +18,10 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; @@ -69,7 +67,10 @@ public LogicalCTEProducer visitLogicalCTEProducer(LogicalCTEProducer> producers) { List> childProducers = Lists.newArrayList(); Plan child = cteProducer.child().accept(this, childProducers); - LogicalCTEProducer newProducer = (LogicalCTEProducer) cteProducer.withChildren(child); + LogicalCTEProducer newProducer = (LogicalCTEProducer) cteProducer; + if (child != cteProducer.child()) { + newProducer = (LogicalCTEProducer) cteProducer.withChildren(child); + } // because current producer relay on it child's producers, so add current producer first. producers.add(newProducer); producers.addAll(childProducers); @@ -79,17 +80,7 @@ public LogicalCTEProducer visitLogicalCTEProducer(LogicalCTEProducer apply, List> producers) { - SubqueryExpr subqueryExpr = apply.getSubqueryExpr(); - PullUpCteAnchor pullSubqueryExpr = new PullUpCteAnchor(); - List> subqueryExprProducers = Lists.newArrayList(); - Plan newPlanInExpr = pullSubqueryExpr.rewriteRoot(subqueryExpr.getQueryPlan(), subqueryExprProducers); - while (newPlanInExpr instanceof LogicalCTEAnchor) { - newPlanInExpr = ((LogicalCTEAnchor) newPlanInExpr).right(); - } - SubqueryExpr newSubqueryExpr = subqueryExpr.withSubquery((LogicalPlan) newPlanInExpr); - Plan newApplyLeft = apply.left().accept(this, producers); - Plan applyRight = apply.right(); PullUpCteAnchor pullApplyRight = new PullUpCteAnchor(); List> childProducers = Lists.newArrayList(); @@ -98,7 +89,10 @@ public Plan visitLogicalApply(LogicalApply apply newApplyRight = ((LogicalCTEAnchor) newApplyRight).right(); } producers.addAll(childProducers); - return apply.withSubqueryExprAndChildren(newSubqueryExpr, - ImmutableList.of(newApplyLeft, newApplyRight)); + if (newApplyLeft != apply.left() || newApplyRight != apply.right()) { + return apply.withChildren(ImmutableList.of(newApplyLeft, newApplyRight)); + } else { + return apply; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderApply.java index b2398ee3b566dc..e428e02e1e09f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpProjectUnderApply.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; @@ -60,7 +59,7 @@ public Rule build() { LogicalProject project = apply.right(); Plan newCorrelate = apply.withChildren(apply.left(), project.child()); List newProjects = new ArrayList<>(apply.left().getOutput()); - if (apply.getSubqueryExpr() instanceof ScalarSubquery) { + if (apply.isScalar()) { // unnest correlated scalar subquery may add count(*) and any_value() to project list // the previous SubqueryToApply rule will make sure of it. So the output column // may be 1 or 2, we add a check here. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java index 5cb11914b66d6b..34bb5723a727a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ScalarApplyToJoin.java @@ -58,7 +58,7 @@ public Rule build() { private Plan unCorrelatedToJoin(LogicalApply apply) { LogicalAssertNumRows assertNumRows = new LogicalAssertNumRows<>(new AssertNumRowsElement(1, - apply.getSubqueryExpr().toString(), AssertNumRowsElement.Assertion.EQ), + apply.right().toString(), AssertNumRowsElement.Assertion.EQ), (LogicalPlan) apply.right()); return new LogicalJoin<>(JoinType.CROSS_JOIN, ExpressionUtils.EMPTY_CONDITION, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java index f4acd8476528c5..258698b1f7f8e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyAggregateFilter.java @@ -115,7 +115,8 @@ public List buildRules() { correlatedPredicate = ExpressionUtils.replace(correlatedPredicate, unCorrelatedExprToSlot); LogicalAggregate newAgg = new LogicalAggregate<>(newGroupby, newAggOutput, PlanUtils.filterOrSelf(ImmutableSet.copyOf(unCorrelatedPredicate), filter.child())); - return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), + return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryType(), apply.isNot(), + apply.getCompareExpr(), apply.getTypeCoercionExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(), apply.isNeedAddSubOutputToProjects(), apply.isInProject(), apply.isMarkJoinSlotNotNull(), apply.left(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java index 30b5cfcef3ee36..b5732a604cad1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyFilter.java @@ -66,7 +66,8 @@ public Rule build() { } Plan child = PlanUtils.filterOrSelf(ImmutableSet.copyOf(unCorrelatedPredicate), filter.child()); - return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), + return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryType(), apply.isNot(), + apply.getCompareExpr(), apply.getTypeCoercionExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(), apply.isNeedAddSubOutputToProjects(), apply.isInProject(), apply.isMarkJoinSlotNotNull(), apply.left(), child); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java index 82950e14dc7b0a..4f31d672a16fa1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/UnCorrelatedApplyProjectFilter.java @@ -87,7 +87,8 @@ public Rule build() { .map(NamedExpression.class::cast) .forEach(projects::add); LogicalProject newProject = project.withProjectsAndChild(projects, child); - return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryExpr(), + return new LogicalApply<>(apply.getCorrelationSlot(), apply.getSubqueryType(), apply.isNot(), + apply.getCompareExpr(), apply.getTypeCoercionExpr(), ExpressionUtils.optionalAnd(correlatedPredicate), apply.getMarkJoinSlotReference(), apply.isNeedAddSubOutputToProjects(), apply.isInProject(), apply.isMarkJoinSlotNotNull(), apply.left(), newProject); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index c499c2dddb9407..e7b755a6d53ebf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; -import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; @@ -129,13 +128,16 @@ public Plan visitLogicalApply(LogicalApply apply List correlationSlot = apply.getCorrelationSlot().stream() .map(s -> ExpressionDeepCopier.INSTANCE.deepCopy(s, context)) .collect(ImmutableList.toImmutableList()); - SubqueryExpr subqueryExpr = (SubqueryExpr) ExpressionDeepCopier.INSTANCE - .deepCopy(apply.getSubqueryExpr(), context); + Optional compareExpr = apply.getCompareExpr() + .map(f -> ExpressionDeepCopier.INSTANCE.deepCopy(f, context)); + Optional typeCoercionExpr = apply.getTypeCoercionExpr() + .map(f -> ExpressionDeepCopier.INSTANCE.deepCopy(f, context)); Optional correlationFilter = apply.getCorrelationFilter() .map(f -> ExpressionDeepCopier.INSTANCE.deepCopy(f, context)); Optional markJoinSlotReference = apply.getMarkJoinSlotReference() .map(m -> (MarkJoinSlotReference) ExpressionDeepCopier.INSTANCE.deepCopy(m, context)); - return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter, + return new LogicalApply<>(correlationSlot, apply.getSubqueryType(), apply.isNot(), + compareExpr, typeCoercionExpr, correlationFilter, markJoinSlotReference, apply.isNeedAddSubOutputToProjects(), apply.isInProject(), apply.isMarkJoinSlotNotNull(), left, right); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java index 517048c209aba4..0b12e225311668 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalApply.java @@ -19,13 +19,9 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.InSubquery; import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference; -import org.apache.doris.nereids.trees.expressions.ScalarSubquery; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; @@ -46,11 +42,26 @@ */ public class LogicalApply extends LogicalBinary implements PropagateFuncDeps { + /** + * SubQueryType + */ + public enum SubQueryType { + IN_SUBQUERY, + EXITS_SUBQUERY, + SCALAR_SUBQUERY + } + + private final SubQueryType subqueryType; + private final boolean isNot; + + // only for InSubquery + private final Optional compareExpr; + + // only for InSubquery + private final Optional typeCoercionExpr; // correlation column private final List correlationSlot; - // original subquery - private final SubqueryExpr subqueryExpr; // correlation Conjunction private final Optional correlationFilter; // The slot replaced by the subquery in MarkJoin @@ -72,16 +83,23 @@ public class LogicalApply groupExpression, Optional logicalProperties, - List correlationSlot, - SubqueryExpr subqueryExpr, Optional correlationFilter, + List correlationSlot, SubQueryType subqueryType, boolean isNot, + Optional compareExpr, Optional typeCoercionExpr, + Optional correlationFilter, Optional markJoinSlotReference, boolean needAddSubOutputToProjects, boolean inProject, boolean isMarkJoinSlotNotNull, LEFT_CHILD_TYPE leftChild, RIGHT_CHILD_TYPE rightChild) { super(PlanType.LOGICAL_APPLY, groupExpression, logicalProperties, leftChild, rightChild); + if (subqueryType == SubQueryType.IN_SUBQUERY) { + Preconditions.checkArgument(compareExpr.isPresent(), "InSubquery must have compareExpr"); + } this.correlationSlot = correlationSlot == null ? ImmutableList.of() : ImmutableList.copyOf(correlationSlot); - this.subqueryExpr = Objects.requireNonNull(subqueryExpr, "subquery can not be null"); + this.subqueryType = subqueryType; + this.isNot = isNot; + this.compareExpr = compareExpr; + this.typeCoercionExpr = typeCoercionExpr; this.correlationFilter = correlationFilter; this.markJoinSlotReference = markJoinSlotReference; this.needAddSubOutputToProjects = needAddSubOutputToProjects; @@ -89,13 +107,26 @@ private LogicalApply(Optional groupExpression, this.isMarkJoinSlotNotNull = isMarkJoinSlotNotNull; } - public LogicalApply(List correlationSlot, SubqueryExpr subqueryExpr, + public LogicalApply(List correlationSlot, SubQueryType subqueryType, boolean isNot, + Optional compareExpr, Optional typeCoercionExpr, Optional correlationFilter, Optional markJoinSlotReference, boolean needAddSubOutputToProjects, boolean inProject, boolean isMarkJoinSlotNotNull, LEFT_CHILD_TYPE input, RIGHT_CHILD_TYPE subquery) { - this(Optional.empty(), Optional.empty(), correlationSlot, subqueryExpr, correlationFilter, - markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, input, - subquery); + this(Optional.empty(), Optional.empty(), correlationSlot, subqueryType, isNot, compareExpr, typeCoercionExpr, + correlationFilter, markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, + input, subquery); + } + + public Optional getCompareExpr() { + return compareExpr; + } + + public Optional getTypeCoercionExpr() { + return typeCoercionExpr; + } + + public Expression getSubqueryOutput() { + return typeCoercionExpr.orElseGet(() -> right().getOutput().get(0)); } public List getCorrelationSlot() { @@ -106,20 +137,24 @@ public Optional getCorrelationFilter() { return correlationFilter; } - public SubqueryExpr getSubqueryExpr() { - return subqueryExpr; - } - public boolean isScalar() { - return this.subqueryExpr instanceof ScalarSubquery; + return subqueryType == SubQueryType.SCALAR_SUBQUERY; } public boolean isIn() { - return this.subqueryExpr instanceof InSubquery; + return subqueryType == SubQueryType.IN_SUBQUERY; } public boolean isExist() { - return this.subqueryExpr instanceof Exists; + return subqueryType == SubQueryType.EXITS_SUBQUERY; + } + + public SubQueryType getSubqueryType() { + return subqueryType; + } + + public boolean isNot() { + return isNot; } public boolean isCorrelated() { @@ -181,19 +216,22 @@ public boolean equals(Object o) { } LogicalApply that = (LogicalApply) o; return Objects.equals(correlationSlot, that.getCorrelationSlot()) - && Objects.equals(subqueryExpr, that.getSubqueryExpr()) + && Objects.equals(subqueryType, that.subqueryType) + && Objects.equals(compareExpr, that.compareExpr) + && Objects.equals(typeCoercionExpr, that.typeCoercionExpr) && Objects.equals(correlationFilter, that.getCorrelationFilter()) && Objects.equals(markJoinSlotReference, that.getMarkJoinSlotReference()) && needAddSubOutputToProjects == that.needAddSubOutputToProjects && inProject == that.inProject - && isMarkJoinSlotNotNull == that.isMarkJoinSlotNotNull; + && isMarkJoinSlotNotNull == that.isMarkJoinSlotNotNull + && isNot == that.isNot; } @Override public int hashCode() { return Objects.hash( - correlationSlot, subqueryExpr, correlationFilter, - markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull); + correlationSlot, subqueryType, compareExpr, typeCoercionExpr, correlationFilter, + markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, isNot); } @Override @@ -215,33 +253,27 @@ public List getExpressions() { } } - public LogicalApply withSubqueryExprAndChildren(SubqueryExpr subqueryExpr, List children) { - return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter, - markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, - children.get(0), children.get(1)); - } - @Override public LogicalApply withChildren(List children) { Preconditions.checkArgument(children.size() == 2); - return new LogicalApply<>(correlationSlot, subqueryExpr, correlationFilter, - markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, + return new LogicalApply<>(correlationSlot, subqueryType, isNot, compareExpr, typeCoercionExpr, + correlationFilter, markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, children.get(0), children.get(1)); } @Override public Plan withGroupExpression(Optional groupExpression) { return new LogicalApply<>(groupExpression, Optional.of(getLogicalProperties()), - correlationSlot, subqueryExpr, correlationFilter, markJoinSlotReference, - needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, left(), right()); + correlationSlot, subqueryType, isNot, compareExpr, typeCoercionExpr, correlationFilter, + markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, left(), right()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { Preconditions.checkArgument(children.size() == 2); - return new LogicalApply<>(groupExpression, logicalProperties, correlationSlot, subqueryExpr, - correlationFilter, markJoinSlotReference, + return new LogicalApply<>(groupExpression, logicalProperties, correlationSlot, subqueryType, isNot, + compareExpr, typeCoercionExpr, correlationFilter, markJoinSlotReference, needAddSubOutputToProjects, inProject, isMarkJoinSlotNotNull, children.get(0), children.get(1)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java index 5f617eaacc41b2..597eadf7a55b0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ExistsApplyToJoinTest.java @@ -18,7 +18,6 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.Exists; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalApply; @@ -44,10 +43,10 @@ public void testCorrelatedExists() { LogicalOlapScan right = PlanConstructor.newLogicalOlapScan(0, "t2", 1); List rightSlots = right.getOutput(); EqualTo equalTo = new EqualTo(leftSlots.get(0), rightSlots.get(0)); - Exists exists = new Exists(right, false); LogicalApply apply = new LogicalApply<>(ImmutableList.of(leftSlots.get(0), rightSlots.get(0)), - exists, Optional.of(equalTo), Optional.empty(), + LogicalApply.SubQueryType.EXITS_SUBQUERY, false, Optional.empty(), Optional.empty(), + Optional.of(equalTo), Optional.empty(), false, false, false, left, right); PlanChecker.from(MemoTestUtils.createConnectContext(), apply) .applyTopDown(new ExistsApplyToJoin()) @@ -64,10 +63,10 @@ public void testUnCorrelatedExists() { LogicalOlapScan right = PlanConstructor.newLogicalOlapScan(0, "t2", 1); List rightSlots = right.getOutput(); EqualTo equalTo = new EqualTo(leftSlots.get(0), rightSlots.get(0)); - Exists exists = new Exists(right, false); LogicalApply apply = new LogicalApply<>(Collections.emptyList(), - exists, Optional.of(equalTo), Optional.empty(), + LogicalApply.SubQueryType.EXITS_SUBQUERY, false, Optional.empty(), Optional.empty(), + Optional.of(equalTo), Optional.empty(), false, false, false, left, right); PlanChecker.from(MemoTestUtils.createConnectContext(), apply) .applyTopDown(new ExistsApplyToJoin()) @@ -84,10 +83,10 @@ public void testUnCorrelatedNotExists() { LogicalOlapScan right = PlanConstructor.newLogicalOlapScan(0, "t2", 1); List rightSlots = right.getOutput(); EqualTo equalTo = new EqualTo(leftSlots.get(0), rightSlots.get(0)); - Exists exists = new Exists(right, true); LogicalApply apply = new LogicalApply<>(Collections.emptyList(), - exists, Optional.of(equalTo), Optional.empty(), + LogicalApply.SubQueryType.EXITS_SUBQUERY, true, Optional.empty(), Optional.empty(), + Optional.of(equalTo), Optional.empty(), false, false, false, left, right); PlanChecker.from(MemoTestUtils.createConnectContext(), apply) .applyTopDown(new ExistsApplyToJoin()) @@ -105,10 +104,10 @@ public void testCorrelatedNotExists() { LogicalOlapScan right = PlanConstructor.newLogicalOlapScan(0, "t2", 1); List rightSlots = right.getOutput(); EqualTo equalTo = new EqualTo(leftSlots.get(0), rightSlots.get(0)); - Exists exists = new Exists(right, true); LogicalApply apply = new LogicalApply<>(ImmutableList.of(leftSlots.get(0), rightSlots.get(0)), - exists, Optional.of(equalTo), Optional.empty(), + LogicalApply.SubQueryType.EXITS_SUBQUERY, true, Optional.empty(), Optional.empty(), + Optional.of(equalTo), Optional.empty(), false, false, false, left, right); PlanChecker.from(MemoTestUtils.createConnectContext(), apply) .applyTopDown(new ExistsApplyToJoin()) diff --git a/regression-test/data/nereids_p0/test_cte_subquery_many.out b/regression-test/data/nereids_p0/test_cte_subquery_many.out new file mode 100644 index 00000000000000..a95e8f993c91e0 --- /dev/null +++ b/regression-test/data/nereids_p0/test_cte_subquery_many.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 10 BASE + diff --git a/regression-test/suites/nereids_p0/test_cte_subquery_many.groovy b/regression-test/suites/nereids_p0/test_cte_subquery_many.groovy new file mode 100644 index 00000000000000..fbd6a93e251e39 --- /dev/null +++ b/regression-test/suites/nereids_p0/test_cte_subquery_many.groovy @@ -0,0 +1,696 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cte_subquery_many") { + multi_sql """ + drop database if exists test_cte_subquery_many_db; + create database test_cte_subquery_many_db; + use test_cte_subquery_many_db; + CREATE TABLE IF NOT EXISTS tbl_1 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_2 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_3 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_4 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_5 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_6 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_7 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_8 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_9 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_10 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_11 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_12 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_13 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_14 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_15 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_16 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_17 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_18 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_19 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + CREATE TABLE IF NOT EXISTS tbl_20 ( + id BIGINT, + col1 VARCHAR(255), + col2 INT, + col3 DECIMAL(10,2), + col4 DATE, + col5 BOOLEAN, + related_id BIGINT, + join_key VARCHAR(50) + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES ("replication_num" = "1"); + INSERT INTO tbl_1 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_2 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_3 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_4 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_5 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_6 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_7 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_8 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_9 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_10 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_11 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_12 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_13 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_14 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_15 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_16 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_17 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_18 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_19 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + INSERT INTO tbl_20 VALUES (1, 'data1-1', 10, 15.0, '2023-01-01', TRUE, 1, 'key1'); + """ + qt_sql """ + WITH cte AS ( + SELECT + t0.id, + SUM(t0.col2) AS agg_value, + CASE + WHEN SUM(t0.col2) > 0 THEN 'BASE' + WHEN SUM(t0.col2) BETWEEN 0 AND 0 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_1 t0 + INNER JOIN tbl_6 j0 + ON t0.join_key = j0.join_key + WHERE EXISTS (SELECT sq0.id + FROM (SELECT + t1.id, + SUM(t1.col2) AS agg_value, + CASE + WHEN SUM(t1.col2) > 100 THEN 'BASE' + WHEN SUM(t1.col2) BETWEEN 50 AND 100 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_2 t1 + LEFT JOIN tbl_7 j1 + ON t1.join_key = j1.join_key + WHERE NOT EXISTS (SELECT sq1.id + FROM (SELECT + t2.id, + SUM(t2.col2) AS agg_value, + CASE + WHEN SUM(t2.col2) > 200 THEN 'BASE' + WHEN SUM(t2.col2) BETWEEN 100 AND 200 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_3 t2 + RIGHT JOIN tbl_8 j2 + ON t2.join_key = j2.join_key + WHERE t2.id IN (SELECT sq2.id + FROM (SELECT + t3.id, + SUM(t3.col2) AS agg_value, + CASE + WHEN SUM(t3.col2) > 300 THEN 'BASE' + WHEN SUM(t3.col2) BETWEEN 150 AND 300 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_4 t3 + INNER JOIN tbl_9 j3 + ON t3.join_key = j3.join_key + WHERE t3.id NOT IN (SELECT sq3.id + FROM (SELECT + t4.id, + SUM(t4.col2) AS agg_value, + CASE + WHEN SUM(t4.col2) > 400 THEN 'BASE' + WHEN SUM(t4.col2) BETWEEN 200 AND 400 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_5 t4 + LEFT JOIN tbl_10 j4 + ON t4.join_key = j4.join_key + WHERE EXISTS (SELECT sq4.id + FROM (SELECT + t5.id, + SUM(t5.col2) AS agg_value, + CASE + WHEN SUM(t5.col2) > 500 THEN 'BASE' + WHEN SUM(t5.col2) BETWEEN 250 AND 500 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_6 t5 + RIGHT JOIN tbl_11 j5 + ON t5.join_key = j5.join_key + WHERE NOT EXISTS (SELECT sq5.id + FROM (SELECT + t6.id, + SUM(t6.col2) AS agg_value, + CASE + WHEN SUM(t6.col2) > 600 THEN 'BASE' + WHEN SUM(t6.col2) BETWEEN 300 AND 600 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_7 t6 + INNER JOIN tbl_12 j6 + ON t6.join_key = j6.join_key + WHERE t6.id IN (SELECT sq6.id + FROM (SELECT + t7.id, + SUM(t7.col2) AS agg_value, + CASE + WHEN SUM(t7.col2) > 700 THEN 'BASE' + WHEN SUM(t7.col2) BETWEEN 350 AND 700 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_8 t7 + LEFT JOIN tbl_13 j7 + ON t7.join_key = j7.join_key + WHERE t7.id NOT IN (SELECT sq7.id + FROM (SELECT + t8.id, + SUM(t8.col2) AS agg_value, + CASE + WHEN SUM(t8.col2) > 800 THEN 'BASE' + WHEN SUM(t8.col2) BETWEEN 400 AND 800 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_9 t8 + RIGHT JOIN tbl_14 j8 + ON t8.join_key = j8.join_key + WHERE EXISTS (SELECT sq8.id + FROM (SELECT + t9.id, + SUM(t9.col2) AS agg_value, + CASE + WHEN SUM(t9.col2) > 900 THEN 'BASE' + WHEN SUM(t9.col2) BETWEEN 450 AND 900 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_10 t9 + INNER JOIN tbl_15 j9 + ON t9.join_key = j9.join_key + WHERE NOT EXISTS (SELECT sq9.id + FROM (SELECT + t10.id, + SUM(t10.col2) AS agg_value, + CASE + WHEN SUM(t10.col2) > 1000 THEN 'BASE' + WHEN SUM(t10.col2) BETWEEN 500 AND 1000 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_11 t10 + LEFT JOIN tbl_16 j10 + ON t10.join_key = j10.join_key + WHERE t10.id IN (SELECT sq10.id + FROM (SELECT + t11.id, + SUM(t11.col2) AS agg_value, + CASE + WHEN SUM(t11.col2) > 1100 THEN 'BASE' + WHEN SUM(t11.col2) BETWEEN 550 AND 1100 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_12 t11 + RIGHT JOIN tbl_17 j11 + ON t11.join_key = j11.join_key + WHERE t11.id NOT IN (SELECT sq11.id + FROM (SELECT + t12.id, + SUM(t12.col2) AS agg_value, + CASE + WHEN SUM(t12.col2) > 1200 THEN 'BASE' + WHEN SUM(t12.col2) BETWEEN 600 AND 1200 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_13 t12 + INNER JOIN tbl_18 j12 + ON t12.join_key = j12.join_key + WHERE EXISTS (SELECT sq12.id + FROM (SELECT + t13.id, + SUM(t13.col2) AS agg_value, + CASE + WHEN SUM(t13.col2) > 1300 THEN 'BASE' + WHEN SUM(t13.col2) BETWEEN 650 AND 1300 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_14 t13 + LEFT JOIN tbl_19 j13 + ON t13.join_key = j13.join_key + WHERE NOT EXISTS (SELECT sq13.id + FROM (SELECT + t14.id, + SUM(t14.col2) AS agg_value, + CASE + WHEN SUM(t14.col2) > 1400 THEN 'BASE' + WHEN SUM(t14.col2) BETWEEN 700 AND 1400 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_15 t14 + RIGHT JOIN tbl_20 j14 + ON t14.join_key = j14.join_key + WHERE t14.id IN (SELECT sq14.id + FROM (SELECT + t15.id, + SUM(t15.col2) AS agg_value, + CASE + WHEN SUM(t15.col2) > 1500 THEN 'BASE' + WHEN SUM(t15.col2) BETWEEN 750 AND 1500 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_16 t15 + INNER JOIN tbl_1 j15 + ON t15.join_key = j15.join_key + WHERE t15.id NOT IN (SELECT sq15.id + FROM (SELECT + t16.id, + SUM(t16.col2) AS agg_value, + CASE + WHEN SUM(t16.col2) > 1600 THEN 'BASE' + WHEN SUM(t16.col2) BETWEEN 800 AND 1600 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_17 t16 + LEFT JOIN tbl_2 j16 + ON t16.join_key = j16.join_key + WHERE EXISTS (SELECT sq16.id + FROM (SELECT + t17.id, + SUM(t17.col2) AS agg_value, + CASE + WHEN SUM(t17.col2) > 1700 THEN 'BASE' + WHEN SUM(t17.col2) BETWEEN 850 AND 1700 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_18 t17 + RIGHT JOIN tbl_3 j17 + ON t17.join_key = j17.join_key + WHERE NOT EXISTS (SELECT sq17.id + FROM (SELECT + t18.id, + SUM(t18.col2) AS agg_value, + CASE + WHEN SUM(t18.col2) > 1800 THEN 'BASE' + WHEN SUM(t18.col2) BETWEEN 900 AND 1800 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_19 t18 + INNER JOIN tbl_4 j18 + ON t18.join_key = j18.join_key + WHERE t18.id IN (SELECT sq18.id + FROM (SELECT + t19.id, + SUM(t19.col2) AS agg_value, + CASE + WHEN SUM(t19.col2) > 1900 THEN 'BASE' + WHEN SUM(t19.col2) BETWEEN 950 AND 1900 THEN 'BASE' + ELSE 'BASE' + END AS case_label + FROM tbl_20 t19 + LEFT JOIN tbl_5 j19 + ON t19.join_key = j19.join_key + WHERE t19.id NOT IN (SELECT sq19.id + FROM (SELECT + id, + col2 AS agg_value, + 'BASE' AS case_label + FROM tbl_1 + WHERE col3 > 10 + LIMIT 1) sq19 + WHERE sq19.agg_value = j19.col2 + AND sq19.case_label LIKE '%BASE%' + AND j19.col2 BETWEEN 100 AND 1000) + AND t19.col5 = true + AND j19.col1 LIKE '%data%' + AND t19.col3 IN (SELECT col3 FROM tbl_3 WHERE id > 0) + GROUP BY t19.id) sq18 + WHERE sq18.agg_value = j18.col2 + AND sq18.case_label LIKE '%BASE%' + AND j18.col2 BETWEEN 10 AND 100) + AND t18.col5 = true + AND j18.col1 LIKE '%data%' + AND t18.col3 IN (SELECT col3 FROM tbl_2 WHERE id > 0) + GROUP BY t18.id) sq17 + WHERE sq17.agg_value = j17.col2 + AND sq17.case_label LIKE '%BASE%' + AND j17.col2 BETWEEN 100 AND 1000) + AND t17.col5 = true + AND j17.col1 LIKE '%data%' + AND t17.col3 IN (SELECT col3 FROM tbl_1 WHERE id > 0) + GROUP BY t17.id) sq16 + WHERE sq16.agg_value = j16.col2 + AND sq16.case_label LIKE '%BASE%' + AND j16.col2 BETWEEN 10 AND 100) + AND t16.col5 = true + AND j16.col1 LIKE '%data%' + AND t16.col3 IN (SELECT col3 FROM tbl_20 WHERE id > 0) + GROUP BY t16.id) sq15 + WHERE sq15.agg_value = j15.col2 + AND sq15.case_label LIKE '%BASE%' + AND j15.col2 BETWEEN 100 AND 1000) + AND t15.col5 = true + AND j15.col1 LIKE '%data%' + AND t15.col3 IN (SELECT col3 FROM tbl_19 WHERE id > 0) + GROUP BY t15.id) sq14 + WHERE sq14.agg_value = j14.col2 + AND sq14.case_label LIKE '%BASE%' + AND j14.col2 BETWEEN 10 AND 100) + AND t14.col5 = true + AND j14.col1 LIKE '%data%' + AND t14.col3 IN (SELECT col3 FROM tbl_18 WHERE id > 0) + GROUP BY t14.id) sq13 + WHERE sq13.agg_value = j13.col2 + AND sq13.case_label LIKE '%BASE%' + AND j13.col2 BETWEEN 100 AND 1000) + AND t13.col5 = true + AND j13.col1 LIKE '%data%' + AND t13.col3 IN (SELECT col3 FROM tbl_17 WHERE id > 0) + GROUP BY t13.id) sq12 + WHERE sq12.agg_value = j12.col2 + AND sq12.case_label LIKE '%BASE%' + AND j12.col2 BETWEEN 10 AND 100) + AND t12.col5 = true + AND j12.col1 LIKE '%data%' + AND t12.col3 IN (SELECT col3 FROM tbl_16 WHERE id > 0) + GROUP BY t12.id) sq11 + WHERE sq11.agg_value = j11.col2 + AND sq11.case_label LIKE '%BASE%' + AND j11.col2 BETWEEN 100 AND 1000) + AND t11.col5 = true + AND j11.col1 LIKE '%data%' + AND t11.col3 IN (SELECT col3 FROM tbl_15 WHERE id > 0) + GROUP BY t11.id) sq10 + WHERE sq10.agg_value = j10.col2 + AND sq10.case_label LIKE '%BASE%' + AND j10.col2 BETWEEN 10 AND 100) + AND t10.col5 = true + AND j10.col1 LIKE '%data%' + AND t10.col3 IN (SELECT col3 FROM tbl_14 WHERE id > 0) + GROUP BY t10.id) sq9 + WHERE sq9.agg_value = j9.col2 + AND sq9.case_label LIKE '%BASE%' + AND j9.col2 BETWEEN 100 AND 1000) + AND t9.col5 = true + AND j9.col1 LIKE '%data%' + AND t9.col3 IN (SELECT col3 FROM tbl_13 WHERE id > 0) + GROUP BY t9.id) sq8 + WHERE sq8.agg_value = j8.col2 + AND sq8.case_label LIKE '%BASE%' + AND j8.col2 BETWEEN 10 AND 100) + AND t8.col5 = true + AND j8.col1 LIKE '%data%' + AND t8.col3 IN (SELECT col3 FROM tbl_12 WHERE id > 0) + GROUP BY t8.id) sq7 + WHERE sq7.agg_value = j7.col2 + AND sq7.case_label LIKE '%BASE%' + AND j7.col2 BETWEEN 100 AND 1000) + AND t7.col5 = true + AND j7.col1 LIKE '%data%' + AND t7.col3 IN (SELECT col3 FROM tbl_11 WHERE id > 0) + GROUP BY t7.id) sq6 + WHERE sq6.agg_value = j6.col2 + AND sq6.case_label LIKE '%BASE%' + AND j6.col2 BETWEEN 10 AND 100) + AND t6.col5 = true + AND j6.col1 LIKE '%data%' + AND t6.col3 IN (SELECT col3 FROM tbl_10 WHERE id > 0) + GROUP BY t6.id) sq5 + WHERE sq5.agg_value = j5.col2 + AND sq5.case_label LIKE '%BASE%' + AND j5.col2 BETWEEN 100 AND 1000) + AND t5.col5 = true + AND j5.col1 LIKE '%data%' + AND t5.col3 IN (SELECT col3 FROM tbl_9 WHERE id > 0) + GROUP BY t5.id) sq4 + WHERE sq4.agg_value = j4.col2 + AND sq4.case_label LIKE '%BASE%' + AND j4.col2 BETWEEN 10 AND 100) + AND t4.col5 = true + AND j4.col1 LIKE '%data%' + AND t4.col3 IN (SELECT col3 FROM tbl_8 WHERE id > 0) + GROUP BY t4.id) sq3 + WHERE sq3.agg_value = j3.col2 + AND sq3.case_label LIKE '%BASE%' + AND j3.col2 BETWEEN 100 AND 1000) + AND t3.col5 = true + AND j3.col1 LIKE '%data%' + AND t3.col3 IN (SELECT col3 FROM tbl_7 WHERE id > 0) + GROUP BY t3.id) sq2 + WHERE sq2.agg_value = j2.col2 + AND sq2.case_label LIKE '%BASE%' + AND j2.col2 BETWEEN 10 AND 100) + AND t2.col5 = true + AND j2.col1 LIKE '%data%' + AND t2.col3 IN (SELECT col3 FROM tbl_6 WHERE id > 0) + GROUP BY t2.id) sq1 + WHERE sq1.agg_value = j1.col2 + AND sq1.case_label LIKE '%BASE%' + AND j1.col2 BETWEEN 100 AND 1000) + AND t1.col5 = true + AND j1.col1 LIKE '%data%' + AND t1.col3 IN (SELECT col3 FROM tbl_5 WHERE id > 0) + GROUP BY t1.id) sq0 + WHERE sq0.agg_value = j0.col2 + AND sq0.case_label LIKE '%BASE%' + AND j0.col2 BETWEEN 10 AND 100) + AND t0.col5 = true + AND j0.col1 LIKE '%data%' + AND t0.col3 IN (SELECT col3 FROM tbl_4 WHERE id > 0) + GROUP BY t0.id + ) + SELECT * FROM cte; + """ +} From 3e30aa64410a98ec3964fba378120ba70a203821 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 19:51:48 +0800 Subject: [PATCH 299/572] branch-3.0: [opt](ci) adjust threshold for perfmance test #53650 (#53777) Cherry-picked from #53650 Co-authored-by: Dongyang Li --- regression-test/pipeline/performance/run-clickbench.sh | 6 ++++++ regression-test/pipeline/performance/run-load.sh | 10 ++++++++++ regression-test/pipeline/performance/run-tpcds.sh | 6 ++++++ regression-test/pipeline/performance/run-tpch.sh | 6 ++++++ 4 files changed, 28 insertions(+) diff --git a/regression-test/pipeline/performance/run-clickbench.sh b/regression-test/pipeline/performance/run-clickbench.sh index 6ed29a464f9e57..286ce1df9a2884 100644 --- a/regression-test/pipeline/performance/run-clickbench.sh +++ b/regression-test/pipeline/performance/run-clickbench.sh @@ -260,6 +260,12 @@ exit_flag=0 if [[ "${target_branch}" == "branch-2.0" ]]; then cold_run_time_threshold=${cold_run_time_threshold_branch20:-110} # 单位 秒 hot_run_time_threshold=${hot_run_time_threshold_branch20:-34} # 单位 秒 + elif [[ "${target_branch}" == "branch-3.1" ]]; then + cold_run_time_threshold=${cold_run_time_threshold_branch31:-120} # 单位 秒 + hot_run_time_threshold=${hot_run_time_threshold_branch31:-34} # 单位 秒 + elif [[ "${target_branch}" == "branch-3.0" ]]; then + cold_run_time_threshold=${cold_run_time_threshold_branch30:-120} # 单位 秒 + hot_run_time_threshold=${hot_run_time_threshold_branch30:-34} # 单位 秒 fi echo "INFO: cold_run_time_threshold is ${cold_run_time_threshold}, hot_run_time_threshold is ${hot_run_time_threshold}" # result.csv 来自 run-clickbench-queries.sh 的产出 diff --git a/regression-test/pipeline/performance/run-load.sh b/regression-test/pipeline/performance/run-load.sh index bd67102f2030a1..156024ffceec76 100644 --- a/regression-test/pipeline/performance/run-load.sh +++ b/regression-test/pipeline/performance/run-load.sh @@ -671,6 +671,16 @@ exit_flag=0 stream_load_orc_speed_threshold=${stream_load_orc_speed_threshold_branch20:-15} # 单位 MB/s stream_load_parquet_speed_threshold=${stream_load_parquet_speed_threshold_branch20:-22} # 单位 MB/s insert_into_select_speed_threshold=${insert_into_select_speed_threshold_branch20:-410} # 单位 Krows/s + elif [[ "${target_branch}" == "branch-3.1" ]]; then + stream_load_json_speed_threshold=${stream_load_json_speed_threshold_branch31:-115} # 单位 MB/s + stream_load_orc_speed_threshold=${stream_load_orc_speed_threshold_branch31:-15} # 单位 MB/s + stream_load_parquet_speed_threshold=${stream_load_parquet_speed_threshold_branch31:-22} # 单位 MB/s + insert_into_select_speed_threshold=${insert_into_select_speed_threshold_branch31:-700} # 单位 Krows/s + elif [[ "${target_branch}" == "branch-3.0" ]]; then + stream_load_json_speed_threshold=${stream_load_json_speed_threshold_branch30:-115} # 单位 MB/s + stream_load_orc_speed_threshold=${stream_load_orc_speed_threshold_branch30:-15} # 单位 MB/s + stream_load_parquet_speed_threshold=${stream_load_parquet_speed_threshold_branch30:-22} # 单位 MB/s + insert_into_select_speed_threshold=${insert_into_select_speed_threshold_branch30:-700} # 单位 Krows/s fi if [[ ${stream_load_json_speed} -lt ${stream_load_json_speed_threshold} ]]; then echo "ERROR: stream_load_json_speed ${stream_load_json_speed} is less than the threshold ${stream_load_json_speed_threshold}" && exit 1; fi if [[ ${stream_load_orc_speed} -lt ${stream_load_orc_speed_threshold} ]]; then echo "ERROR: stream_load_orc_speed ${stream_load_orc_speed} is less than the threshold ${stream_load_orc_speed_threshold}" && exit 1; fi diff --git a/regression-test/pipeline/performance/run-tpcds.sh b/regression-test/pipeline/performance/run-tpcds.sh index 478d822059a2c3..d40c75fe186df1 100644 --- a/regression-test/pipeline/performance/run-tpcds.sh +++ b/regression-test/pipeline/performance/run-tpcds.sh @@ -130,6 +130,12 @@ exit_flag=0 if [[ "${target_branch}" == "branch-2.0" ]]; then cold_run_time_threshold=${cold_run_time_threshold_branch20:-370000} # ms hot_run_time_threshold=${hot_run_time_threshold_branch20:-260000} # ms + elif [[ "${target_branch}" == "branch-3.1" ]]; then + cold_run_time_threshold=${cold_run_time_threshold_branch31:-315000} # ms + hot_run_time_threshold=${hot_run_time_threshold_branch31:-190000} # ms + elif [[ "${target_branch}" == "branch-3.0" ]]; then + cold_run_time_threshold=${cold_run_time_threshold_branch30:-315000} # ms + hot_run_time_threshold=${hot_run_time_threshold_branch30:-190000} # ms fi echo "INFO: cold_run_time_threshold is ${cold_run_time_threshold}, hot_run_time_threshold is ${hot_run_time_threshold}" if ! check_tpcds_result "${teamcity_build_checkoutDir}"/run-tpcds-queries.log; then diff --git a/regression-test/pipeline/performance/run-tpch.sh b/regression-test/pipeline/performance/run-tpch.sh index 36df5d575cef1c..120b5e91ada338 100644 --- a/regression-test/pipeline/performance/run-tpch.sh +++ b/regression-test/pipeline/performance/run-tpch.sh @@ -122,6 +122,12 @@ exit_flag=0 if [[ "${target_branch}" == "branch-2.0" ]]; then cold_run_time_threshold=${cold_run_time_threshold_branch20:-130000} # ms hot_run_time_threshold=${hot_run_time_threshold_branch20:-55000} # ms + elif [[ "${target_branch}" == "branch-3.1" ]]; then + cold_run_time_threshold=${cold_run_time_threshold_branch31:-120000} # ms + hot_run_time_threshold=${hot_run_time_threshold_branch31:-42000} # ms + elif [[ "${target_branch}" == "branch-3.0" ]]; then + cold_run_time_threshold=${cold_run_time_threshold_branch30:-120000} # ms + hot_run_time_threshold=${hot_run_time_threshold_branch30:-42000} # ms fi echo "INFO: cold_run_time_threshold is ${cold_run_time_threshold}, hot_run_time_threshold is ${hot_run_time_threshold}" if ! check_tpch_result "${teamcity_build_checkoutDir}"/run-tpch-queries.log; then From 909ed0efebdafd175c8129120b5046e36e2899f7 Mon Sep 17 00:00:00 2001 From: deardeng Date: Wed, 23 Jul 2025 20:04:58 +0800 Subject: [PATCH 300/572] [fix](vcg docker case) Fix case due to dockerAwaitUntil func removed (#53758) --- .../virtual_compute_group/test_vcg.groovy | 16 ++++++++-------- .../test_vcg_metrics.groovy | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy index 74aef9e628e3b6..88cb8ed4e1dd90 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg.groovy @@ -198,7 +198,7 @@ suite('test_vcg', 'multi_cluster,docker') { // test manual cancel warm up job, generate new jobs sql """CANCEL WARM UP JOB WHERE ID=${showWarmup[0].JobId}""" - dockerAwaitUntil(50, 3) { + awaitUntil(50, 3) { showWarmup = sql_return_maparray """SHOW WARM UP JOB""" // cancel 2, generate 2 showWarmup.size() == 4 @@ -295,7 +295,7 @@ suite('test_vcg', 'multi_cluster,docker') { assertTrue(json.code.equalsIgnoreCase("OK")) } - dockerAwaitUntil(20) { + awaitUntil(20) { showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group after alter {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } @@ -307,7 +307,7 @@ suite('test_vcg', 'multi_cluster,docker') { assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) // alter active -> sync to fe -> cancel old jobs -> generate new jobs // -> sync to ms -> sync to fe -> save new jobs - dockerAwaitUntil(50, 3) { + awaitUntil(50, 3) { showWarmup = sql_return_maparray """SHOW WARM UP JOB""" // cancel 2, generate 2 showWarmup.size() == 6 @@ -326,7 +326,7 @@ suite('test_vcg', 'multi_cluster,docker') { cluster.stopBackends(6) // test warm up job destory and generate new jobs - dockerAwaitUntil(50, 3) { + awaitUntil(50, 3) { sql """USE @${normalVclusterName}""" sql """select count(*) from ${tbl}""" showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ @@ -337,7 +337,7 @@ suite('test_vcg', 'multi_cluster,docker') { assertEquals(showWarmup.size(), 6) cluster.startBackends(6) - dockerAwaitUntil(50, 3) { + awaitUntil(50, 3) { showWarmup = sql_return_maparray """SHOW WARM UP JOB""" showWarmup.size() == 8 } @@ -350,7 +350,7 @@ suite('test_vcg', 'multi_cluster,docker') { // test rename vcg def newNormalVclusterName = "newNormalVirtualClusterName" rename_cloud_cluster.call(newNormalVclusterName, normalVclusterId, ms) - dockerAwaitUntil(20) { + awaitUntil(20) { showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group after rename {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } @@ -365,7 +365,7 @@ suite('test_vcg', 'multi_cluster,docker') { // rename back to rename_cloud_cluster.call(normalVclusterName, normalVclusterId, ms) - dockerAwaitUntil(20) { + awaitUntil(20) { showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ log.info("show compute group after rename back {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } @@ -393,7 +393,7 @@ suite('test_vcg', 'multi_cluster,docker') { jsonObject = jsonSlurper.parseText(tag) cloudClusterId = jsonObject.compute_group_id drop_cluster(clusterName2, cloudClusterId, ms) - dockerAwaitUntil(20) { + awaitUntil(20) { def showRet = sql """SHOW COMPUTE GROUPS""" log.info("show cgs: {}", showRet) showRet.size() == 2 diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy index 9939539568c46e..b698799b753cab 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_metrics.groovy @@ -139,7 +139,7 @@ suite('test_vcg_metrics', 'multi_cluster,docker') { cluster.stopBackends(4, 5) // test warm up job destory and generate new jobs - dockerAwaitUntil(50, 3) { + awaitUntil(50, 3) { sql """USE @${normalVclusterName}""" sql """select count(*) from ${tbl}""" showComputeGroup = sql_return_maparray """ SHOW COMPUTE GROUPS """ From 93dc6e7600c56182600093d7b37791089e76b25e Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 23 Jul 2025 20:06:31 +0800 Subject: [PATCH 301/572] [fix](vcg) use "vcg cancel" as cancel message for warm up jobs (#53752) cherry-pick #53751 --- .../java/org/apache/doris/cloud/CacheHotspotManager.java | 6 +++++- .../doris/cloud/catalog/CloudInstanceStatusChecker.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java index 417f1b8d233e55..ed51961151e334 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java @@ -840,11 +840,15 @@ public void cancel(CancelCloudWarmUpStmt stmt) throws DdlException { } public void cancel(long jobId) throws DdlException { + cancel(jobId, "user cancel"); + } + + public void cancel(long jobId, String msg) throws DdlException { CloudWarmUpJob job = cloudWarmUpJobs.get(jobId); if (job == null) { throw new DdlException("job id: " + jobId + " does not exist."); } - if (!job.cancel("user cancel", true)) { + if (!job.cancel(msg, true)) { throw new DdlException("job can not be cancelled. State: " + job.getJobState()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java index e40ee50ae58feb..9f0c51709e0411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java @@ -152,7 +152,7 @@ private void cancelCacheJobs(ComputeGroup vcgInFe, List jobIds) { try { if (Env.getCurrentEnv().isMaster()) { // cancel old jobId, will write editlog, so just master can do - cacheHotspotManager.cancel(Long.parseLong(jobId)); + cacheHotspotManager.cancel(Long.parseLong(jobId), "vcg cancel"); LOG.info("virtual compute group {}, cancel jobId {}", vcgInFe.getName(), jobId); } } catch (DdlException e) { From e66150e539ac7a3e15dab24352951473a3454917 Mon Sep 17 00:00:00 2001 From: HappenLee Date: Wed, 23 Jul 2025 20:12:57 +0800 Subject: [PATCH 302/572] [Log](topn) Log query id in topn two read and change the timeout to uery timeout (#52717) cherry pick #53450 --- be/src/common/config.cpp | 1 - be/src/common/config.h | 1 - be/src/exec/rowid_fetcher.cpp | 11 ++++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 0c83fc76e1318c..9ee475aacaf7ab 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -312,7 +312,6 @@ DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1"); DEFINE_Int32(doris_scanner_thread_pool_queue_size, "102400"); // default thrift client connect timeout(in seconds) DEFINE_mInt32(thrift_connect_timeout_seconds, "3"); -DEFINE_mInt32(fetch_rpc_timeout_seconds, "30"); // default thrift client retry interval (in milliseconds) DEFINE_mInt64(thrift_client_retry_interval_ms, "1000"); diff --git a/be/src/common/config.h b/be/src/common/config.h index a6fe08af1f626a..f8312c3adb372b 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -355,7 +355,6 @@ DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num); DECLARE_Int32(doris_scanner_thread_pool_queue_size); // default thrift client connect timeout(in seconds) DECLARE_mInt32(thrift_connect_timeout_seconds); -DECLARE_mInt32(fetch_rpc_timeout_seconds); // default thrift client retry interval (in milliseconds) DECLARE_mInt64(thrift_client_retry_interval_ms); // max message size of thrift request diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp index d6c47201dadf0a..e0b81283df15ed 100644 --- a/be/src/exec/rowid_fetcher.cpp +++ b/be/src/exec/rowid_fetcher.cpp @@ -231,7 +231,7 @@ Status RowIDFetcher::fetch(const vectorized::ColumnPtr& column_row_ids, std::vector cntls(_stubs.size()); bthread::CountdownEvent counter(_stubs.size()); for (size_t i = 0; i < _stubs.size(); ++i) { - cntls[i].set_timeout_ms(config::fetch_rpc_timeout_seconds * 1000); + cntls[i].set_timeout_ms(_fetch_option.runtime_state->execution_timeout() * 1000); auto callback = brpc::NewCallback(fetch_callback, &counter); _stubs[i]->multiget_data(&cntls[i], &mget_req, &resps[i], callback); } @@ -465,16 +465,17 @@ Status RowIdStorageReader::read_by_rowids(const PMultiGetRequest& request, LOG(INFO) << "Query stats: " << fmt::format( + "query_id:{}, " "hit_cached_pages:{}, total_pages_read:{}, compressed_bytes_read:{}, " "io_latency:{}ns, " "uncompressed_bytes_read:{}," "bytes_read:{}," "acquire_tablet_ms:{}, acquire_rowsets_ms:{}, acquire_segments_ms:{}, " "lookup_row_data_ms:{}", - stats.cached_pages_num, stats.total_pages_num, stats.compressed_bytes_read, - stats.io_ns, stats.uncompressed_bytes_read, stats.bytes_read, - acquire_tablet_ms, acquire_rowsets_ms, acquire_segments_ms, - lookup_row_data_ms); + print_id(request.query_id()), stats.cached_pages_num, + stats.total_pages_num, stats.compressed_bytes_read, stats.io_ns, + stats.uncompressed_bytes_read, stats.bytes_read, acquire_tablet_ms, + acquire_rowsets_ms, acquire_segments_ms, lookup_row_data_ms); return Status::OK(); } From 2a7b910fbe7543689414135c560d6ff7ecec304b Mon Sep 17 00:00:00 2001 From: airborne12 Date: Wed, 23 Jul 2025 20:15:52 +0800 Subject: [PATCH 303/572] branch-3.0: [test](inverted index) enhance test_mow_table_with_format_v2 regression case (#53762) (#53781) pick from #53762 --- .../test_mow_table_with_format_v2.groovy | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index 65827d038bde17..f1b7f076ba7e8f 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -163,7 +163,7 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - trigger_and_wait_compaction(tableName, "cumulative") + // trigger_and_wait_compaction(tableName, "cumulative") // check indexes for (def tablet in tablets) { boolean running = true @@ -171,6 +171,34 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { backend_id = tablet.BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) + be_show_tablet_status(ip, port, tablet_id) + (code, out, err) = be_show_tablet_status(ip, port, tablet_id) + logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-3]")) + assertTrue(out.contains("[4-4]")) + assertTrue(out.contains("[5-5]")) + assertTrue(out.contains("[6-6]")) + assertTrue(out.contains("[7-7]")) + assertTrue(out.contains("[8-8]")) + assertTrue(out.contains("[9-9]")) + logger.info("run compaction:" + tablet_id) + (code, out, err) = be_run_cumulative_compaction(ip, port, tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + do { + Thread.sleep(100) + (code, out, err) = be_get_compaction_status(ip, port, tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + (code, out, err) = be_show_tablet_status(ip, port, tablet_id) + logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-9]")) check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") } From 16d6e3662087ff62ff8a53e6011a2433da31a5fd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 20:17:17 +0800 Subject: [PATCH 304/572] branch-3.0: [test](inverted index) make test_cumulative_compaction_with_format_v2 regression case more stable #53756 (#53769) Cherry-picked from #53756 Co-authored-by: airborne12 --- ...umulative_compaction_with_format_v2.groovy | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy index eff05b1fd8daa4..6e39fa09fcd7fc 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy @@ -163,13 +163,42 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { def tablets = sql_return_maparray """ show tablets from ${tableName}; """ // trigger compactions for all tablets in ${tableName} - trigger_and_wait_compaction(tableName, "cumulative") + //trigger_and_wait_compaction(tableName, "cumulative") // wait for all compactions done for (def tablet in tablets) { String tablet_id = tablet.TabletId backend_id = tablet.BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) + be_show_tablet_status(ip, port, tablet_id) + (code, out, err) = be_show_tablet_status(ip, port, tablet_id) + logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertTrue(out.contains("[3-3]")) + assertTrue(out.contains("[4-4]")) + assertTrue(out.contains("[5-5]")) + assertTrue(out.contains("[6-6]")) + assertTrue(out.contains("[7-7]")) + assertTrue(out.contains("[8-8]")) + assertTrue(out.contains("[9-9]")) + logger.info("run compaction:" + tablet_id) + (code, out, err) = be_run_cumulative_compaction(ip, port, tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + boolean running = true + do { + Thread.sleep(100) + (code, out, err) = be_get_compaction_status(ip, port, tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + (code, out, err) = be_show_tablet_status(ip, port, tablet_id) + logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-9]")) int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") From a4d31bbc636f7274a6715f41675945b68d97abd1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 20:19:50 +0800 Subject: [PATCH 305/572] branch-3.0: [fix](test) fix p2 case test_show_data_with_compaction #53558 (#53767) Cherry-picked from #53558 Co-authored-by: zzzxl --- regression-test/suites/inverted_index_p2/test_show_data.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/regression-test/suites/inverted_index_p2/test_show_data.groovy b/regression-test/suites/inverted_index_p2/test_show_data.groovy index 657b982c683556..c48d87d3d3edb0 100644 --- a/regression-test/suites/inverted_index_p2/test_show_data.groovy +++ b/regression-test/suites/inverted_index_p2/test_show_data.groovy @@ -804,6 +804,8 @@ suite("test_show_data_with_compaction", "p2") { load_httplogs_data.call(tableWithOutIndexCompaction, '9', 'true', 'json', 'documents-1000.json') load_httplogs_data.call(tableWithOutIndexCompaction, '10', 'true', 'json', 'documents-1000.json') + sql "sync" + def another_with_index_size = wait_for_show_data_finish(tableWithOutIndexCompaction, 60000, 0) assertTrue(another_with_index_size != "wait_timeout") trigger_and_wait_compaction(tableWithOutIndexCompaction, "full") From 461114eab2c947f6966a6635744be24ce26bd95a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 20:29:29 +0800 Subject: [PATCH 306/572] branch-3.0: [fix](p0) Fix `test_array_map` #53710 (#53745) Cherry-picked from #53710 Co-authored-by: Gabriel --- regression-test/suites/function_p0/test_array_map.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/function_p0/test_array_map.groovy b/regression-test/suites/function_p0/test_array_map.groovy index 9b4b0526fcf1c7..15360ed873aca3 100644 --- a/regression-test/suites/function_p0/test_array_map.groovy +++ b/regression-test/suites/function_p0/test_array_map.groovy @@ -20,6 +20,7 @@ suite("test_array_map") { drop table if exists mock_table; """ + sql """ DROP FUNCTION IF EXISTS clean_html_entity_test(string) """ sql """ CREATE ALIAS FUNCTION clean_html_entity_test(string) WITH PARAMETER(html) AS REPLACE( From 341d3cb9f80c593f1a7647b38f9c47a9a717bf3c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 24 Jul 2025 10:11:41 +0800 Subject: [PATCH 307/572] branch-3.0: [fix](nereids)should distinguish hash and other conjuncts for outer join in MultiJoin #53051 (#53090) Cherry-picked from #53051 Co-authored-by: starocean999 --- .../nereids/rules/rewrite/MultiJoin.java | 24 +- .../nereids/rules/rewrite/ReorderJoin.java | 17 +- .../join/test_many_multi_join.groovy | 906 ++++++++++++++++++ 3 files changed, 932 insertions(+), 15 deletions(-) create mode 100644 regression-test/suites/nereids_p0/join/test_many_multi_join.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java index 83e849086ab737..bc6ea17d8a661b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MultiJoin.java @@ -69,14 +69,16 @@ public class MultiJoin extends AbstractLogicalPlan implements BlockFuncDepsPropa // MultiJoin just contains one OUTER/SEMI/ANTI. private final JoinType joinType; // When contains one OUTER/SEMI/ANTI join, keep separately its condition. - private final List notInnerJoinConditions; + private final List notInnerJoinHashConditions; + private final List notInnerJoinOtherConditions; public MultiJoin(List inputs, List joinFilter, JoinType joinType, - List notInnerJoinConditions) { + List notInnerJoinHashConditions, List notInnerJoinOtherConditions) { super(PlanType.LOGICAL_MULTI_JOIN, inputs); this.joinFilter = Objects.requireNonNull(joinFilter); this.joinType = joinType; - this.notInnerJoinConditions = Objects.requireNonNull(notInnerJoinConditions); + this.notInnerJoinHashConditions = Objects.requireNonNull(notInnerJoinHashConditions); + this.notInnerJoinOtherConditions = Objects.requireNonNull(notInnerJoinOtherConditions); } public JoinType getJoinType() { @@ -87,13 +89,17 @@ public List getJoinFilter() { return joinFilter; } - public List getNotInnerJoinConditions() { - return notInnerJoinConditions; + public List getNotInnerHashJoinConditions() { + return notInnerJoinHashConditions; + } + + public List getNotInnerOtherJoinConditions() { + return notInnerJoinOtherConditions; } @Override public MultiJoin withChildren(List children) { - return new MultiJoin(children, joinFilter, joinType, notInnerJoinConditions); + return new MultiJoin(children, joinFilter, joinType, notInnerJoinHashConditions, notInnerJoinOtherConditions); } @Override @@ -160,7 +166,8 @@ public R accept(PlanVisitor visitor, C context) { public List getExpressions() { return new Builder() .addAll(joinFilter) - .addAll(notInnerJoinConditions) + .addAll(notInnerJoinHashConditions) + .addAll(notInnerJoinOtherConditions) .build(); } @@ -180,7 +187,8 @@ public String toString() { return Utils.toSqlString("MultiJoin", "joinType", joinType, "joinFilter", joinFilter, - "notInnerJoinConditions", notInnerJoinConditions + "notInnerHashJoinConditions", notInnerJoinHashConditions, + "notInnerOtherJoinConditions", notInnerJoinOtherConditions ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java index e4af85b92c9c9c..37765dd36239fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java @@ -111,7 +111,8 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) List inputs = Lists.newArrayList(); List joinFilter = Lists.newArrayList(); - List notInnerJoinConditions = Lists.newArrayList(); + List notInnerHashJoinConditions = Lists.newArrayList(); + List notInnerOtherJoinConditions = Lists.newArrayList(); LogicalJoin join; // Implicit rely on {rule: MergeFilters}, so don't exist filter--filter--join. @@ -127,8 +128,8 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) joinFilter.addAll(join.getHashJoinConjuncts()); joinFilter.addAll(join.getOtherJoinConjuncts()); } else { - notInnerJoinConditions.addAll(join.getHashJoinConjuncts()); - notInnerJoinConditions.addAll(join.getOtherJoinConjuncts()); + notInnerHashJoinConditions.addAll(join.getHashJoinConjuncts()); + notInnerOtherJoinConditions.addAll(join.getOtherJoinConjuncts()); } // recursively convert children. @@ -161,7 +162,8 @@ public Plan joinToMultiJoin(Plan plan, Map planToHintType) inputs, joinFilter, join.getJoinType(), - notInnerJoinConditions); + notInnerHashJoinConditions, + notInnerOtherJoinConditions); } /** @@ -253,7 +255,7 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planT multiJoinHandleChildren.children().subList(0, multiJoinHandleChildren.arity() - 1), pushedFilter, JoinType.INNER_JOIN, - ExpressionUtils.EMPTY_CONDITION), planToHintType); + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION), planToHintType); } else if (multiJoinHandleChildren.getJoinType().isRightJoin()) { left = multiJoinHandleChildren.child(0); Set leftOutputExprIdSet = left.getOutputExprIdSet(); @@ -267,7 +269,7 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planT multiJoinHandleChildren.children().subList(1, multiJoinHandleChildren.arity()), pushedFilter, JoinType.INNER_JOIN, - ExpressionUtils.EMPTY_CONDITION), planToHintType); + ExpressionUtils.EMPTY_CONDITION, ExpressionUtils.EMPTY_CONDITION), planToHintType); } else { remainingFilter = multiJoin.getJoinFilter(); Preconditions.checkState(multiJoinHandleChildren.arity() == 2); @@ -284,7 +286,8 @@ public Plan multiJoinToJoin(MultiJoin multiJoin, Map planT return PlanUtils.filterOrSelf(ImmutableSet.copyOf(remainingFilter), new LogicalJoin<>( multiJoinHandleChildren.getJoinType(), - ExpressionUtils.EMPTY_CONDITION, multiJoinHandleChildren.getNotInnerJoinConditions(), + multiJoinHandleChildren.getNotInnerHashJoinConditions(), + multiJoinHandleChildren.getNotInnerOtherJoinConditions(), planToHintType.getOrDefault(right, new DistributeHint(DistributeType.NONE)), Optional.empty(), left, right, null)); diff --git a/regression-test/suites/nereids_p0/join/test_many_multi_join.groovy b/regression-test/suites/nereids_p0/join/test_many_multi_join.groovy new file mode 100644 index 00000000000000..b560cddbc3f5a3 --- /dev/null +++ b/regression-test/suites/nereids_p0/join/test_many_multi_join.groovy @@ -0,0 +1,906 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_many_multi_join", "nereids_p0") { + def DBname = "nereids_regression_test_many_multi_join" + sql "DROP DATABASE IF EXISTS ${DBname}" + sql "CREATE DATABASE IF NOT EXISTS ${DBname}" + sql "use ${DBname}" + + multi_sql """ + drop table if exists table1; + drop table if exists table2; + drop table if exists table3; + drop table if exists table4; + drop table if exists table5; + drop table if exists table6; + drop table if exists table7; + drop table if exists table8; + drop table if exists table9; + drop table if exists table10; + drop table if exists table11; + drop table if exists table12; + drop table if exists table13; + drop table if exists table14; + drop table if exists table15; + drop table if exists table16; + drop table if exists table17; + drop table if exists table18; + drop table if exists table19; + drop table if exists table20; + drop table if exists table21; + + drop table if exists seq; + CREATE TABLE seq (number INT) ENGINE=OLAP UNIQUE KEY(number) + DISTRIBUTED BY HASH(number) BUCKETS 1 + PROPERTIES("replication_num" = "1"); + + INSERT INTO seq VALUES (1),(2),(3),(4),(5); + + + CREATE TABLE IF NOT EXISTS table1 ( + id BIGINT NOT NULL, + value1 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table2 ( + id BIGINT NOT NULL, + value2 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table3 ( + id BIGINT NOT NULL, + value3 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table4 ( + id BIGINT NOT NULL, + value4 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table5 ( + id BIGINT NOT NULL, + value5 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table6 ( + id BIGINT NOT NULL, + value6 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table7 ( + id BIGINT NOT NULL, + value7 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table8 ( + id BIGINT NOT NULL, + value8 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table9 ( + id BIGINT NOT NULL, + value9 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table10 ( + id BIGINT NOT NULL, + value10 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table11 ( + id BIGINT NOT NULL, + value11 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table12 ( + id BIGINT NOT NULL, + value12 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table13 ( + id BIGINT NOT NULL, + value13 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table14 ( + id BIGINT NOT NULL, + value14 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table15 ( + id BIGINT NOT NULL, + value15 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table16 ( + id BIGINT NOT NULL, + value16 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table17 ( + id BIGINT NOT NULL, + value17 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table18 ( + id BIGINT NOT NULL, + value18 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table19 ( + id BIGINT NOT NULL, + value19 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table20 ( + id BIGINT NOT NULL, + value20 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + CREATE TABLE IF NOT EXISTS table21 ( + id BIGINT NOT NULL, + value21 DECIMAL(20,6), + related_id BIGINT, + ts DATETIME + ) + ENGINE=OLAP + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 16 + PROPERTIES ( + "replication_num" = "1" + ); + + + INSERT INTO table1 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table2 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table3 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table4 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table5 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table6 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table7 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table8 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table9 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table10 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table11 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table12 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table13 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table14 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table15 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table16 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table17 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table18 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table19 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table20 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + INSERT INTO table21 + SELECT + number, + RAND()*2, + CAST(RAND()*2 AS BIGINT), + NOW() - INTERVAL CAST(RAND()*365 AS INT) DAY + FROM seq WHERE number = 2; + + + INSERT INTO table1 (id, value1, related_id, ts) VALUES + (1, -9.432227, 114, '2025-06-17 17:28:08'), + (2, -2.109207, 130, '2025-06-05 17:28:08'), + (3, -4.487591, 118, '2025-06-15 17:28:08'), + (4, -8.823451, 133, '2025-06-13 17:28:08'), + (5, 7.741022, 138, '2025-06-08 17:28:08'); + """ + + sql """ + SELECT * FROM (SELECT + t1.id, + (SELECT sum(ratio) FROM (SELECT + t2.id, + (SELECT sum(ratio) FROM (SELECT + t3.id, + (SELECT sum(ratio) FROM (SELECT + t4.id, + (SELECT sum(ratio) FROM (SELECT + t5.id, + (SELECT sum(ratio) FROM (SELECT + t6.id, + (SELECT sum(ratio) FROM (SELECT + t7.id, + (SELECT sum(ratio) FROM (SELECT + t8.id, + (SELECT sum(ratio) FROM (SELECT + t9.id, + (SELECT sum(ratio) FROM (SELECT + t10.id, + (SELECT sum(ratio) FROM (SELECT + t11.id, + (SELECT sum(ratio) FROM (SELECT + t12.id, + (SELECT sum(ratio) FROM (SELECT + t13.id, + (SELECT sum(ratio) FROM (SELECT + t14.id, + (SELECT sum(ratio) FROM (SELECT + t15.id, + (SELECT sum(ratio) FROM (SELECT + t16.id, + (SELECT sum(ratio) FROM (SELECT + t17.id, + (SELECT sum(ratio) FROM (SELECT + t18.id, + (SELECT sum(ratio) FROM (SELECT + t19.id, + (SELECT sum(ratio) FROM (SELECT + t20.id, + (SELECT sum(ratio) FROM (SELECT + t21.id, + (SELECT sum(ratio) FROM (SELECT + t22.id, + (SELECT sum(ratio) FROM (SELECT + t23.id, + (SELECT sum(ratio) FROM (SELECT + t24.id, + (SELECT sum(ratio) FROM (SELECT + t25.id, + (SELECT sum(ratio) FROM (SELECT + t26.id, + (SELECT sum(ratio) FROM (SELECT + t27.id, + (SELECT sum(ratio) FROM (SELECT + t28.id, + (SELECT sum(ratio) FROM (SELECT + t29.id, + (SELECT sum(ratio) FROM (SELECT + t30.id, + (SELECT sum(ratio) FROM (SELECT + t31.id, + (SELECT sum(ratio) FROM (SELECT + t32.id, + (SELECT sum(ratio) FROM (SELECT + t33.id, + (SELECT sum(ratio) FROM (SELECT + t34.id, + (SELECT sum(ratio) FROM (SELECT + t35.id, + (SELECT sum(ratio) FROM (SELECT + t36.id, + (SELECT sum(ratio) FROM (SELECT + t37.id, + (SELECT sum(ratio) FROM (SELECT + t38.id, + (SELECT sum(ratio) FROM (SELECT + t39.id, + (SELECT sum(ratio) FROM (SELECT + t40.id, + (SELECT sum(ratio) FROM (SELECT + t41.id, + (SELECT sum(ratio) FROM (SELECT + t42.id, + (SELECT sum(ratio) FROM (SELECT + t43.id, + (SELECT sum(ratio) FROM (SELECT + t44.id, + (SELECT sum(ratio) FROM (SELECT + t45.id, + (SELECT sum(ratio) FROM (SELECT + t46.id, + (SELECT sum(ratio) FROM (SELECT + t47.id, + (SELECT sum(ratio) FROM (SELECT + t48.id, + (SELECT sum(ratio) FROM (SELECT + t49.id, + (SELECT total FROM (SELECT + SUM(t50.value11) AS total + FROM table11 t50 + WHERE t50.related_id = t49.id + ) AS layer50) * 1.0 / COUNT(*) AS ratio + FROM table10 t49 + WHERE EXISTS ( + SELECT 1 FROM table11 + WHERE id = t49.related_id + ) + GROUP BY t49.id) AS layer49) * 1.0 / COUNT(*) AS ratio + FROM table9 t48 + WHERE EXISTS ( + SELECT 1 FROM table10 + WHERE id = t48.related_id + ) + GROUP BY t48.id) AS layer48) * 1.0 / COUNT(*) AS ratio + FROM table8 t47 + WHERE EXISTS ( + SELECT 1 FROM table9 + WHERE id = t47.related_id + ) + GROUP BY t47.id) AS layer47) * 1.0 / COUNT(*) AS ratio + FROM table7 t46 + WHERE EXISTS ( + SELECT 1 FROM table8 + WHERE id = t46.related_id + ) + GROUP BY t46.id) AS layer46) * 1.0 / COUNT(*) AS ratio + FROM table6 t45 + WHERE EXISTS ( + SELECT 1 FROM table7 + WHERE id = t45.related_id + ) + GROUP BY t45.id) AS layer45) * 1.0 / COUNT(*) AS ratio + FROM table5 t44 + WHERE EXISTS ( + SELECT 1 FROM table6 + WHERE id = t44.related_id + ) + GROUP BY t44.id) AS layer44) * 1.0 / COUNT(*) AS ratio + FROM table4 t43 + WHERE EXISTS ( + SELECT 1 FROM table5 + WHERE id = t43.related_id + ) + GROUP BY t43.id) AS layer43) * 1.0 / COUNT(*) AS ratio + FROM table3 t42 + WHERE EXISTS ( + SELECT 1 FROM table4 + WHERE id = t42.related_id + ) + GROUP BY t42.id) AS layer42) * 1.0 / COUNT(*) AS ratio + FROM table2 t41 + WHERE EXISTS ( + SELECT 1 FROM table3 + WHERE id = t41.related_id + ) + GROUP BY t41.id) AS layer41) * 1.0 / COUNT(*) AS ratio + FROM table1 t40 + WHERE EXISTS ( + SELECT 1 FROM table2 + WHERE id = t40.related_id + ) + GROUP BY t40.id) AS layer40) * 1.0 / COUNT(*) AS ratio + FROM table20 t39 + WHERE EXISTS ( + SELECT 1 FROM table1 + WHERE id = t39.related_id + ) + GROUP BY t39.id) AS layer39) * 1.0 / COUNT(*) AS ratio + FROM table19 t38 + WHERE EXISTS ( + SELECT 1 FROM table20 + WHERE id = t38.related_id + ) + GROUP BY t38.id) AS layer38) * 1.0 / COUNT(*) AS ratio + FROM table18 t37 + WHERE EXISTS ( + SELECT 1 FROM table19 + WHERE id = t37.related_id + ) + GROUP BY t37.id) AS layer37) * 1.0 / COUNT(*) AS ratio + FROM table17 t36 + WHERE EXISTS ( + SELECT 1 FROM table18 + WHERE id = t36.related_id + ) + GROUP BY t36.id) AS layer36) * 1.0 / COUNT(*) AS ratio + FROM table16 t35 + WHERE EXISTS ( + SELECT 1 FROM table17 + WHERE id = t35.related_id + ) + GROUP BY t35.id) AS layer35) * 1.0 / COUNT(*) AS ratio + FROM table15 t34 + WHERE EXISTS ( + SELECT 1 FROM table16 + WHERE id = t34.related_id + ) + GROUP BY t34.id) AS layer34) * 1.0 / COUNT(*) AS ratio + FROM table14 t33 + WHERE EXISTS ( + SELECT 1 FROM table15 + WHERE id = t33.related_id + ) + GROUP BY t33.id) AS layer33) * 1.0 / COUNT(*) AS ratio + FROM table13 t32 + WHERE EXISTS ( + SELECT 1 FROM table14 + WHERE id = t32.related_id + ) + GROUP BY t32.id) AS layer32) * 1.0 / COUNT(*) AS ratio + FROM table12 t31 + WHERE EXISTS ( + SELECT 1 FROM table13 + WHERE id = t31.related_id + ) + GROUP BY t31.id) AS layer31) * 1.0 / COUNT(*) AS ratio + FROM table11 t30 + WHERE EXISTS ( + SELECT 1 FROM table12 + WHERE id = t30.related_id + ) + GROUP BY t30.id) AS layer30) * 1.0 / COUNT(*) AS ratio + FROM table10 t29 + WHERE EXISTS ( + SELECT 1 FROM table11 + WHERE id = t29.related_id + ) + GROUP BY t29.id) AS layer29) * 1.0 / COUNT(*) AS ratio + FROM table9 t28 + WHERE EXISTS ( + SELECT 1 FROM table10 + WHERE id = t28.related_id + ) + GROUP BY t28.id) AS layer28) * 1.0 / COUNT(*) AS ratio + FROM table8 t27 + WHERE EXISTS ( + SELECT 1 FROM table9 + WHERE id = t27.related_id + ) + GROUP BY t27.id) AS layer27) * 1.0 / COUNT(*) AS ratio + FROM table7 t26 + WHERE EXISTS ( + SELECT 1 FROM table8 + WHERE id = t26.related_id + ) + GROUP BY t26.id) AS layer26) * 1.0 / COUNT(*) AS ratio + FROM table6 t25 + WHERE EXISTS ( + SELECT 1 FROM table7 + WHERE id = t25.related_id + ) + GROUP BY t25.id) AS layer25) * 1.0 / COUNT(*) AS ratio + FROM table5 t24 + WHERE EXISTS ( + SELECT 1 FROM table6 + WHERE id = t24.related_id + ) + GROUP BY t24.id) AS layer24) * 1.0 / COUNT(*) AS ratio + FROM table4 t23 + WHERE EXISTS ( + SELECT 1 FROM table5 + WHERE id = t23.related_id + ) + GROUP BY t23.id) AS layer23) * 1.0 / COUNT(*) AS ratio + FROM table3 t22 + WHERE EXISTS ( + SELECT 1 FROM table4 + WHERE id = t22.related_id + ) + GROUP BY t22.id) AS layer22) * 1.0 / COUNT(*) AS ratio + FROM table2 t21 + WHERE EXISTS ( + SELECT 1 FROM table3 + WHERE id = t21.related_id + ) + GROUP BY t21.id) AS layer21) * 1.0 / COUNT(*) AS ratio + FROM table1 t20 + WHERE EXISTS ( + SELECT 1 FROM table2 + WHERE id = t20.related_id + ) + GROUP BY t20.id) AS layer20) * 1.0 / COUNT(*) AS ratio + FROM table20 t19 + WHERE EXISTS ( + SELECT 1 FROM table1 + WHERE id = t19.related_id + ) + GROUP BY t19.id) AS layer19) * 1.0 / COUNT(*) AS ratio + FROM table19 t18 + WHERE EXISTS ( + SELECT 1 FROM table20 + WHERE id = t18.related_id + ) + GROUP BY t18.id) AS layer18) * 1.0 / COUNT(*) AS ratio + FROM table18 t17 + WHERE EXISTS ( + SELECT 1 FROM table19 + WHERE id = t17.related_id + ) + GROUP BY t17.id) AS layer17) * 1.0 / COUNT(*) AS ratio + FROM table17 t16 + WHERE EXISTS ( + SELECT 1 FROM table18 + WHERE id = t16.related_id + ) + GROUP BY t16.id) AS layer16) * 1.0 / COUNT(*) AS ratio + FROM table16 t15 + WHERE EXISTS ( + SELECT 1 FROM table17 + WHERE id = t15.related_id + ) + GROUP BY t15.id) AS layer15) * 1.0 / COUNT(*) AS ratio + FROM table15 t14 + WHERE EXISTS ( + SELECT 1 FROM table16 + WHERE id = t14.related_id + ) + GROUP BY t14.id) AS layer14) * 1.0 / COUNT(*) AS ratio + FROM table14 t13 + WHERE EXISTS ( + SELECT 1 FROM table15 + WHERE id = t13.related_id + ) + GROUP BY t13.id) AS layer13) * 1.0 / COUNT(*) AS ratio + FROM table13 t12 + WHERE EXISTS ( + SELECT 1 FROM table14 + WHERE id = t12.related_id + ) + GROUP BY t12.id) AS layer12) * 1.0 / COUNT(*) AS ratio + FROM table12 t11 + WHERE EXISTS ( + SELECT 1 FROM table13 + WHERE id = t11.related_id + ) + GROUP BY t11.id) AS layer11) * 1.0 / COUNT(*) AS ratio + FROM table11 t10 + WHERE EXISTS ( + SELECT 1 FROM table12 + WHERE id = t10.related_id + ) + GROUP BY t10.id) AS layer10) * 1.0 / COUNT(*) AS ratio + FROM table10 t9 + WHERE EXISTS ( + SELECT 1 FROM table11 + WHERE id = t9.related_id + ) + GROUP BY t9.id) AS layer9) * 1.0 / COUNT(*) AS ratio + FROM table9 t8 + WHERE EXISTS ( + SELECT 1 FROM table10 + WHERE id = t8.related_id + ) + GROUP BY t8.id) AS layer8) * 1.0 / COUNT(*) AS ratio + FROM table8 t7 + WHERE EXISTS ( + SELECT 1 FROM table9 + WHERE id = t7.related_id + ) + GROUP BY t7.id) AS layer7) * 1.0 / COUNT(*) AS ratio + FROM table7 t6 + WHERE EXISTS ( + SELECT 1 FROM table8 + WHERE id = t6.related_id + ) + GROUP BY t6.id) AS layer6) * 1.0 / COUNT(*) AS ratio + FROM table6 t5 + WHERE EXISTS ( + SELECT 1 FROM table7 + WHERE id = t5.related_id + ) + GROUP BY t5.id) AS layer5) * 1.0 / COUNT(*) AS ratio + FROM table5 t4 + WHERE EXISTS ( + SELECT 1 FROM table6 + WHERE id = t4.related_id + ) + GROUP BY t4.id) AS layer4) * 1.0 / COUNT(*) AS ratio + FROM table4 t3 + WHERE EXISTS ( + SELECT 1 FROM table5 + WHERE id = t3.related_id + ) + GROUP BY t3.id) AS layer3) * 1.0 / COUNT(*) AS ratio + FROM table3 t2 + WHERE EXISTS ( + SELECT 1 FROM table4 + WHERE id = t2.related_id + ) + GROUP BY t2.id) AS layer2) * 1.0 / COUNT(*) AS ratio + FROM table2 t1 + WHERE EXISTS ( + SELECT 1 FROM table3 + WHERE id = t1.related_id + ) + GROUP BY t1.id) AS layer1; + """ +} + From 44b6e4803f04e21dca2d9303a1fd5f7863bf3817 Mon Sep 17 00:00:00 2001 From: Luwei Date: Thu, 24 Jul 2025 10:50:04 +0800 Subject: [PATCH 308/572] [fix](regression) fix multi-az failover regression case (#53805) --- .../use_default_vcg_read_write.out | 18 + .../use_vcg_read_write_routine_load.out | 373 ++++++++++++++++++ .../default_vcg_auto_failover.groovy | 8 +- ...failover_standby_disable_compaction.groovy | 28 +- .../standby_disable_compaction.groovy | 3 +- .../use_default_vcg_read_write.groovy | 11 +- ...se_vcg_read_write_unhealthy_node_50.groovy | 18 +- 7 files changed, 448 insertions(+), 11 deletions(-) create mode 100644 regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.out diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out index 3900f080bd0b57..0abde7ca45bf71 100644 --- a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.out @@ -17,3 +17,21 @@ -- !all12 -- 33 +-- !all11 -- +20 + +-- !all12 -- +11 + +-- !all11 -- +40 + +-- !all12 -- +22 + +-- !all11 -- +60 + +-- !all12 -- +33 + diff --git a/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.out b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.out new file mode 100644 index 00000000000000..6006a85a705d18 --- /dev/null +++ b/regression-test/data/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_routine_load.out @@ -0,0 +1,373 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q1 -- +0 abc 0 +1 abc 2 +10 abc 20 +11 abc 22 +12 abc 24 +13 abc 26 +14 abc 28 +15 abc 30 +16 abc 32 +17 abc 34 +18 abc 36 +19 abc 38 +2 abc 4 +20 abc 40 +21 abc 42 +22 abc 44 +23 abc 46 +24 abc 48 +25 abc 50 +26 abc 52 +27 abc 54 +28 abc 56 +29 abc 58 +3 abc 6 +4 abc 8 +5 abc 10 +6 abc 12 +7 abc 14 +8 abc 16 +9 abc 18 + +-- !q1 -- +0 abc 0 +0 abc 0 +1 abc 2 +1 abc 2 +10 abc 20 +10 abc 20 +11 abc 22 +11 abc 22 +12 abc 24 +12 abc 24 +13 abc 26 +13 abc 26 +14 abc 28 +14 abc 28 +15 abc 30 +15 abc 30 +16 abc 32 +16 abc 32 +17 abc 34 +17 abc 34 +18 abc 36 +18 abc 36 +19 abc 38 +19 abc 38 +2 abc 4 +2 abc 4 +20 abc 40 +20 abc 40 +21 abc 42 +21 abc 42 +22 abc 44 +22 abc 44 +23 abc 46 +23 abc 46 +24 abc 48 +24 abc 48 +25 abc 50 +25 abc 50 +26 abc 52 +26 abc 52 +27 abc 54 +27 abc 54 +28 abc 56 +28 abc 56 +29 abc 58 +29 abc 58 +3 abc 6 +3 abc 6 +4 abc 8 +4 abc 8 +5 abc 10 +5 abc 10 +6 abc 12 +6 abc 12 +7 abc 14 +7 abc 14 +8 abc 16 +8 abc 16 +9 abc 18 +9 abc 18 + +-- !q1 -- +0 abc 0 +0 abc 0 +0 abc 0 +1 abc 2 +1 abc 2 +1 abc 2 +10 abc 20 +10 abc 20 +10 abc 20 +11 abc 22 +11 abc 22 +11 abc 22 +12 abc 24 +12 abc 24 +12 abc 24 +13 abc 26 +13 abc 26 +13 abc 26 +14 abc 28 +14 abc 28 +14 abc 28 +15 abc 30 +15 abc 30 +15 abc 30 +16 abc 32 +16 abc 32 +16 abc 32 +17 abc 34 +17 abc 34 +17 abc 34 +18 abc 36 +18 abc 36 +18 abc 36 +19 abc 38 +19 abc 38 +19 abc 38 +2 abc 4 +2 abc 4 +2 abc 4 +20 abc 40 +20 abc 40 +20 abc 40 +21 abc 42 +21 abc 42 +21 abc 42 +22 abc 44 +22 abc 44 +22 abc 44 +23 abc 46 +23 abc 46 +23 abc 46 +24 abc 48 +24 abc 48 +24 abc 48 +25 abc 50 +25 abc 50 +25 abc 50 +26 abc 52 +26 abc 52 +26 abc 52 +27 abc 54 +27 abc 54 +27 abc 54 +28 abc 56 +28 abc 56 +28 abc 56 +29 abc 58 +29 abc 58 +29 abc 58 +3 abc 6 +3 abc 6 +3 abc 6 +4 abc 8 +4 abc 8 +4 abc 8 +5 abc 10 +5 abc 10 +5 abc 10 +6 abc 12 +6 abc 12 +6 abc 12 +7 abc 14 +7 abc 14 +7 abc 14 +8 abc 16 +8 abc 16 +8 abc 16 +9 abc 18 +9 abc 18 +9 abc 18 + +-- !q1 -- +0 abc 0 +1 abc 2 +10 abc 20 +11 abc 22 +12 abc 24 +13 abc 26 +14 abc 28 +15 abc 30 +16 abc 32 +17 abc 34 +18 abc 36 +19 abc 38 +2 abc 4 +20 abc 40 +21 abc 42 +22 abc 44 +23 abc 46 +24 abc 48 +25 abc 50 +26 abc 52 +27 abc 54 +28 abc 56 +29 abc 58 +3 abc 6 +4 abc 8 +5 abc 10 +6 abc 12 +7 abc 14 +8 abc 16 +9 abc 18 + +-- !q1 -- +0 abc 0 +0 abc 0 +1 abc 2 +1 abc 2 +10 abc 20 +10 abc 20 +11 abc 22 +11 abc 22 +12 abc 24 +12 abc 24 +13 abc 26 +13 abc 26 +14 abc 28 +14 abc 28 +15 abc 30 +15 abc 30 +16 abc 32 +16 abc 32 +17 abc 34 +17 abc 34 +18 abc 36 +18 abc 36 +19 abc 38 +19 abc 38 +2 abc 4 +2 abc 4 +20 abc 40 +20 abc 40 +21 abc 42 +21 abc 42 +22 abc 44 +22 abc 44 +23 abc 46 +23 abc 46 +24 abc 48 +24 abc 48 +25 abc 50 +25 abc 50 +26 abc 52 +26 abc 52 +27 abc 54 +27 abc 54 +28 abc 56 +28 abc 56 +29 abc 58 +29 abc 58 +3 abc 6 +3 abc 6 +4 abc 8 +4 abc 8 +5 abc 10 +5 abc 10 +6 abc 12 +6 abc 12 +7 abc 14 +7 abc 14 +8 abc 16 +8 abc 16 +9 abc 18 +9 abc 18 + +-- !q1 -- +0 abc 0 +0 abc 0 +0 abc 0 +1 abc 2 +1 abc 2 +1 abc 2 +10 abc 20 +10 abc 20 +10 abc 20 +11 abc 22 +11 abc 22 +11 abc 22 +12 abc 24 +12 abc 24 +12 abc 24 +13 abc 26 +13 abc 26 +13 abc 26 +14 abc 28 +14 abc 28 +14 abc 28 +15 abc 30 +15 abc 30 +15 abc 30 +16 abc 32 +16 abc 32 +16 abc 32 +17 abc 34 +17 abc 34 +17 abc 34 +18 abc 36 +18 abc 36 +18 abc 36 +19 abc 38 +19 abc 38 +19 abc 38 +2 abc 4 +2 abc 4 +2 abc 4 +20 abc 40 +20 abc 40 +20 abc 40 +21 abc 42 +21 abc 42 +21 abc 42 +22 abc 44 +22 abc 44 +22 abc 44 +23 abc 46 +23 abc 46 +23 abc 46 +24 abc 48 +24 abc 48 +24 abc 48 +25 abc 50 +25 abc 50 +25 abc 50 +26 abc 52 +26 abc 52 +26 abc 52 +27 abc 54 +27 abc 54 +27 abc 54 +28 abc 56 +28 abc 56 +28 abc 56 +29 abc 58 +29 abc 58 +29 abc 58 +3 abc 6 +3 abc 6 +3 abc 6 +4 abc 8 +4 abc 8 +4 abc 8 +5 abc 10 +5 abc 10 +5 abc 10 +6 abc 12 +6 abc 12 +6 abc 12 +7 abc 14 +7 abc 14 +7 abc 14 +8 abc 16 +8 abc 16 +8 abc 16 +9 abc 18 +9 abc 18 +9 abc 18 + diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy index cf9934469c6243..06a52a31b8bd5f 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/default_vcg_auto_failover.groovy @@ -130,7 +130,13 @@ suite('default_vcg_auto_failover', 'multi_cluster,docker') { def reconnectFe = { sleep(10000) logger.info("Reconnecting to a new frontend...") - def newFe = cluster.getMasterFe() + def newFe + if (options.connectToFollower) { + newFe = cluster.getOneFollowerFe() + } else { + newFe = cluster.getMasterFe() + } + if (newFe) { logger.info("New frontend found: ${newFe.host}:${newFe.httpPort}") def url = String.format( diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy index a92d5b7a92030d..1404b6c1e2ee3b 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/failover_standby_disable_compaction.groovy @@ -153,6 +153,31 @@ suite('failover_standby_disable_compaction', 'multi_cluster,docker') { """ cluster.stopBackends(4, 5) + + def reconnectFe = { + sleep(10000) + logger.info("Reconnecting to a new frontend...") + def newFe + if (options.connectToFollower) { + newFe = cluster.getOneFollowerFe() + } else { + newFe = cluster.getMasterFe() + } + if (newFe) { + logger.info("New frontend found: ${newFe.host}:${newFe.httpPort}") + def url = String.format( + "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", + newFe.host, newFe.queryPort) + url = context.config.buildUrlWithDb(url, context.dbName) + context.connectTo(url, context.config.jdbcUser, context.config.jdbcPassword) + logger.info("Successfully reconnected to the new frontend") + } else { + logger.error("No new frontend found to reconnect") + } + } + + reconnectFe() + sleep(30000) sql """ insert into ${tbl} (k1, k2) values (1, "10") """ sql """ SELECT count(*) FROM ${tableName} """ @@ -274,9 +299,8 @@ suite('failover_standby_disable_compaction', 'multi_cluster,docker') { assertTrue(before_cluster2_be0_compaction < after_cluster2_be0_compaction || before_cluster2_be1_compaction < after_cluster2_be1_compaction ) - def set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + def addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set, false) if (options.connectToFollower) { checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) } else { diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy index dc39526bfe0c04..2393e27ef0e8de 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/standby_disable_compaction.groovy @@ -261,9 +261,8 @@ suite('standby_disable_compaction', 'multi_cluster,docker') { assertTrue(before_cluster2_be0_compaction < after_cluster2_be0_compaction || before_cluster2_be1_compaction < after_cluster2_be1_compaction ) - def set = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set + def addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set, false) if (options.connectToFollower) { checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) } else { diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy index 707ea42ce89113..77080ee3711916 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_default_vcg_read_write.groovy @@ -129,7 +129,12 @@ suite('use_default_vcg_read_write', 'multi_cluster,docker') { def reconnectFe = { sleep(10000) logger.info("Reconnecting to a new frontend...") - def newFe = cluster.getMasterFe() + def newFe + if (options.connectToFollower) { + newFe = cluster.getOneFollowerFe() + } else { + newFe = cluster.getMasterFe() + } if (newFe) { logger.info("New frontend found: ${newFe.host}:${newFe.httpPort}") def url = String.format( @@ -428,7 +433,7 @@ suite('use_default_vcg_read_write', 'multi_cluster,docker') { checkProfileNew.call(cluster.getMasterFe(), addrSet) } - sleep(16000) + sleep(21000) sql """ insert into ${tbl} (k1, k2) values (1, "10"); @@ -439,7 +444,7 @@ suite('use_default_vcg_read_write', 'multi_cluster,docker') { log.info("show compute group {}", showComputeGroup) vcgInShow = showComputeGroup.find { it.Name == normalVclusterName } assertNotNull(vcgInShow) - assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster1","standbyComputeGroup":"newcluster2"')) + assertTrue(vcgInShow.Policy.contains('"activeComputeGroup":"newcluster2","standbyComputeGroup":"newcluster1"')) } // connect to follower, run again options.connectToFollower = true diff --git a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy index e1f2923814e30f..40a875f64b87a5 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/use_vcg_read_write_unhealthy_node_50.groovy @@ -243,7 +243,11 @@ suite('use_vcg_read_write_unhealthy_node_50', 'multi_cluster,docker') { def addrSet = [cluster1Ips[0] + ":" + "8060", cluster1Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(addrSet) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } cluster.stopBackends(4) sleep(10000) @@ -333,7 +337,11 @@ suite('use_vcg_read_write_unhealthy_node_50', 'multi_cluster,docker') { addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(addrSet) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(16000) sql """ @@ -406,7 +414,11 @@ suite('use_vcg_read_write_unhealthy_node_50', 'multi_cluster,docker') { addrSet = [cluster2Ips[0] + ":" + "8060", cluster2Ips[1] + ":" + "8060"] as Set sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(addrSet) + if (options.connectToFollower) { + checkProfileNew.call(cluster.getOneFollowerFe(), addrSet) + } else { + checkProfileNew.call(cluster.getMasterFe(), addrSet) + } sleep(16000) // show cluster From a0aeec3622fe87825533803a34598a29cfb0ad52 Mon Sep 17 00:00:00 2001 From: zfr95 Date: Thu, 24 Jul 2025 10:58:52 +0800 Subject: [PATCH 309/572] [fix](auth)fix auth test case in cloud mode (#53785) pick from #53646 [fix](auth)fix auth test case in cloud mode --- .../test_account_management_grant_auth.groovy | 16 ++++++++-------- .../test_account_management_role_auth.groovy | 16 ++++++++-------- .../test_account_management_user_auth.groovy | 16 ++++++++-------- .../auth_call/test_assistant_command_auth.groovy | 14 +++++++------- .../test_cluster_management_auth.groovy | 8 ++++---- .../test_database_management_auth.groovy | 15 ++++++++------- .../auth_call/test_ddl_catalog_auth.groovy | 16 ++++++++-------- .../auth_call/test_ddl_database_auth.groovy | 12 ++++++------ .../auth_call/test_ddl_encryptkey_auth.groovy | 12 ++++++------ .../suites/auth_call/test_ddl_file_auth.groovy | 11 ++++++----- .../auth_call/test_ddl_function_auth.groovy | 16 ++++++++-------- .../auth_call/test_ddl_mask_view_auth.groovy | 7 ++++--- .../suites/auth_call/test_ddl_mv_auth.groovy | 7 +++---- .../auth_call/test_ddl_part_table_auth.groovy | 8 +++++--- .../auth_call/test_ddl_row_policy_auth.groovy | 9 +++++---- .../test_ddl_sql_block_rule_auth.groovy | 9 ++++----- .../suites/auth_call/test_ddl_view_auth.groovy | 7 +++---- .../auth_call/test_dml_broker_load_auth.groovy | 7 ++++--- .../test_dml_cancel_profile_auth.groovy | 5 ++--- .../auth_call/test_dml_delete_table_auth.groovy | 7 +++---- .../auth_call/test_dml_export_table_auth.groovy | 7 +++---- .../suites/auth_call/test_dml_insert_auth.groovy | 7 +++---- .../test_dml_multi_routine_load_auth.groovy | 7 +++---- .../auth_call/test_dml_mysql_load_auth.groovy | 7 +++---- .../auth_call/test_dml_outfile_auth.groovy | 7 +++---- .../auth_call/test_dml_routine_load_auth.groovy | 7 +++---- .../auth_call/test_dml_select_udf_auth.groovy | 7 +++---- .../auth_call/test_dml_stream_load_auth.groovy | 7 +++---- .../auth_call/test_dml_update_table_auth.groovy | 7 +++---- .../test_grant_show_view_priv_auth.groovy | 11 +++++------ .../auth_call/test_hive_base_case_auth.groovy | 15 +++++++-------- .../auth_call/test_show_backend_auth.groovy | 5 ++--- .../auth_call/test_show_broker_auth.groovy | 4 ++-- .../auth_call/test_show_charset_auth.groovy | 13 +++++++------ .../test_show_convert_light_sc_auth.groovy | 5 ++--- .../auth_call/test_show_create_table_auth.groovy | 7 +++---- .../suites/auth_call/test_show_data_auth.groovy | 7 +++---- .../auth_call/test_show_database_id_auth.groovy | 7 +++---- .../test_show_dynamic_table_auth.groovy | 7 +++---- .../auth_call/test_show_frontend_auth.groovy | 5 ++--- .../suites/auth_call/test_show_grant_auth.groovy | 5 ++--- .../suites/auth_call/test_show_proc_auth.groovy | 5 ++--- .../auth_call/test_show_query_stats_auth.groovy | 5 ++--- .../auth_call/test_show_tablet_auth.groovy | 7 +++---- .../auth_call/test_show_typecast_auth.groovy | 5 ++--- 45 files changed, 187 insertions(+), 207 deletions(-) diff --git a/regression-test/suites/auth_call/test_account_management_grant_auth.groovy b/regression-test/suites/auth_call/test_account_management_grant_auth.groovy index a1e1f68916f90a..b957713d937309 100644 --- a/regression-test/suites/auth_call/test_account_management_grant_auth.groovy +++ b/regression-test/suites/auth_call/test_account_management_grant_auth.groovy @@ -25,14 +25,6 @@ suite("test_account_management_grant_auth","p0,auth_call") { String dbName = 'test_account_management_grant_auth_db' String user_derive = 'test_account_management_grant_user_derive_role' String role_derive = 'test_account_management_grant_role_derive_role' - - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } try_sql("DROP USER ${user}") try_sql("DROP USER ${user_derive}") @@ -42,6 +34,14 @@ suite("test_account_management_grant_auth","p0,auth_call") { sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + connect(user, "${pwd}", context.config.jdbcUrl) { test { sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';""" diff --git a/regression-test/suites/auth_call/test_account_management_role_auth.groovy b/regression-test/suites/auth_call/test_account_management_role_auth.groovy index dab9596b8c2ae5..d1b58662f2c74d 100644 --- a/regression-test/suites/auth_call/test_account_management_role_auth.groovy +++ b/regression-test/suites/auth_call/test_account_management_role_auth.groovy @@ -27,14 +27,6 @@ suite("test_account_management_role_auth","p0,auth_call") { String role_derive = 'test_account_management_role_auth_derive_role' - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } - try_sql("DROP USER ${user}") try_sql("DROP role ${role}") try_sql("DROP role ${role_derive}") @@ -45,6 +37,14 @@ suite("test_account_management_role_auth","p0,auth_call") { sql """create database ${dbName}""" sql """GRANT '${role}' TO ${user};""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + connect(user, "${pwd}", context.config.jdbcUrl) { test { sql """CREATE ROLE ${role_derive}""" diff --git a/regression-test/suites/auth_call/test_account_management_user_auth.groovy b/regression-test/suites/auth_call/test_account_management_user_auth.groovy index 9cbb489615b41e..4d9aa6eed3f6b6 100644 --- a/regression-test/suites/auth_call/test_account_management_user_auth.groovy +++ b/regression-test/suites/auth_call/test_account_management_user_auth.groovy @@ -25,14 +25,6 @@ suite("test_account_management_user_auth","p0,auth_call") { String dbName = 'test_account_management_user_auth_db' String user_derive = 'test_account_management_user_derive_user' - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } - try_sql("DROP USER ${user}") try_sql("DROP USER ${user_derive}") try_sql """drop database if exists ${dbName}""" @@ -41,6 +33,14 @@ suite("test_account_management_user_auth","p0,auth_call") { sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + connect(user, "${pwd}", context.config.jdbcUrl) { test { sql """CREATE USER ${user_derive} IDENTIFIED BY '${pwd}';""" diff --git a/regression-test/suites/auth_call/test_assistant_command_auth.groovy b/regression-test/suites/auth_call/test_assistant_command_auth.groovy index ba539f129d7bc9..1f150d196e13af 100644 --- a/regression-test/suites/auth_call/test_assistant_command_auth.groovy +++ b/regression-test/suites/auth_call/test_assistant_command_auth.groovy @@ -26,6 +26,13 @@ suite("test_assistant_command_auth","p0,auth_call") { String tableName = 'test_assistant_command_auth_tb' String catalogName = 'test_assistant_command_auth_catalog' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + sql """create database ${dbName}""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -34,13 +41,6 @@ suite("test_assistant_command_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" - sql """grant select_priv on regression_test to ${user}""" - sql """create database ${dbName}""" - sql """create table ${dbName}.${tableName} ( id BIGINT, username VARCHAR(20) diff --git a/regression-test/suites/auth_call/test_cluster_management_auth.groovy b/regression-test/suites/auth_call/test_cluster_management_auth.groovy index c3c1692c405109..afef32b78ca596 100644 --- a/regression-test/suites/auth_call/test_cluster_management_auth.groovy +++ b/regression-test/suites/auth_call/test_cluster_management_auth.groovy @@ -61,6 +61,10 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") { String user = 'test_cluster_management_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -69,10 +73,6 @@ suite ("test_cluster_management_auth","nonConcurrent,p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" - sql """grant select_priv on regression_test to ${user}""" - // pipeline can't support delete node, it can affect other case if (is_exists_follower()) { connect(user, "${pwd}", context.config.jdbcUrl) { diff --git a/regression-test/suites/auth_call/test_database_management_auth.groovy b/regression-test/suites/auth_call/test_database_management_auth.groovy index cbf979abc1e3d5..5c572f98b76140 100644 --- a/regression-test/suites/auth_call/test_database_management_auth.groovy +++ b/regression-test/suites/auth_call/test_database_management_auth.groovy @@ -24,13 +24,6 @@ suite("test_database_management_auth","p0,auth_call") { String pwd = 'C123_567p' String dbName = 'test_database_management_auth_db' - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } try_sql("DROP USER ${user}") try_sql """drop database if exists ${dbName}""" @@ -39,6 +32,14 @@ suite("test_database_management_auth","p0,auth_call") { sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + connect(user, "${pwd}", context.config.jdbcUrl) { test { sql """SHOW FRONTEND CONFIG""" diff --git a/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy b/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy index 8d6b31d35c9217..89ad3b4c1043d0 100644 --- a/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_catalog_auth.groovy @@ -24,14 +24,6 @@ suite("test_ddl_catalog_auth","p0,auth_call") { String catalogNameNew = 'test_ddl_catalog_auth_catalog_new' String catalogNameOther = 'test_ddl_catalog_auth_catalog_other' - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } - sql """create catalog if not exists ${catalogNameOther} properties ( 'type'='hms' );""" @@ -42,6 +34,14 @@ suite("test_ddl_catalog_auth","p0,auth_call") { sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + // ddl create connect(user, "${pwd}", context.config.jdbcUrl) { test { diff --git a/regression-test/suites/auth_call/test_ddl_database_auth.groovy b/regression-test/suites/auth_call/test_ddl_database_auth.groovy index bdcf5ff7c39be7..982fd5ca966d0e 100644 --- a/regression-test/suites/auth_call/test_ddl_database_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_database_auth.groovy @@ -25,6 +25,12 @@ suite("test_ddl_database_auth","p0,auth_call") { String tableName = 'test_ddl_database_auth_tb' String tableNameNew = 'test_ddl_database_auth_tb_new' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + try_sql """drop database if exists ${dbNameNew}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -33,12 +39,6 @@ suite("test_ddl_database_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - try_sql """drop database if exists ${dbNameNew}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" - sql """grant select_priv on regression_test to ${user}""" - // ddl create connect(user, "${pwd}", context.config.jdbcUrl) { test { diff --git a/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy b/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy index dfa469bfce0028..951c2421dc7213 100644 --- a/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_encryptkey_auth.groovy @@ -23,6 +23,12 @@ suite("test_ddl_encryptkey_auth","p0,auth_call") { String dbName = 'test_ddl_encryptkey_auth_db' String encryptkeyName = 'test_ddl_encryptkey_auth_ecyk' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + sql """create database ${dbName}""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,12 +37,6 @@ suite("test_ddl_encryptkey_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" - sql """grant select_priv on regression_test to ${user}""" - sql """create database ${dbName}""" - // ddl create,show,drop connect(user, "${pwd}", context.config.jdbcUrl) { test { diff --git a/regression-test/suites/auth_call/test_ddl_file_auth.groovy b/regression-test/suites/auth_call/test_ddl_file_auth.groovy index 2cc604734064cb..335b206ea08706 100644 --- a/regression-test/suites/auth_call/test_ddl_file_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_file_auth.groovy @@ -23,6 +23,12 @@ suite("test_ddl_file_auth","p0,auth_call") { String dbName = 'test_ddl_file_auth_db' String fileName = 'test_ddl_file_auth_file' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + sql """create database ${dbName}""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,11 +37,6 @@ suite("test_ddl_file_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" - sql """grant select_priv on regression_test to ${user}""" - sql """create database ${dbName}""" String s3_endpoint = getS3Endpoint() String bucket = context.config.otherConfigs.get("s3BucketName"); diff --git a/regression-test/suites/auth_call/test_ddl_function_auth.groovy b/regression-test/suites/auth_call/test_ddl_function_auth.groovy index a2e38f0eb6d6c8..e14f8a259111ea 100644 --- a/regression-test/suites/auth_call/test_ddl_function_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_function_auth.groovy @@ -23,14 +23,6 @@ suite("test_ddl_function_auth","p0,auth_call") { String dbName = 'test_ddl_function_auth_db' String functionName = 'test_ddl_function_auth_fct' - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } - try_sql("DROP USER ${user}") try_sql """drop database if exists ${dbName}""" try_sql("""DROP FUNCTION ${dbName}.${functionName}(INT)""") @@ -39,6 +31,14 @@ suite("test_ddl_function_auth","p0,auth_call") { sql """create database ${dbName}""" sql """grant select_priv on ${dbName}.* to ${user}""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + // ddl create,show,drop connect(user, "${pwd}", context.config.jdbcUrl) { test { diff --git a/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy b/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy index 4a8bc2787dc62c..0f97477461f937 100644 --- a/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_mask_view_auth.groovy @@ -24,6 +24,10 @@ suite("test_ddl_mask_view_auth","p0,auth_call") { String tableName = 'test_ddl_mask_view_auth_tb' String viewName = 'test_ddl_mask_view_auth_view' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -32,9 +36,6 @@ suite("test_ddl_mask_view_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy index 4b4810604409fe..cf466167daf3aa 100644 --- a/regression-test/suites/auth_call/test_ddl_mv_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_mv_auth.groovy @@ -25,6 +25,9 @@ suite("test_ddl_mv_auth","p0,auth_call") { String mvName = 'test_ddl_mv_auth_mv' String rollupName = 'test_ddl_mv_auth_rollup' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -32,10 +35,6 @@ suite("test_ddl_mv_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy b/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy index 5217dc2d051860..797b0ae05ff8be 100644 --- a/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_part_table_auth.groovy @@ -23,6 +23,10 @@ suite("test_ddl_part_table_auth","p0,auth_call") { String dbName = 'test_ddl_part_table_auth_db' String tableName = 'test_ddl_part_table_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,9 +35,7 @@ suite("test_ddl_part_table_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy b/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy index f9ac109f7f377f..477c553853c243 100644 --- a/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_row_policy_auth.groovy @@ -24,6 +24,11 @@ suite("test_ddl_row_policy_auth","p0,auth_call") { String tableName = 'test_ddl_row_policy_auth_tb' String rowPolicyName = 'test_ddl_row_policy_auth_rp' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + try_sql """DROP ROW POLICY ${rowPolicyName} on ${dbName}.${tableName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -32,10 +37,6 @@ suite("test_ddl_row_policy_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - try_sql """DROP ROW POLICY ${rowPolicyName} on ${dbName}.${tableName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy b/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy index 568ea9723d58c3..e519465f385709 100644 --- a/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_sql_block_rule_auth.groovy @@ -24,6 +24,10 @@ suite("test_ddl_sql_block_rule_auth","p0,auth_call") { String tableName = 'test_ddl_sbr_auth_tb' String sqlBlockRuleName = 'test_ddl_sbr_auth_sbr' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + try_sql("""DROP SQL_BLOCK_RULE ${sqlBlockRuleName};""") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,11 +35,6 @@ suite("test_ddl_sql_block_rule_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - try_sql("""DROP SQL_BLOCK_RULE ${sqlBlockRuleName};""") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_ddl_view_auth.groovy b/regression-test/suites/auth_call/test_ddl_view_auth.groovy index 1a915acdb3cd6f..db17c25e7867c2 100644 --- a/regression-test/suites/auth_call/test_ddl_view_auth.groovy +++ b/regression-test/suites/auth_call/test_ddl_view_auth.groovy @@ -24,6 +24,9 @@ suite("test_ddl_view_auth","p0,auth_call") { String tableName = 'test_ddl_view_auth_tb' String viewName = 'test_ddl_view_auth_view' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,10 +34,6 @@ suite("test_ddl_view_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy b/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy index 4eb9202f6ee8af..5777d3f3077a3d 100644 --- a/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_broker_load_auth.groovy @@ -37,6 +37,10 @@ suite("test_dml_broker_load_auth","p0,auth_call") { String region = getS3Region() String bucket = getS3BucketName() + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -45,9 +49,6 @@ suite("test_dml_broker_load_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy b/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy index 0b4d05ef88c970..64a42b1217f675 100644 --- a/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_cancel_profile_auth.groovy @@ -23,6 +23,8 @@ suite("test_dml_cancel_profile_auth","p0,auth_call,nonConcurrent") { String user = 'test_dml_cancel_profile_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -30,9 +32,6 @@ suite("test_dml_cancel_profile_auth","p0,auth_call,nonConcurrent") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { diff --git a/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy b/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy index f956e626103add..030ab2f83c9401 100644 --- a/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_delete_table_auth.groovy @@ -25,6 +25,9 @@ suite("test_dml_delete_table_auth","p0,auth_call") { String dbName = 'test_dml_delete_table_auth_db' String tableName = 'test_dml_delete_table_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -32,10 +35,6 @@ suite("test_dml_delete_table_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_export_table_auth.groovy b/regression-test/suites/auth_call/test_dml_export_table_auth.groovy index 0112e86fee639d..eddfe473cc5722 100644 --- a/regression-test/suites/auth_call/test_dml_export_table_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_export_table_auth.groovy @@ -31,6 +31,9 @@ suite("test_dml_export_table_auth","p0,auth_call") { String tableName = 'test_dml_export_table_auth_tb' String exportLabel = 'test_dml_export_table_auth_label' + hashCode.toString() + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -38,10 +41,6 @@ suite("test_dml_export_table_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_insert_auth.groovy b/regression-test/suites/auth_call/test_dml_insert_auth.groovy index 5b8a20e18f9f70..3e1ba7e4b3b978 100644 --- a/regression-test/suites/auth_call/test_dml_insert_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_insert_auth.groovy @@ -26,6 +26,9 @@ suite("test_dml_insert_auth","p0,auth_call") { String tableName = 'test_dml_insert_auth_tb' String srcTableName = 'test_dml_insert_auth_tb_src' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -33,10 +36,6 @@ suite("test_dml_insert_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy b/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy index 5b8ecef7e57eae..42d771950def0f 100644 --- a/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_multi_routine_load_auth.groovy @@ -29,6 +29,9 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") { String tableName2 = 'test_dml_multi_routine_load_auth_tb2' String labelName = 'test_dml_multi_routine_load_auth_label' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -36,10 +39,6 @@ suite("test_dml_multi_routine_load_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy b/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy index 97151a0d001e85..acb6bbea6912bf 100644 --- a/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_mysql_load_auth.groovy @@ -25,6 +25,9 @@ suite("test_dml_mysql_load_auth","p0,auth_call") { String dbName = 'test_dml_mysql_load_auth_db' String tableName = 'test_dml_mysql_load_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -32,10 +35,6 @@ suite("test_dml_mysql_load_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_outfile_auth.groovy b/regression-test/suites/auth_call/test_dml_outfile_auth.groovy index 5f4096344b8624..0649430d1e2dfe 100644 --- a/regression-test/suites/auth_call/test_dml_outfile_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_outfile_auth.groovy @@ -29,6 +29,9 @@ suite("test_dml_outfile_auth","p0,auth_call") { String dbName = 'test_dml_outfile_auth_db' String tableName = 'test_dml_outfile_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -36,10 +39,6 @@ suite("test_dml_outfile_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy b/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy index 025ac555356aa7..5b4f6b720f21ac 100644 --- a/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_routine_load_auth.groovy @@ -28,6 +28,9 @@ suite("test_dml_routine_load_auth","p0,auth_call") { String tableName = 'test_dml_routine_load_auth_tb' String labelName = 'test_dml_routine_load_auth_label' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -35,10 +38,6 @@ suite("test_dml_routine_load_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy index 6aa74784969e30..f09d28b167c4ca 100644 --- a/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_select_udf_auth.groovy @@ -29,6 +29,9 @@ suite("test_dml_select_udf_auth","p0,auth_call") { String tableName = 'test_dml_select_udf_auth_tb' String udfName = 'test_dml_select_udf_auth_udf' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -36,10 +39,6 @@ suite("test_dml_select_udf_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy index 4a4f19641b2a50..8531a193da7b4e 100644 --- a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy @@ -24,6 +24,9 @@ suite("test_dml_stream_load_auth","p0,auth_call") { String dbName = 'test_dml_stream_load_auth_db' String tableName = 'test_dml_stream_load_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,10 +34,6 @@ suite("test_dml_stream_load_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_dml_update_table_auth.groovy b/regression-test/suites/auth_call/test_dml_update_table_auth.groovy index eb258df2df2800..f323645231fa9f 100644 --- a/regression-test/suites/auth_call/test_dml_update_table_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_update_table_auth.groovy @@ -25,6 +25,9 @@ suite("test_dml_update_table_auth","p0,auth_call") { String dbName = 'test_dml_update_table_auth_db' String tableName = 'test_dml_update_table_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -32,10 +35,6 @@ suite("test_dml_update_table_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy b/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy index c05da39ce2e06c..070d4f5f2b34e0 100644 --- a/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy +++ b/regression-test/suites/auth_call/test_grant_show_view_priv_auth.groovy @@ -27,6 +27,11 @@ suite("test_grant_show_view_priv_auth","p0,auth_call") { String tableName = 'test_grant_show_view_priv_auth_tb' String viewName = 'test_grant_show_view_priv_auth_view' + try_sql("DROP USER ${user}") + try_sql("DROP USER ${user_derive}") + try_sql """drop database if exists ${dbName}""" + + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -34,12 +39,6 @@ suite("test_grant_show_view_priv_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql("DROP USER ${user_derive}") - try_sql """drop database if exists ${dbName}""" - - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy index 0fb0d422ebd2c0..a02c620862b9a2 100644 --- a/regression-test/suites/auth_call/test_hive_base_case_auth.groovy +++ b/regression-test/suites/auth_call/test_hive_base_case_auth.groovy @@ -24,14 +24,6 @@ suite("test_hive_base_case_auth", "p0,auth_call") { String tableName = 'test_hive_base_case_auth_tb' String tableNameNew = 'test_hive_base_case_auth_tb_new' - //cloud-mode - if (isCloudMode()) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; - } - String enabled = context.config.otherConfigs.get("enableHiveTest") if (enabled == null || !enabled.equalsIgnoreCase("true")) { logger.info("diable Hive test.") @@ -49,6 +41,13 @@ suite("test_hive_base_case_auth", "p0,auth_call") { try_sql """drop catalog if exists ${catalogName}""" try_sql """drop database if exists ${dbName}""" sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } sql """grant select_priv on regression_test to ${user}""" // create catalog diff --git a/regression-test/suites/auth_call/test_show_backend_auth.groovy b/regression-test/suites/auth_call/test_show_backend_auth.groovy index 627e28c57d1bd7..a28acfb5db748a 100644 --- a/regression-test/suites/auth_call/test_show_backend_auth.groovy +++ b/regression-test/suites/auth_call/test_show_backend_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_backend_auth","p0,auth_call") { String user = 'test_show_backend_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_backend_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """grant select_priv on internal.information_schema.* to ${user}""" def show_grants_result = sql """show grants for ${user}""" diff --git a/regression-test/suites/auth_call/test_show_broker_auth.groovy b/regression-test/suites/auth_call/test_show_broker_auth.groovy index a364ad5b33146d..ee0560a2026907 100644 --- a/regression-test/suites/auth_call/test_show_broker_auth.groovy +++ b/regression-test/suites/auth_call/test_show_broker_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_broker_auth","p0,auth_call") { String user = 'test_show_broker_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -29,8 +31,6 @@ suite("test_show_broker_auth","p0,auth_call") { sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { diff --git a/regression-test/suites/auth_call/test_show_charset_auth.groovy b/regression-test/suites/auth_call/test_show_charset_auth.groovy index 14991bc99f9e38..d1b69a519928cc 100644 --- a/regression-test/suites/auth_call/test_show_charset_auth.groovy +++ b/regression-test/suites/auth_call/test_show_charset_auth.groovy @@ -22,20 +22,21 @@ suite("test_show_no_auth","p0,auth_call") { String user1 = 'test_show_charset_auth_user1' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + try_sql("DROP USER ${user1}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """CREATE USER '${user1}' IDENTIFIED BY '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user1}"""; } - try_sql("DROP USER ${user}") - try_sql("DROP USER ${user1}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" - sql """CREATE USER '${user1}' IDENTIFIED BY '${pwd}'""" - sql """grant select_priv on regression_test to ${user}""" - connect(user, "${pwd}", context.config.jdbcUrl) { sql """SHOW CHARSET""" sql """SHOW DATA TYPES""" diff --git a/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy b/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy index 6f8387d892925b..d43ee604554d01 100644 --- a/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy +++ b/regression-test/suites/auth_call/test_show_convert_light_sc_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_convert_light_sc_auth","p0,auth_call") { String user = 'test_show_convert_light_sc_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_convert_light_sc_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { diff --git a/regression-test/suites/auth_call/test_show_create_table_auth.groovy b/regression-test/suites/auth_call/test_show_create_table_auth.groovy index 166179bae5f6be..58c0915b86b359 100644 --- a/regression-test/suites/auth_call/test_show_create_table_auth.groovy +++ b/regression-test/suites/auth_call/test_show_create_table_auth.groovy @@ -23,6 +23,9 @@ suite("test_show_create_table_auth","p0,auth_call") { String dbName = 'test_show_create_table_auth_db' String tableName = 'test_show_create_table_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -30,10 +33,6 @@ suite("test_show_create_table_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_show_data_auth.groovy b/regression-test/suites/auth_call/test_show_data_auth.groovy index 951ba564d42fcf..98ae37b88b175c 100644 --- a/regression-test/suites/auth_call/test_show_data_auth.groovy +++ b/regression-test/suites/auth_call/test_show_data_auth.groovy @@ -24,6 +24,9 @@ suite("test_show_data_auth","p0,auth_call") { String tableName = 'test_show_data_auth_tb' String tableName2 = 'test_show_data_auth_tb2' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -31,10 +34,6 @@ suite("test_show_data_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_show_database_id_auth.groovy b/regression-test/suites/auth_call/test_show_database_id_auth.groovy index d9d131ee7793b4..120979eb3a14e9 100644 --- a/regression-test/suites/auth_call/test_show_database_id_auth.groovy +++ b/regression-test/suites/auth_call/test_show_database_id_auth.groovy @@ -23,6 +23,9 @@ suite("test_show_database_id_auth","p0,auth_call") { String dbName = 'test_show_database_id_auth_db' String tableName = 'test_show_database_id_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -30,10 +33,6 @@ suite("test_show_database_id_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" diff --git a/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy b/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy index ccfc42455f6859..a716616989d482 100644 --- a/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy +++ b/regression-test/suites/auth_call/test_show_dynamic_table_auth.groovy @@ -23,6 +23,9 @@ suite("test_show_dynamic_table_auth","p0,auth_call") { String dbName = 'test_show_dynamic_table_auth_db' String tableName = 'test_show_dynamic_table_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -30,10 +33,6 @@ suite("test_show_dynamic_table_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_show_frontend_auth.groovy b/regression-test/suites/auth_call/test_show_frontend_auth.groovy index 120698039705b2..0d0aa0fc2e7ead 100644 --- a/regression-test/suites/auth_call/test_show_frontend_auth.groovy +++ b/regression-test/suites/auth_call/test_show_frontend_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_frontend_auth","p0,auth_call") { String user = 'test_show_frontend_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_frontend_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """grant select_priv on internal.information_schema.* to ${user}""" def show_grants_result = sql """show grants for ${user}""" diff --git a/regression-test/suites/auth_call/test_show_grant_auth.groovy b/regression-test/suites/auth_call/test_show_grant_auth.groovy index efd2e84e38a9f7..de7ef41e5099db 100644 --- a/regression-test/suites/auth_call/test_show_grant_auth.groovy +++ b/regression-test/suites/auth_call/test_show_grant_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_grant_auth","p0,auth_call") { String user = 'test_show_grant_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_grant_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { diff --git a/regression-test/suites/auth_call/test_show_proc_auth.groovy b/regression-test/suites/auth_call/test_show_proc_auth.groovy index 4608c64ca67016..d5b672d315556d 100644 --- a/regression-test/suites/auth_call/test_show_proc_auth.groovy +++ b/regression-test/suites/auth_call/test_show_proc_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_proc_auth","p0,auth_call") { String user = 'test_show_proc_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_proc_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { diff --git a/regression-test/suites/auth_call/test_show_query_stats_auth.groovy b/regression-test/suites/auth_call/test_show_query_stats_auth.groovy index 7552038a7432b2..829271572616f7 100644 --- a/regression-test/suites/auth_call/test_show_query_stats_auth.groovy +++ b/regression-test/suites/auth_call/test_show_query_stats_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_query_stats_auth","p0,auth_call") { String user = 'test_show_query_stats_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_query_stats_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" logger.info("context.config.jdbcUrl: " + context.config.jdbcUrl) diff --git a/regression-test/suites/auth_call/test_show_tablet_auth.groovy b/regression-test/suites/auth_call/test_show_tablet_auth.groovy index b3201d4b8af1ce..0183e36add78d0 100644 --- a/regression-test/suites/auth_call/test_show_tablet_auth.groovy +++ b/regression-test/suites/auth_call/test_show_tablet_auth.groovy @@ -23,6 +23,9 @@ suite("test_show_tablet_auth","p0,auth_call") { String dbName = 'test_show_tablet_auth_db' String tableName = 'test_show_tablet_auth_tb' + try_sql("DROP USER ${user}") + try_sql """drop database if exists ${dbName}""" + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -30,10 +33,6 @@ suite("test_show_tablet_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - try_sql """drop database if exists ${dbName}""" - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" sql """create database ${dbName}""" sql """create table ${dbName}.${tableName} ( diff --git a/regression-test/suites/auth_call/test_show_typecast_auth.groovy b/regression-test/suites/auth_call/test_show_typecast_auth.groovy index 9e5d7186f01e17..237834267ca966 100644 --- a/regression-test/suites/auth_call/test_show_typecast_auth.groovy +++ b/regression-test/suites/auth_call/test_show_typecast_auth.groovy @@ -21,6 +21,8 @@ suite("test_show_typecast_auth","p0,auth_call") { String user = 'test_show_typecast_auth_user' String pwd = 'C123_567p' + try_sql("DROP USER ${user}") + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" //cloud-mode if (isCloudMode()) { def clusters = sql " SHOW CLUSTERS; " @@ -28,9 +30,6 @@ suite("test_show_typecast_auth","p0,auth_call") { def validCluster = clusters[0][0] sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; } - - try_sql("DROP USER ${user}") - sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" sql """grant select_priv on regression_test to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { From 03a026943b620a945193b396cbe29402cf90093c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 24 Jul 2025 17:05:39 +0800 Subject: [PATCH 310/572] branch-3.0: [test](inverted index) enhance compaction assertion compatibility for format v2 regression cases #53808 (#53832) Cherry-picked from #53808 Co-authored-by: airborne12 --- ...t_cumulative_compaction_with_format_v2.groovy | 16 ++++++++++++++-- .../test_mow_table_with_format_v2.groovy | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy index 6e39fa09fcd7fc..6af721010ad4c4 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_cumulative_compaction_with_format_v2.groovy @@ -198,10 +198,22 @@ suite("test_cumulative_compaction_with_format_v2", "inverted_index_format_v2") { (code, out, err) = be_show_tablet_status(ip, port, tablet_id) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) - assertTrue(out.contains("[2-9]")) + // Parse the tablet status to get rowset count after compaction + def tabletJson = parseJson(out.trim()) + def rowsets = tabletJson.rowsets + int activeRowsetCount = rowsets.size() + // After compaction, we should have fewer rowsets than before (originally 9 rowsets: [0-1], [2-2], ..., [9-9]) + // The exact number depends on compaction strategy, but should be less than 9 + assertTrue(activeRowsetCount < 9, "Expected fewer rowsets after compaction, got: ${activeRowsetCount}") + assertTrue(activeRowsetCount >= 2, "Expected at least 2 rowsets after compaction, got: ${activeRowsetCount}") + // Verify we still have [0-1] and some compacted rowsets starting from version 2 + boolean hasBaseRowset = rowsets.any { it.contains("[0-1]") } + boolean hasCompactedRowsets = rowsets.any { it.contains("[2-") } + assertTrue(hasBaseRowset, "Should have base rowset [0-1]") + assertTrue(hasCompactedRowsets, "Should have compacted rowsets starting from version 2") int segment_count = calc_segment_count(tablet) logger.info("TabletId: " + tablet_id + ", segment_count: " + segment_count) - check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") + check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3, "V2") } int segmentsCount = 0 diff --git a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy index f1b7f076ba7e8f..7fa1a2f93d1fb7 100644 --- a/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy +++ b/regression-test/suites/inverted_index_p0/index_format_v2/test_mow_table_with_format_v2.groovy @@ -198,8 +198,20 @@ suite("test_mow_table_with_format_v2", "inverted_index_format_v2") { (code, out, err) = be_show_tablet_status(ip, port, tablet_id) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) - assertTrue(out.contains("[2-9]")) - check_nested_index_file(ip, port, tablet_id, 2, 3, "V2") + // Parse the tablet status to get rowset count after compaction + def tabletJson = parseJson(out.trim()) + def rowsets = tabletJson.rowsets + int activeRowsetCount = rowsets.size() + // After compaction, we should have fewer rowsets than before (originally 9 rowsets: [0-1], [2-2], ..., [9-9]) + // The exact number depends on compaction strategy, but should be less than 9 + assertTrue(activeRowsetCount < 9, "Expected fewer rowsets after compaction, got: ${activeRowsetCount}") + assertTrue(activeRowsetCount >= 2, "Expected at least 2 rowsets after compaction, got: ${activeRowsetCount}") + // Verify we still have [0-1] and some compacted rowsets starting from version 2 + boolean hasBaseRowset = rowsets.any { it.contains("[0-1]") } + boolean hasCompactedRowsets = rowsets.any { it.contains("[2-") } + assertTrue(hasBaseRowset, "Should have base rowset [0-1]") + assertTrue(hasCompactedRowsets, "Should have compacted rowsets starting from version 2") + check_nested_index_file(ip, port, tablet_id, activeRowsetCount, 3, "V2") } int segmentsCount = 0 From e8c5b33489babc5b3129a0aaeb9cea4243c47300 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 24 Jul 2025 19:26:11 +0800 Subject: [PATCH 311/572] branch-3.0: [test](p2) reduce broker_load_batch_size in compaction_width_array_column #53797 (#53809) Cherry-picked from #53797 Co-authored-by: Kaijie Chen --- .../suites/compaction/compaction_width_array_column.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/compaction/compaction_width_array_column.groovy b/regression-test/suites/compaction/compaction_width_array_column.groovy index 0204b9946c0217..d80e67c6cd8442 100644 --- a/regression-test/suites/compaction/compaction_width_array_column.groovy +++ b/regression-test/suites/compaction/compaction_width_array_column.groovy @@ -16,6 +16,7 @@ // under the License. suite('compaction_width_array_column', "p2") { + sql """set global broker_load_batch_size = 4062;""" String backend_id; def backendId_to_backendIP = [:] def backendId_to_backendHttpPort = [:] From ab1fccd53bfc75654477ed70e8c1299c7f7792c1 Mon Sep 17 00:00:00 2001 From: hui lai Date: Thu, 24 Jul 2025 20:10:34 +0800 Subject: [PATCH 312/572] branch-3.0: [fix](load) fix multi table load plan fail after restart master Fe or leader change (#53799) (#53829) pick (#53799) multi table load plan fail after restart master Fe or leader change: ``` mysql> show routine load for test_multi_table\G *************************** Id: 1753247186255 Name: test2 CreateTime: 2025-07-23 13:06:53 PauseTime: NULL EndTime: NULL DbName: db TableName: IsMultiTable: true State: RUNNING DataSourceType: KAFKA CurrentTaskNum: 1 JobProperties: {"max_batch_rows": "3000000","timezone":"Asia/Shanghai","send_batch_parallelism":"1","loadd_to_single_tablet":"false","column_separator":";'''","line_delimiter":"\n","delete":"*"," current_concurrent_number":"1","partial_columns":"false","merge_type":"APPEND","exec_mem_limit":"2147483648","strict_mode":"false","max_batch_interval": 20","max_batch_size": "209715200","esscape":"\u 0000","enclose":"\u0000","partitions":"**","columnToColumnExpr":"","whereExpr":"*****'',"desired_concurrent_number":"256","precedingFilter":"*","format":"csv","max_error_number":"0","max_filter_ratio":"1. 0","sequence_col":"****} DataSourceProperties: {"topic":"my-topic","currentkafkaPartitions": "0", "brokerList": "10.16.10.10.10.77:19092"} CustomProperties: {"kafka_default_offsets":"OFFSET_BEGINNING","group.id": "test2_7f6143d8-f270-4667-851a-e8fb87c27d32"} Statistic: {"receivedBytes":89,"runningTxns": [1542060502549504],"errorRows":0, "committedTaskNum":0, "loadedRows":1,"LoadRowsRate":0,"abortedTaskNum":7,"errorRowsAfterResumed":0,"totalRows" :1,"unselectedRows":0,"receivedBytesRate":1,"taskExecuteTimeMs":51588} Progress: {"0":"0"} Lag: {"0":1} ReasonOfStateChanged: ErrorLogUrls: OtherMsg: 2025-07-23 13:08:07: [INTERNAL_ERROR]TStatus:AnalysisException: errCode = 2, detailMessage = , connect context's user is null, ComputeGroupException: CURRENT_USER_NO_AUTH_TO_US E_DEFAULT_COMPUTE_GROUP, you can contact the system admministrator and request that they grant you the defaultcompute group permissions, use SQL 'SHOW PROPERTY like'default_compute_group'` and NT USAGE_PRIV ON COMPUTE GROUP {compute_group_name}TO{user} GRA 0# # doris::Status doris::Status::create(doris::TStatus const&) at /mnt/disk1/laihui/build/ldb_toolchain/bin/../lib/gcc/x86_64-pc-linux-gnu/114/include/g++-v14/bits/basic_string.h:228 1# doris::io::MultiTablePipe::request_and_exec_plans() at /mnt/disk1/laihui/doris/be/src/common/status.h:522 2# doris: RoutineLoadTaskExecutor::exec_task(std::shared_ptr, doris::DataConsumerPool*, std::function)>) at /mnt/di sk1/laihui/doris/be/src/runtime/routine_load/routine_load_task_executor.cpp:0 3# std::_Function_handler - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/load/routineload/KafkaTaskInfo.java | 2 +- .../load/routineload/RoutineLoadJob.java | 6 - .../test_multi_table_load_restart.groovy | 148 ++++++++++++++++++ 3 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java index e3292dc671f8b4..0474f0d4fdc956 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/KafkaTaskInfo.java @@ -109,7 +109,7 @@ public TRoutineLoadTask createRoutineLoadTask() throws UserException { tRoutineLoadTask.setFormat(TFileFormatType.FORMAT_CSV_PLAIN); } tRoutineLoadTask.setMemtableOnSinkNode(routineLoadJob.isMemtableOnSinkNode()); - tRoutineLoadTask.setQualifiedUser(routineLoadJob.getQualifiedUser()); + tRoutineLoadTask.setQualifiedUser(routineLoadJob.getUserIdentity().getQualifiedUser()); tRoutineLoadTask.setCloudCluster(routineLoadJob.getCloudCluster()); return tRoutineLoadTask; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 30ef3f54f1aec2..9c83bb6e9f9971 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -292,7 +292,6 @@ public boolean isFinalState() { protected byte escape = 0; // use for cloud cluster mode - protected String qualifiedUser; protected String cloudCluster; public void setTypeRead(boolean isTypeRead) { @@ -343,7 +342,6 @@ public RoutineLoadJob(Long id, String name, SessionVariable var = ConnectContext.get().getSessionVariable(); sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode())); this.memtableOnSinkNode = ConnectContext.get().getSessionVariable().enableMemtableOnSinkNode; - this.qualifiedUser = ConnectContext.get().getQualifiedUser(); try { this.cloudCluster = ConnectContext.get().getCloudCluster(); } catch (ComputeGroupException e) { @@ -789,10 +787,6 @@ public void setComment(String comment) { this.comment = comment; } - public String getQualifiedUser() { - return qualifiedUser; - } - public String getCloudCluster() { return cloudCluster; } diff --git a/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy b/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy new file mode 100644 index 00000000000000..d89f513eb5d0e6 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +suite("test_multi_table_load_restart","docker") { + def options = new ClusterOptions() + options.cloudMode = true + docker(options) { + def kafkaCsvTpoics = [ + "test_multi_table_load_restart", + ] + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") + for (String kafkaCsvTopic in kafkaCsvTpoics) { + def testData = [ + "test_multi_table_load_restart|1,test_data_1,2023-01-01,value1,2023-01-01 10:00:00,extra1" + ] + testData.each { line -> + logger.info("Sending data to kafka: ${line}") + def record = new ProducerRecord<>(kafkaCsvTopic, null, line) + producer.send(record) + } + } + + def tableName = "test_multi_table_load_restart" + def job = "test_multi_table_load_restart" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(20) NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + try { + sql """ + CREATE ROUTINE LOAD ${job} + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${kafkaCsvTpoics[0]}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + sql "sync" + + cluster.restartFrontends() + sleep(30000) + context.reconnectFe() + + for (String kafkaCsvTopic in kafkaCsvTpoics) { + def testData = [ + "test_multi_table_load_restart|2,test_data_1,2023-01-01,value1,2023-01-01 10:00:00,extra1" + ] + testData.each { line -> + logger.info("Sending data to kafka: ${line}") + def record = new ProducerRecord<>(kafkaCsvTopic, null, line) + producer.send(record) + } + } + producer.close() + + def count = 0 + def maxWaitCount = 60 + while (count < maxWaitCount) { + def state = sql "show routine load for ${job}" + def routineLoadState = state[0][8].toString() + def statistic = state[0][14].toString() + logger.info("Routine load state: ${routineLoadState}") + logger.info("Routine load statistic: ${statistic}") + def rowCount = sql "select count(*) from ${tableName}" + if (routineLoadState == "RUNNING" && rowCount[0][0] == 2) { + break + } + sleep(1000) + count++ + } + } catch (Exception e) { + logger.error("Test failed with exception: ${e.message}") + } finally { + try { + sql "stop routine load for ${job}" + } catch (Exception e) { + logger.warn("Failed to stop routine load job: ${e.message}") + } + } + } + } +} \ No newline at end of file From 66d1c3344e26ff76159dccb67ab154cf978c1a65 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 25 Jul 2025 09:50:38 +0800 Subject: [PATCH 313/572] [fix](warmup) fix warm up jobs missing last batch (#53860) (#53861) ### What problem does this PR solve? Issue Number: DORIS-21591 Related PR: #53860 Problem Summary: 1. `_pending_job_metas` may be cleared by a CLEAR_JOB request, so we need to check it again. 2. We can not call pop_front before the job is finished, because GET_CURRENT_JOB_STATE_AND_LEASE is relying on the pending job size. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/cloud/cloud_warm_up_manager.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index f9a256d2b97e4a..57e0a9ad27568c 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -186,7 +186,6 @@ void CloudWarmUpManager::handle_jobs() { if (_closed) break; if (!_pending_job_metas.empty()) { cur_job = _pending_job_metas.front(); - _pending_job_metas.pop_front(); } } @@ -292,6 +291,13 @@ void CloudWarmUpManager::handle_jobs() { { std::unique_lock lock(_mtx); _finish_job.push_back(cur_job); + // _pending_job_metas may be cleared by a CLEAR_JOB request + // so we need to check it again. + if (!_pending_job_metas.empty()) { + // We can not call pop_front before the job is finished, + // because GET_CURRENT_JOB_STATE_AND_LEASE is relying on the pending job size. + _pending_job_metas.pop_front(); + } } } #endif From 9ce24edf36fa3fdc0e0a97121bd28c63d44022fc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 25 Jul 2025 09:52:14 +0800 Subject: [PATCH 314/572] branch-3.0: [fix](ci) vault_p0 start minio #53772 (#53846) Cherry-picked from #53772 Co-authored-by: Dongyang Li --- regression-test/pipeline/vault_p0/run.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/regression-test/pipeline/vault_p0/run.sh b/regression-test/pipeline/vault_p0/run.sh index 7cf48010f95bb0..a292884ae2da74 100644 --- a/regression-test/pipeline/vault_p0/run.sh +++ b/regression-test/pipeline/vault_p0/run.sh @@ -70,7 +70,12 @@ run() { # start minio docker to run case test_rountine_load sed -i "s/^CONTAINER_UID=\"doris--\"/CONTAINER_UID=\"doris-external--\"/" "${teamcity_build_checkoutDir}"/docker/thirdparties/custom_settings.env - if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh -c minio; then echo; else echo "ERROR: start minio docker failed"; fi + if bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh -c minio || + bash "${teamcity_build_checkoutDir}"/docker/thirdparties/run-thirdparties-docker.sh -c minio; then + echo "INFO: start minio docker success" + else + echo "ERROR: start minio docker twice failed" && return 1 + fi # used to set up HDFS docker docker_compose_hdfs_yaml=' From bf989bf0bbf608a1e379691c55efbfe71a85c8b0 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Fri, 25 Jul 2025 17:08:16 +0800 Subject: [PATCH 315/572] [fix](test) enlarge timeout for test_autoinc_broker_load (#53744) (#53885) pick #53744 --- .../external_table_p0/hive/test_autoinc_broker_load.groovy | 2 +- .../hive/test_partial_update_broker_load.groovy | 2 +- regression-test/suites/variant_github_events_new_p0/load.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy index 106263203fcd74..8a98f891f86b2d 100644 --- a/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy +++ b/regression-test/suites/external_table_p0/hive/test_autoinc_broker_load.groovy @@ -46,7 +46,7 @@ suite("test_autoinc_broker_load", "p0,external,hive,external_docker,external_doc } def wait_for_load_result = {checklabel, testTable -> - def max_try_milli_secs = 10000 + def max_try_milli_secs = 60000 while(max_try_milli_secs) { def result = sql "show load where label = '${checklabel}'" if(result[0][2] == "FINISHED") { diff --git a/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy b/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy index f9b31a3d8a8d92..f49a11efc27a2e 100644 --- a/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy +++ b/regression-test/suites/external_table_p0/hive/test_partial_update_broker_load.groovy @@ -47,7 +47,7 @@ suite("test_primary_key_partial_update_broker_load", "p0,external,hive,external_ } def wait_for_load_result = {checklabel, testTable -> - def max_try_milli_secs = 10000 + def max_try_milli_secs = 2000000 while(max_try_milli_secs) { def result = sql "show load where label = '${checklabel}'" if(result[0][2] == "FINISHED") { diff --git a/regression-test/suites/variant_github_events_new_p0/load.groovy b/regression-test/suites/variant_github_events_new_p0/load.groovy index 582f3bec5652ed..7f8e979ac3160d 100644 --- a/regression-test/suites/variant_github_events_new_p0/load.groovy +++ b/regression-test/suites/variant_github_events_new_p0/load.groovy @@ -125,7 +125,7 @@ suite("regression_test_variant_github_events_p0", "p0"){ def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='github_events' ORDER BY createtime DESC LIMIT 1 """ return jobStateResult[0][9] } - int max_try_time = 200 + int max_try_time = 1000 while (max_try_time--){ String result = getJobState("github_events") if (result == "FINISHED") { From 726ab9d522be670ca6f67597295183b0dcf2e496 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Fri, 25 Jul 2025 17:08:45 +0800 Subject: [PATCH 316/572] [ci](cloud) align pipeline to check leaks (#53085) ### What problem does this PR solve? Issue Number: close #xxx Related PR: pick #52886 #52725 --- regression-test/pipeline/cloud_p0/clean.sh | 2 +- regression-test/pipeline/cloud_p0/run.sh | 20 ++-- .../pipeline/common/doris-utils.sh | 100 ++++++++++++++++-- regression-test/pipeline/vault_p0/clean.sh | 3 +- regression-test/pipeline/vault_p0/run.sh | 20 ++-- 5 files changed, 116 insertions(+), 29 deletions(-) diff --git a/regression-test/pipeline/cloud_p0/clean.sh b/regression-test/pipeline/cloud_p0/clean.sh index 2b67ae9349566f..9af014bb6e3d44 100644 --- a/regression-test/pipeline/cloud_p0/clean.sh +++ b/regression-test/pipeline/cloud_p0/clean.sh @@ -26,5 +26,5 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "#### Run tpcds test on Doris ####" DORIS_HOME="${teamcity_build_checkoutDir}/output" export DORIS_HOME -export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace clean_fdb "cloud_instance_0" +exit diff --git a/regression-test/pipeline/cloud_p0/run.sh b/regression-test/pipeline/cloud_p0/run.sh index 55f0c41901372c..6cca8b92af8af5 100644 --- a/regression-test/pipeline/cloud_p0/run.sh +++ b/regression-test/pipeline/cloud_p0/run.sh @@ -111,15 +111,19 @@ if print_running_pipeline_tasks; then :; fi # shellcheck source=/dev/null source "$(cd "${teamcity_build_checkoutDir}" && bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')" -echo "#### 5. check if need backup doris logs" +check_if_need_gcore "${exit_flag}" +if stop_doris_grace; then + echo "INFO: stop doris grace success." +else + echo "ERROR: stop grace failed." && exit_flag=2 +fi +if core_file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then + reporting_build_problem "coredump" + print_doris_fe_log + print_doris_be_log +fi +echo "#### check if need backup doris logs ####" if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then - check_if_need_gcore "${exit_flag}" - if core_file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then - reporting_build_problem "coredump" - print_doris_fe_log - print_doris_be_log - fi - export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace if log_file_name=$(archive_doris_logs "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_logs.tar.gz"); then if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then reporting_messages_error "${log_info##*logs.tar.gz to }" diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh index 44f075e14e1855..06e971201c5223 100644 --- a/regression-test/pipeline/common/doris-utils.sh +++ b/regression-test/pipeline/common/doris-utils.sh @@ -222,14 +222,87 @@ function stop_doris() { function stop_doris_grace() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi - if "${DORIS_HOME}"/be/bin/stop_be.sh --grace && "${DORIS_HOME}"/fe/bin/stop_fe.sh --grace; then - echo "INFO: normally stoped doris --grace" + local ret=0 + local keywords="detected memory leak|undefined-behavior" + sudo mkdir -p /tmp/be/bin && cp -rf "${DORIS_HOME}"/be/bin/be.pid /tmp/be/bin/be.pid + if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash "${DORIS_HOME}"/be/bin/stop_be.sh --grace; then + echo "INFO: doris be stopped gracefully." + if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" && "${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then + echo "INFO: try to find keywords ${keywords} in be.out" + if [[ -f "${DORIS_HOME}"/be/log/be.out ]]; then + if grep -E "${keywords}" "${DORIS_HOME}"/be/log/be.out; then + echo "##teamcity[buildProblem description='Ubsan or Lsan fail']" + echo "====================================head -n 200 be/log/be.out====================================" + head -n 200 "${DORIS_HOME}"/be/log/be.out + echo "=================================================================================================" + echo "ERROR: found memory leaks or undefined behavior in be.out" && ret=1 + else + echo "INFO: no memory leaks or undefined behavior found in be.out" + fi + else + echo "##teamcity[buildProblem description='Stop BE grace fail']" + echo "ERROR: be.out not find, which is not expected" && ret=1 + fi + fi else - pgrep -fi doris | xargs kill -9 &>/dev/null - echo "WARNING: force stoped doris" + echo "ERROR: doris be stop grace failed." && ret=1 fi - if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then bash "${DORIS_HOME}"/ms/bin/stop.sh --grace; fi - if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then bash "${DORIS_HOME}"/recycler/bin/stop.sh --grace; fi + if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash "${DORIS_HOME}"/fe/bin/stop_fe.sh --grace; then + echo "INFO: doris fe stopped gracefully." + else + echo "ERROR: doris fe stop grace failed." && ret=1 + fi + if [[ -f "${DORIS_HOME}"/ms/bin/stop.sh ]]; then + sudo mkdir -p /tmp/ms/bin && cp -rf "${DORIS_HOME}"/ms/bin/doris_cloud.pid /tmp/ms/bin/doris_cloud.pid + if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash "${DORIS_HOME}"/ms/bin/stop.sh --grace; then + echo "INFO: doris ms stopped gracefully." + if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" && "${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then + echo "INFO: try to find keywords ${keywords} in doris_cloud.out" + if [[ -f "${DORIS_HOME}"/ms/log/doris_cloud.out ]]; then + if grep -E "${keywords}" "${DORIS_HOME}"/ms/log/doris_cloud.out; then + echo "##teamcity[buildProblem description='Ubsan or Lsan fail']" + echo "====================================head -n 200 ms/log/doris_cloud.out====================================" + head -n 200 "${DORIS_HOME}"/ms/log/doris_cloud.out + echo "==========================================================================================================" + echo "ERROR: found memory leaks or undefined behavior in ms/log/doris_cloud.out" && ret=1 + else + echo "INFO: no memory leaks or undefined behavior found in ms/log/doris_cloud.out" + fi + else + echo "ERROR: ms/log/doris_cloud.out not find, which is not expected" && ret=1 + fi + fi + else + echo "##teamcity[buildProblem description='Stop MS grace fail']" + echo "ERROR: doris ms stop grace failed." && ret=1 + fi + fi + if [[ -f "${DORIS_HOME}"/recycler/bin/stop.sh ]]; then + sudo mkdir -p /tmp/recycler/bin && cp -rf "${DORIS_HOME}"/recycler/bin/doris_cloud.pid /tmp/recycler/bin/doris_cloud.pid + if timeout -v "${DORIS_STOP_GRACE_TIMEOUT:-"10m"}" bash "${DORIS_HOME}"/recycler/bin/stop.sh --grace; then + echo "INFO: doris recycler stopped gracefully." + # if [[ -n "${DORIS_STOP_GRACE_CHECK_KEYWORD:=''}" && "${DORIS_STOP_GRACE_CHECK_KEYWORD,,}" == "true" ]]; then + # echo "INFO: try to find keywords ${keywords} in doris_cloud.out" + # if [[ -f "${DORIS_HOME}"/recycler/log/doris_cloud.out ]]; then + # if grep -E "${keywords}" "${DORIS_HOME}"/recycler/log/doris_cloud.out; then + # echo "##teamcity[buildProblem description='Ubsan or Lsan fail']" + # echo "=================================head -n 200 recycler/log/doris_cloud.out=================================" + # head -n 200 "${DORIS_HOME}"/recycler/log/doris_cloud.out + # echo "==========================================================================================================" + # echo "ERROR: found memory leaks or undefined behavior in recycler/log/doris_cloud.out" && ret=1 + # else + # echo "INFO: no memory leaks or undefined behavior found in recycler/log/doris_cloud.out" + # fi + # else + # echo "ERROR: recycler/log/doris_cloud.out not find, which is not expected" && ret=1 + # fi + # fi + else + echo "##teamcity[buildProblem description='Stop RECYCLER grace fail']" + echo "ERROR: doris recycler stop grace failed." && ret=1 + fi + fi + return "${ret}" } function clean_fdb() { @@ -299,7 +372,7 @@ deploy_doris_sql_converter() { fi } -function restart_doris() { +function _restart_doris() { if stop_doris; then echo; fi if ! start_doris_fe; then return 1; fi if ! start_doris_be; then return 1; fi @@ -319,6 +392,11 @@ function restart_doris() { sleep 10s } +function restart_doris() { + # restart BE may block on JVM_MonitorWait() for a long time, here try twice + _restart_doris || _restart_doris +} + function check_tpch_table_rows() { if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi db_name="$1" @@ -663,9 +741,9 @@ archive_doris_coredump() { rm -rf "${DORIS_HOME:?}/${archive_dir}" mkdir -p "${DORIS_HOME}/${archive_dir}" declare -A pids - pids['be']="$(cat "${DORIS_HOME}"/be/bin/be.pid)" - pids['ms']="$(cat "${DORIS_HOME}"/ms/bin/doris_cloud.pid)" - pids['recycler']="$(cat "${DORIS_HOME}"/recycler/bin/doris_cloud.pid)" + pids['be']="$(cat /tmp/be/bin/be.pid)" + pids['ms']="$(cat /tmp/ms/bin/doris_cloud.pid)" + pids['recycler']="$(cat /tmp/recycler/bin/doris_cloud.pid)" local has_core=false for p in "${!pids[@]}"; do pid="${pids[${p}]}" @@ -877,7 +955,7 @@ function check_if_need_gcore() { sleep 10 fi else - echo "ERROR: unknown exit_flag ${exit_flag}" && return 1 + echo "ERROR: exit_flag ${exit_flag} is not 124(timeout), no need to gcore" && return 1 fi } diff --git a/regression-test/pipeline/vault_p0/clean.sh b/regression-test/pipeline/vault_p0/clean.sh index 1500bd48bf4821..d72767e158a13a 100644 --- a/regression-test/pipeline/vault_p0/clean.sh +++ b/regression-test/pipeline/vault_p0/clean.sh @@ -26,8 +26,9 @@ if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "#### stop doris and clean fdb ####" DORIS_HOME="${teamcity_build_checkoutDir}/output" export DORIS_HOME -export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace clean_fdb "cloud_instance_0" echo "#### docker-compose down ####" docker-compose down + +exit diff --git a/regression-test/pipeline/vault_p0/run.sh b/regression-test/pipeline/vault_p0/run.sh index a292884ae2da74..e6e7217a78913b 100644 --- a/regression-test/pipeline/vault_p0/run.sh +++ b/regression-test/pipeline/vault_p0/run.sh @@ -153,15 +153,19 @@ if print_running_pipeline_tasks; then :; fi # shellcheck source=/dev/null source "$(cd "${teamcity_build_checkoutDir}" && bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')" -echo "#### 5. check if need backup doris logs" +check_if_need_gcore "${exit_flag}" +if stop_doris_grace; then + echo "INFO: stop doris grace success." +else + echo "ERROR: stop grace failed." && exit_flag=2 +fi +if core_file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then + reporting_build_problem "coredump" + print_doris_fe_log + print_doris_be_log +fi +echo "#### check if need backup doris logs ####" if [[ ${exit_flag} != "0" ]] || ${need_collect_log}; then - check_if_need_gcore "${exit_flag}" - if core_file_name=$(archive_doris_coredump "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_coredump.tar.gz"); then - reporting_build_problem "coredump" - print_doris_fe_log - print_doris_be_log - fi - export -f stop_doris_grace && timeout -v 20m bash -cx stop_doris_grace if log_file_name=$(archive_doris_logs "${pr_num_from_trigger}_${commit_id_from_trigger}_$(date +%Y%m%d%H%M%S)_doris_logs.tar.gz"); then if log_info="$(upload_doris_log_to_oss "${log_file_name}")"; then reporting_messages_error "${log_info##*logs.tar.gz to }" From 61dd708a66d5d62ef15bd36c08d6e2edd46bacad Mon Sep 17 00:00:00 2001 From: zhengyu Date: Sat, 26 Jul 2025 10:36:39 +0800 Subject: [PATCH 317/572] [regression](filecache) fix regression failures part2 (#53783) (#53915) --- .../cache/http/test_reset_capacity.groovy | 2 +- .../cluster/test_warm_up_cluster.groovy | 7 +-- .../cluster/test_warm_up_cluster_batch.groovy | 7 +-- .../test_warm_up_cluster_bigsize.groovy | 7 +-- .../cluster/test_warm_up_cluster_empty.groovy | 7 +-- .../cluster/test_warm_up_compute_group.groovy | 7 +-- .../table/test_warm_up_partition.groovy | 7 +-- ...test_warm_up_same_table_multi_times.groovy | 14 ++--- .../warm_up/table/test_warm_up_table.groovy | 12 ++-- .../warm_up/table/test_warm_up_tables.groovy | 13 ++--- .../cloud_p0/cache/ttl/alter_ttl_1.groovy | 56 ++++++++----------- .../cloud_p0/cache/ttl/alter_ttl_4.groovy | 8 ++- .../cache/ttl/create_table_as_select.groovy | 23 -------- .../cache/ttl/test_ttl_preempt.groovy | 9 ++- 14 files changed, 76 insertions(+), 103 deletions(-) diff --git a/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy b/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy index 2d04caaa0786c1..72904a7bcee585 100644 --- a/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy +++ b/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy @@ -254,7 +254,7 @@ suite("test_reset_capacity") { continue } def i = line.indexOf(' ') - ttl_cache_size = line.substring(i).toLong() + def ttl_cache_size = line.substring(i).toLong() logger.info("current ttl_cache_size " + ttl_cache_size); assertTrue(ttl_cache_size <= 1073741824) flag1 = true diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy index b6ae1be881e2fa..99050350c51b87 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy @@ -157,12 +157,11 @@ suite("test_warm_up_cluster") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy index 9a2aff3393333f..315f9b8f892bf2 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy @@ -120,12 +120,11 @@ suite("test_warm_up_cluster_batch") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy index 4f97116e120e7e..ad80e053d2074c 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy @@ -130,12 +130,11 @@ suite("test_warm_up_cluster_bigsize") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy index d86238c5be9ad4..f50ec580c2cccd 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy @@ -126,12 +126,11 @@ suite("test_warm_up_cluster_empty") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy index 5a16e92b36bb5d..710f10d855b825 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy @@ -157,12 +157,11 @@ suite("test_warm_up_compute_group") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy index c6819ad58ec20b..d4bf55441bdb22 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy @@ -121,12 +121,11 @@ suite("test_warm_up_partition") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy index fd28dec7ddd75c..027c064e91f09d 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy @@ -124,12 +124,11 @@ suite("test_warm_up_same_table_multi_times") { int j = 0 for (; j < retryTime; j++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } @@ -187,12 +186,11 @@ suite("test_warm_up_same_table_multi_times") { j = 0 for (; j < retryTime; j++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy index 258e9e87ef6121..c8d7325f9c0d28 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy @@ -25,7 +25,7 @@ suite("test_warm_up_table") { } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][9] + return jobStateResult[0] } List ipList = new ArrayList<>(); @@ -145,12 +145,11 @@ suite("test_warm_up_table") { int j = 0 for (; j < retryTime; j++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } @@ -159,7 +158,8 @@ suite("test_warm_up_table") { assertTrue(false); } def tablesString = getTablesFromShowCommand(jobId[0][0]) - assertTrue(tablesString.contains("customer"), tablesString) + + assertTrue(tablesString.any { it != null && it.contains("customer") }) sleep(30000) long ttl_cache_size = 0 getMetricsMethod.call(ipList[0], brpcPortList[0]) { diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy index 03d45f1cce8e1e..c37fa61dcd3c4d 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy @@ -25,7 +25,7 @@ suite("test_warm_up_tables") { } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][9] + return jobStateResult[0] } List ipList = new ArrayList<>(); @@ -149,12 +149,11 @@ suite("test_warm_up_tables") { int i = 0 for (; i < retryTime; i++) { sleep(1000) - def status = getJobState(jobId[0][0]) - logger.info(status) - if (status.equals("CANCELLED")) { + def statuses = getJobState(jobId[0][0]) + if (statuses.any { it != null && it.equals("CANCELLED") }) { assertTrue(false); } - if (status.equals("FINISHED")) { + if (statuses.any { it != null && it.equals("FINISHED") }) { break; } } @@ -169,8 +168,8 @@ suite("test_warm_up_tables") { waitJobDone(jobId_); def tablesString = getTablesFromShowCommand(jobId_[0][0]) - assertTrue(tablesString.contains("customer.p3"), tablesString) - assertTrue(tablesString.contains("supplier"), tablesString) + assertTrue(tablesString.any { it != null && it.contains("customer") }) + assertTrue(tablesString.any { it != null && it.contains("supplier") }) sleep(30000) long ttl_cache_size = 0 diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy index 6a07df14922408..0ec671603a8313 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy @@ -18,7 +18,17 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("alter_ttl_1") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + sql "set global enable_auto_analyze = false" + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="90") """ String[][] backends = sql """ show backends """ String backendId; @@ -103,12 +113,14 @@ suite("alter_ttl_1") { sql """ select count(*) from customer_ttl """ sleep(30000) long ttl_cache_size = 0 + long normal_cache_size = 0 getMetricsMethod.call() { respCode, body -> assertEquals("${respCode}".toString(), "200") String out = "${body}".toString() def strs = out.split('\n') Boolean flag1 = false; + Boolean flag2 = false; for (String line in strs) { if (flag1) break; if (line.contains("ttl_cache_size")) { @@ -119,67 +131,47 @@ suite("alter_ttl_1") { ttl_cache_size = line.substring(i).toLong() flag1 = true } - } - assertTrue(flag1) - } - sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="140") """ - sleep(80000) - // after 110s, the first load has translate to normal - getMetricsMethod.call() { - respCode, body -> - assertEquals("${respCode}".toString(), "200") - String out = "${body}".toString() - def strs = out.split('\n') - Boolean flag1 = false; - for (String line in strs) { - if (flag1) break; - if (line.contains("ttl_cache_size")) { - if (line.startsWith("#")) { - continue - } - def i = line.indexOf(' ') - assertEquals(line.substring(i).toLong(), 0) - - } - if (line.contains("normal_queue_cache_size")) { if (line.startsWith("#")) { continue } def i = line.indexOf(' ') - assertEquals(line.substring(i).toLong(), ttl_cache_size) - flag1 = true + normal_cache_size = line.substring(i).toLong() + flag2 = true } } - assertTrue(flag1) + assertTrue(flag1 && flag2) } - // wait for ttl timeout - sleep(50000) + sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="100") """ + sleep(80000) + // after 110s, the first load has translate to normal getMetricsMethod.call() { respCode, body -> assertEquals("${respCode}".toString(), "200") String out = "${body}".toString() def strs = out.split('\n') Boolean flag1 = false; - Boolean flag2 = false; for (String line in strs) { - if (flag1 && flag2) break; + if (flag1) break; if (line.contains("ttl_cache_size")) { if (line.startsWith("#")) { continue } def i = line.indexOf(' ') assertEquals(line.substring(i).toLong(), 0) + } + if (line.contains("normal_queue_cache_size")) { if (line.startsWith("#")) { continue } def i = line.indexOf(' ') - assertEquals(line.substring(i).toLong(), ttl_cache_size) + assertEquals(line.substring(i).toLong(), ttl_cache_size + normal_cache_size) flag1 = true } } assertTrue(flag1) } + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy index 6edda04994c632..03ae1dfa5cb523 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy @@ -148,7 +148,7 @@ suite("alter_ttl_4") { } sleep(60000) // one customer table would take about 1.3GB, the total cache size is 20GB - // the following would take 20.8G all + // the following would take 20G all // evict customer_ttl load_customer_once("customer") load_customer_once("customer") @@ -166,6 +166,12 @@ suite("alter_ttl_4") { load_customer_once("customer") load_customer_once("customer") load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") // some datas in s3 and will download them sql """ select C_CUSTKEY from customer_ttl order by C_CUSTKEY limit 1""" diff --git a/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy b/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy index 689c6faa168d87..97159ac0db596c 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy @@ -135,29 +135,6 @@ def clearFileCache = { check_func -> DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 32 PROPERTIES("file_cache_ttl_seconds"="120","disable_auto_compaction" = "true") as select * from customer_ttl""" - sleep(30000) // 30s - getMetricsMethod.call() { - respCode, body -> - assertEquals("${respCode}".toString(), "200") - String out = "${body}".toString() - def strs = out.split('\n') - Boolean flag1 = false; - Boolean flag2 = false; - for (String line in strs) { - if (flag1 && flag2) break; - if (line.contains("ttl_cache_size")) { - if (line.startsWith("#")) { - continue - } - def i = line.indexOf(' ') - long cur_ttl_cache_size = line.substring(i).toLong() - assertTrue(Math.abs(2* ttl_cache_size - cur_ttl_cache_size) < 10000) - flag1 = true - } - } - assertTrue(flag1) - } - sleep(150000) getMetricsMethod.call() { respCode, body -> diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy index e8008a05e1334f..f1d10b13556b58 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy @@ -123,7 +123,14 @@ suite("test_ttl_preempt") { } // one customer table would take about 1.3GB, the total cache size is 20GB - // the following would take 19.5G all + // the following would take 20G all + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") + load_customer_once("customer") load_customer_once("customer") load_customer_once("customer") load_customer_once("customer") From a2b813ffde3bdf84beb45869259e2565e2ba2d87 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 27 Jul 2025 20:32:55 +0800 Subject: [PATCH 318/572] branch-3.0: [fix](be) core dump because of invalid bitmap data #53088 (#53934) Cherry-picked from #53088 Co-authored-by: JinYang <130516674+gohalo@users.noreply.github.com> --- be/src/util/bitmap_value.h | 7 ++++++- be/test/util/bitmap_value_test.cpp | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h index 2d15ac99611274..e38a5650e0dfab 100644 --- a/be/src/util/bitmap_value.h +++ b/be/src/util/bitmap_value.h @@ -1909,7 +1909,12 @@ class BitmapValue { case BitmapTypeCode::BITMAP64_V2: _type = BITMAP; _is_shared = false; - _bitmap = std::make_shared(detail::Roaring64Map::read(src)); + try { + _bitmap = std::make_shared(detail::Roaring64Map::read(src)); + } catch (const std::runtime_error& e) { + LOG(ERROR) << "Decode roaring bitmap failed, " << e.what(); + return false; + } break; case BitmapTypeCode::SET: { _type = SET; diff --git a/be/test/util/bitmap_value_test.cpp b/be/test/util/bitmap_value_test.cpp index 2c08161450ecc3..74980ee90913e6 100644 --- a/be/test/util/bitmap_value_test.cpp +++ b/be/test/util/bitmap_value_test.cpp @@ -1188,4 +1188,10 @@ TEST(BitmapValueTest, bitmap_value_iterator_test) { } } } + +TEST(BitmapValueTest, invalid_data) { + BitmapValue bitmap; + char data[] = {0x02, static_cast(0xff), 0x03}; + EXPECT_FALSE(bitmap.deserialize(data)); +} } // namespace doris From 44ad9ce1adbacec83416714ad09d29ea0da3f608 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 09:45:34 +0800 Subject: [PATCH 319/572] branch-3.0: [Opt](config) Add a config for min buffer size to flush for partial update #53841 (#53931) Cherry-picked from #53841 Co-authored-by: bobhan1 --- be/src/common/config.cpp | 2 ++ be/src/common/config.h | 2 ++ be/src/olap/memtable.cpp | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9ee475aacaf7ab..4bddb99e0a413b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -670,6 +670,8 @@ DEFINE_mInt32(memory_gc_sleep_time_ms, "500"); DEFINE_mInt64(write_buffer_size, "209715200"); // max buffer size used in memtable for the aggregated table, default 400MB DEFINE_mInt64(write_buffer_size_for_agg, "419430400"); + +DEFINE_mInt64(min_write_buffer_size_for_partial_update, "1048576"); // max parallel flush task per memtable writer DEFINE_mInt32(memtable_flush_running_count_limit, "2"); diff --git a/be/src/common/config.h b/be/src/common/config.h index f8312c3adb372b..c664fd75f0d2d6 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -713,6 +713,8 @@ DECLARE_mInt32(memory_gc_sleep_time_ms); DECLARE_mInt64(write_buffer_size); // max buffer size used in memtable for the aggregated table, default 400MB DECLARE_mInt64(write_buffer_size_for_agg); + +DECLARE_mInt64(min_write_buffer_size_for_partial_update); // max parallel flush task per memtable writer DECLARE_mInt32(memtable_flush_running_count_limit); diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 1bc3960dd6dfcc..f16d6d0aad0877 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -483,8 +483,9 @@ bool MemTable::need_flush() const { auto max_size = config::write_buffer_size; if (_is_partial_update) { auto update_columns_size = _num_columns; + auto min_buffer_size = config::min_write_buffer_size_for_partial_update; max_size = max_size * update_columns_size / _tablet_schema->num_columns(); - max_size = max_size > 1048576 ? max_size : 1048576; + max_size = max_size > min_buffer_size ? max_size : min_buffer_size; } return memory_usage() >= max_size; } From 6090a12ae4c24e991cd9164cdfffbc7d88c3a4dc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 15:29:16 +0800 Subject: [PATCH 320/572] branch-3.0: [fix](ci) fix compile error 'Unknown host repo.maven.apache.org' #53900 (#53905) Cherry-picked from #53900 Co-authored-by: Dongyang Li --- regression-test/pipeline/performance/compile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh index 34f49db0c100d8..b0ad82a0f25d31 100644 --- a/regression-test/pipeline/performance/compile.sh +++ b/regression-test/pipeline/performance/compile.sh @@ -117,6 +117,7 @@ set -x # shellcheck disable=SC2086 sudo docker run -i --rm \ --name "${docker_name}" \ + --network=host \ -e TZ=Asia/Shanghai \ ${mount_swapfile} \ -v /etc/localtime:/etc/localtime:ro \ From daa97ed54526e0b42cc75bee1f964c4f7f297256 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Mon, 28 Jul 2025 20:30:15 -0700 Subject: [PATCH 321/572] branch-3.0: [fix](audit) fix wrong column separator of audit log (#53966) wrong pick, typo error --- .../doris/plugin/audit/AuditLoader.java | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java index c1047bec1b13ee..89b6e58f7a1643 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java @@ -147,34 +147,34 @@ private void assembleAudit(AuditEvent event) { private void fillLogBuffer(AuditEvent event, StringBuilder logBuffer) { // should be same order as InternalSchema.AUDIT_SCHEMA - logBuffer.append(event.queryId).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(TimeUtils.longToTimeStringWithms(event.timestamp)).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.clientIp).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.user).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.ctl).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.db).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.state).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.errorCode).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.errorMessage).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.queryTime).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.scanBytes).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.scanRows).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.returnRows).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.shuffleSendRows).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.shuffleSendBytes).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.scanBytesFromLocalStorage).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.scanBytesFromRemoteStorage).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.stmtId).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.stmtType).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.isQuery ? 1 : 0).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.isNereids ? 1 : 0).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.feIp).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.cpuTimeMs).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.sqlHash).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.sqlDigest).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.peakMemoryBytes).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.workloadGroup).append("AUDIT_TABLE_COL_SEPARATOR"); - logBuffer.append(event.cloudClusterName).append("AUDIT_TABLE_COL_SEPARATOR"); + logBuffer.append(event.queryId).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(TimeUtils.longToTimeStringWithms(event.timestamp)).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.clientIp).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.user).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.ctl).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.db).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.state).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.errorCode).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.errorMessage).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.queryTime).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.scanBytes).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.scanRows).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.returnRows).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.shuffleSendRows).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.shuffleSendBytes).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.scanBytesFromLocalStorage).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.scanBytesFromRemoteStorage).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.stmtId).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.stmtType).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.isQuery ? 1 : 0).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.isNereids ? 1 : 0).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.feIp).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.cpuTimeMs).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.sqlHash).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.sqlDigest).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.peakMemoryBytes).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.workloadGroup).append(AUDIT_TABLE_COL_SEPARATOR); + logBuffer.append(event.cloudClusterName).append(AUDIT_TABLE_COL_SEPARATOR); // already trim the query in org.apache.doris.qe.AuditLogHelper#logAuditLog String stmt = event.stmt; if (LOG.isDebugEnabled()) { From ad6e93af3cb3ead2f2ec4455b162039e98ed8391 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 29 Jul 2025 13:59:45 +0800 Subject: [PATCH 322/572] [chore](release) bump to 3.0.7 (#53946) --- gensrc/script/gen_build_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index a25d121bdc24b0..438f28d79b5963 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -30,7 +30,7 @@ set -eo pipefail build_version_prefix="doris" build_version_major=3 build_version_minor=0 -build_version_patch=6 +build_version_patch=7 build_version_hotfix=0 build_version_rc_version="rc01" From 8c486d010656cb1ad368afbbf0c7c981f87a02f4 Mon Sep 17 00:00:00 2001 From: deardeng Date: Tue, 29 Jul 2025 23:01:27 +0800 Subject: [PATCH 323/572] branch-3.0: [fix](cloud) add log for warm up task #52946 (#53959) cherry pick from #52946 --- be/src/cloud/cloud_backend_service.cpp | 29 +++++++++++++++ be/src/cloud/cloud_internal_service.cpp | 3 ++ .../io/cache/block_file_cache_downloader.cpp | 35 +++++++++++++++++-- 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/be/src/cloud/cloud_backend_service.cpp b/be/src/cloud/cloud_backend_service.cpp index a50d0e36419f0c..06617bcbbacafb 100644 --- a/be/src/cloud/cloud_backend_service.cpp +++ b/be/src/cloud/cloud_backend_service.cpp @@ -167,6 +167,16 @@ void CloudBackendService::warm_up_tablets(TWarmUpTabletsResponse& response, void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& response, const TWarmUpCacheAsyncRequest& request) { + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < request.tablet_ids.size() && i < 10; ++i) { + if (i > 0) oss << ","; + oss << request.tablet_ids[i]; + } + oss << "]"; + LOG(INFO) << "warm_up_cache_async: enter, request=" << request.host << ":" << request.brpc_port + << ", tablets num=" << request.tablet_ids.size() << ", tablet_ids=" << oss.str(); + std::string host = request.host; auto dns_cache = ExecEnv::GetInstance()->dns_cache(); if (dns_cache == nullptr) { @@ -186,6 +196,7 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons _exec_env->brpc_internal_client_cache()->get_new_client_no_cache(brpc_addr); if (!brpc_stub) { st = Status::RpcError("Address {} is wrong", brpc_addr); + LOG(WARNING) << "warm_up_cache_async: failed to get brpc_stub for addr " << brpc_addr; return; } brpc::Controller cntl; @@ -193,7 +204,10 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons std::for_each(request.tablet_ids.cbegin(), request.tablet_ids.cend(), [&](int64_t tablet_id) { brpc_request.add_tablet_ids(tablet_id); }); PGetFileCacheMetaResponse brpc_response; + brpc_stub->get_file_cache_meta_by_tablet_id(&cntl, &brpc_request, &brpc_response, nullptr); + VLOG_DEBUG << "warm_up_cache_async: request=" << brpc_request.DebugString() + << ", response=" << brpc_response.DebugString(); if (!cntl.Failed()) { g_file_cache_warm_up_cache_async_submitted_segment_num << brpc_response.file_cache_block_metas().size(); @@ -201,6 +215,8 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons std::move(*brpc_response.mutable_file_cache_block_metas())); } else { st = Status::RpcError("{} isn't connected", brpc_addr); + LOG(WARNING) << "warm_up_cache_async: brpc call failed, addr=" << brpc_addr + << ", error=" << cntl.ErrorText(); } st.to_thrift(&t_status); response.status = t_status; @@ -208,6 +224,15 @@ void CloudBackendService::warm_up_cache_async(TWarmUpCacheAsyncResponse& respons void CloudBackendService::check_warm_up_cache_async(TCheckWarmUpCacheAsyncResponse& response, const TCheckWarmUpCacheAsyncRequest& request) { + std::ostringstream oss; + oss << "["; + for (size_t i = 0; i < request.tablets.size() && i < 10; ++i) { + if (i > 0) oss << ","; + oss << request.tablets[i]; + } + oss << "]"; + LOG(INFO) << "check_warm_up_cache_async: enter, request tablets num=" << request.tablets.size() + << ", tablet_ids=" << oss.str(); std::map task_done; _engine.file_cache_block_downloader().check_download_task(request.tablets, &task_done); DBUG_EXECUTE_IF("CloudBackendService.check_warm_up_cache_async.return_task_false", { @@ -217,6 +242,10 @@ void CloudBackendService::check_warm_up_cache_async(TCheckWarmUpCacheAsyncRespon }); response.__set_task_done(task_done); + for (const auto& [tablet_id, done] : task_done) { + VLOG_DEBUG << "check_warm_up_cache_async: tablet_id=" << tablet_id << ", done=" << done; + } + Status st = Status::OK(); TStatus t_status; st.to_thrift(&t_status); diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 72267252aa63b8..02dab747c33984 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -73,6 +73,7 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id( LOG_WARNING("try to access tablet file cache meta, but file cache not enabled"); return; } + LOG(INFO) << "warm up get meta from this be, tablets num=" << request->tablet_ids().size(); for (const auto& tablet_id : request->tablet_ids()) { auto res = _engine.tablet_mgr().get_tablet(tablet_id); if (!res.has_value()) { @@ -105,6 +106,8 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id( } }); } + VLOG_DEBUG << "warm up get meta request=" << request->DebugString() + << ", response=" << response->DebugString(); } bvar::Adder g_file_cache_event_driven_warm_up_submitted_segment_num( diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index 1732197e5b407c..96f507816a6b5b 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -45,6 +45,7 @@ bvar::Adder g_file_cache_download_finished_size("file_cache_download_f bvar::Adder g_file_cache_download_submitted_num("file_cache_download_submitted_num"); bvar::Adder g_file_cache_download_finished_num("file_cache_download_finished_num"); bvar::Adder g_file_cache_download_failed_num("file_cache_download_failed_num"); +bvar::Adder block_file_cache_downloader_task_total("file_cache_downloader_queue_total"); FileCacheBlockDownloader::FileCacheBlockDownloader(CloudStorageEngine& engine) : _engine(engine) { _poller = std::thread(&FileCacheBlockDownloader::polling_download_task, this); @@ -81,6 +82,8 @@ void FileCacheBlockDownloader::submit_download_task(DownloadTask task) { std::lock_guard lock(_inflight_mtx); for (auto& meta : std::get<0>(task.task_message)) { ++_inflight_tablets[meta.tablet_id()]; + LOG(INFO) << "submit_download_task: inflight_tablets[" << meta.tablet_id() + << "] = " << _inflight_tablets[meta.tablet_id()]; if (meta.size() > 0) { g_file_cache_download_submitted_size << meta.size(); } @@ -103,9 +106,14 @@ void FileCacheBlockDownloader::submit_download_task(DownloadTask task) { } g_file_cache_download_failed_num << 1; } + LOG(INFO) << "submit_download_task: task queue full, pop front"; _task_queue.pop_front(); // Eliminate the earliest task in the queue + block_file_cache_downloader_task_total << -1; } + VLOG_DEBUG << "submit_download_task: push task, queue size before push: " + << _task_queue.size(); _task_queue.push_back(std::move(task)); + block_file_cache_downloader_task_total << 1; _empty.notify_all(); } g_file_cache_download_submitted_num << 1; @@ -119,16 +127,21 @@ void FileCacheBlockDownloader::polling_download_task() { std::unique_lock lock(_mtx); _empty.wait(lock, [this]() { return !_task_queue.empty() || _closed; }); if (_closed) { + LOG(INFO) << "polling_download_task: downloader closed, exit polling"; break; } task = std::move(_task_queue.front()); _task_queue.pop_front(); + block_file_cache_downloader_task_total << -1; + VLOG_DEBUG << "polling_download_task: pop task, queue size after pop: " + << _task_queue.size(); } if (std::chrono::duration_cast(std::chrono::steady_clock::now() - task.atime) .count() < hot_interval) { + VLOG_DEBUG << "polling_download_task: submit download_blocks to thread pool"; auto st = _workers->submit_func( [this, task_ = std::move(task)]() mutable { download_blocks(task_); }); if (!st.ok()) { @@ -159,6 +172,9 @@ std::unordered_map snapshot_rs_metas(BaseTable void FileCacheBlockDownloader::download_file_cache_block( const DownloadTask::FileCacheBlockMetaVec& metas) { std::ranges::for_each(metas, [&](const FileCacheBlockMeta& meta) { + VLOG_DEBUG << "download_file_cache_block: start, tablet_id=" << meta.tablet_id() + << ", rowset_id=" << meta.rowset_id() << ", segment_id=" << meta.segment_id() + << ", offset=" << meta.offset() << ", size=" << meta.size(); CloudTabletSPtr tablet; if (auto res = _engine.tablet_mgr().get_tablet(meta.tablet_id(), false); !res.has_value()) { LOG(INFO) << "failed to find tablet " << meta.tablet_id() << " : " << res.error(); @@ -170,6 +186,8 @@ void FileCacheBlockDownloader::download_file_cache_block( auto id_to_rowset_meta_map = snapshot_rs_metas(tablet.get()); auto find_it = id_to_rowset_meta_map.find(meta.rowset_id()); if (find_it == id_to_rowset_meta_map.end()) { + LOG(WARNING) << "download_file_cache_block: tablet_id=" << meta.tablet_id() + << "rowset_id not found, rowset_id=" << meta.rowset_id(); return; } @@ -179,7 +197,7 @@ void FileCacheBlockDownloader::download_file_cache_block( return; } - auto download_done = [&, tablet_id = meta.tablet_id()](Status) { + auto download_done = [&, tablet_id = meta.tablet_id()](Status st) { std::lock_guard lock(_inflight_mtx); auto it = _inflight_tablets.find(tablet_id); TEST_SYNC_POINT_CALLBACK("FileCacheBlockDownloader::download_file_cache_block"); @@ -187,11 +205,17 @@ void FileCacheBlockDownloader::download_file_cache_block( LOG(WARNING) << "inflight ref cnt not exist, tablet id " << tablet_id; } else { it->second--; + VLOG_DEBUG << "download_file_cache_block: inflight_tablets[" << tablet_id + << "] = " << it->second; if (it->second <= 0) { DCHECK_EQ(it->second, 0) << it->first; _inflight_tablets.erase(it); + VLOG_DEBUG << "download_file_cache_block: erase inflight_tablets[" << tablet_id + << "]"; } } + LOG(INFO) << "download_file_cache_block: download_done, tablet_Id=" << tablet_id + << "status=" << st.to_string(); }; DownloadFileMeta download_meta { @@ -215,6 +239,8 @@ void FileCacheBlockDownloader::download_file_cache_block( } void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& meta) { + LOG(INFO) << "download_segment_file: start, path=" << meta.path << ", offset=" << meta.offset + << ", download_size=" << meta.download_size << ", file_size=" << meta.file_size; FileReaderSPtr file_reader; FileReaderOptions opts { .cache_type = FileCachePolicy::FILE_BLOCK_CACHE, @@ -224,7 +250,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met }; auto st = meta.file_system->open_file(meta.path, &file_reader, &opts); if (!st.ok()) { - LOG(WARNING) << "failed to download file: " << st; + LOG(WARNING) << "failed to download file path=" << meta.path << ", st=" << st; if (meta.download_done) { meta.download_done(std::move(st)); } @@ -244,13 +270,15 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met size_t size = std::min(one_single_task_size, static_cast(meta.download_size - offset)); size_t bytes_read; + VLOG_DEBUG << "download_segment_file, path=" << meta.path << ", read_at offset=" << offset + << ", size=" << size; // TODO(plat1ko): // 1. Directly append buffer data to file cache // 2. Provide `FileReader::async_read()` interface DCHECK(meta.ctx.is_dryrun == config::enable_reader_dryrun_when_download_file_cache); auto st = file_reader->read_at(offset, {buffer.get(), size}, &bytes_read, &meta.ctx); if (!st.ok()) { - LOG(WARNING) << "failed to download file: " << st; + LOG(WARNING) << "failed to download file path=" << meta.path << ", st=" << st; if (meta.download_done) { meta.download_done(std::move(st)); } @@ -261,6 +289,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met } if (meta.download_done) { + LOG(INFO) << "download_segment_file: download finished, path=" << meta.path; meta.download_done(Status::OK()); } g_file_cache_download_finished_num << 1; From d8f734a14d467c09086d6f939164c2e40849df09 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 30 Jul 2025 09:30:47 +0800 Subject: [PATCH 324/572] =?UTF-8?q?branch-3.0:[fix](audit)Fixed=20an=20iss?= =?UTF-8?q?ue=20that=20the=20audit=20log=20would=20record=20the=20previo?= =?UTF-8?q?=E2=80=A6=20(#54008)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …us queryId when parseSQL fails. (#53107) pick: https://github.com/apache/doris/pull/53107 --- .../doris/common/NereidsSqlCacheManager.java | 2 ++ .../apache/doris/nereids/SqlCacheContext.java | 9 ++++--- .../doris/nereids/StatementContext.java | 4 ++-- .../org/apache/doris/qe/ConnectContext.java | 7 ++++++ .../org/apache/doris/qe/ConnectProcessor.java | 3 +++ .../apache/doris/qe/ConnectContextTest.java | 24 +++++++++++++++++++ .../org/apache/doris/qe/SqlCacheTest.java | 6 ++--- 7 files changed, 47 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index ba4b465d4b0852..f9836ccd192898 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -144,6 +144,7 @@ public void tryAddFeSqlCache(ConnectContext connectContext, String sql) { } SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get(); + sqlCacheContext.setQueryId(connectContext.queryId()); String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL ? generateCacheKey(connectContext, normalizeSql(sql)) : generateCacheKey(connectContext, DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5())); @@ -171,6 +172,7 @@ public void tryAddBeCache(ConnectContext connectContext, String sql, CacheAnalyz return; } SqlCacheContext sqlCacheContext = sqlCacheContextOpt.get(); + sqlCacheContext.setQueryId(connectContext.queryId()); String key = sqlCacheContext.getCacheKeyType() == CacheKeyType.SQL ? generateCacheKey(connectContext, normalizeSql(sql)) : generateCacheKey(connectContext, DebugUtil.printId(sqlCacheContext.getOrComputeCacheKeyMd5())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java index 0794d0aca0b2cd..458cdae3b739f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java @@ -60,7 +60,7 @@ public class SqlCacheContext { private static final Logger LOG = LogManager.getLogger(SqlCacheContext.class); private final UserIdentity userIdentity; - private final TUniqueId queryId; + private volatile TUniqueId queryId; // if contains udf/udaf/tableValuesFunction we can not process it and skip use sql cache private volatile boolean cannotProcessExpression; private volatile String originSql; @@ -99,9 +99,8 @@ public class SqlCacheContext { private volatile CacheKeyType cacheKeyType = CacheKeyType.SQL; - public SqlCacheContext(UserIdentity userIdentity, TUniqueId queryId) { + public SqlCacheContext(UserIdentity userIdentity) { this.userIdentity = Objects.requireNonNull(userIdentity, "userIdentity cannot be null"); - this.queryId = Objects.requireNonNull(queryId, "queryId cannot be null"); } public String getPhysicalPlan() { @@ -437,6 +436,10 @@ public void setCacheKeyType(CacheKeyType cacheKeyType) { this.cacheKeyType = cacheKeyType; } + public void setQueryId(TUniqueId queryId) { + this.queryId = queryId; + } + /** FullTableName */ @lombok.Data @lombok.AllArgsConstructor diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 9401ec102dac9f..ec372f863f89cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -254,10 +254,10 @@ public StatementContext(ConnectContext connectContext, OriginStatement originSta this.originStatement = originStatement; exprIdGenerator = ExprId.createGenerator(initialId); if (connectContext != null && connectContext.getSessionVariable() != null - && connectContext.queryId() != null && CacheAnalyzer.canUseSqlCache(connectContext.getSessionVariable())) { + // cannot set the queryId here because the queryId for the current query is set in the subsequent steps. this.sqlCacheContext = new SqlCacheContext( - connectContext.getCurrentUserIdentity(), connectContext.queryId()); + connectContext.getCurrentUserIdentity()); if (originStatement != null) { this.sqlCacheContext.setOriginSql(originStatement.originStmt); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 8b0bdf385718b7..bf5d7e37ffbde0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -867,6 +867,13 @@ public void setQueryId(TUniqueId queryId) { } } + public void resetQueryId() { + if (this.queryId != null) { + this.lastQueryId = this.queryId.deepCopy(); + } + this.queryId = null; + } + public void setTraceId(String traceId) { this.traceId = traceId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index cb3c3478fd01ff..168eaab9f6fc22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -236,6 +236,9 @@ protected void auditAfterExec(String origStmt, StatementBase parsedStmt, // only throw an exception when there is a problem interacting with the requesting client protected void handleQuery(String originStmt) throws ConnectionException { + // Before executing the query, the queryId should be set to empty. + // Otherwise, if SQL parsing fails, the audit log will record the queryId from the previous query. + ctx.resetQueryId(); if (Config.isCloudMode()) { if (!ctx.getCurrentUserIdentity().isRootUser() && ((CloudSystemInfoService) Env.getCurrentSystemInfo()).getInstanceStatus() diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java index 6ad9532a29784b..70ef82dbcd1be9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ConnectContextTest.java @@ -31,6 +31,7 @@ import java.nio.channels.SocketChannel; import java.util.List; +import java.util.UUID; public class ConnectContextTest { @Mocked @@ -272,4 +273,27 @@ public void testInsertQueryTimeoutS() { result = context.getInsertTimeoutS(); Assert.assertEquals(propertyValue, result); } + + @Test + public void testResetQueryId() { + ConnectContext context = new ConnectContext(); + Assert.assertNull(context.queryId); + Assert.assertNull(context.lastQueryId); + + UUID uuid = UUID.randomUUID(); + TUniqueId queryId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits()); + context.setQueryId(queryId); + Assert.assertEquals(queryId, context.queryId); + Assert.assertNull(context.lastQueryId); + + context.resetQueryId(); + Assert.assertNull(context.queryId); + Assert.assertEquals(queryId, context.lastQueryId); + + UUID uuid2 = UUID.randomUUID(); + TUniqueId queryId2 = new TUniqueId(uuid2.getMostSignificantBits(), uuid2.getLeastSignificantBits()); + context.setQueryId(queryId2); + Assert.assertEquals(queryId2, context.queryId); + Assert.assertEquals(queryId, context.lastQueryId); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SqlCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SqlCacheTest.java index afe95a49bde6c2..f67d037fdf64fb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/SqlCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SqlCacheTest.java @@ -37,11 +37,11 @@ public void testCacheKey() { queryId.setLo(uuid.getLeastSignificantBits()); UserIdentity admin = new UserIdentity("admin", "127.0.0.1"); - SqlCacheContext cacheContext = new SqlCacheContext(admin, queryId); + SqlCacheContext cacheContext = new SqlCacheContext(admin); cacheContext.setOriginSql("SELECT * FROM tbl"); PUniqueId key1 = cacheContext.doComputeCacheKeyMd5(ImmutableSet.of()); - SqlCacheContext cacheContext2 = new SqlCacheContext(admin, queryId); + SqlCacheContext cacheContext2 = new SqlCacheContext(admin); cacheContext2.setOriginSql( "-- Same query with comments and extra spaces\n" + "/* Comment */ SELECT * FROM tbl " @@ -49,7 +49,7 @@ public void testCacheKey() { PUniqueId key2 = cacheContext2.doComputeCacheKeyMd5(ImmutableSet.of()); Assertions.assertEquals(key1, key2); - SqlCacheContext cacheContext3 = new SqlCacheContext(admin, queryId); + SqlCacheContext cacheContext3 = new SqlCacheContext(admin); cacheContext3.setOriginSql( "-- Same query with comments and extra spaces\n" + "/* Comment */ SELeCT * FROM tbl " From 063767ad2922f406dc00e768bcdbcff404d81b95 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Tue, 29 Jul 2025 19:58:50 -0700 Subject: [PATCH 325/572] branch-3.0: [fix](audit) fix invalid audit log column and line separator #52968 (#54059) bp #52968 --- .../java/org/apache/doris/plugin/audit/AuditLoader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java index 89b6e58f7a1643..50e95eed9aa1b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java @@ -46,9 +46,9 @@ public class AuditLoader extends Plugin implements AuditPlugin { public static final String AUDIT_LOG_TABLE = "audit_log"; - // the "\\u001F" and "\\u001E" are used to separate columns and lines in audit log data - public static final String AUDIT_TABLE_COL_SEPARATOR = "\\u001F"; - public static final String AUDIT_TABLE_LINE_DELIMITER = "\\u001E"; + // the "0x1F" and "0x1E" are used to separate columns and lines in audit log data + public static final char AUDIT_TABLE_COL_SEPARATOR = 0x1F; + public static final char AUDIT_TABLE_LINE_DELIMITER = 0x1E; // the "\\x1F" and "\\x1E" are used to specified column and line delimiter in stream load request // which is corresponding to the "\\u001F" and "\\u001E" in audit log data. public static final String AUDIT_TABLE_COL_SEPARATOR_STR = "\\x1F"; From 7f5da93a651dd72fd5240fd52014a90226264268 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 16:30:24 +0800 Subject: [PATCH 326/572] branch-3.0: [regression-test](variant) Add variant upgrade case #53842 (#53880) Cherry-picked from #53842 Co-authored-by: XueYuhai <382297540@qq.com> --- .../sql/affinityByIssuesAndPRs1.out | 12 + .../sql/affinityByIssuesAndPRs2.out | 3 + .../sql/authorsWithTheMostPushes.out | 53 + .../sql/countingStar1.out | 4 + .../sql/countingStar2.out | 4 + .../sql/countingStar3.out | 3 + .../distributionOfRepositoriesByStarCount.out | 4 + .../sql/githubRoulette.out | 53 + ...tOfTopRepositoriesChangedOverTheYears1.out | 3 + ...tOfTopRepositoriesChangedOverTheYears2.out | 3 + ...tOfTopRepositoriesChangedOverTheYears3.out | 3 + ...tOfTopRepositoriesChangedOverTheYears4.out | 3 + ...tOfTopRepositoriesChangedOverTheYears5.out | 3 + ...tOfTopRepositoriesChangedOverTheYears6.out | 3 + ...asTheTotalNumberOfStarsChangedOverTime.out | 4 + .../sql/issuesWithTheMostComments1.out | 4 + .../sql/issuesWithTheMostComments2.out | 53 + .../sql/issuesWithTheMostComments3.out | 53 + .../sql/issuesWithTheMostComments4.out | 53 + .../sql/issuesWithTheMostComments5.out | 53 + .../sql/issuesWithTheMostComments6.out | 4 + .../sql/issuesWithTheMostComments7.out | 53 + .../sql/mostForkedRepositories.out | 53 + .../sql/mostPopularCommentsOnGithub.out | 53 + ...organizationsByTheNumberOfRepositories.out | 3 + .../sql/organizationsByTheNumberOfStars.out | 53 + .../sql/proportionsBetweenStarsAndForks1.out | 53 + .../sql/proportionsBetweenStarsAndForks2.out | 3 + .../sql/proportionsBetweenStarsAndForks3.out | 3 + .../sql/proportionsBetweenStarsAndForks4.out | 4 + .../sql/proportionsBetweenStarsAndForks5.out | 4 + .../repositoriesByAmountOfModifiedCode.out | 53 + .../sql/repositoriesByTheNumberOfPushes.out | 23 + ...toriesWithClickhouse_related_comments1.out | 4 + ...toriesWithClickhouse_related_comments2.out | 4 + ...epositoriesWithDoris_related_comments1.out | 4 + ...epositoriesWithDoris_related_comments2.out | 4 + .../repositoriesWithTheHighestGrowthYoY.out | 3 + ...ositoriesWithTheMaximumAmountOfIssues1.out | 53 + ...ositoriesWithTheMaximumAmountOfIssues2.out | 53 + ...ositoriesWithTheMaximumAmountOfIssues3.out | 3 + ...ositoriesWithTheMaximumAmountOfIssues4.out | 53 + ...iesWithTheMaximumAmountOfPullRequests1.out | 53 + ...iesWithTheMaximumAmountOfPullRequests2.out | 53 + ...hTheMaximumNumberOfAcceptedInvitations.out | 15 + ...iesWithTheMostPeopleWhoHavePushAccess1.out | 53 + ...epositoriesWithTheMostStarsOverOneDay1.out | 53 + ...itoriesWithTheMostSteadyGrowthOverTime.out | 53 + ...positoriesWithTheWorstStagnation_order.out | 3 + .../sql/repositoryAffinityList1.out | 3 + .../sql/repositoryAffinityList2.out | 3 + .../sql/starsFromHeavyGithubUsers1.out | 3 + .../sql/starsFromHeavyGithubUsers2.out | 3 + .../sql/theLongestRepositoryNames1.out | 53 + .../sql/theLongestRepositoryNames2.out | 53 + .../sql/theMostToughCodeReviews.out | 53 + .../theTotalNumberOfRepositoriesOnGithub.out | 4 + .../sql/theTotalNumberOfUsersOnGithub1.out | 4 + .../sql/theTotalNumberOfUsersOnGithub2.out | 4 + .../sql/theTotalNumberOfUsersOnGithub3.out | 4 + .../sql/theTotalNumberOfUsersOnGithub4.out | 4 + .../sql/topRepositoriesByStars.out | 53 + .../whatIsTheBestDayOfTheWeekToCatchAStar.out | 4 + .../sql/whoAreAllThosePeopleGivingStars1.out | 53 + .../sql/whoAreAllThosePeopleGivingStars2.out | 3 + .../sql/whoAreAllThosePeopleGivingStars3.out | 3 + .../variant_p2/tpch_upgrade/sql/q01_trans.out | 31 + .../variant_p2/tpch_upgrade/sql/q02_trans.out | 66 + .../variant_p2/tpch_upgrade/sql/q03_trans.out | 13 + .../variant_p2/tpch_upgrade/sql/q05_trans.out | 8 + .../variant_p2/tpch_upgrade/sql/q06_trans.out | 4 + .../variant_p2/tpch_upgrade/sql/q08_trans.out | 5 + .../variant_p2/tpch_upgrade/sql/q09_trans.out | 178 ++ .../variant_p2/tpch_upgrade/sql/q10_trans.out | 23 + .../variant_p2/tpch_upgrade/sql/q12_trans.out | 5 + .../variant_p2/tpch_upgrade/sql/q13_trans.out | 40 + .../variant_p2/tpch_upgrade/sql/q14_trans.out | 4 + .../variant_p2/tpch_upgrade/sql/q16_trans.out | 2765 +++++++++++++++++ .../load.groovy | 105 + .../sql/affinityByIssuesAndPRs1.sql | 19 + .../sql/affinityByIssuesAndPRs2.sql | 14 + .../sql/authorsWithTheMostPushes.sql | 9 + .../sql/countingStar1.sql | 1 + .../sql/countingStar2.sql | 1 + .../sql/countingStar3.sql | 1 + .../distributionOfRepositoriesByStarCount.sql | 14 + .../sql/githubRoulette.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears1.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears2.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears3.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears4.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears5.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears6.sql | 1 + ...tOfTopRepositoriesChangedOverTheYears7.sql | 30 + ...asTheTotalNumberOfStarsChangedOverTime.sql | 2 + .../sql/issuesWithTheMostComments1.sql | 1 + .../sql/issuesWithTheMostComments2.sql | 1 + .../sql/issuesWithTheMostComments3.sql | 17 + .../sql/issuesWithTheMostComments4.sql | 9 + .../sql/issuesWithTheMostComments5.sql | 9 + .../sql/issuesWithTheMostComments6.sql | 11 + .../sql/issuesWithTheMostComments7.sql | 9 + .../sql/issuesWithTheMostComments8.sql | 13 + .../sql/mostForkedRepositories.sql | 1 + .../sql/mostPopularCommentsOnGithub.sql | 1 + ...organizationsByTheNumberOfRepositories.sql | 14 + .../sql/organizationsByTheNumberOfStars.sql | 8 + .../sql/organizationsByTheSizeOfCommunity.sql | 23 + .../sql/proportionsBetweenStarsAndForks1.sql | 17 + .../sql/proportionsBetweenStarsAndForks2.sql | 18 + .../sql/proportionsBetweenStarsAndForks3.sql | 18 + .../sql/proportionsBetweenStarsAndForks4.sql | 13 + .../sql/proportionsBetweenStarsAndForks5.sql | 21 + .../repositoriesByAmountOfModifiedCode.sql | 12 + .../sql/repositoriesByTheNumberOfPushes.sql | 17 + ...toriesWithClickhouse_related_comments1.sql | 1 + ...toriesWithClickhouse_related_comments2.sql | 17 + ...epositoriesWithDoris_related_comments1.sql | 1 + ...epositoriesWithDoris_related_comments2.sql | 17 + .../repositoriesWithTheHighestGrowthYoY.sql | 20 + ...ositoriesWithTheMaximumAmountOfIssues1.sql | 1 + ...ositoriesWithTheMaximumAmountOfIssues2.sql | 18 + ...ositoriesWithTheMaximumAmountOfIssues3.sql | 19 + ...ositoriesWithTheMaximumAmountOfIssues4.sql | 18 + ...iesWithTheMaximumAmountOfPullRequests1.sql | 1 + ...iesWithTheMaximumAmountOfPullRequests2.sql | 1 + ...hTheMaximumNumberOfAcceptedInvitations.sql | 17 + ...iesWithTheMostPeopleWhoHavePushAccess1.sql | 13 + ...iesWithTheMostPeopleWhoHavePushAccess2.sql | 13 + ...iesWithTheMostPeopleWhoHavePushAccess3.sql | 16 + ...epositoriesWithTheMostStarsOverOneDay1.sql | 25 + ...epositoriesWithTheMostStarsOverOneDay2.sql | 25 + ...epositoriesWithTheMostStarsOverOneDay3.sql | 1 + ...itoriesWithTheMostSteadyGrowthOverTime.sql | 20 + ...positoriesWithTheWorstStagnation_order.sql | 20 + .../sql/repositoryAffinityList1.sql | 13 + .../sql/repositoryAffinityList2.sql | 23 + .../sql/starsFromHeavyGithubUsers1.sql | 13 + .../sql/starsFromHeavyGithubUsers2.sql | 15 + .../sql/theLongestRepositoryNames1.sql | 1 + .../sql/theLongestRepositoryNames2.sql | 1 + .../sql/theMostToughCodeReviews.sql | 10 + .../theTotalNumberOfRepositoriesOnGithub.sql | 1 + .../sql/theTotalNumberOfUsersOnGithub1.sql | 1 + .../sql/theTotalNumberOfUsersOnGithub2.sql | 1 + .../sql/theTotalNumberOfUsersOnGithub3.sql | 1 + .../sql/theTotalNumberOfUsersOnGithub4.sql | 1 + .../sql/topLabels1.sql | 9 + .../sql/topLabels2.sql | 9 + .../sql/topLabels3.sql | 14 + .../sql/topRepositoriesByStars.sql | 1 + .../whatIsTheBestDayOfTheWeekToCatchAStar.sql | 1 + .../sql/whoAreAllThosePeopleGivingStars1.sql | 1 + .../sql/whoAreAllThosePeopleGivingStars2.sql | 1 + .../sql/whoAreAllThosePeopleGivingStars3.sql | 13 + .../variant_p2/tpch_upgrade/load.groovy | 91 + .../variant_p2/tpch_upgrade/sql/q01_trans.sql | 32 + .../variant_p2/tpch_upgrade/sql/q02_trans.sql | 42 + .../variant_p2/tpch_upgrade/sql/q03_trans.sql | 25 + .../variant_p2/tpch_upgrade/sql/q04_trans.sql | 21 + .../variant_p2/tpch_upgrade/sql/q05_trans.sql | 25 + .../variant_p2/tpch_upgrade/sql/q06_trans.sql | 11 + .../variant_p2/tpch_upgrade/sql/q07_trans.sql | 41 + .../variant_p2/tpch_upgrade/sql/q08_trans.sql | 39 + .../variant_p2/tpch_upgrade/sql/q09_trans.sql | 33 + .../variant_p2/tpch_upgrade/sql/q10_trans.sql | 32 + .../variant_p2/tpch_upgrade/sql/q11_trans.sql | 29 + .../variant_p2/tpch_upgrade/sql/q12_trans.sql | 28 + .../variant_p2/tpch_upgrade/sql/q13_trans.sql | 20 + .../variant_p2/tpch_upgrade/sql/q14_trans.sql | 13 + .../variant_p2/tpch_upgrade/sql/q15_trans.sql | 18 + .../variant_p2/tpch_upgrade/sql/q16_trans.sql | 29 + .../variant_p2/tpch_upgrade/sql/q17_trans.sql | 16 + .../variant_p2/tpch_upgrade/sql/q18_trans.sql | 34 + .../variant_p2/tpch_upgrade/sql/q19_trans.sql | 36 + .../variant_p2/tpch_upgrade/sql/q20_trans.sql | 33 + .../variant_p2/tpch_upgrade/sql/q21_trans.sql | 39 + .../variant_p2/tpch_upgrade/sql/q22_trans.sql | 35 + 178 files changed, 6201 insertions(+) create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.out create mode 100644 regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q01_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q02_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q03_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q05_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q06_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q08_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q09_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q10_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q12_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q13_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q14_trans.out create mode 100644 regression-test/data/variant_p2/tpch_upgrade/sql/q16_trans.out create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments8.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheSizeOfCommunity.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels3.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.sql create mode 100644 regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/load.groovy create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q01_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q02_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q03_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q04_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q05_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q06_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q07_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q08_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q09_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q10_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q11_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q12_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q13_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q14_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q15_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q16_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q17_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q18_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q19_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q20_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q21_trans.sql create mode 100644 regression-test/suites/variant_p2/tpch_upgrade/sql/q22_trans.sql diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.out new file mode 100644 index 00000000000000..af6af3eb73d014 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !affinityByIssuesAndPRs1 -- +10000 + +-- !affinityByIssuesAndPRs1_2 -- +0 + +-- !affinityByIssuesAndPRs1_3 -- + +-- !affinityByIssuesAndPRs1_4 -- +0 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.out new file mode 100644 index 00000000000000..c352737724a011 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !affinityByIssuesAndPRs2 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.out new file mode 100644 index 00000000000000..04a92f3915cbdd --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !authorsWithTheMostPushes -- +github-actions[bot] 459 380 +seryozha1989 242 12 +cageyjames 119 1 +renovate[bot] 78 40 +LombiqBot 76 5 +freecall2019 63 1 +hotspotlab 62 1 +hotspot2023 61 1 +proxylist-to-bot 58 1 +PacoReinaCampo 37 37 +milesholt 37 1 +Chourouk-Zioud 31 1 +apaolacci 28 1 +pull[bot] 24 24 +geonosis-bot 21 21 +brokjad 19 1 +fpoirotte 18 18 +himobi 18 1 +rdower 16 16 +ropensci-bot 15 7 +dependabot[bot] 14 9 +supervpnops 14 2 +Vamsireddy90 13 1 +robocdnjs 12 1 +B74LABgit 11 3 +billybobza 11 1 +rmayr 11 2 +BlackSnowDot 10 1 +infaautovcs 10 1 +vpnsuperapp 10 1 +codership-jenkins 9 1 +direwolf-github 9 5 +ivy-root 9 1 +lannai1 9 1 +renovate-bot 9 9 +SuperSonicHub1 8 1 +azure-sdk 8 4 +robodoo 8 3 +NathaliaGreiffo 7 1 +SvenRoederer 7 6 +WolseyBankWitness 7 1 +oyvindberg 7 7 +rahqueu 7 1 +wangbadan3306 7 7 +EllisWaterman 6 1 +Hall-1910 6 1 +Mahesh6702 6 1 +Riley-Pritchard 6 1 +Uclic 6 1 +everyday-cc 6 3 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.out new file mode 100644 index 00000000000000..6e3cf19a28cf66 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !countingStar1 -- +329 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.out new file mode 100644 index 00000000000000..bc6e25bb384481 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !countingStar2 -- +started 329 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.out new file mode 100644 index 00000000000000..5942e8fbb3fb15 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !countingStar3 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.out new file mode 100644 index 00000000000000..8f8fa1373d0e9c --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !distributionOfRepositoriesByStarCount -- +1.0 317 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.out new file mode 100644 index 00000000000000..7ebfa8277881b8 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !githubRoulette -- +rsashka/newlang +matheusoms/grupo_2_2_semestre_2022 +Thiagoalves150/CURR-CULO +miquelmq20/qb-phone +1uphealth/fhir-react +hanjuku-kaso/awesome-offline-rl +joweich/chat-miner +shieldfy/API-Security-Checklist +zzzgydi/clash-verge +martinothamar/Mediator +spacecloud-io/space-cloud +Mushy96/Mushy_BankRobbery +the-aws-terraform-samples/terraform-eks-jumphost +shishkebab/shishkebab.github.io +BryanEstrada003/Tarea_AOP +blackpjotr/azure-sdk +camptocamp/oapi-poc +janovetz/remez-exchange +eilishmcmaster/eilishmcmaster.github.io +wechaty/wechaty +breck7/pldb +jakewvincent/mkdnflow.nvim +picoxr/Photon-state-sync +audreyandoy/task-list-api +catppuccin/catppuccin +learn-anything/blogs +saper150/remporium +IntelliTect/Coalesce +LilPoppy/Hackintosh-opencore-Z490-Asus +PierreMarion23/ipypivot +RyuGeunHwan/code_J +hirosystems/stacks.js +kronion/sts-playground +saper150/remporium +NVIDIA-Merlin/Transformers4Rec +aatharvauti/portfolio +google/guava +gremo/react-directus +Azure/azure-sdk +dotnet/razor +pureconfig/pureconfig +signavio/react-stick +mrrosh/NoErrorSounds +retorquere/zotero-better-bibtex +espanso/espanso +nopnop2002/esp-idf-smtp-camera +quartznet/quartznet +Zyflx/PsychEngine-Source-Code-Test +RajeshKumar1331/yt-home-page-clone +pulumi/pulumi + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out new file mode 100644 index 00000000000000..411783179221f9 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheListOfTopRepositoriesChangedOverTheYears1 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out new file mode 100644 index 00000000000000..ef231a83cb4f5d --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheListOfTopRepositoriesChangedOverTheYears2 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out new file mode 100644 index 00000000000000..b09f15ca076bd4 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheListOfTopRepositoriesChangedOverTheYears3 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out new file mode 100644 index 00000000000000..008eddda38f627 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheListOfTopRepositoriesChangedOverTheYears4 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out new file mode 100644 index 00000000000000..de580149bc37c1 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheListOfTopRepositoriesChangedOverTheYears5 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out new file mode 100644 index 00000000000000..cbe87efdd1cbf8 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheListOfTopRepositoriesChangedOverTheYears6 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out new file mode 100644 index 00000000000000..037fb3ac1f8900 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !howHasTheTotalNumberOfStarsChangedOverTime -- +2022 329 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.out new file mode 100644 index 00000000000000..ed75a908fb048c --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments1 -- +501 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.out new file mode 100644 index 00000000000000..91668f820d48c0 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments2 -- +GovAlta/ui-components 68 +dotnet/roslyn 8 +scientist-softserv/adventist-dl 6 +unifyai/ivy 6 +davesag/traverse-folders 5 +googleapis/python-logging 5 +gafusion/regression_notifications 4 +nezhivar/nezhOS 4 +pcwiese/opentelemetry-collector-contrib 4 +romeno-moreno/mxmrig 4 +xbrianlee/liferay-portal 4 +Firehed/php-project-template 3 +aiyuekuang/ant-design 3 +flutter/flutter 3 +jzhao62/ant-design 3 +kmalakoff/sequelize-parse-url 3 +kubernetes/kubernetes 3 +liferay-uniform/liferay-portal 3 +opensearch-project/common-utils 3 +ray-project/ray 3 +tmcclung/mastodon 3 +zerai/cf 3 +ACDguide/BigData 2 +Expensify/App 2 +MPAS-Dev/compass 2 +MicrosoftDocs/data-integration 2 +OutOfSyncStudios/object-key-cache 2 +a-r-j/graphein 2 +abigblueball/mastodon 2 +apache/airflow 2 +bhargavaman/meraType 2 +cockroachdb/cockroach 2 +coradoya/dicomviewer 2 +corgibytes/freshli-lib 2 +covid-projections/act-now-packages 2 +dolthub/dolt 2 +flipkart-incubator/zjsonpatch 2 +fullcalendar/fullcalendar 2 +jeffsawatzky/python-jsonapi 2 +karan-vk/Burger-R 2 +kiegroup/kogito-apps 2 +kstolte/kstolte.github.io 2 +kubermatic/kubermatic 2 +p4bl1t0/utn_tup_lab_comp_3 2 +patrickcate/dutch-art-daily 2 +raiden-network/explorer 2 +spack/spack 2 +tamuratak/vscode 2 +usdigitalresponse/univaf 2 +220926NET/P2-ADGX 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.out new file mode 100644 index 00000000000000..2e42184c3726a4 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments3 -- +GovAlta/ui-components 68 68 1.0 +dotnet/roslyn 8 8 1.0 +scientist-softserv/adventist-dl 6 1 6.0 +unifyai/ivy 6 6 1.0 +davesag/traverse-folders 5 5 1.0 +googleapis/python-logging 5 5 1.0 +gafusion/regression_notifications 4 2 2.0 +nezhivar/nezhOS 4 1 4.0 +pcwiese/opentelemetry-collector-contrib 4 4 1.0 +romeno-moreno/mxmrig 4 3 1.33 +xbrianlee/liferay-portal 4 2 2.0 +Firehed/php-project-template 3 3 1.0 +aiyuekuang/ant-design 3 1 3.0 +flutter/flutter 3 3 1.0 +jzhao62/ant-design 3 1 3.0 +kmalakoff/sequelize-parse-url 3 3 1.0 +kubernetes/kubernetes 3 2 1.5 +liferay-uniform/liferay-portal 3 1 3.0 +opensearch-project/common-utils 3 3 1.0 +ray-project/ray 3 2 1.5 +tmcclung/mastodon 3 3 1.0 +zerai/cf 3 3 1.0 +ACDguide/BigData 2 1 2.0 +Expensify/App 2 2 1.0 +MPAS-Dev/compass 2 1 2.0 +MicrosoftDocs/data-integration 2 1 2.0 +OutOfSyncStudios/object-key-cache 2 2 1.0 +a-r-j/graphein 2 1 2.0 +abigblueball/mastodon 2 2 1.0 +apache/airflow 2 1 2.0 +bhargavaman/meraType 2 2 1.0 +cockroachdb/cockroach 2 2 1.0 +coradoya/dicomviewer 2 2 1.0 +corgibytes/freshli-lib 2 2 1.0 +covid-projections/act-now-packages 2 2 1.0 +dolthub/dolt 2 1 2.0 +flipkart-incubator/zjsonpatch 2 2 1.0 +fullcalendar/fullcalendar 2 2 1.0 +jeffsawatzky/python-jsonapi 2 2 1.0 +karan-vk/Burger-R 2 2 1.0 +kiegroup/kogito-apps 2 1 2.0 +kstolte/kstolte.github.io 2 2 1.0 +kubermatic/kubermatic 2 1 2.0 +p4bl1t0/utn_tup_lab_comp_3 2 2 1.0 +patrickcate/dutch-art-daily 2 2 1.0 +raiden-network/explorer 2 2 1.0 +spack/spack 2 2 1.0 +tamuratak/vscode 2 1 2.0 +usdigitalresponse/univaf 2 2 1.0 +220926NET/P2-ADGX 1 1 1.0 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.out new file mode 100644 index 00000000000000..b1de957538a555 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments4 -- +scientist-softserv/adventist-dl 117 6 +nezhivar/nezhOS 190 4 +gafusion/regression_notifications 1 3 +xbrianlee/liferay-portal 671 3 +liferay-uniform/liferay-portal 770 3 +aiyuekuang/ant-design 1332 3 +jzhao62/ant-design 1699 3 +tamuratak/vscode 1 2 +romeno-moreno/mxmrig 2 2 +ACDguide/BigData 81 2 +MicrosoftDocs/data-integration 187 2 +a-r-j/graphein 233 2 +MPAS-Dev/compass 446 2 +kiegroup/kogito-apps 1497 2 +dolthub/dolt 4717 2 +kubermatic/kubermatic 11314 2 +apache/airflow 27477 2 +ray-project/ray 28683 2 +kubernetes/kubernetes 113702 2 +AlexBurneikis/SoT-DLC 1 1 +Balikuddembe/ToDoList-Code-Review 1 1 +JesseFarebro/Arcade-Learning-Environment 1 1 +NVlabs/edm 1 1 +ObelusFamily/Anythink-Market-p8atrkrs 1 1 +edakn/CepTelefonu_KayitTakipProgram 1 1 +luckelectricity/get_up 1 1 +monikag199/Test-repository-2 1 1 +rafacODEZ/minesweeperGUI 1 1 +DefenceX/vivoe-media-framework 2 1 +Earnkhay/nk-portfolio 2 1 +GovAlta/ui-components 2 1 +GullianLanguage/Gullian 2 1 +JasperVanEsveld/discordeno-audio-plugin 2 1 +jebl8843/CSCI3308-Final-Project-SciQuiz 2 1 +michaelandrew/machadwick.com 2 1 +vegasmorph/CS 2 1 +Flutter-Bounty-Hunters/inception 3 1 +GovAlta/ui-components 3 1 +p4bl1t0/utn_tup_lab_comp_3 3 1 +romeno-moreno/mxmrig 3 1 +vicplusplus/farm-paragon 3 1 +vivi90/python-vmc 3 1 +AngelPazzo/ProyectBackEnd 4 1 +GovAlta/ui-components 4 1 +djpowers/close-duplicate-cve-issues 4 1 +gafusion/regression_notifications 4 1 +romeno-moreno/mxmrig 4 1 +GovAlta/ui-components 5 1 +RelativelyFine/COMPSA-Frontend 5 1 +purothemes/polestar 5 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.out new file mode 100644 index 00000000000000..5c8ce924a0fa34 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments5 -- +scientist-softserv/adventist-dl 117 6 +nezhivar/nezhOS 190 4 +aiyuekuang/ant-design 1332 3 +jzhao62/ant-design 1699 3 +liferay-uniform/liferay-portal 770 3 +xbrianlee/liferay-portal 671 3 +ACDguide/BigData 81 2 +MPAS-Dev/compass 446 2 +MicrosoftDocs/data-integration 187 2 +a-r-j/graphein 233 2 +apache/airflow 27477 2 +dolthub/dolt 4717 2 +kiegroup/kogito-apps 1497 2 +kubermatic/kubermatic 11314 2 +kubernetes/kubernetes 113702 2 +ray-project/ray 28683 2 +220926NET/P2-ADGX 18 1 +ADVRHumanoids/awesome_leg 12 1 +ActiveState/cli 2225 1 +AlexxIT/go2rtc 66 1 +AllTheMods/ATM-6 3155 1 +Alluxio/alluxio 16487 1 +Automattic/wp-calypso 69836 1 +AzerothWarsLR/WarcraftLegacies 140 1 +Azure/ResourceModules 1727 1 +Azure/azure-sdk-for-c 2400 1 +AzureAD/microsoft-authentication-library-for-dotnet 3790 1 +Badger-Finance/badger-rewards 1000 1 +BlissRoms-x86/manifest 40 1 +Bworld-Studio/oph-server-node 183 1 +Cazadorsniper/PO3 1710 1 +ChainSafe/lodestar 4735 1 +Chatterino/chatterino2 4117 1 +City-of-Helsinki/events-helsinki-monorepo 48 1 +CodeDredd/pinia-orm 540 1 +CommunityToolkit/Maui 735 1 +EugenMayer/docker-sync 826 1 +Expensify/App 11711 1 +Expensify/App 12251 1 +Firehed/php-project-template 16 1 +Firehed/php-project-template 20 1 +Firehed/php-project-template 24 1 +FlyingDiver/Indigo-radiora2 57 1 +GTBitsOfGood/helping-mamas 130 1 +GiuseppeFilingeri/upgraded-symmetrical-waddle 38434 1 +GoogleCloudPlatform/magic-modules 6529 1 +GoogleCloudPlatform/terraform-google-secured-data-warehouse 35 1 +GovAlta/ui-components 11 1 +GovAlta/ui-components 12 1 +GovAlta/ui-components 13 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.out new file mode 100644 index 00000000000000..189897a9ecf552 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments6 -- +nezhivar/nezhOS 190 4 4 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.out new file mode 100644 index 00000000000000..8cd1a830b5c96d --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !issuesWithTheMostComments7 -- +hashicorp/boundary-ui 3 1 +AnyelinaMarte/saeamt-docente 2 1 +cdandrango3/facturas 2 1 +george1410/daily-mix-saver 2 1 +manifoldmarkets/manifold 2 1 +sghome/landing-luminas 2 1 +turja-c/personal-website 2 1 +zignini/exercicios 2 1 +CaioPires92/Aluratube 1 1 +CsarChvz/EventosPoli-Nxt 1 1 +EdmundLT/the-goods-store 1 1 +Esteban-hernandez/tienda 1 1 +Joalor64GH/Chocolate-Engine 1 1 +JoeyLicht/joey-fritter-front 1 1 +KATT/useSuspense-next-prisma-starter 1 1 +MarceloMachadoxD/alutratube 1 1 +Meland-Inc/status-pages 1 1 +MorrisonCole/morrisoncole.co.uk 1 1 +Neelam-Nishad/my-app 1 1 +NelieTchat/Python_Project 1 1 +NishatFariza/jk-portfolio 1 1 +Olawale06/Star 1 1 +OrdinaNederland/Stichting-NUTwente 1 1 +Prasetyonh/ReactJS-eNgaji 1 1 +PrettyLegit0101/spudify-server 1 1 +ThalisFernandes/RPG-Javascript 1 1 +UXMcCauley/portfolio 1 1 +Victor-Gabriel7/crud 1 1 +aggre/aggre.dev 1 1 +arthores/weather-check 1 1 +callmekungfu/yonglin.io 1 1 +carloshs1/task-manager 1 1 +cypress-io/cypress 1 1 +diegohpezet/Pokemon-API-Game 1 1 +ealfla8/ByteSized 1 1 +edugomess/Projeto_1 1 1 +edwinyi516/biy-frontend 1 1 +fedirko-pro/archery-telegram-bot 1 1 +flow2000/vercel-api 1 1 +galenczk/cs340-agenda-master-newui 1 1 +h1xten/tic-tac-toe-hopr 1 1 +hirosystems/connect 1 1 +hylansilva/warthena-project 1 1 +ivanatias/express-typescript-todos-API 1 1 +j-fu/ExtendableSparse.jl 1 1 +jdrew153/stream 1 1 +jupitersh/jupitersh.github.io 1 1 +kad200/nextjs-blog 1 1 +kperusi/famous-exam-project 1 1 +leonardmoses/ct-eyeCare-calendar-1.1 1 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.out new file mode 100644 index 00000000000000..bb6c08a632a35b --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mostForkedRepositories -- +informatika-19/latihan-backend-19421040-bagasdwijayanto 2 +joinpursuit/8-0-react-hooks-lab 2 +learn-co-curriculum/phase-1-practice-toy-tale 2 +neutron-org/testnets 2 +420-7A4-FE/exercice-gh-pages 1 +4creators/jxrlib 1 +ArcBees/GWTP 1 +CS234124/ex0_introtogit 1 +CSE110-FA22/Lab6_Starter 1 +Egorze111/cse210-04 1 +Genymobile/scrcpy 1 +GovReady/dataqa 1 +JEWELFUND/STN 1 +Karltheram/ME 1 +LOWPROKB/zmk-config-zen-2 1 +Launch-X-Latam/MisionBackend 1 +OLIMEX/Ultra-Sound-Levitation 1 +SwitchOS/switchdex 1 +TEnLOcODE/win-11-rdp-server-1 1 +The-Good-Stores/the-goods-store 1 +TheJoeFin/Text-Grab 1 +TheOdinProject/css-exercises 1 +Thinkful-Ed/starter-mobile-first-development 1 +abhisheknaiidu/abhisheknaiidu 1 +academicpages/academicpages.github.io 1 +africommerce/africommerce-sass-backend 1 +aloisorozco/SOEN287Assignment1 1 +angular/angular 1 +appacademy/practice-for-week-12-csrf-exercise 1 +atolVerderben/tentsuyu 1 +aws-samples/route-53-application-recovery-controller-codepipeline-with-terraform 1 +aziz/atom-plastic-code-wrap-syntax 1 +bbpanzu/bb-fnf-mods 1 +calamity-inc/Stand-for-FH5 1 +cassianobrexbit/dio-live-athena 1 +choishingwan/PRSice 1 +curiousmockingbird/coinCounter 1 +danielgindi/Charts 1 +davidkpiano/useEffectReducer 1 +davidtchiu/cs455-hwk-bcnf 1 +distributed-system-analysis/.github 1 +distributed-system-analysis/dsa-round-table 1 +earlephilhower/arduino-pico 1 +evilsocket/veryfied 1 +facebook/metro 1 +filiphsps/filiphsps 1 +firstcontributions/first-contributions 1 +florinpop17/app-ideas 1 +forcedotcom/postman-salesforce-apis 1 +gem-pasteur/Integron_Finder 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.out new file mode 100644 index 00000000000000..6d0c31d5b56070 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mostPopularCommentsOnGithub -- +参考資料ありがとうございます。\n 1 +yey 1 +will fix 1 +should we actually print something out here? 1 +same here, should we print here? 1 +same as above. 1 +rerun tests 1 +other chagnges made 1 +nit: should we use `^2.50.0` to get latest version? 1 +nice catch! 1 +nice catch 1 +message: n must be a >= 0 - so more explict 1 +looks like it is 👌🏼. 1 +label this as nsfw. 1 +https://github.com/opensearch-project/common-utils/pull/311\r\ncovered this backport 1 +https://github.com/opensearch-project/common-utils/pull/289\r\ncovered this change 1 +https://github.com/opensearch-project/common-utils/pull/273\r\n\r\ncovered these changes. 1 +https://github.com/gpuweb/gpuweb/issues/332 1 +hmmmm, reverted it back.. I forgot I was editing some files before :( 1 +hi @timmo001 I've corrected the incorrect upload. Would you be able to look it over? 1 +during -> while 1 +corp-pass-fork: test succeeded 1 +closing pull commit 1 +ci:test:upgrade 1 +ci:test:security 1 +ci:test:relevant 1 +ci:test:publications-environments 1 +ci:test:publications 1 +changed to [filter_ability] 1 +but the sentence seems to read a bit awkward if using Weight. 1 +all my friends have the same error (5 people) 1 +```suggestion\r\nstatic const std::string tmpdir{get_tmpdir()};\r\n``` 1 +Yes, it is correct 👍 \r\n 1 +Wonder if we will make it to 10 years 😂 1 +Will follow up with more realistic workloads (`ray.init()`, GPUs, multinode with resource requests) 1 +Why does the CRL updater talk to the readonly instance? 1 +We're using `CREATE_NO_WINDOW` quite a few times; it's probably time to put it into a constant. 1 +We shouldn't need to update for VN should we? 1 +We need to copy database to avoid its corruption. 1 +Wait, the wg is to make sure the thread has started, not to make sure it has ended? 1 +Updated. 1 +Update 1 +Tyson uploaded a SQL DB in commit: 2f5516a9b482c24b2e256486d21d7c86fe43336a 1 +This link is not working 1 +This is false by default. Can't accept until that's done and checks pass. 1 +This import is unused 1 +These should be `UInt`. see: https://github.com/MoarVM/MoarVM/pull/1725 1 +The dot is missing.\r\n 1 +The commit does not matter. I added a comment. 1 +That doesn't change the fact that what's stated in documentation is wrong. 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.out new file mode 100644 index 00000000000000..f7a79352ba0e25 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !organizationsByTheNumberOfRepositories -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.out new file mode 100644 index 00000000000000..da2952f7be634c --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !organizationsByTheNumberOfStars -- +desenvjonathan 5 +google 4 +iheyfy2 4 +ossu 3 +coderaiser 2 +djyanyan 2 +h5bp 2 +jlevy 2 +koltigin 2 +kubevious 2 +lxc 2 +martinothamar 2 +mesos 2 +milanm 2 +mrrosh 2 +saper150 2 +suadelabs 2 +the-aws-terraform-samples 2 +thealgorithms 2 +typicode 2 +veggiemonk 2 +zyflx 2 +0x192 1 +1uphealth 1 +42wim 1 +aatharvauti 1 +abdelrhmanhamouda 1 +adelolmo 1 +adrianhajdin 1 +afterlogic 1 +agrodan 1 +ahoshaiyan 1 +alandtse 1 +alexjoverm 1 +allthingssmitty 1 +amplab 1 +anilist 1 +antfu 1 +antoinepassemiers 1 +aosoft 1 +arcadesdude 1 +audreyandoy 1 +automattic 1 +avelino 1 +azure 1 +azure-sdk 1 +baba-s 1 +babel 1 +backstopmedia 1 +bbuchfink 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.out new file mode 100644 index 00000000000000..98c4fc911f9c24 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !proportionsBetweenStarsAndForks1 -- +informatika-19/latihan-backend-19421040-bagasdwijayanto 2 0 0.0 +joinpursuit/8-0-react-hooks-lab 2 0 0.0 +learn-co-curriculum/phase-1-practice-toy-tale 2 0 0.0 +neutron-org/testnets 2 0 0.0 +420-7A4-FE/exercice-gh-pages 1 0 0.0 +4creators/jxrlib 1 0 0.0 +ArcBees/GWTP 1 0 0.0 +CS234124/ex0_introtogit 1 0 0.0 +CSE110-FA22/Lab6_Starter 1 0 0.0 +Egorze111/cse210-04 1 0 0.0 +Genymobile/scrcpy 1 0 0.0 +GovReady/dataqa 1 0 0.0 +JEWELFUND/STN 1 0 0.0 +Karltheram/ME 1 0 0.0 +LOWPROKB/zmk-config-zen-2 1 0 0.0 +Launch-X-Latam/MisionBackend 1 1 1.0 +OLIMEX/Ultra-Sound-Levitation 1 1 1.0 +SwitchOS/switchdex 1 0 0.0 +TEnLOcODE/win-11-rdp-server-1 1 0 0.0 +The-Good-Stores/the-goods-store 1 0 0.0 +TheJoeFin/Text-Grab 1 0 0.0 +TheOdinProject/css-exercises 1 1 1.0 +Thinkful-Ed/starter-mobile-first-development 1 0 0.0 +abhisheknaiidu/abhisheknaiidu 1 0 0.0 +academicpages/academicpages.github.io 1 0 0.0 +africommerce/africommerce-sass-backend 1 0 0.0 +aloisorozco/SOEN287Assignment1 1 0 0.0 +angular/angular 1 0 0.0 +appacademy/practice-for-week-12-csrf-exercise 1 0 0.0 +atolVerderben/tentsuyu 1 0 0.0 +aws-samples/route-53-application-recovery-controller-codepipeline-with-terraform 1 0 0.0 +aziz/atom-plastic-code-wrap-syntax 1 0 0.0 +bbpanzu/bb-fnf-mods 1 0 0.0 +calamity-inc/Stand-for-FH5 1 0 0.0 +cassianobrexbit/dio-live-athena 1 0 0.0 +choishingwan/PRSice 1 0 0.0 +curiousmockingbird/coinCounter 1 1 1.0 +danielgindi/Charts 1 0 0.0 +davidkpiano/useEffectReducer 1 0 0.0 +davidtchiu/cs455-hwk-bcnf 1 0 0.0 +distributed-system-analysis/.github 1 0 0.0 +distributed-system-analysis/dsa-round-table 1 0 0.0 +earlephilhower/arduino-pico 1 0 0.0 +evilsocket/veryfied 1 0 0.0 +facebook/metro 1 0 0.0 +filiphsps/filiphsps 1 0 0.0 +firstcontributions/first-contributions 1 0 0.0 +florinpop17/app-ideas 1 0 0.0 +forcedotcom/postman-salesforce-apis 1 0 0.0 +gem-pasteur/Integron_Finder 1 0 0.0 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.out new file mode 100644 index 00000000000000..75a95a6165e316 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !proportionsBetweenStarsAndForks2 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.out new file mode 100644 index 00000000000000..140358cdc0bf95 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !proportionsBetweenStarsAndForks3 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.out new file mode 100644 index 00000000000000..d293d9fe05e6e9 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !proportionsBetweenStarsAndForks4 -- +102 329 3.23 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.out new file mode 100644 index 00000000000000..d158cfe90cebc2 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !proportionsBetweenStarsAndForks5 -- +\N \N \N + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.out new file mode 100644 index 00000000000000..4183aacebd828a --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesByAmountOfModifiedCode -- +webmd-health-services/BuildMasterAutomation 1 1 6968 4646 +volcain-io/exercism.io 1 1 5797 5192 +Macaulay2/M2 1 1 6311 3485 +cypress-io/cypress-documentation 1 1 4824 4584 +UmamiDAO/metrics-api 1 1 3853 3417 +kyungphill/practice_vue 1 1 4606 2663 +jinqshen/greatEffort 1 1 4433 2767 +esparzou/site_tiledesign 1 1 4466 2663 +mikeyhodl/kubernetes 1 1 3893 2415 +harrisonho99/react-native-windows-samples 2 1 4352 1564 +hto-projects/be-sound 1 1 3904 1598 +nf-core/tools 1 1 4189 1100 +jdhanotaGU/CRAPS-Game 1 1 2244 2837 +mheap/action-router 1 1 1073 3193 +vdavalon01/kotlin 1 1 1520 1701 +FNNDSC/ami 1 1 1555 1182 +ADCP1/airbnb-backend 1 1 2173 257 +Humba01/wertsfy 2 1 1379 869 +vortesnail/webpack-demo 1 1 1945 269 +christopher-caldwell/serverless-gcp 1 1 1452 341 +archanpatkar/tvl 1 1 630 1079 +sanity-io/sanity 1 1 956 719 +unicef/etools-datamart 1 1 803 608 +Superalgos/Governance-Plugins 1 1 716 614 +Mbed-TLS/mbedtls 1 1 1046 258 +dekim2324/darksky 1 1 101 1182 +DanielDonato/alura_pic_vue 1 1 140 1125 +wei/pull 1 1 534 626 +nextstrain/nextstrain.org 1 1 638 441 +akngs/d3-boxplot 1 1 495 549 +grygikrz/webpack-nodejs 1 1 207 770 +CatoRaymond-FS/WDV3322ProgWebApps 1 1 880 96 +microsoft/appcenter-cli 1 1 248 699 +mscststs/webEasy 1 1 155 716 +nicoproto/chat-redux 1 1 572 292 +sorokya/eo_protocol 1 1 458 386 +bsartain/AlgaeCal-Test 1 1 456 358 +saileshbro/DevConnector 1 1 412 397 +stripe/stripe-android 1 1 600 196 +Amiiiiiiiin/Escaping-Hell 1 1 493 277 +matt-riley/graphql_examples 5 1 395 356 +nordic96/portfolio_website 1 1 622 129 +dima17502/2019-2-Atom-Frontend-D-Volkov 1 1 426 322 +jsorkin24/github-battle 1 1 425 321 +shimaxu/Learn-JS 1 1 427 316 +wojtekmaj/get-user-locale 5 1 492 243 +cross-rs/cross 1 1 348 350 +fizietechD/optimism 1 1 259 427 +bitwarden/clients 1 1 385 279 +enonic/starter-react4xp 5 1 320 344 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.out new file mode 100644 index 00000000000000..bc59f62fee61ce --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesByTheNumberOfPushes -- +ValcambiSuisseNFT/verify-info 2 1 +spotlessmind1975/ugbasic 2 1 +sul-dlss/cybersource-rest-ruby 2 1 +yuukoamamiya/yuukoamamiya.github.io 2 1 +AntoinePassemiers/ArchMM 1 1 +DonDebonair/daan.fyi 1 1 +LorenaCoheneBaez/LorenaCoheneBaez 1 1 +c-rutter/imabc 1 1 +callmekungfu/yonglin.io 1 1 +darenmalfait/daren.be 1 1 +davep/trMandelbrot 1 1 +elisandrosp/gerenciador-tarefas-next-88aoj 1 1 +gavinleroy/aquascope 1 1 +hidratarse/PMDM-2223 1 1 +huggingface/transformers 1 1 +mgaertne/minqlx-plugin-tests 1 1 +poonchoi/nft-generator 1 1 +veit/python-basics-tutorial-de 1 1 +warrant-dev/warrant-node 1 1 +yapily/helm-charts 1 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.out new file mode 100644 index 00000000000000..893fc5459672b4 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithClickhouse_related_comments1 -- +rstudio/pins-r 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.out new file mode 100644 index 00000000000000..e7a9371634358b --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithClickhouse_related_comments2 -- +rstudio/pins-r 0 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.out new file mode 100644 index 00000000000000..441b0917b6d017 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithDoris_related_comments1 -- +rstudio/pins-r 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.out new file mode 100644 index 00000000000000..f18e9420593301 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithDoris_related_comments2 -- +rstudio/pins-r 0 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.out new file mode 100644 index 00000000000000..9a0abc202eeea6 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheHighestGrowthYoY -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out new file mode 100644 index 00000000000000..b141955f3b5d8f --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumAmountOfIssues1 -- +DsooRadha/CDMX013-md-links 6 1 +Shyenaia/prework-study-guide 4 1 +hzinner/lab-agile-planning 4 1 +pddemo/demo 4 1 +Wel-Alves/lab-agile-planning 3 1 +DefenceX/vivoe-media-framework 2 1 +GDSC-IGDTUW-Autumn-of-Code-2022/ar-filters 2 1 +faker-js/faker 2 1 +mrozycki/rustmas 2 1 +Adalab/da-promoc-m1-grupo4-gadea-lourdes-sila-sonia 1 1 +AlgebraicJulia/ASKEM-demos 1 1 +AmplifyCreations/AmplifyShaderEditor-Feedback 1 1 +Byron/google-apis-rs 1 1 +DDMAL/CantusDB 1 1 +Daily-Illini/wpgu 1 1 +DragonStuff/apts-jp-to-sql 1 1 +ESNFranceG33kTeam/sAPI 1 1 +EsmailELBoBDev2/upptime 1 1 +Firehed/php-project-template 1 1 +FrenchYeti/dexcalibur 1 1 +GiuseppeFilingeri/upgraded-symmetrical-waddle 1 1 +GoogleChrome/developer.chrome.com 1 1 +IronWillGames/Dransik 1 1 +IsmailAlamKhan/adb_ui 1 1 +MelvorIdle/melvoridle.github.io 1 1 +MicrosoftDocs/azure-docs 1 1 +MystenLabs/fastcrypto 1 1 +NationalSecurityAgency/ghidra 1 1 +NewGraphEnvironment/fish_passage_elk_2022_reporting 1 1 +PDAL/PDAL 1 1 +PipedreamHQ/pipedream 1 1 +Procurement-PoE/Procurement 1 1 +RMolania/TCGA_PanCancer_UnwantedVariation 1 1 +SistemasTecTlaxiaco/actividad-6-practica-de-entrevista-cuestionarios-etc-equipo-xd-1 1 1 +SunbirdAI/noise-sensors-monitoring 1 1 +The-Three-Dudes/one-chart 1 1 +TingluoHuang/proxy-test 1 1 +TinkerStorm/shuffle-theory 1 1 +VaibhavEkambaram/Penguin 1 1 +aptos-labs/aptos-core 1 1 +boneIO-eu/boneIO 1 1 +bounswe/bounswe2022group6 1 1 +brberis/ai-dream-io 1 1 +cadpage/cadpage-parsers 1 1 +casman300/my_website_upptimes 1 1 +cheeaun/hackerweb-native-2 1 1 +cheminfo/nmrium 1 1 +classilla/tenfourfox 1 1 +cloudbip/upptime 1 1 +djpowers/close-duplicate-cve-issues 1 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out new file mode 100644 index 00000000000000..4f14eea2a82853 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumAmountOfIssues2 -- +DsooRadha/CDMX013-md-links 6 1 0 +Shyenaia/prework-study-guide 4 1 0 +hzinner/lab-agile-planning 4 1 0 +pddemo/demo 4 1 0 +Wel-Alves/lab-agile-planning 3 1 0 +DefenceX/vivoe-media-framework 2 1 0 +GDSC-IGDTUW-Autumn-of-Code-2022/ar-filters 2 1 0 +faker-js/faker 2 1 0 +mrozycki/rustmas 2 1 0 +Adalab/da-promoc-m1-grupo4-gadea-lourdes-sila-sonia 1 1 0 +AlgebraicJulia/ASKEM-demos 1 1 0 +AmplifyCreations/AmplifyShaderEditor-Feedback 1 1 0 +Byron/google-apis-rs 1 1 0 +DDMAL/CantusDB 1 1 0 +Daily-Illini/wpgu 1 1 0 +DragonStuff/apts-jp-to-sql 1 1 0 +ESNFranceG33kTeam/sAPI 1 1 0 +EsmailELBoBDev2/upptime 1 1 0 +Firehed/php-project-template 1 1 0 +FrenchYeti/dexcalibur 1 1 0 +GiuseppeFilingeri/upgraded-symmetrical-waddle 1 1 0 +GoogleChrome/developer.chrome.com 1 1 0 +IronWillGames/Dransik 1 1 0 +IsmailAlamKhan/adb_ui 1 1 0 +MelvorIdle/melvoridle.github.io 1 1 0 +MicrosoftDocs/azure-docs 1 1 0 +MystenLabs/fastcrypto 1 1 0 +NationalSecurityAgency/ghidra 1 1 0 +NewGraphEnvironment/fish_passage_elk_2022_reporting 1 1 0 +PDAL/PDAL 1 1 0 +PipedreamHQ/pipedream 1 1 0 +Procurement-PoE/Procurement 1 1 0 +RMolania/TCGA_PanCancer_UnwantedVariation 1 1 0 +SistemasTecTlaxiaco/actividad-6-practica-de-entrevista-cuestionarios-etc-equipo-xd-1 1 1 0 +SunbirdAI/noise-sensors-monitoring 1 1 0 +The-Three-Dudes/one-chart 1 1 0 +TingluoHuang/proxy-test 1 1 0 +TinkerStorm/shuffle-theory 1 1 0 +VaibhavEkambaram/Penguin 1 1 0 +aptos-labs/aptos-core 1 1 0 +boneIO-eu/boneIO 1 1 0 +bounswe/bounswe2022group6 1 1 0 +brberis/ai-dream-io 1 1 0 +cadpage/cadpage-parsers 1 1 0 +casman300/my_website_upptimes 1 1 0 +cheeaun/hackerweb-native-2 1 1 0 +cheminfo/nmrium 1 1 0 +classilla/tenfourfox 1 1 0 +cloudbip/upptime 1 1 0 +djpowers/close-duplicate-cve-issues 1 1 0 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out new file mode 100644 index 00000000000000..19c9f84cef03b6 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumAmountOfIssues3 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out new file mode 100644 index 00000000000000..4de87ee12ff4c8 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumAmountOfIssues4 -- +SuadeLabs/rattr 0 0 2 +Zyflx/PsychEngine-Source-Code-Test 0 0 2 +h5bp/Front-end-Developer-Interview-Questions 0 0 2 +iheyfy2/Scripts-1 0 0 2 +iheyfy2/cloud189app-action 0 0 2 +jlevy/the-art-of-command-line 0 0 2 +martinothamar/Mediator 0 0 2 +mesos/chronos 0 0 2 +milanm/DevOps-Roadmap 0 0 2 +ossu/computer-science 0 0 2 +saper150/remporium 0 0 2 +veggiemonk/awesome-docker 0 0 2 +0x192/universal-android-debloater 0 0 1 +1uphealth/fhir-react 0 0 1 +42wim/matterbridge 0 0 1 +AbdelrhmanHamouda/locust-k8s-operator 0 0 1 +AgroDan/golden-kali 0 0 1 +AllThingsSmitty/css-protips 0 0 1 +AniList/discord-search-bot 0 0 1 +AntoinePassemiers/ArchMM 0 0 1 +Automattic/pocket-casts-android 0 0 1 +Azure/azure-sdk 0 0 1 +BenK93/MessagingAPI 0 0 1 +BetterDiscord/Installer 0 0 1 +BishopFox/sliver 0 0 1 +BlueAndi/esp-rgb-led-matrix 0 0 1 +BlueRaja/Weighted-Item-Randomizer-for-C-Sharp 0 0 1 +BryanEstrada003/Tarea_AOP 0 0 1 +Byron/gitoxide 0 0 1 +CommanderQ/SmartThings-Edge-Drivers 0 0 1 +CosminPOP/Outfitter 0 0 1 +DJYanYan/Banhaw-Landing-Page 0 0 1 +DJYanYan/Simon-Game 0 0 1 +DennisWG/QuickBind 0 0 1 +DonDebonair/daan.fyi 0 0 1 +EffiSciencesResearch/hackathon42 0 0 1 +ElJeloos/repasoGitHub 0 0 1 +EmilyEdna/SS-SSR-V2RAY 0 0 1 +Evavic44/portfolio-ideas 0 0 1 +GarnetSunset/rgg_save_tools 0 0 1 +HLC-Lewis/st_freecam 0 0 1 +Hailey-Ross/vorp_goldexchange 0 0 1 +HakCat-Tech/WiFi-Nugget 0 0 1 +IanLunn/Hover 0 0 1 +IntelliTect/Coalesce 0 0 1 +Josee9988/project-template 0 0 1 +JunkFood02/Seal 0 0 1 +KhronosGroup/Vulkan-Guide 0 0 1 +KinDR007/VictronMPPT-ESPHOME 0 0 1 +Launch-X-Latam/MisionBackend 0 0 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out new file mode 100644 index 00000000000000..95344913044dc5 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumAmountOfPullRequests1 -- +0xMoJo7/Snapchain 1 1 +102/eslint-plugin-comment-annotations 1 1 +ADCP1/airbnb-backend 1 1 +AbiaEssienRepos/auto-price-estimation-project 1 1 +AcalaNetwork/safe-config-service 1 1 +AdamHidvegi/CurrencyC 1 1 +AhmadTanvir/vue_lara 1 1 +Alan-love/language-server-protocol 1 1 +AlexLazareva/sarafan 1 1 +Alttaab/19.3-flask-survey-exercise 1 1 +Amiiiiiiiin/Escaping-Hell 1 1 +Arda-Computer/Arda 1 1 +AristotelesDantas/cadastro-de-motociclistas 1 1 +ArtemBilas/minimal-landing 1 1 +ArthurDoom/webapp 1 1 +AstarNetwork/safe-config-service 1 1 +Ategon/AuborealGameOff2022 1 1 +Automattic/wp-calypso 1 1 +Azure/react-azure-maps 1 1 +BSData/horus-heresy 1 1 +BarberoPablo/prode 1 1 +Bayusaiinbasran/bayusaiinbasrantugas7 1 1 +BeOrNot2Be/MEDILOV-back 1 1 +Blockhead4/React_JS-Tutorial 1 1 +Bworld-Studio/oph-server-node 1 1 +C-Lodder/music-player 1 1 +C-Shrestha/newpantry 1 1 +CFC-Servers/gm_express_service 1 1 +CatoRaymond-FS/WDV3322ProgWebApps 1 1 +Christoph551/prework-study-guide 1 1 +ColtAllen/btyd 1 1 +CrowdStrike/ember-headless-table 1 1 +DanielDonato/alura_pic_vue 1 1 +DennisOSRM/toolbox-rs 1 1 +DeyvyMY/SICPAV2 1 1 +Dithn/windows-itpro-docs 1 1 +ED1978/countries_lab 1 1 +Earnkhay/nk-portfolio 1 1 +Earnkhay/vue-login-page 1 1 +EncryptEx/myhackupc 1 1 +Eric-LV/cv 1 1 +FAForever/fa 1 1 +FAIRDataTeam/FAIRDataPoint 1 1 +FF0X00/anti-AD 1 1 +FNNDSC/ami 1 1 +FalkenDev/SparkRentals-Mobile-App 1 1 +Fejiro-A/flutter_web_helper_services 1 1 +FerNanDoAnc/sistema-musica 1 1 +Figuritas-Unlam/Figuritas 1 1 +Frefreak/opentelemetry-collector-contrib 1 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out new file mode 100644 index 00000000000000..88c26b04a0eae6 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumAmountOfPullRequests2 -- +mhutchinson/mhutchinson-distributor 3 3 +neutron-org/testnets 3 3 +azelezovs/auto-bootcamp-setup 2 2 +pcwiese/opentelemetry-collector-contrib 9 1 +char0n/ramda-adjunct 5 1 +enonic/starter-react4xp 5 1 +matt-riley/graphql_examples 5 1 +razee-io/WatchKeeper 5 1 +wojtekmaj/get-user-locale 5 1 +davesag/traverse-folders 4 1 +ergebnis/php-package-template 4 1 +freaktechnik/twaese 4 1 +swagger-api/swagger-js 4 1 +wojtekmaj/react-calendar 4 1 +Gotensfer/Taller-VI 3 1 +abigblueball/mastodon 3 1 +cycjimmy/h5-video-player 3 1 +freaktechnik/advanced-github-notifier 3 1 +kmalakoff/sequelize-parse-url 3 1 +mock-server/mockserver 3 1 +ngarbezza/collection_filter 3 1 +oli-ver/datatables-ssp-java 3 1 +userfrosting/vinyl-fs-vpath 3 1 +Aiko-IT-Systems/DisCatSharp.ProjectTemplates 2 1 +GabrielVicente-GT/kodonote 2 1 +Humba01/wertsfy 2 1 +IBMResearch/stylelint-config-ibmresearch 2 1 +OutOfSyncStudios/object-key-cache 2 1 +andiadammalik/H181600617_andiadammalik_3D 2 1 +aniskhan001/napi 2 1 +byulmaru/mastodon 2 1 +corgibytes/freshli-lib 2 1 +elastic/opbeans-python 2 1 +freaktechnik/my-deprecations 2 1 +harrisonho99/react-native-windows-samples 2 1 +j5ik2o/docker-controller-scala 2 1 +jeffsawatzky/python-jsonapi 2 1 +kstolte/kstolte.github.io 2 1 +leaflet-extras/leaflet-providers 2 1 +list-abstraction/doubly-linked-list 2 1 +modernpoacher/halacious 2 1 +monkriz1991/arhiterm 2 1 +neilmfrench/homelab-prod 2 1 +patrickcate/dutch-art-daily 2 1 +rpoole444/ideabox 2 1 +samip5/k8s-cluster 2 1 +streetsidesoftware/cspell 2 1 +thefrosty/wp-upgrade-task-runner 2 1 +tool3/docsify-action 2 1 +whelk-io/asciidoc-template-maven-plugin 2 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out new file mode 100644 index 00000000000000..d22ec198218887 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMaximumNumberOfAcceptedInvitations -- +SuadeLabs/rattr 0 2 +Zyflx/PsychEngine-Source-Code-Test 0 2 +h5bp/Front-end-Developer-Interview-Questions 0 2 +iheyfy2/Scripts-1 0 2 +iheyfy2/cloud189app-action 0 2 +jlevy/the-art-of-command-line 0 2 +martinothamar/Mediator 0 2 +mesos/chronos 0 2 +milanm/DevOps-Roadmap 0 2 +ossu/computer-science 0 2 +saper150/remporium 0 2 +veggiemonk/awesome-docker 0 2 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out new file mode 100644 index 00000000000000..7507336d2570d5 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMostPeopleWhoHavePushAccess1 -- +zzzgydi/clash-verge 0 1 +zorzalerrante/tsundoku 0 1 +zhoudaxiaa/vpn- 0 1 +zhongyang219/trafficmonitor 0 1 +yingdev/tickeys 0 1 +ycd/manage-fastapi 0 1 +yassineb14/android-staticbroadcastreceiver 0 1 +yangshun/tech-interview-handbook 0 1 +xrayfree/free-ssr-ss-v2ray-vpn-clash 0 1 +xiaohulugo/3dlinedetection 0 1 +wfxr/forgit 0 1 +wechaty/wechaty 0 1 +webacademyufac/programacao-avancada-backend-t2 0 1 +vuejs/vue 0 1 +visjs/vis-network 0 1 +victorsdm/curriculo-cmd 0 1 +victoriametrics/victoriametrics 0 1 +viatorus/compile-time-printer 0 1 +v2ray/v2ray-core 0 1 +udokah/veecam 0 1 +typicode/json-server 0 1 +typicode/hotel 0 1 +typestrong/ts-loader 0 1 +trustedsec/social-engineer-toolkit 0 1 +transmission/transmission 0 1 +torvalds/linux 0 1 +timescale/timescaledb 0 1 +thiagoalves150/curr-culo 0 1 +thesilentdefender/cubesatjs 0 1 +theodinproject/css-exercises 0 1 +theaquarium/quizletmatch-autosolver 0 1 +thealgorithms/jupyter 0 1 +thealgorithms/java 0 1 +the-aws-terraform-samples/terraform-route-53-application-recovery-controller-codepipeline-with-terraform 0 1 +the-aws-terraform-samples/terraform-eks-jumphost 0 1 +telegrammessenger/telegram-ios 0 1 +teamnewpipe/newpipe 0 1 +t3-oss/create-t3-app 0 1 +sveltejs/realworld 0 1 +stormwave/saveeditor 0 1 +starkscan/starkscan-verifier 0 1 +ssut/payload-dumper-go 0 1 +spotdl/spotify-downloader 0 1 +spierala/local-redux-store 0 1 +spacecloud-io/space-cloud 0 1 +snesrev/zelda3 0 1 +skatejs/skatejs 0 1 +sindresorhus/awesome 0 1 +signavio/react-stick 0 1 +shuttle-hq/shuttle 0 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out new file mode 100644 index 00000000000000..06ec21c90eccf1 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMostStarsOverOneDay1 -- +SuadeLabs/rattr 2022-11-08 2 +Zyflx/PsychEngine-Source-Code-Test 2022-11-08 2 +h5bp/Front-end-Developer-Interview-Questions 2022-11-08 2 +iheyfy2/Scripts-1 2022-11-08 2 +iheyfy2/cloud189app-action 2022-11-08 2 +jlevy/the-art-of-command-line 2022-11-08 2 +martinothamar/Mediator 2022-11-08 2 +mesos/chronos 2022-11-08 2 +milanm/DevOps-Roadmap 2022-11-08 2 +ossu/computer-science 2022-11-08 2 +saper150/remporium 2022-11-08 2 +veggiemonk/awesome-docker 2022-11-08 2 +0x192/universal-android-debloater 2022-11-08 1 +1uphealth/fhir-react 2022-11-08 1 +42wim/matterbridge 2022-11-08 1 +AbdelrhmanHamouda/locust-k8s-operator 2022-11-08 1 +AgroDan/golden-kali 2022-11-08 1 +AllThingsSmitty/css-protips 2022-11-08 1 +AniList/discord-search-bot 2022-11-08 1 +AntoinePassemiers/ArchMM 2022-11-08 1 +Automattic/pocket-casts-android 2022-11-08 1 +Azure/azure-sdk 2022-11-08 1 +BenK93/MessagingAPI 2022-11-08 1 +BetterDiscord/Installer 2022-11-08 1 +BishopFox/sliver 2022-11-08 1 +BlueAndi/esp-rgb-led-matrix 2022-11-08 1 +BlueRaja/Weighted-Item-Randomizer-for-C-Sharp 2022-11-08 1 +BryanEstrada003/Tarea_AOP 2022-11-08 1 +Byron/gitoxide 2022-11-08 1 +CommanderQ/SmartThings-Edge-Drivers 2022-11-08 1 +CosminPOP/Outfitter 2022-11-08 1 +DJYanYan/Banhaw-Landing-Page 2022-11-08 1 +DJYanYan/Simon-Game 2022-11-08 1 +DennisWG/QuickBind 2022-11-08 1 +DonDebonair/daan.fyi 2022-11-08 1 +EffiSciencesResearch/hackathon42 2022-11-08 1 +ElJeloos/repasoGitHub 2022-11-08 1 +EmilyEdna/SS-SSR-V2RAY 2022-11-08 1 +Evavic44/portfolio-ideas 2022-11-08 1 +GarnetSunset/rgg_save_tools 2022-11-08 1 +HLC-Lewis/st_freecam 2022-11-08 1 +Hailey-Ross/vorp_goldexchange 2022-11-08 1 +HakCat-Tech/WiFi-Nugget 2022-11-08 1 +IanLunn/Hover 2022-11-08 1 +IntelliTect/Coalesce 2022-11-08 1 +Josee9988/project-template 2022-11-08 1 +JunkFood02/Seal 2022-11-08 1 +KhronosGroup/Vulkan-Guide 2022-11-08 1 +KinDR007/VictronMPPT-ESPHOME 2022-11-08 1 +Launch-X-Latam/MisionBackend 2022-11-08 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out new file mode 100644 index 00000000000000..ff3c61d2e99bee --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheMostSteadyGrowthOverTime -- +0x192/universal-android-debloater 1 1 1.0 +1uphealth/fhir-react 1 1 1.0 +42wim/matterbridge 1 1 1.0 +AbdelrhmanHamouda/locust-k8s-operator 1 1 1.0 +AgroDan/golden-kali 1 1 1.0 +AllThingsSmitty/css-protips 1 1 1.0 +AniList/discord-search-bot 1 1 1.0 +AntoinePassemiers/ArchMM 1 1 1.0 +Automattic/pocket-casts-android 1 1 1.0 +Azure/azure-sdk 1 1 1.0 +BenK93/MessagingAPI 1 1 1.0 +BetterDiscord/Installer 1 1 1.0 +BishopFox/sliver 1 1 1.0 +BlueAndi/esp-rgb-led-matrix 1 1 1.0 +BlueRaja/Weighted-Item-Randomizer-for-C-Sharp 1 1 1.0 +BryanEstrada003/Tarea_AOP 1 1 1.0 +Byron/gitoxide 1 1 1.0 +CommanderQ/SmartThings-Edge-Drivers 1 1 1.0 +CosminPOP/Outfitter 1 1 1.0 +DJYanYan/Banhaw-Landing-Page 1 1 1.0 +DJYanYan/Simon-Game 1 1 1.0 +DennisWG/QuickBind 1 1 1.0 +DonDebonair/daan.fyi 1 1 1.0 +EffiSciencesResearch/hackathon42 1 1 1.0 +ElJeloos/repasoGitHub 1 1 1.0 +EmilyEdna/SS-SSR-V2RAY 1 1 1.0 +Evavic44/portfolio-ideas 1 1 1.0 +GarnetSunset/rgg_save_tools 1 1 1.0 +HLC-Lewis/st_freecam 1 1 1.0 +Hailey-Ross/vorp_goldexchange 1 1 1.0 +HakCat-Tech/WiFi-Nugget 1 1 1.0 +IanLunn/Hover 1 1 1.0 +IntelliTect/Coalesce 1 1 1.0 +Josee9988/project-template 1 1 1.0 +JunkFood02/Seal 1 1 1.0 +KhronosGroup/Vulkan-Guide 1 1 1.0 +KinDR007/VictronMPPT-ESPHOME 1 1 1.0 +Launch-X-Latam/MisionBackend 1 1 1.0 +LianjiaTech/bruno 1 1 1.0 +LilPoppy/Hackintosh-opencore-Z490-Asus 1 1 1.0 +LorenaCoheneBaez/LorenaCoheneBaez 1 1 1.0 +MagnoEfren/gui_python_tkinter 1 1 1.0 +Martinhh13/Practica8 1 1 1.0 +Mushy96/Mushy_BankRobbery 1 1 1.0 +NVIDIA-Merlin/Transformers4Rec 1 1 1.0 +OLIMEX/Ultra-Sound-Levitation 1 1 1.0 +PierreMarion23/ipypivot 1 1 1.0 +Pinguwien/DotNetCoreFakeAuth 1 1 1.0 +PlantSimulationLab/Helios 1 1 1.0 +PyGithub/PyGithub 1 1 1.0 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.out new file mode 100644 index 00000000000000..39b21a2439f33e --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoriesWithTheWorstStagnation_order -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.out new file mode 100644 index 00000000000000..9c258135b4be7f --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoryAffinityList1 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.out new file mode 100644 index 00000000000000..70e24a8edfc9a8 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !repositoryAffinityList2 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.out new file mode 100644 index 00000000000000..fd3791088d772b --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !starsFromHeavyGithubUsers1 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.out new file mode 100644 index 00000000000000..e5c1178d400de6 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !starsFromHeavyGithubUsers2 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.out new file mode 100644 index 00000000000000..aa4ae1f202049b --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theLongestRepositoryNames1 -- +1 the-aws-terraform-samples/terraform-route-53-application-recovery-controller-codepipeline-with-terraform +1 mikeycal/the-video-editors-render-script-for-blender +1 the-aws-terraform-samples/terraform-eks-jumphost +1 python-semantic-release/python-semantic-release +1 rafaelsilverioit/twitter-django-rest-framework +1 webacademyufac/programacao-avancada-backend-t2 +1 BlueRaja/Weighted-Item-Randomizer-for-C-Sharp +1 adrianhajdin/project_modern_ui_ux_restaurant +2 h5bp/Front-end-Developer-Interview-Questions +1 baba-s/awesome-unity-open-source-on-github +1 elisandrosp/gerenciador-tarefas-next-88aoj +1 practical-tutorials/project-based-learning +1 yassineb14/Android-StaticBroadcastReceiver +1 eilishmcmaster/eilishmcmaster.github.io +1 koltigin/Restake-Turkce-Kurulum-Rehberi +1 LilPoppy/Hackintosh-opencore-Z490-Asus +1 cornehoskam/Umbraco9-Blazor-Starterkit +1 AbdelrhmanHamouda/locust-k8s-operator +1 alexjoverm/typescript-library-starter +1 brsbrc/Restake-Turkce-Kurulum-Rehberi +1 chobits/ngx_http_proxy_connect_module +1 michael-rubel/livewire-best-practices +1 gabors-data-analysis/da_case_studies +1 jpatel98/instock-api-silentdebuggers +1 xrayfree/free-ssr-ss-v2ray-vpn-clash +1 CommanderQ/SmartThings-Edge-Drivers +1 desenvjonathan/nlw-esports-explorer +1 igorescodro/version-catalog-example +1 jwasham/coding-interview-university +1 koltigin/Inery-Node-Kurulum-Rehberi +1 theaquarium/quizletmatch-autosolver +1 yuukoamamiya/yuukoamamiya.github.io +1 RajeshKumar1331/yt-home-page-clone +2 Zyflx/PsychEngine-Source-Code-Test +1 chrisdl/Django-QuerySet-Cheatsheet +1 estevam5s/pedacinho-do-ceu-website +1 fknaesel/exercicios-de-programacao +1 fr0gger/Awesome_Malware_Techniques +1 hideuvpn/android-google-play-store +1 matheusoms/grupo_2_2_semestre_2022 +1 trustedsec/social-engineer-toolkit +1 0x192/universal-android-debloater +1 LorenaCoheneBaez/LorenaCoheneBaez +1 chenweize1998/fully-hyperbolic-nn +1 leonardoH263/curr-culo.-Leonardo- +1 EffiSciencesResearch/hackathon42 +1 Scrawk/Marching-Cubes-On-The-GPU +1 codecrafters-io/build-your-own-x +1 encrypted-def/basic-algo-lecture +1 jdonnelly36/Deformable-ProtoPNet + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.out new file mode 100644 index 00000000000000..e69f3ca593af37 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theLongestRepositoryNames2 -- +lxc/lxc 1 +lxc/lxd 1 +antfu/ni 1 +golang/go 1 +ossu/math 1 +prql/prql 1 +vuejs/vue 1 +nektos/act 1 +amplab/snap 1 +breck7/pldb 1 +mli/autocut 1 +numpy/numpy 1 +wfxr/forgit 1 +bufbuild/buf 1 +dotnet/razor 1 +google/guava 1 +google/paxml 1 +jsb/RingMenu 1 +IanLunn/Hover 1 +Udokah/veecam 1 +grafana/thema 1 +httpie/httpie 1 +jhawthorn/fzy 1 +kubevious/cli 1 +mesos/chronos 2 +pulumi/pulumi 1 +rcaos/TVToday 1 +Byron/gitoxide 1 +alandtse/tesla 1 +bloomberg/foml 1 +c-rutter/imabc 1 +dosisod/refurb 1 +github/explore 1 +mono/SkiaSharp 1 +openai/whisper 1 +snesrev/zelda3 1 +torvalds/linux 1 +typicode/hotel 1 +Azure/azure-sdk 1 +JunkFood02/Seal 1 +Questie/Questie 1 +SuadeLabs/rattr 2 +arcadesdude/BRU 1 +broxus/locklift 1 +caozhiyi/CppNet 1 +espanso/espanso 1 +eunjin2118/GoGo 1 +lirantal/dockly 1 +mrrosh/OpenClam 1 +netdata/netdata 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.out new file mode 100644 index 00000000000000..a77c61e62ca67c --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theMostToughCodeReviews -- +https://github.com/kubernetes/kubernetes/pull/ 3 +https://github.com/napari/napari/pull/ 2 +https://github.com/percona/pmm/pull/ 2 +https://github.com/ADCP1/airbnb-backend/pull/ 1 +https://github.com/Azure-Samples/communication-services-dotnet-quickstarts/pull/ 1 +https://github.com/CDOT-CV/jpo-cvdp/pull/ 1 +https://github.com/DataDog/dd-trace-js/pull/ 1 +https://github.com/DavidSM64/Diddy-Kong-Racing/pull/ 1 +https://github.com/GuruCICDCanary-Prod/CICDCanary/pull/ 1 +https://github.com/MCreator/MCreator/pull/ 1 +https://github.com/MetaMask/metamask-mobile/pull/ 1 +https://github.com/NewGraphEnvironment/fish_passage_elk_2022_reporting/pull/ 1 +https://github.com/Open-Systems-Pharmacology/PK-Sim/pull/ 1 +https://github.com/Spacechild1/pure-data/pull/ 1 +https://github.com/Tatsuro-fastball14/phileo_1210/pull/ 1 +https://github.com/Terracotta-OSS/terracotta-core/pull/ 1 +https://github.com/WebAssembly/wabt/pull/ 1 +https://github.com/akitasoftware/akita-cli/pull/ 1 +https://github.com/apache/airflow/pull/ 1 +https://github.com/apache/beam/pull/ 1 +https://github.com/apache/flink-kubernetes-operator/pull/ 1 +https://github.com/apache/maven-resolver/pull/ 1 +https://github.com/aptos-labs/aptos-core/pull/ 1 +https://github.com/arangodb/arangodb/pull/ 1 +https://github.com/autonomousapps/gradle-best-practices-plugin/pull/ 1 +https://github.com/aws-amplify/amplify-ui/pull/ 1 +https://github.com/aws/amazon-genomics-cli/pull/ 1 +https://github.com/breannly/java-explore-with-me/pull/ 1 +https://github.com/darchlabs/synchronizer-v2/pull/ 1 +https://github.com/dart-lang/linter/pull/ 1 +https://github.com/dotnet/runtime/pull/ 1 +https://github.com/duckduckgo/Android/pull/ 1 +https://github.com/ethereum/solidity/pull/ 1 +https://github.com/facebookincubator/velox/pull/ 1 +https://github.com/flutter/devtools/pull/ 1 +https://github.com/forcedotcom/sfdx-scanner/pull/ 1 +https://github.com/frollous/frollo-swift-sdk/pull/ 1 +https://github.com/halide/Halide/pull/ 1 +https://github.com/hashicorp/design-system/pull/ 1 +https://github.com/home-assistant/core/pull/ 1 +https://github.com/insertStringHere/MineBound/pull/ 1 +https://github.com/intel-iot-devkit/rtsf-at-checkout-reference-design/pull/ 1 +https://github.com/irods/irods/pull/ 1 +https://github.com/jacktrip/jacktrip/pull/ 1 +https://github.com/jitsi/jitsi-meet/pull/ 1 +https://github.com/k8ssandra/cass-operator/pull/ 1 +https://github.com/kubernetes-sigs/gateway-api/pull/ 1 +https://github.com/latis-data/latis3/pull/ 1 +https://github.com/letsencrypt/boulder/pull/ 1 +https://github.com/libfuse/libfuse/pull/ 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.out new file mode 100644 index 00000000000000..b7ce929fc4d23e --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theTotalNumberOfRepositoriesOnGithub -- +5619 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.out new file mode 100644 index 00000000000000..61ea9003ab08b5 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theTotalNumberOfUsersOnGithub1 -- +4600 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.out new file mode 100644 index 00000000000000..a7393c692bd992 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theTotalNumberOfUsersOnGithub2 -- +228 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.out new file mode 100644 index 00000000000000..f9223a342cd52a --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theTotalNumberOfUsersOnGithub3 -- +3157 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.out new file mode 100644 index 00000000000000..a46b96cc0028ab --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !theTotalNumberOfUsersOnGithub4 -- +247 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.out new file mode 100644 index 00000000000000..346989999490bb --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !topRepositoriesByStars -- +SuadeLabs/rattr 2 +Zyflx/PsychEngine-Source-Code-Test 2 +h5bp/Front-end-Developer-Interview-Questions 2 +iheyfy2/Scripts-1 2 +iheyfy2/cloud189app-action 2 +jlevy/the-art-of-command-line 2 +martinothamar/Mediator 2 +mesos/chronos 2 +milanm/DevOps-Roadmap 2 +ossu/computer-science 2 +saper150/remporium 2 +veggiemonk/awesome-docker 2 +0x192/universal-android-debloater 1 +1uphealth/fhir-react 1 +42wim/matterbridge 1 +AbdelrhmanHamouda/locust-k8s-operator 1 +AgroDan/golden-kali 1 +AllThingsSmitty/css-protips 1 +AniList/discord-search-bot 1 +AntoinePassemiers/ArchMM 1 +Automattic/pocket-casts-android 1 +Azure/azure-sdk 1 +BenK93/MessagingAPI 1 +BetterDiscord/Installer 1 +BishopFox/sliver 1 +BlueAndi/esp-rgb-led-matrix 1 +BlueRaja/Weighted-Item-Randomizer-for-C-Sharp 1 +BryanEstrada003/Tarea_AOP 1 +Byron/gitoxide 1 +CommanderQ/SmartThings-Edge-Drivers 1 +CosminPOP/Outfitter 1 +DJYanYan/Banhaw-Landing-Page 1 +DJYanYan/Simon-Game 1 +DennisWG/QuickBind 1 +DonDebonair/daan.fyi 1 +EffiSciencesResearch/hackathon42 1 +ElJeloos/repasoGitHub 1 +EmilyEdna/SS-SSR-V2RAY 1 +Evavic44/portfolio-ideas 1 +GarnetSunset/rgg_save_tools 1 +HLC-Lewis/st_freecam 1 +Hailey-Ross/vorp_goldexchange 1 +HakCat-Tech/WiFi-Nugget 1 +IanLunn/Hover 1 +IntelliTect/Coalesce 1 +Josee9988/project-template 1 +JunkFood02/Seal 1 +KhronosGroup/Vulkan-Guide 1 +KinDR007/VictronMPPT-ESPHOME 1 +Launch-X-Latam/MisionBackend 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out new file mode 100644 index 00000000000000..7f34c4b76f7db3 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !whatIsTheBestDayOfTheWeekToCatchAStar -- +3 329 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.out new file mode 100644 index 00000000000000..173fadbff27f2d --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.out @@ -0,0 +1,53 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !whoAreAllThosePeopleGivingStars1 -- +daweedkob 34 +Github5201314 7 +co-sh 6 +JosephCastro 5 +MedG1 5 +guilhermedamaral 5 +bbjornstad 4 +developerspro 4 +iheyfy2 4 +jesusignaciofueyogonzalez 4 +SlowpokeVG 3 +YerPalPandora 3 +blackpjotr 3 +brsbrc 3 +proKress 3 +BolaGhaly 2 +BrewingWeasel 2 +IvanIsay 2 +JCariveau 2 +LNshuti 2 +Ta2shera 2 +burketanner 2 +chrisxjh 2 +codegeek1001 2 +coontiect 2 +coskuncinar 2 +cvpcasada 2 +dbsdm 2 +diffblogbot 2 +itxDeeni 2 +jaroes 2 +jbmoorhouse 2 +kprav33n 2 +riyadomf 2 +saper150 2 +shfshanyue 2 +taftadahir 2 +wukinnin 2 +0m15 1 +8ightfold 1 +Al3n70rn 1 +AntonioGB1 1 +Apple1D 1 +ChiaraManeo 1 +CorruptComputer 1 +CyndaZ42 1 +DTIV 1 +DrumondGitHub 1 +EHilly 1 +Edilsonlimatec 1 + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.out new file mode 100644 index 00000000000000..87cf6d8f87c61d --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !whoAreAllThosePeopleGivingStars2 -- + diff --git a/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.out b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.out new file mode 100644 index 00000000000000..a75ee0fde45ee8 --- /dev/null +++ b/regression-test/data/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !whoAreAllThosePeopleGivingStars3 -- + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q01_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q01_trans.out new file mode 100644 index 00000000000000..eb103e759f24d2 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q01_trans.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01_trans -- +15000 + +-- !q01_trans_2 -- +600572 + +-- !q01_trans_3 -- +25 + +-- !q01_trans_4 -- +150000 + +-- !q01_trans_5 -- +20000 + +-- !q01_trans_6 -- +80000 + +-- !q01_trans_7 -- +5 + +-- !q01_trans_8 -- +1000 + +-- !q01_trans_9 -- +A F 3774200.0 5.320753880690001E9 5.0540962666828375E9 5.256751331449239E9 25.537587116854997 36002.12382901415 0.05014459706345391 147790 +N F 95257.0 1.3373779584000012E8 1.2713237265120006E8 1.3228629122944506E8 25.30066401062417 35521.32691633469 0.04939442231075727 3765 +N O 7459297.0 1.051227000889988E10 9.986238338384867E9 1.038557837658565E10 25.545537671232875 36000.924688013285 0.05009595890418442 292000 +R F 3785523.0 5.337950526469928E9 5.071818532941938E9 5.274405503049396E9 25.5259438574251 35994.02921403044 0.04998927856189716 148301 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q02_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q02_trans.out new file mode 100644 index 00000000000000..628af0a0def0b5 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q02_trans.out @@ -0,0 +1,66 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q02_trans -- +9828.21 Supplier#000000647 UNITED KINGDOM 13120 Manufacturer#5 x5U7MBZmwfG9 33-258-202-4782 s the slyly even ideas poach fluffily +9759.38 Supplier#000000044 GERMANY 17242 Manufacturer#4 kERxlLDnlIZJdN66zAPHklyL 17-713-930-5667 x. carefully quiet account +9508.37 Supplier#000000070 FRANCE 3563 Manufacturer#1 INWNH2w,OOWgNDq0BRCcBwOMQc6PdFDc4 16-821-608-1166 ests sleep quickly express ideas. ironic ideas haggle about the final T +9508.37 Supplier#000000070 FRANCE 17268 Manufacturer#4 INWNH2w,OOWgNDq0BRCcBwOMQc6PdFDc4 16-821-608-1166 ests sleep quickly express ideas. ironic ideas haggle about the final T +9453.01 Supplier#000000802 ROMANIA 10021 Manufacturer#5 ,6HYXb4uaHITmtMBj4Ak57Pd 29-342-882-6463 gular frets. permanently special multipliers believe blithely alongs +9453.01 Supplier#000000802 ROMANIA 13275 Manufacturer#4 ,6HYXb4uaHITmtMBj4Ak57Pd 29-342-882-6463 gular frets. permanently special multipliers believe blithely alongs +9198.31 Supplier#000000025 RUSSIA 12238 Manufacturer#1 RCQKONXMFnrodzz6w7fObFVV6CUm2q 32-431-945-3541 ely regular deposits. carefully regular sauternes engage furiously above the regular accounts. idly +9192.1 Supplier#000000115 UNITED KINGDOM 13325 Manufacturer#1 nJ 2t0f7Ve,wL1,6WzGBJLNBUCKlsV 33-597-248-1220 es across the carefully express accounts boost caref +9032.15 Supplier#000000959 GERMANY 4958 Manufacturer#4 8grA EHBnwOZhO 17-108-642-3106 nding dependencies nag furiou +8702.02 Supplier#000000333 RUSSIA 11810 Manufacturer#3 MaVf XgwPdkiX4nfJGOis8Uu2zKiIZH 32-508-202-6136 oss the deposits cajole carefully even pinto beans. regular foxes detect alo +8615.5 Supplier#000000812 FRANCE 10551 Manufacturer#2 8qh4tezyScl5bidLAysvutB,,ZI2dn6xP 16-585-724-6633 y quickly regular deposits? quickly pending packages after the caref +8615.5 Supplier#000000812 FRANCE 13811 Manufacturer#4 8qh4tezyScl5bidLAysvutB,,ZI2dn6xP 16-585-724-6633 y quickly regular deposits? quickly pending packages after the caref +8488.53 Supplier#000000367 RUSSIA 6854 Manufacturer#4 E Sv9brQVf43Mzz 32-458-198-9557 ages. carefully final excuses nag finally. carefully ironic deposits abov +8430.52 Supplier#000000646 FRANCE 11384 Manufacturer#3 IUzsmT,2oBgjhWP2TlXTL6IkJH,4h,1SJRt 16-601-220-5489 ites among the always final ideas kindle according to the theodolites. notornis in +8271.39 Supplier#000000146 RUSSIA 4637 Manufacturer#5 rBDNgCr04x0sfdzD5,gFOutCiG2 32-792-619-3155 s cajole quickly special requests. quickly enticing theodolites h +8096.98 Supplier#000000574 RUSSIA 323 Manufacturer#4 2O8 sy9g2mlBOuEjzj0pA2pevk, 32-866-246-8752 ully after the regular requests. slyly final dependencies wake slyly along the busy deposit +8096.98 Supplier#000000574 RUSSIA 13784 Manufacturer#4 2O8 sy9g2mlBOuEjzj0pA2pevk, 32-866-246-8752 ully after the regular requests. slyly final dependencies wake slyly along the busy deposit +8069.74 Supplier#000000656 ROMANIA 7655 Manufacturer#2 mQXqRMgstvOI 29-633-362-8481 ronic packages integrate. even excuses integrate carefully ruthlessly bold packages. regular ideas a +7448.46 Supplier#000000690 ROMANIA 9430 Manufacturer#2 nK6Lv WWUh59jE525 29-330-952-4018 nic pinto beans doubt blithely b +7431.0 Supplier#000000311 RUSSIA 13784 Manufacturer#4 yjGDnCKi4Wmtim H3n9p 32-445-679-8585 uriously final requests integrate. sheaves against the furiously final accounts are evenly abo +7392.78 Supplier#000000170 UNITED KINGDOM 7655 Manufacturer#2 RtsXQ,SunkA XHy9 33-803-340-5398 ake carefully across the quickly +7205.2 Supplier#000000477 GERMANY 10956 Manufacturer#5 VtaNKN5Mqui5yh7j2ldd5waf 17-180-144-7991 excuses wake express deposits. furiously careful asymptotes according to the carefull +7144.78 Supplier#000000276 FRANCE 13275 Manufacturer#4 KdVDs6EGfWVsPdjuCh9iep 16-752-344-8255 cial, ironic theodolites against the decoys cajole slyly ironic foxes. carefull +6820.35 Supplier#000000007 UNITED KINGDOM 13217 Manufacturer#5 s,4TicNGB4uO6PaSqNBUq 33-990-965-2201 s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit +6721.7 Supplier#000000954 FRANCE 4191 Manufacturer#3 P3O5p UFz1QsLmZX 16-537-341-8517 ect blithely blithely final acco +6329.9 Supplier#000000996 GERMANY 10735 Manufacturer#2 Wx4dQwOAwWjfSCGupfrM 17-447-811-3282 ironic forges cajole blithely agai +6177.35 Supplier#000000053 GERMANY 5797 Manufacturer#1 i9v3 EsYCfLKFU6PIt8iihBOHBB37yR7b3GD7Rt 17-886-101-6083 onic, special deposits wake furio +6173.87 Supplier#000000408 RUSSIA 18139 Manufacturer#1 qcor1u,vJXAokjnL5,dilyYNmh 32-858-724-2950 blithely pending packages cajole furiously slyly pending notornis. slyly final +5733.61 Supplier#000000121 FRANCE 13120 Manufacturer#5 CWGri,tKI 7gDcDsI 16-275-849-2485 against the ironic, permanent pinto beans. doggedly pending deposits sleep agai +5408.07 Supplier#000000623 GERMANY 6854 Manufacturer#4 dSSQ3dTYwThbLppbetVUeuPfBIUF 17-593-337-7365 ial frays use. carefully special foxes wake carefully slyly pending deposits-- final requests a +5364.99 Supplier#000000785 RUSSIA 13784 Manufacturer#4 W VkHBpQyD3qjQjWGpWicOpmILFehmEdWy67kUGY 32-297-653-2203 packages boost carefully. express ideas along +5322.35 Supplier#000000587 GERMANY 3080 Manufacturer#2 58,gb EuMperMCg2lv XUQ9vi4GzhO2a 17-128-699-9949 thin pinto beans boost silently. ruthless deposits haggle quickly above the slyly unusual th +5069.27 Supplier#000000328 GERMANY 16327 Manufacturer#1 SMm24d WG62 17-231-513-5721 he unusual ideas. slyly final packages a +4941.88 Supplier#000000321 ROMANIA 7320 Manufacturer#5 pLngFl5yeMcHyov 29-573-279-1406 y final requests impress s +4680.75 Supplier#000000326 GERMANY 13325 Manufacturer#1 9kFiCwhcBldg4xwm 17-390-604-7483 quests could use furiously across the ironic, even f +4672.25 Supplier#000000239 RUSSIA 12238 Manufacturer#1 XO101kgHrJagK2FL1U6QCaTE ncCsMbeuTgK6o8 32-396-654-6826 arls wake furiously deposits. even, regular depen +4586.49 Supplier#000000680 RUSSIA 5679 Manufacturer#3 UhvDfdEfJh,Qbe7VZb8uSGO2TU 0jEa6nXZXE 32-522-382-1620 the regularly regular dependencies. carefully bold excuses under th +4518.31 Supplier#000000149 FRANCE 18344 Manufacturer#5 pVyWsjOidpHKp4NfKU4yLeym 16-660-553-2456 ts detect along the foxes. final Tiresias are. idly pending deposits haggle; even, blithe pin +4324.51 Supplier#000000957 UNITED KINGDOM 10956 Manufacturer#5 mSpFa,4jJ5R40k10YOvGEtl4KYjo 33-616-674-6155 hily after the fluffily regular dependencies. deposits nag regular, silent accounts. i +4315.15 Supplier#000000509 FRANCE 18972 Manufacturer#2 SF7dR8V5pK 16-298-154-3365 ronic orbits are furiously across the requests. quickly express ideas across the special, bold +3526.53 Supplier#000000553 FRANCE 8036 Manufacturer#4 a,liVofXbCJ 16-599-552-3755 lar dinos nag slyly brave +3526.53 Supplier#000000553 FRANCE 17018 Manufacturer#3 a,liVofXbCJ 16-599-552-3755 lar dinos nag slyly brave +3294.68 Supplier#000000350 GERMANY 4841 Manufacturer#4 KIFxV73eovmwhh 17-113-181-4017 e slyly special foxes. furiously unusual deposits detect carefully carefully ruthless foxes. quick +2972.26 Supplier#000000016 RUSSIA 1015 Manufacturer#4 YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh 32-822-502-4215 ously express ideas haggle quickly dugouts? fu +2963.09 Supplier#000000840 ROMANIA 3080 Manufacturer#2 iYzUIypKhC0Y 29-781-337-5584 eep blithely regular dependencies. blithely regular platelets sublate alongside o +2221.25 Supplier#000000771 ROMANIA 13981 Manufacturer#2 lwZ I15rq9kmZXUNhl 29-986-304-9006 nal foxes eat slyly about the fluffily permanent id +1381.97 Supplier#000000104 FRANCE 18103 Manufacturer#3 Dcl4yGrzqv3OPeRO49bKh78XmQEDR7PBXIs0m 16-434-972-6922 gular ideas. bravely bold deposits haggle through the carefully final deposits. slyly unusual idea +1342.17 Supplier#000000384 GERMANY 13120 Manufacturer#5 zMr51gtJ0Vu83Dk 17-554-428-8511 taphs cajole furiously blithely final +906.07 Supplier#000000138 ROMANIA 8363 Manufacturer#4 utbplAm g7RmxVfYoNdhcrQGWuzRqPe0qHSwbKw 29-533-434-6776 ickly unusual requests cajole. accounts above the furiously special excuses +765.69 Supplier#000000799 RUSSIA 11276 Manufacturer#2 jwFN7ZB3T9sMF 32-579-339-1495 nusual requests. furiously unusual epitaphs integrate. slyly +747.88 Supplier#000000243 FRANCE 17242 Manufacturer#4 8aQ3HGeOXxgYeMAXZQe B5y2RKEF5jdmN3Qb 16-554-376-5494 kly silent requests among the blithely regular foxes use fu +727.89 Supplier#000000470 ROMANIA 6213 Manufacturer#3 XckbzsAgBLbUkdfjgJEPjmUMTM8ebSMEvI 29-165-289-1523 gular excuses. furiously regular excuses sleep slyly caref +704.83 Supplier#000000323 RUSSIA 3563 Manufacturer#1 0LEOmcTTomY1F0y 32-563-275-6438 accounts. unusual requests haggle slyly special packages. always silent instructions e +683.07 Supplier#000000651 RUSSIA 4888 Manufacturer#4 oWekiBV6s,1g 32-181-426-4490 ly regular requests cajole abou +167.56 Supplier#000000290 FRANCE 2037 Manufacturer#1 6Bk06GVtwZaKqg01 16-675-286-5102 the theodolites. ironic, ironic deposits above +165.76 Supplier#000000769 FRANCE 1015 Manufacturer#4 ak2320fUkG 16-655-591-2134 ly ironic ideas. quickly ironic platelets hag +91.39 Supplier#000000949 UNITED KINGDOM 9430 Manufacturer#2 a,UE,6nRVl2fCphkOoetR1ajIzAEJ1Aa1G1HV 33-332-697-2768 pinto beans. carefully express requests hagg +-314.06 Supplier#000000510 ROMANIA 17242 Manufacturer#4 VmXQl ,vY8JiEseo8Mv4zscvNCfsY 29-207-852-3454 bold deposits. carefully even d +-435.02 Supplier#000000295 UNITED KINGDOM 8036 Manufacturer#4 gpm7fahY9j6YyTr Dozul 33-998-989-3147 en requests according to the +-820.89 Supplier#000000409 GERMANY 2156 Manufacturer#5 LyXUYFz7aXrvy65kKAbTatGzGS,NDBcdtD 17-719-517-9836 y final, slow theodolites. furiously regular req +-845.44 Supplier#000000704 ROMANIA 9926 Manufacturer#5 hQvlBqbqqnA5Dgo1BffRBX78tkkRu 29-300-896-5991 ctions. carefully sly requ +-942.73 Supplier#000000563 GERMANY 5797 Manufacturer#1 Rc7U1cRUhYs03JD 17-108-537-2691 slyly furiously final decoys; silent, special realms poach f +-963.79 Supplier#000000065 RUSSIA 13275 Manufacturer#4 BsAnHUmSFArppKrM 32-444-835-2434 l ideas wake carefully around the regular packages. furiously ruthless pinto bea + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q03_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q03_trans.out new file mode 100644 index 00000000000000..abee8c3e676538 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q03_trans.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q03_trans -- +223140 355369.0698 1995-03-14 0 +584291 354494.73180000007 1995-02-21 0 +405063 353125.4577 1995-03-03 0 +573861 351238.277 1995-03-09 0 +554757 349181.7426 1995-03-14 0 +506021 321075.581 1995-03-10 0 +121604 318576.41539999994 1995-03-07 0 +108514 314967.0754 1995-02-20 0 +462502 312604.54199999996 1995-03-08 0 +178727 309728.93059999996 1995-02-25 0 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q05_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q05_trans.out new file mode 100644 index 00000000000000..5f7680a2ff4611 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q05_trans.out @@ -0,0 +1,8 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q05_trans -- +CHINA 7822102.999999998 +INDIA 6376121.5084999995 +JAPAN 6000077.2184000015 +INDONESIA 5580475.402700002 +VIETNAM 4497840.546599999 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q06_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q06_trans.out new file mode 100644 index 00000000000000..174241a7cd2579 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q06_trans.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q06_trans -- +1.1803420253399998E7 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q08_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q08_trans.out new file mode 100644 index 00000000000000..9b4bfafa8e5507 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q08_trans.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q08_trans -- +1995 0.028648741305617564 +1996 0.01825027910796215 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q09_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q09_trans.out new file mode 100644 index 00000000000000..111c4709dfec6a --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q09_trans.out @@ -0,0 +1,178 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q09_trans -- +ALGERIA 1998 2321785.3682000004 +ALGERIA 1997 3685016.8589000003 +ALGERIA 1996 4276597.425300001 +ALGERIA 1995 4418370.415399998 +ALGERIA 1994 3864849.9520999985 +ALGERIA 1993 3541051.386499999 +ALGERIA 1992 4310013.348199999 +ARGENTINA 1998 2685983.800500001 +ARGENTINA 1997 4242147.812399997 +ARGENTINA 1996 3907867.010300001 +ARGENTINA 1995 4605921.501099997 +ARGENTINA 1994 3542096.1564000016 +ARGENTINA 1993 3949965.9387999997 +ARGENTINA 1992 4521180.469499999 +BRAZIL 1998 2778730.3931000014 +BRAZIL 1997 4642037.468700008 +BRAZIL 1996 4530304.603400003 +BRAZIL 1995 4502344.865699998 +BRAZIL 1994 4875806.501499998 +BRAZIL 1993 4687478.653099999 +BRAZIL 1992 5035200.0464 +CANADA 1998 2194509.0464999997 +CANADA 1997 3482197.952099998 +CANADA 1996 3712231.2813999993 +CANADA 1995 4014814.847599999 +CANADA 1994 4145304.4855000004 +CANADA 1993 3787069.604500001 +CANADA 1992 4168009.4200999993 +CHINA 1998 3398578.0000999975 +CHINA 1997 6358959.333799999 +CHINA 1996 6435158.322899999 +CHINA 1995 6174776.211300004 +CHINA 1994 6385751.0812 +CHINA 1993 5765034.119400002 +CHINA 1992 6324034.237900004 +EGYPT 1998 2333148.3334000004 +EGYPT 1997 3661244.2731000003 +EGYPT 1996 3765371.2368 +EGYPT 1995 4094744.2924999967 +EGYPT 1994 3566508.0817999984 +EGYPT 1993 3725283.7746999995 +EGYPT 1992 3373762.333500001 +ETHIOPIA 1998 1953927.2681999994 +ETHIOPIA 1997 3285786.326600001 +ETHIOPIA 1996 3525028.795200001 +ETHIOPIA 1995 3781674.891100001 +ETHIOPIA 1994 3037409.435999999 +ETHIOPIA 1993 3008978.2676999997 +ETHIOPIA 1992 2721203.2355 +FRANCE 1998 2604373.880499999 +FRANCE 1997 3982872.048799999 +FRANCE 1996 3622479.2413 +FRANCE 1995 4479939.7020000005 +FRANCE 1994 3531013.1980999988 +FRANCE 1993 4086437.310200002 +FRANCE 1992 3637792.1333 +GERMANY 1998 3291023.2965 +GERMANY 1997 5139337.344300005 +GERMANY 1996 4799810.457699997 +GERMANY 1995 5405785.7978 +GERMANY 1994 4555556.459199995 +GERMANY 1993 4428195.101900003 +GERMANY 1992 4656148.420400001 +INDIA 1998 2591288.1873999992 +INDIA 1997 5159562.7032999955 +INDIA 1996 5307258.304899997 +INDIA 1995 5148208.790199998 +INDIA 1994 5164001.958199999 +INDIA 1993 4321398.438800001 +INDIA 1992 5297703.693499999 +INDONESIA 1998 3094900.1597 +INDONESIA 1997 5719773.035800001 +INDONESIA 1996 6037238.599299995 +INDONESIA 1995 5266783.489899999 +INDONESIA 1994 5470762.872900003 +INDONESIA 1993 6189826.661299995 +INDONESIA 1992 4414623.154899998 +IRAN 1998 3214864.1208999995 +IRAN 1997 3688049.0690999976 +IRAN 1996 3621649.224699998 +IRAN 1995 4420783.4205 +IRAN 1994 4373984.652299998 +IRAN 1993 3731301.7814000007 +IRAN 1992 4417133.366199994 +IRAQ 1998 2338859.4098999994 +IRAQ 1997 3622681.564300003 +IRAQ 1996 4762291.872199997 +IRAQ 1995 4558092.735899999 +IRAQ 1994 4951604.169900003 +IRAQ 1993 3830077.9911000016 +IRAQ 1992 3938636.4873999995 +JAPAN 1998 1849535.0801999997 +JAPAN 1997 4068688.8536999985 +JAPAN 1996 4044774.7597000008 +JAPAN 1995 4793005.802699999 +JAPAN 1994 4114717.056800001 +JAPAN 1993 3614468.748500002 +JAPAN 1992 4266694.470000001 +JORDAN 1998 1811488.0719 +JORDAN 1997 2951297.867800001 +JORDAN 1996 3302528.3066999987 +JORDAN 1995 3221813.9990000012 +JORDAN 1994 2417892.0921000005 +JORDAN 1993 3107641.7661 +JORDAN 1992 3316379.0585000003 +KENYA 1998 2579075.419000001 +KENYA 1997 2929194.231699999 +KENYA 1996 3569129.5619 +KENYA 1995 3542889.1086999993 +KENYA 1994 3983095.3993999995 +KENYA 1993 3713988.9707999993 +KENYA 1992 3304641.833999999 +MOROCCO 1998 1815334.8179999997 +MOROCCO 1997 3693214.844699999 +MOROCCO 1996 4116175.9229999995 +MOROCCO 1995 3515127.140199997 +MOROCCO 1994 4003072.111999999 +MOROCCO 1993 3599199.6678999993 +MOROCCO 1992 3958335.422400001 +MOZAMBIQUE 1998 1620428.7345999999 +MOZAMBIQUE 1997 2802166.6473000003 +MOZAMBIQUE 1996 2409955.175499999 +MOZAMBIQUE 1995 2771602.6274000006 +MOZAMBIQUE 1994 2548226.2158000013 +MOZAMBIQUE 1993 2843748.9052999993 +MOZAMBIQUE 1992 2556501.094300002 +PERU 1998 2036430.3602 +PERU 1997 4064142.4090999993 +PERU 1996 4068678.5670999996 +PERU 1995 4657694.841200002 +PERU 1994 4731959.4655 +PERU 1993 4144006.6609999975 +PERU 1992 3754635.0077999993 +ROMANIA 1998 1992773.6811000004 +ROMANIA 1997 2854639.8679999993 +ROMANIA 1996 3139337.302899999 +ROMANIA 1995 3222153.3775999993 +ROMANIA 1994 3222844.319000001 +ROMANIA 1993 3488994.028799999 +ROMANIA 1992 3029274.4420000017 +RUSSIA 1998 2339865.6635 +RUSSIA 1997 4153619.5424 +RUSSIA 1996 3772067.4040999995 +RUSSIA 1995 4704988.860700001 +RUSSIA 1994 4479082.869399998 +RUSSIA 1993 4767719.9791 +RUSSIA 1992 4533465.559 +SAUDI ARABIA 1998 3386948.956400002 +SAUDI ARABIA 1997 5425980.3373 +SAUDI ARABIA 1996 5227607.167699995 +SAUDI ARABIA 1995 4506731.641100001 +SAUDI ARABIA 1994 4698658.742500002 +SAUDI ARABIA 1993 5493626.5285 +SAUDI ARABIA 1992 4573560.015000002 +UNITED KINGDOM 1998 2252021.513699999 +UNITED KINGDOM 1997 4343926.802599999 +UNITED KINGDOM 1996 4189476.306499999 +UNITED KINGDOM 1995 4469569.882899999 +UNITED KINGDOM 1994 4410094.626399998 +UNITED KINGDOM 1993 4054677.1050000004 +UNITED KINGDOM 1992 3978688.8831 +UNITED STATES 1998 2238771.5581000005 +UNITED STATES 1997 4135581.5734000015 +UNITED STATES 1996 3624013.2660000008 +UNITED STATES 1995 3892244.5171999987 +UNITED STATES 1994 3289224.1137999995 +UNITED STATES 1993 3626170.2028 +UNITED STATES 1992 3993973.4997000005 +VIETNAM 1998 1924313.4861999997 +VIETNAM 1997 3436195.3709 +VIETNAM 1996 4017288.892700001 +VIETNAM 1995 3644054.137200002 +VIETNAM 1994 4141277.6665000007 +VIETNAM 1993 2556114.1693000016 +VIETNAM 1992 4090524.4904999994 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q10_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q10_trans.out new file mode 100644 index 00000000000000..d72918439a1265 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q10_trans.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q10_trans -- +8242 Customer#000008242 622786.7297 6322.09 ETHIOPIA P2n4nJhy,UqSo2s43YfSvYJDZ6lk 15-792-676-1184 slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i +7714 Customer#000007714 557400.3053 9799.98 IRAN SnnIGB,SkmnWpX3 20-922-418-6024 arhorses according to the blithely express re +11032 Customer#000011032 512500.9641 8496.93 UNITED KINGDOM WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly 33-102-772-3533 posits-- furiously ironic accounts are again +2455 Customer#000002455 481592.4053 2070.99 GERMANY RVn1ZSRtLqPlJLIZxvpmsbgC02 17-946-225-9977 al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro +12106 Customer#000012106 479414.2133 5342.11 UNITED STATES wth3twOmu6vy 34-905-346-4472 ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas. +8530 Customer#000008530 457855.94670000003 9734.95 MOROCCO GMQyte94oDM7eD7exnkj 4hH9yq3 25-736-932-5850 slyly asymptotes. quickly final deposits in +13984 Customer#000013984 446316.5104 3482.28 IRAN qZXwuapCHvxbX 20-981-264-2952 y unusual courts could wake furiously +1966 Customer#000001966 444059.0382 1937.72 ALGERIA jPv1 UHra5JLALR5Isci5u0636RoAu7t vH 10-973-269-8886 the blithely even accounts. final deposits cajole around the blithely final packages. +11026 Customer#000011026 417913.4142 7738.76 ALGERIA XorIktoJOAEJkpNNMx 10-184-163-4632 ly even dolphins eat along the blithely even instructions. express attainments cajole slyly. busy dolphins in +8501 Customer#000008501 412797.51 6906.7 ARGENTINA 776af4rOa mZ66hczs 11-317-552-5840 y final deposits after the fluffily even accounts are slyly final, regular +1565 Customer#000001565 412506.00619999995 1820.03 BRAZIL EWQO5Ck,nMuHVQimqL8dLrixRP6QKveXcz9QgorW 12-402-178-2007 ously regular accounts wake slyly ironic idea +14398 Customer#000014398 408575.3599999999 -602.24 UNITED STATES GWRCgIPHajtU21vICVvbJJerFu2cUk 34-814-111-5424 s. blithely even accounts cajole blithely. even foxes doubt-- +1465 Customer#000001465 405055.34569999995 9365.93 INDIA tDRaTC7UgFbBX7VF6cVXYQA0 18-807-487-1074 s lose blithely ironic, regular packages. regular, final foxes haggle c +12595 Customer#000012595 401402.2391 -6.92 INDIA LmeaX5cR,w9NqKugl yRm98 18-186-132-3352 o the busy accounts. blithely special gifts maintain a +961 Customer#000000961 401198.17370000004 6963.68 JAPAN 5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn 22-989-463-6089 e final requests: busily final accounts believe a +14299 Customer#000014299 400968.3751 6595.97 RUSSIA 7lFczTya0iM1bhEWT 32-156-618-1224 carefully regular requests. quickly ironic accounts against the ru +623 Customer#000000623 399883.42569999996 7887.6 INDONESIA HXiFb9oWlgqZXrJPUCEJ6zZIPxAM4m6 19-113-202-7085 requests. dolphins above the busily regular dependencies cajole after +9151 Customer#000009151 396562.02950000006 5691.95 IRAQ 7gIdRdaxB91EVdyx8DyPjShpMD 21-834-147-4906 ajole fluffily. furiously regular accounts are special, silent account +14819 Customer#000014819 396271.1036 7308.39 FRANCE w8StIbymUXmLCcUag6sx6LUIp8E3pA,Ux 16-769-398-7926 ss, final asymptotes use furiously slyly ironic dependencies. special, express dugouts according to the dep +13478 Customer#000013478 395513.13580000005 -778.11 KENYA 9VIsvIeZrJpC6OOdYheMC2vdtq8Ai0Rt 24-983-202-8240 r theodolites. slyly unusual pinto beans sleep fluffily against the asymptotes. quickly r + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q12_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q12_trans.out new file mode 100644 index 00000000000000..2432e646c92247 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q12_trans.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q12_trans -- +MAIL 647 945 +SHIP 620 943 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q13_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q13_trans.out new file mode 100644 index 00000000000000..b433c1697ca725 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q13_trans.out @@ -0,0 +1,40 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q13_trans -- +0 5000 +10 665 +9 657 +11 621 +12 567 +8 564 +13 492 +18 482 +7 480 +20 456 +14 456 +16 449 +19 447 +15 432 +17 423 +21 412 +22 371 +6 337 +23 323 +24 256 +25 204 +5 204 +26 155 +27 141 +28 97 +4 94 +29 64 +3 48 +30 27 +31 26 +32 14 +33 11 +2 11 +34 6 +35 5 +1 2 +36 1 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q14_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q14_trans.out new file mode 100644 index 00000000000000..447f13165c9da9 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q14_trans.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q14_trans -- +16.283855689005982 + diff --git a/regression-test/data/variant_p2/tpch_upgrade/sql/q16_trans.out b/regression-test/data/variant_p2/tpch_upgrade/sql/q16_trans.out new file mode 100644 index 00000000000000..d24fcb3693fb36 --- /dev/null +++ b/regression-test/data/variant_p2/tpch_upgrade/sql/q16_trans.out @@ -0,0 +1,2765 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q16_trans -- +Brand#14 SMALL ANODIZED NICKEL 45 12 +Brand#22 SMALL BURNISHED BRASS 19 12 +Brand#25 PROMO POLISHED COPPER 14 12 +Brand#35 LARGE ANODIZED STEEL 45 12 +Brand#35 PROMO BRUSHED COPPER 9 12 +Brand#51 ECONOMY ANODIZED STEEL 9 12 +Brand#53 LARGE BRUSHED NICKEL 45 12 +Brand#11 ECONOMY POLISHED COPPER 14 8 +Brand#11 LARGE PLATED STEEL 23 8 +Brand#11 PROMO POLISHED STEEL 23 8 +Brand#11 STANDARD ANODIZED COPPER 9 8 +Brand#12 ECONOMY BURNISHED BRASS 9 8 +Brand#12 LARGE ANODIZED BRASS 14 8 +Brand#12 SMALL ANODIZED TIN 23 8 +Brand#12 SMALL BRUSHED NICKEL 23 8 +Brand#12 STANDARD ANODIZED BRASS 3 8 +Brand#12 STANDARD BURNISHED TIN 23 8 +Brand#13 ECONOMY POLISHED BRASS 9 8 +Brand#13 LARGE BURNISHED COPPER 45 8 +Brand#13 MEDIUM ANODIZED STEEL 23 8 +Brand#13 MEDIUM PLATED NICKEL 3 8 +Brand#13 PROMO BURNISHED BRASS 9 8 +Brand#13 PROMO POLISHED BRASS 3 8 +Brand#13 PROMO POLISHED TIN 36 8 +Brand#13 SMALL BURNISHED STEEL 23 8 +Brand#13 STANDARD BRUSHED STEEL 9 8 +Brand#14 ECONOMY BRUSHED TIN 3 8 +Brand#14 ECONOMY BURNISHED TIN 23 8 +Brand#14 PROMO BRUSHED STEEL 9 8 +Brand#14 PROMO PLATED TIN 45 8 +Brand#15 ECONOMY PLATED TIN 9 8 +Brand#15 STANDARD BRUSHED COPPER 14 8 +Brand#15 STANDARD PLATED TIN 3 8 +Brand#21 ECONOMY POLISHED TIN 3 8 +Brand#21 PROMO POLISHED COPPER 9 8 +Brand#21 PROMO POLISHED TIN 49 8 +Brand#21 SMALL POLISHED STEEL 3 8 +Brand#21 STANDARD PLATED BRASS 49 8 +Brand#21 STANDARD PLATED NICKEL 49 8 +Brand#22 ECONOMY ANODIZED TIN 49 8 +Brand#22 ECONOMY BRUSHED BRASS 14 8 +Brand#22 LARGE BURNISHED TIN 36 8 +Brand#22 MEDIUM ANODIZED STEEL 36 8 +Brand#22 MEDIUM PLATED STEEL 9 8 +Brand#22 PROMO POLISHED NICKEL 9 8 +Brand#22 SMALL ANODIZED STEEL 19 8 +Brand#22 STANDARD ANODIZED COPPER 23 8 +Brand#23 ECONOMY BRUSHED NICKEL 23 8 +Brand#23 LARGE ANODIZED BRASS 9 8 +Brand#23 LARGE ANODIZED STEEL 23 8 +Brand#23 SMALL BRUSHED COPPER 23 8 +Brand#23 STANDARD BRUSHED TIN 3 8 +Brand#23 STANDARD BURNISHED NICKEL 49 8 +Brand#23 STANDARD PLATED NICKEL 36 8 +Brand#24 ECONOMY ANODIZED BRASS 19 8 +Brand#24 ECONOMY POLISHED BRASS 36 8 +Brand#24 LARGE BURNISHED STEEL 14 8 +Brand#24 MEDIUM PLATED NICKEL 36 8 +Brand#25 ECONOMY BRUSHED STEEL 49 8 +Brand#25 MEDIUM BURNISHED TIN 3 8 +Brand#25 PROMO ANODIZED TIN 36 8 +Brand#25 PROMO PLATED NICKEL 3 8 +Brand#25 SMALL BURNISHED BRASS 3 8 +Brand#31 LARGE ANODIZED BRASS 3 8 +Brand#31 SMALL ANODIZED COPPER 3 8 +Brand#31 SMALL ANODIZED NICKEL 9 8 +Brand#31 SMALL ANODIZED STEEL 14 8 +Brand#32 MEDIUM ANODIZED STEEL 49 8 +Brand#32 MEDIUM BURNISHED COPPER 19 8 +Brand#32 SMALL BURNISHED STEEL 23 8 +Brand#32 STANDARD BURNISHED STEEL 45 8 +Brand#34 ECONOMY ANODIZED NICKEL 49 8 +Brand#34 LARGE BURNISHED TIN 49 8 +Brand#34 MEDIUM BURNISHED NICKEL 3 8 +Brand#34 PROMO ANODIZED TIN 3 8 +Brand#34 SMALL BRUSHED TIN 3 8 +Brand#34 STANDARD BURNISHED TIN 23 8 +Brand#35 MEDIUM BRUSHED STEEL 45 8 +Brand#35 PROMO BURNISHED STEEL 14 8 +Brand#35 SMALL BURNISHED STEEL 23 8 +Brand#35 SMALL POLISHED COPPER 14 8 +Brand#35 STANDARD PLATED COPPER 9 8 +Brand#41 ECONOMY BRUSHED BRASS 23 8 +Brand#41 LARGE BURNISHED STEEL 23 8 +Brand#41 PROMO BURNISHED TIN 14 8 +Brand#41 PROMO PLATED STEEL 36 8 +Brand#41 PROMO POLISHED TIN 19 8 +Brand#41 SMALL BURNISHED COPPER 23 8 +Brand#42 LARGE POLISHED TIN 14 8 +Brand#42 MEDIUM ANODIZED TIN 49 8 +Brand#42 MEDIUM BRUSHED TIN 14 8 +Brand#42 MEDIUM BURNISHED NICKEL 23 8 +Brand#42 MEDIUM PLATED COPPER 45 8 +Brand#42 MEDIUM PLATED TIN 45 8 +Brand#42 SMALL PLATED COPPER 36 8 +Brand#43 ECONOMY BRUSHED STEEL 45 8 +Brand#43 LARGE BRUSHED COPPER 19 8 +Brand#43 PROMO BRUSHED BRASS 36 8 +Brand#43 SMALL BURNISHED TIN 45 8 +Brand#43 SMALL PLATED COPPER 45 8 +Brand#44 PROMO POLISHED TIN 23 8 +Brand#44 SMALL POLISHED NICKEL 14 8 +Brand#44 SMALL POLISHED TIN 45 8 +Brand#44 STANDARD BURNISHED COPPER 3 8 +Brand#51 LARGE ANODIZED BRASS 19 8 +Brand#51 LARGE POLISHED COPPER 23 8 +Brand#51 MEDIUM ANODIZED TIN 9 8 +Brand#51 MEDIUM ANODIZED TIN 14 8 +Brand#51 MEDIUM BURNISHED NICKEL 23 8 +Brand#51 SMALL ANODIZED COPPER 45 8 +Brand#51 SMALL ANODIZED COPPER 49 8 +Brand#51 SMALL BRUSHED COPPER 45 8 +Brand#51 SMALL BRUSHED TIN 36 8 +Brand#51 STANDARD POLISHED TIN 3 8 +Brand#52 ECONOMY ANODIZED STEEL 3 8 +Brand#52 ECONOMY PLATED TIN 19 8 +Brand#52 LARGE PLATED TIN 3 8 +Brand#52 MEDIUM ANODIZED TIN 19 8 +Brand#52 MEDIUM BURNISHED COPPER 3 8 +Brand#52 PROMO POLISHED BRASS 23 8 +Brand#52 SMALL PLATED COPPER 36 8 +Brand#52 SMALL POLISHED NICKEL 9 8 +Brand#52 STANDARD POLISHED NICKEL 45 8 +Brand#53 ECONOMY POLISHED STEEL 45 8 +Brand#53 LARGE POLISHED NICKEL 3 8 +Brand#53 SMALL BRUSHED COPPER 14 8 +Brand#53 STANDARD PLATED STEEL 45 8 +Brand#54 ECONOMY POLISHED BRASS 49 8 +Brand#54 ECONOMY POLISHED TIN 23 8 +Brand#54 LARGE ANODIZED NICKEL 49 8 +Brand#54 MEDIUM BRUSHED STEEL 9 8 +Brand#54 SMALL BURNISHED NICKEL 14 8 +Brand#54 SMALL PLATED TIN 14 8 +Brand#54 STANDARD BURNISHED STEEL 14 8 +Brand#54 STANDARD PLATED BRASS 23 8 +Brand#55 MEDIUM BURNISHED TIN 36 8 +Brand#55 PROMO ANODIZED BRASS 14 8 +Brand#55 STANDARD BURNISHED COPPER 45 8 +Brand#15 STANDARD PLATED TIN 36 7 +Brand#23 SMALL POLISHED BRASS 49 7 +Brand#42 STANDARD PLATED COPPER 19 7 +Brand#51 LARGE POLISHED NICKEL 14 7 +Brand#11 ECONOMY ANODIZED BRASS 19 4 +Brand#11 ECONOMY ANODIZED BRASS 45 4 +Brand#11 ECONOMY ANODIZED NICKEL 36 4 +Brand#11 ECONOMY BRUSHED COPPER 3 4 +Brand#11 ECONOMY BRUSHED COPPER 9 4 +Brand#11 ECONOMY BRUSHED STEEL 9 4 +Brand#11 ECONOMY BRUSHED STEEL 36 4 +Brand#11 ECONOMY BURNISHED BRASS 36 4 +Brand#11 ECONOMY BURNISHED COPPER 9 4 +Brand#11 ECONOMY BURNISHED COPPER 49 4 +Brand#11 ECONOMY BURNISHED NICKEL 14 4 +Brand#11 ECONOMY BURNISHED NICKEL 49 4 +Brand#11 ECONOMY PLATED COPPER 19 4 +Brand#11 ECONOMY PLATED NICKEL 45 4 +Brand#11 ECONOMY PLATED TIN 9 4 +Brand#11 ECONOMY POLISHED BRASS 3 4 +Brand#11 ECONOMY POLISHED COPPER 3 4 +Brand#11 ECONOMY POLISHED COPPER 45 4 +Brand#11 ECONOMY POLISHED NICKEL 36 4 +Brand#11 ECONOMY POLISHED STEEL 23 4 +Brand#11 ECONOMY POLISHED TIN 14 4 +Brand#11 LARGE ANODIZED COPPER 23 4 +Brand#11 LARGE ANODIZED NICKEL 9 4 +Brand#11 LARGE ANODIZED STEEL 9 4 +Brand#11 LARGE ANODIZED TIN 45 4 +Brand#11 LARGE BRUSHED STEEL 19 4 +Brand#11 LARGE BRUSHED TIN 3 4 +Brand#11 LARGE BRUSHED TIN 14 4 +Brand#11 LARGE BURNISHED COPPER 9 4 +Brand#11 LARGE BURNISHED COPPER 19 4 +Brand#11 LARGE BURNISHED STEEL 23 4 +Brand#11 LARGE BURNISHED TIN 9 4 +Brand#11 LARGE PLATED COPPER 23 4 +Brand#11 LARGE PLATED TIN 9 4 +Brand#11 LARGE PLATED TIN 14 4 +Brand#11 LARGE PLATED TIN 23 4 +Brand#11 LARGE POLISHED NICKEL 49 4 +Brand#11 MEDIUM ANODIZED BRASS 45 4 +Brand#11 MEDIUM ANODIZED TIN 14 4 +Brand#11 MEDIUM BRUSHED BRASS 14 4 +Brand#11 MEDIUM BRUSHED BRASS 45 4 +Brand#11 MEDIUM BRUSHED NICKEL 14 4 +Brand#11 MEDIUM BRUSHED NICKEL 36 4 +Brand#11 MEDIUM BRUSHED STEEL 19 4 +Brand#11 MEDIUM BURNISHED COPPER 9 4 +Brand#11 MEDIUM BURNISHED TIN 36 4 +Brand#11 MEDIUM PLATED BRASS 3 4 +Brand#11 MEDIUM PLATED TIN 19 4 +Brand#11 PROMO ANODIZED BRASS 3 4 +Brand#11 PROMO ANODIZED BRASS 19 4 +Brand#11 PROMO ANODIZED BRASS 45 4 +Brand#11 PROMO ANODIZED BRASS 49 4 +Brand#11 PROMO ANODIZED STEEL 23 4 +Brand#11 PROMO ANODIZED TIN 45 4 +Brand#11 PROMO BRUSHED BRASS 23 4 +Brand#11 PROMO BRUSHED STEEL 3 4 +Brand#11 PROMO BURNISHED BRASS 23 4 +Brand#11 PROMO BURNISHED BRASS 36 4 +Brand#11 PROMO BURNISHED BRASS 49 4 +Brand#11 PROMO BURNISHED TIN 9 4 +Brand#11 PROMO PLATED BRASS 9 4 +Brand#11 PROMO PLATED BRASS 45 4 +Brand#11 PROMO PLATED NICKEL 19 4 +Brand#11 PROMO POLISHED BRASS 3 4 +Brand#11 PROMO POLISHED BRASS 9 4 +Brand#11 PROMO POLISHED BRASS 19 4 +Brand#11 PROMO POLISHED COPPER 14 4 +Brand#11 PROMO POLISHED COPPER 45 4 +Brand#11 PROMO POLISHED TIN 49 4 +Brand#11 SMALL ANODIZED COPPER 36 4 +Brand#11 SMALL ANODIZED NICKEL 3 4 +Brand#11 SMALL ANODIZED NICKEL 14 4 +Brand#11 SMALL ANODIZED TIN 14 4 +Brand#11 SMALL ANODIZED TIN 19 4 +Brand#11 SMALL ANODIZED TIN 45 4 +Brand#11 SMALL BRUSHED TIN 14 4 +Brand#11 SMALL BRUSHED TIN 23 4 +Brand#11 SMALL BRUSHED TIN 45 4 +Brand#11 SMALL BURNISHED BRASS 49 4 +Brand#11 SMALL BURNISHED COPPER 23 4 +Brand#11 SMALL PLATED COPPER 45 4 +Brand#11 SMALL PLATED NICKEL 3 4 +Brand#11 SMALL PLATED STEEL 36 4 +Brand#11 SMALL PLATED TIN 19 4 +Brand#11 SMALL POLISHED BRASS 14 4 +Brand#11 SMALL POLISHED BRASS 23 4 +Brand#11 SMALL POLISHED COPPER 14 4 +Brand#11 SMALL POLISHED COPPER 36 4 +Brand#11 SMALL POLISHED STEEL 9 4 +Brand#11 STANDARD BRUSHED COPPER 23 4 +Brand#11 STANDARD BRUSHED NICKEL 14 4 +Brand#11 STANDARD BRUSHED TIN 14 4 +Brand#11 STANDARD BURNISHED BRASS 3 4 +Brand#11 STANDARD BURNISHED STEEL 23 4 +Brand#11 STANDARD PLATED BRASS 19 4 +Brand#11 STANDARD PLATED TIN 19 4 +Brand#11 STANDARD POLISHED NICKEL 45 4 +Brand#11 STANDARD POLISHED TIN 14 4 +Brand#11 STANDARD POLISHED TIN 45 4 +Brand#12 ECONOMY ANODIZED BRASS 23 4 +Brand#12 ECONOMY ANODIZED COPPER 14 4 +Brand#12 ECONOMY ANODIZED NICKEL 19 4 +Brand#12 ECONOMY ANODIZED NICKEL 45 4 +Brand#12 ECONOMY ANODIZED STEEL 9 4 +Brand#12 ECONOMY BRUSHED COPPER 36 4 +Brand#12 ECONOMY BRUSHED NICKEL 49 4 +Brand#12 ECONOMY BRUSHED STEEL 49 4 +Brand#12 ECONOMY BURNISHED COPPER 45 4 +Brand#12 ECONOMY PLATED COPPER 23 4 +Brand#12 ECONOMY PLATED STEEL 23 4 +Brand#12 ECONOMY PLATED TIN 36 4 +Brand#12 ECONOMY POLISHED BRASS 14 4 +Brand#12 ECONOMY POLISHED COPPER 45 4 +Brand#12 ECONOMY POLISHED NICKEL 9 4 +Brand#12 LARGE ANODIZED NICKEL 9 4 +Brand#12 LARGE ANODIZED NICKEL 49 4 +Brand#12 LARGE ANODIZED STEEL 49 4 +Brand#12 LARGE ANODIZED TIN 36 4 +Brand#12 LARGE ANODIZED TIN 45 4 +Brand#12 LARGE BURNISHED BRASS 14 4 +Brand#12 LARGE BURNISHED BRASS 19 4 +Brand#12 LARGE BURNISHED COPPER 9 4 +Brand#12 LARGE BURNISHED NICKEL 45 4 +Brand#12 LARGE BURNISHED TIN 36 4 +Brand#12 LARGE PLATED BRASS 3 4 +Brand#12 LARGE PLATED STEEL 36 4 +Brand#12 LARGE PLATED STEEL 45 4 +Brand#12 LARGE PLATED TIN 23 4 +Brand#12 LARGE POLISHED COPPER 14 4 +Brand#12 LARGE POLISHED COPPER 19 4 +Brand#12 LARGE POLISHED COPPER 49 4 +Brand#12 LARGE POLISHED STEEL 3 4 +Brand#12 MEDIUM ANODIZED COPPER 9 4 +Brand#12 MEDIUM ANODIZED COPPER 45 4 +Brand#12 MEDIUM ANODIZED NICKEL 45 4 +Brand#12 MEDIUM BRUSHED BRASS 19 4 +Brand#12 MEDIUM BRUSHED COPPER 9 4 +Brand#12 MEDIUM BRUSHED COPPER 36 4 +Brand#12 MEDIUM BRUSHED COPPER 49 4 +Brand#12 MEDIUM BRUSHED NICKEL 3 4 +Brand#12 MEDIUM BRUSHED NICKEL 14 4 +Brand#12 MEDIUM BRUSHED NICKEL 23 4 +Brand#12 MEDIUM BURNISHED BRASS 3 4 +Brand#12 MEDIUM BURNISHED COPPER 36 4 +Brand#12 MEDIUM BURNISHED NICKEL 19 4 +Brand#12 MEDIUM BURNISHED TIN 14 4 +Brand#12 MEDIUM PLATED BRASS 23 4 +Brand#12 MEDIUM PLATED TIN 19 4 +Brand#12 MEDIUM PLATED TIN 23 4 +Brand#12 MEDIUM PLATED TIN 49 4 +Brand#12 PROMO ANODIZED BRASS 9 4 +Brand#12 PROMO ANODIZED BRASS 45 4 +Brand#12 PROMO ANODIZED NICKEL 14 4 +Brand#12 PROMO ANODIZED STEEL 49 4 +Brand#12 PROMO ANODIZED TIN 3 4 +Brand#12 PROMO ANODIZED TIN 19 4 +Brand#12 PROMO BRUSHED COPPER 14 4 +Brand#12 PROMO BRUSHED COPPER 19 4 +Brand#12 PROMO BRUSHED NICKEL 23 4 +Brand#12 PROMO BRUSHED STEEL 23 4 +Brand#12 PROMO BRUSHED STEEL 36 4 +Brand#12 PROMO BURNISHED BRASS 49 4 +Brand#12 PROMO BURNISHED TIN 9 4 +Brand#12 PROMO BURNISHED TIN 14 4 +Brand#12 PROMO PLATED BRASS 36 4 +Brand#12 PROMO POLISHED COPPER 23 4 +Brand#12 PROMO POLISHED NICKEL 3 4 +Brand#12 PROMO POLISHED NICKEL 9 4 +Brand#12 PROMO POLISHED STEEL 14 4 +Brand#12 PROMO POLISHED TIN 23 4 +Brand#12 PROMO POLISHED TIN 36 4 +Brand#12 SMALL ANODIZED BRASS 36 4 +Brand#12 SMALL ANODIZED COPPER 23 4 +Brand#12 SMALL ANODIZED STEEL 36 4 +Brand#12 SMALL ANODIZED TIN 14 4 +Brand#12 SMALL BRUSHED COPPER 19 4 +Brand#12 SMALL BRUSHED COPPER 36 4 +Brand#12 SMALL BRUSHED TIN 36 4 +Brand#12 SMALL BURNISHED BRASS 14 4 +Brand#12 SMALL BURNISHED COPPER 9 4 +Brand#12 SMALL BURNISHED COPPER 36 4 +Brand#12 SMALL PLATED BRASS 9 4 +Brand#12 SMALL POLISHED BRASS 49 4 +Brand#12 SMALL POLISHED NICKEL 19 4 +Brand#12 SMALL POLISHED TIN 3 4 +Brand#12 STANDARD ANODIZED BRASS 19 4 +Brand#12 STANDARD ANODIZED NICKEL 19 4 +Brand#12 STANDARD ANODIZED STEEL 19 4 +Brand#12 STANDARD BRUSHED COPPER 36 4 +Brand#12 STANDARD BRUSHED NICKEL 23 4 +Brand#12 STANDARD BRUSHED STEEL 49 4 +Brand#12 STANDARD BURNISHED BRASS 23 4 +Brand#12 STANDARD BURNISHED COPPER 14 4 +Brand#12 STANDARD BURNISHED NICKEL 45 4 +Brand#12 STANDARD BURNISHED NICKEL 49 4 +Brand#12 STANDARD BURNISHED TIN 3 4 +Brand#12 STANDARD BURNISHED TIN 14 4 +Brand#12 STANDARD PLATED BRASS 19 4 +Brand#12 STANDARD PLATED NICKEL 45 4 +Brand#12 STANDARD PLATED STEEL 36 4 +Brand#12 STANDARD PLATED STEEL 45 4 +Brand#12 STANDARD PLATED TIN 9 4 +Brand#12 STANDARD POLISHED BRASS 49 4 +Brand#12 STANDARD POLISHED COPPER 3 4 +Brand#12 STANDARD POLISHED NICKEL 23 4 +Brand#12 STANDARD POLISHED TIN 14 4 +Brand#13 ECONOMY ANODIZED NICKEL 14 4 +Brand#13 ECONOMY ANODIZED NICKEL 19 4 +Brand#13 ECONOMY ANODIZED STEEL 45 4 +Brand#13 ECONOMY ANODIZED STEEL 49 4 +Brand#13 ECONOMY BRUSHED BRASS 3 4 +Brand#13 ECONOMY BURNISHED STEEL 14 4 +Brand#13 ECONOMY BURNISHED TIN 19 4 +Brand#13 ECONOMY BURNISHED TIN 45 4 +Brand#13 ECONOMY PLATED COPPER 19 4 +Brand#13 ECONOMY PLATED NICKEL 3 4 +Brand#13 ECONOMY PLATED STEEL 23 4 +Brand#13 ECONOMY PLATED TIN 3 4 +Brand#13 ECONOMY POLISHED BRASS 3 4 +Brand#13 ECONOMY POLISHED COPPER 9 4 +Brand#13 ECONOMY POLISHED COPPER 49 4 +Brand#13 ECONOMY POLISHED STEEL 23 4 +Brand#13 ECONOMY POLISHED STEEL 49 4 +Brand#13 LARGE ANODIZED BRASS 23 4 +Brand#13 LARGE ANODIZED COPPER 19 4 +Brand#13 LARGE ANODIZED NICKEL 9 4 +Brand#13 LARGE ANODIZED STEEL 45 4 +Brand#13 LARGE ANODIZED TIN 19 4 +Brand#13 LARGE BRUSHED BRASS 3 4 +Brand#13 LARGE BRUSHED BRASS 9 4 +Brand#13 LARGE BRUSHED BRASS 19 4 +Brand#13 LARGE BRUSHED COPPER 9 4 +Brand#13 LARGE BRUSHED COPPER 36 4 +Brand#13 LARGE BRUSHED NICKEL 3 4 +Brand#13 LARGE BRUSHED NICKEL 9 4 +Brand#13 LARGE BRUSHED NICKEL 14 4 +Brand#13 LARGE BRUSHED NICKEL 23 4 +Brand#13 LARGE BRUSHED STEEL 19 4 +Brand#13 LARGE BRUSHED TIN 49 4 +Brand#13 LARGE BURNISHED BRASS 49 4 +Brand#13 LARGE BURNISHED TIN 49 4 +Brand#13 LARGE PLATED COPPER 23 4 +Brand#13 LARGE PLATED STEEL 14 4 +Brand#13 LARGE PLATED STEEL 19 4 +Brand#13 LARGE PLATED STEEL 36 4 +Brand#13 LARGE PLATED TIN 14 4 +Brand#13 LARGE PLATED TIN 45 4 +Brand#13 LARGE POLISHED BRASS 3 4 +Brand#13 LARGE POLISHED BRASS 23 4 +Brand#13 LARGE POLISHED BRASS 49 4 +Brand#13 MEDIUM ANODIZED BRASS 3 4 +Brand#13 MEDIUM ANODIZED BRASS 36 4 +Brand#13 MEDIUM ANODIZED COPPER 14 4 +Brand#13 MEDIUM ANODIZED NICKEL 3 4 +Brand#13 MEDIUM ANODIZED STEEL 14 4 +Brand#13 MEDIUM ANODIZED STEEL 19 4 +Brand#13 MEDIUM ANODIZED STEEL 36 4 +Brand#13 MEDIUM BRUSHED BRASS 49 4 +Brand#13 MEDIUM BRUSHED COPPER 23 4 +Brand#13 MEDIUM BRUSHED NICKEL 45 4 +Brand#13 MEDIUM BURNISHED BRASS 9 4 +Brand#13 MEDIUM BURNISHED STEEL 19 4 +Brand#13 MEDIUM BURNISHED STEEL 49 4 +Brand#13 MEDIUM PLATED BRASS 3 4 +Brand#13 MEDIUM PLATED BRASS 23 4 +Brand#13 MEDIUM PLATED BRASS 36 4 +Brand#13 MEDIUM PLATED COPPER 19 4 +Brand#13 MEDIUM PLATED COPPER 23 4 +Brand#13 MEDIUM PLATED STEEL 3 4 +Brand#13 PROMO ANODIZED BRASS 14 4 +Brand#13 PROMO ANODIZED COPPER 9 4 +Brand#13 PROMO ANODIZED COPPER 45 4 +Brand#13 PROMO ANODIZED STEEL 23 4 +Brand#13 PROMO BRUSHED COPPER 49 4 +Brand#13 PROMO BURNISHED COPPER 19 4 +Brand#13 PROMO BURNISHED NICKEL 9 4 +Brand#13 PROMO BURNISHED STEEL 23 4 +Brand#13 PROMO BURNISHED STEEL 45 4 +Brand#13 PROMO BURNISHED TIN 19 4 +Brand#13 PROMO PLATED BRASS 14 4 +Brand#13 PROMO PLATED BRASS 19 4 +Brand#13 PROMO PLATED COPPER 3 4 +Brand#13 PROMO PLATED COPPER 19 4 +Brand#13 PROMO PLATED TIN 19 4 +Brand#13 PROMO POLISHED BRASS 49 4 +Brand#13 PROMO POLISHED STEEL 45 4 +Brand#13 PROMO POLISHED TIN 14 4 +Brand#13 SMALL ANODIZED STEEL 23 4 +Brand#13 SMALL ANODIZED TIN 3 4 +Brand#13 SMALL ANODIZED TIN 45 4 +Brand#13 SMALL BRUSHED COPPER 3 4 +Brand#13 SMALL BRUSHED NICKEL 19 4 +Brand#13 SMALL BRUSHED TIN 9 4 +Brand#13 SMALL BRUSHED TIN 45 4 +Brand#13 SMALL BURNISHED BRASS 19 4 +Brand#13 SMALL BURNISHED BRASS 45 4 +Brand#13 SMALL PLATED BRASS 9 4 +Brand#13 SMALL PLATED TIN 45 4 +Brand#13 SMALL POLISHED NICKEL 19 4 +Brand#13 SMALL POLISHED STEEL 49 4 +Brand#13 STANDARD ANODIZED COPPER 45 4 +Brand#13 STANDARD ANODIZED NICKEL 9 4 +Brand#13 STANDARD ANODIZED NICKEL 19 4 +Brand#13 STANDARD ANODIZED STEEL 14 4 +Brand#13 STANDARD ANODIZED TIN 9 4 +Brand#13 STANDARD ANODIZED TIN 36 4 +Brand#13 STANDARD BRUSHED BRASS 19 4 +Brand#13 STANDARD BRUSHED TIN 9 4 +Brand#13 STANDARD BURNISHED BRASS 9 4 +Brand#13 STANDARD BURNISHED BRASS 14 4 +Brand#13 STANDARD BURNISHED COPPER 45 4 +Brand#13 STANDARD PLATED BRASS 49 4 +Brand#13 STANDARD PLATED COPPER 19 4 +Brand#13 STANDARD PLATED NICKEL 23 4 +Brand#13 STANDARD PLATED TIN 9 4 +Brand#13 STANDARD POLISHED BRASS 49 4 +Brand#13 STANDARD POLISHED COPPER 9 4 +Brand#13 STANDARD POLISHED COPPER 49 4 +Brand#13 STANDARD POLISHED NICKEL 14 4 +Brand#13 STANDARD POLISHED NICKEL 19 4 +Brand#13 STANDARD POLISHED STEEL 23 4 +Brand#14 ECONOMY ANODIZED BRASS 19 4 +Brand#14 ECONOMY ANODIZED COPPER 9 4 +Brand#14 ECONOMY ANODIZED STEEL 19 4 +Brand#14 ECONOMY ANODIZED STEEL 45 4 +Brand#14 ECONOMY BRUSHED BRASS 19 4 +Brand#14 ECONOMY BRUSHED COPPER 45 4 +Brand#14 ECONOMY BRUSHED NICKEL 14 4 +Brand#14 ECONOMY BRUSHED TIN 14 4 +Brand#14 ECONOMY BURNISHED COPPER 9 4 +Brand#14 ECONOMY BURNISHED COPPER 19 4 +Brand#14 ECONOMY BURNISHED STEEL 36 4 +Brand#14 ECONOMY BURNISHED TIN 3 4 +Brand#14 ECONOMY PLATED BRASS 36 4 +Brand#14 ECONOMY PLATED COPPER 49 4 +Brand#14 ECONOMY PLATED STEEL 45 4 +Brand#14 ECONOMY PLATED TIN 9 4 +Brand#14 ECONOMY POLISHED COPPER 3 4 +Brand#14 ECONOMY POLISHED TIN 19 4 +Brand#14 LARGE ANODIZED COPPER 9 4 +Brand#14 LARGE ANODIZED COPPER 23 4 +Brand#14 LARGE ANODIZED NICKEL 3 4 +Brand#14 LARGE ANODIZED NICKEL 9 4 +Brand#14 LARGE ANODIZED NICKEL 19 4 +Brand#14 LARGE ANODIZED TIN 9 4 +Brand#14 LARGE BRUSHED COPPER 14 4 +Brand#14 LARGE BRUSHED NICKEL 45 4 +Brand#14 LARGE PLATED BRASS 3 4 +Brand#14 LARGE PLATED NICKEL 3 4 +Brand#14 LARGE PLATED NICKEL 14 4 +Brand#14 LARGE PLATED NICKEL 49 4 +Brand#14 LARGE PLATED TIN 49 4 +Brand#14 LARGE POLISHED BRASS 9 4 +Brand#14 LARGE POLISHED BRASS 14 4 +Brand#14 LARGE POLISHED BRASS 36 4 +Brand#14 LARGE POLISHED NICKEL 3 4 +Brand#14 LARGE POLISHED NICKEL 14 4 +Brand#14 LARGE POLISHED STEEL 9 4 +Brand#14 LARGE POLISHED STEEL 23 4 +Brand#14 LARGE POLISHED STEEL 36 4 +Brand#14 MEDIUM ANODIZED NICKEL 3 4 +Brand#14 MEDIUM ANODIZED NICKEL 49 4 +Brand#14 MEDIUM ANODIZED STEEL 23 4 +Brand#14 MEDIUM ANODIZED STEEL 36 4 +Brand#14 MEDIUM BRUSHED BRASS 9 4 +Brand#14 MEDIUM BRUSHED COPPER 23 4 +Brand#14 MEDIUM BRUSHED STEEL 14 4 +Brand#14 MEDIUM BURNISHED COPPER 14 4 +Brand#14 MEDIUM BURNISHED STEEL 3 4 +Brand#14 MEDIUM BURNISHED STEEL 49 4 +Brand#14 MEDIUM PLATED BRASS 36 4 +Brand#14 MEDIUM PLATED STEEL 49 4 +Brand#14 MEDIUM PLATED TIN 14 4 +Brand#14 PROMO ANODIZED BRASS 49 4 +Brand#14 PROMO ANODIZED STEEL 36 4 +Brand#14 PROMO BRUSHED STEEL 19 4 +Brand#14 PROMO BURNISHED BRASS 23 4 +Brand#14 PROMO BURNISHED STEEL 36 4 +Brand#14 PROMO PLATED BRASS 9 4 +Brand#14 PROMO PLATED BRASS 45 4 +Brand#14 PROMO PLATED COPPER 45 4 +Brand#14 PROMO PLATED STEEL 3 4 +Brand#14 PROMO POLISHED BRASS 9 4 +Brand#14 PROMO POLISHED COPPER 49 4 +Brand#14 PROMO POLISHED STEEL 19 4 +Brand#14 SMALL ANODIZED STEEL 23 4 +Brand#14 SMALL ANODIZED TIN 23 4 +Brand#14 SMALL BRUSHED BRASS 19 4 +Brand#14 SMALL BRUSHED BRASS 36 4 +Brand#14 SMALL BRUSHED COPPER 9 4 +Brand#14 SMALL BRUSHED TIN 36 4 +Brand#14 SMALL BURNISHED BRASS 45 4 +Brand#14 SMALL BURNISHED COPPER 9 4 +Brand#14 SMALL BURNISHED COPPER 14 4 +Brand#14 SMALL BURNISHED COPPER 45 4 +Brand#14 SMALL BURNISHED NICKEL 36 4 +Brand#14 SMALL BURNISHED STEEL 36 4 +Brand#14 SMALL BURNISHED TIN 23 4 +Brand#14 SMALL PLATED NICKEL 3 4 +Brand#14 SMALL PLATED NICKEL 9 4 +Brand#14 SMALL PLATED STEEL 14 4 +Brand#14 SMALL POLISHED BRASS 36 4 +Brand#14 SMALL POLISHED COPPER 36 4 +Brand#14 SMALL POLISHED NICKEL 9 4 +Brand#14 SMALL POLISHED STEEL 14 4 +Brand#14 SMALL POLISHED TIN 14 4 +Brand#14 STANDARD ANODIZED BRASS 19 4 +Brand#14 STANDARD ANODIZED NICKEL 14 4 +Brand#14 STANDARD ANODIZED STEEL 9 4 +Brand#14 STANDARD BRUSHED COPPER 45 4 +Brand#14 STANDARD BRUSHED NICKEL 45 4 +Brand#14 STANDARD BRUSHED TIN 45 4 +Brand#14 STANDARD BURNISHED BRASS 9 4 +Brand#14 STANDARD BURNISHED BRASS 23 4 +Brand#14 STANDARD BURNISHED BRASS 49 4 +Brand#14 STANDARD BURNISHED NICKEL 9 4 +Brand#14 STANDARD PLATED BRASS 36 4 +Brand#14 STANDARD PLATED COPPER 45 4 +Brand#14 STANDARD POLISHED NICKEL 3 4 +Brand#14 STANDARD POLISHED NICKEL 9 4 +Brand#14 STANDARD POLISHED TIN 19 4 +Brand#15 ECONOMY ANODIZED COPPER 14 4 +Brand#15 ECONOMY ANODIZED STEEL 19 4 +Brand#15 ECONOMY ANODIZED STEEL 36 4 +Brand#15 ECONOMY BRUSHED BRASS 36 4 +Brand#15 ECONOMY BRUSHED COPPER 14 4 +Brand#15 ECONOMY BRUSHED NICKEL 14 4 +Brand#15 ECONOMY BRUSHED STEEL 3 4 +Brand#15 ECONOMY BRUSHED TIN 3 4 +Brand#15 ECONOMY BURNISHED BRASS 14 4 +Brand#15 ECONOMY BURNISHED COPPER 3 4 +Brand#15 ECONOMY BURNISHED COPPER 23 4 +Brand#15 ECONOMY PLATED NICKEL 49 4 +Brand#15 ECONOMY PLATED STEEL 3 4 +Brand#15 ECONOMY PLATED STEEL 19 4 +Brand#15 ECONOMY PLATED STEEL 45 4 +Brand#15 LARGE ANODIZED BRASS 19 4 +Brand#15 LARGE ANODIZED BRASS 36 4 +Brand#15 LARGE ANODIZED BRASS 45 4 +Brand#15 LARGE ANODIZED COPPER 3 4 +Brand#15 LARGE ANODIZED NICKEL 9 4 +Brand#15 LARGE ANODIZED TIN 19 4 +Brand#15 LARGE BRUSHED BRASS 9 4 +Brand#15 LARGE BRUSHED BRASS 19 4 +Brand#15 LARGE BRUSHED COPPER 14 4 +Brand#15 LARGE BRUSHED STEEL 9 4 +Brand#15 LARGE BRUSHED STEEL 14 4 +Brand#15 LARGE BRUSHED STEEL 19 4 +Brand#15 LARGE BRUSHED STEEL 36 4 +Brand#15 LARGE BURNISHED BRASS 14 4 +Brand#15 LARGE BURNISHED BRASS 19 4 +Brand#15 LARGE BURNISHED COPPER 9 4 +Brand#15 LARGE BURNISHED COPPER 45 4 +Brand#15 LARGE BURNISHED TIN 49 4 +Brand#15 LARGE PLATED BRASS 19 4 +Brand#15 LARGE PLATED COPPER 3 4 +Brand#15 LARGE PLATED COPPER 23 4 +Brand#15 LARGE PLATED NICKEL 36 4 +Brand#15 MEDIUM ANODIZED BRASS 23 4 +Brand#15 MEDIUM ANODIZED COPPER 9 4 +Brand#15 MEDIUM ANODIZED NICKEL 3 4 +Brand#15 MEDIUM ANODIZED TIN 19 4 +Brand#15 MEDIUM BRUSHED BRASS 9 4 +Brand#15 MEDIUM BRUSHED TIN 23 4 +Brand#15 MEDIUM BURNISHED COPPER 36 4 +Brand#15 MEDIUM BURNISHED TIN 45 4 +Brand#15 MEDIUM PLATED COPPER 9 4 +Brand#15 MEDIUM PLATED NICKEL 9 4 +Brand#15 MEDIUM PLATED NICKEL 19 4 +Brand#15 MEDIUM PLATED STEEL 36 4 +Brand#15 MEDIUM PLATED STEEL 49 4 +Brand#15 MEDIUM PLATED TIN 9 4 +Brand#15 MEDIUM PLATED TIN 14 4 +Brand#15 MEDIUM PLATED TIN 23 4 +Brand#15 PROMO ANODIZED COPPER 23 4 +Brand#15 PROMO ANODIZED STEEL 14 4 +Brand#15 PROMO ANODIZED TIN 45 4 +Brand#15 PROMO BRUSHED COPPER 14 4 +Brand#15 PROMO BRUSHED COPPER 19 4 +Brand#15 PROMO BRUSHED NICKEL 19 4 +Brand#15 PROMO BRUSHED NICKEL 23 4 +Brand#15 PROMO BRUSHED STEEL 14 4 +Brand#15 PROMO BRUSHED TIN 36 4 +Brand#15 PROMO BURNISHED NICKEL 9 4 +Brand#15 PROMO BURNISHED STEEL 45 4 +Brand#15 PROMO PLATED COPPER 3 4 +Brand#15 PROMO PLATED COPPER 36 4 +Brand#15 PROMO PLATED STEEL 3 4 +Brand#15 PROMO PLATED TIN 49 4 +Brand#15 PROMO POLISHED COPPER 3 4 +Brand#15 PROMO POLISHED NICKEL 36 4 +Brand#15 PROMO POLISHED STEEL 36 4 +Brand#15 PROMO POLISHED TIN 49 4 +Brand#15 SMALL ANODIZED BRASS 14 4 +Brand#15 SMALL ANODIZED BRASS 19 4 +Brand#15 SMALL ANODIZED COPPER 9 4 +Brand#15 SMALL ANODIZED TIN 45 4 +Brand#15 SMALL BRUSHED BRASS 3 4 +Brand#15 SMALL BRUSHED COPPER 19 4 +Brand#15 SMALL BRUSHED STEEL 23 4 +Brand#15 SMALL BRUSHED TIN 45 4 +Brand#15 SMALL BURNISHED BRASS 19 4 +Brand#15 SMALL BURNISHED COPPER 14 4 +Brand#15 SMALL BURNISHED NICKEL 19 4 +Brand#15 SMALL BURNISHED NICKEL 49 4 +Brand#15 SMALL BURNISHED STEEL 9 4 +Brand#15 SMALL BURNISHED TIN 19 4 +Brand#15 SMALL BURNISHED TIN 23 4 +Brand#15 SMALL BURNISHED TIN 36 4 +Brand#15 SMALL PLATED BRASS 3 4 +Brand#15 SMALL PLATED COPPER 23 4 +Brand#15 SMALL PLATED COPPER 49 4 +Brand#15 SMALL PLATED NICKEL 36 4 +Brand#15 SMALL PLATED NICKEL 45 4 +Brand#15 SMALL PLATED STEEL 3 4 +Brand#15 SMALL PLATED TIN 9 4 +Brand#15 SMALL POLISHED COPPER 9 4 +Brand#15 SMALL POLISHED NICKEL 3 4 +Brand#15 SMALL POLISHED STEEL 19 4 +Brand#15 SMALL POLISHED STEEL 36 4 +Brand#15 SMALL POLISHED TIN 19 4 +Brand#15 SMALL POLISHED TIN 49 4 +Brand#15 STANDARD ANODIZED NICKEL 19 4 +Brand#15 STANDARD ANODIZED NICKEL 49 4 +Brand#15 STANDARD ANODIZED TIN 36 4 +Brand#15 STANDARD BRUSHED NICKEL 3 4 +Brand#15 STANDARD BURNISHED BRASS 23 4 +Brand#15 STANDARD BURNISHED STEEL 3 4 +Brand#15 STANDARD BURNISHED STEEL 45 4 +Brand#15 STANDARD PLATED BRASS 36 4 +Brand#15 STANDARD PLATED COPPER 14 4 +Brand#15 STANDARD PLATED COPPER 23 4 +Brand#15 STANDARD PLATED NICKEL 19 4 +Brand#15 STANDARD PLATED TIN 45 4 +Brand#15 STANDARD POLISHED BRASS 14 4 +Brand#15 STANDARD POLISHED COPPER 23 4 +Brand#15 STANDARD POLISHED NICKEL 45 4 +Brand#21 ECONOMY ANODIZED BRASS 3 4 +Brand#21 ECONOMY ANODIZED NICKEL 14 4 +Brand#21 ECONOMY ANODIZED STEEL 19 4 +Brand#21 ECONOMY ANODIZED STEEL 23 4 +Brand#21 ECONOMY ANODIZED STEEL 49 4 +Brand#21 ECONOMY ANODIZED TIN 19 4 +Brand#21 ECONOMY BRUSHED BRASS 9 4 +Brand#21 ECONOMY BRUSHED BRASS 14 4 +Brand#21 ECONOMY BRUSHED BRASS 36 4 +Brand#21 ECONOMY BRUSHED COPPER 49 4 +Brand#21 ECONOMY BRUSHED STEEL 45 4 +Brand#21 ECONOMY BRUSHED TIN 49 4 +Brand#21 ECONOMY BURNISHED BRASS 3 4 +Brand#21 ECONOMY BURNISHED COPPER 45 4 +Brand#21 ECONOMY BURNISHED STEEL 19 4 +Brand#21 ECONOMY BURNISHED STEEL 36 4 +Brand#21 ECONOMY PLATED BRASS 36 4 +Brand#21 ECONOMY PLATED COPPER 3 4 +Brand#21 ECONOMY PLATED COPPER 14 4 +Brand#21 ECONOMY PLATED NICKEL 49 4 +Brand#21 ECONOMY POLISHED NICKEL 3 4 +Brand#21 ECONOMY POLISHED NICKEL 9 4 +Brand#21 LARGE ANODIZED COPPER 3 4 +Brand#21 LARGE ANODIZED COPPER 9 4 +Brand#21 LARGE ANODIZED STEEL 36 4 +Brand#21 LARGE ANODIZED TIN 45 4 +Brand#21 LARGE BRUSHED COPPER 45 4 +Brand#21 LARGE BRUSHED STEEL 23 4 +Brand#21 LARGE BURNISHED BRASS 49 4 +Brand#21 LARGE BURNISHED COPPER 19 4 +Brand#21 LARGE BURNISHED STEEL 49 4 +Brand#21 LARGE BURNISHED TIN 49 4 +Brand#21 LARGE PLATED BRASS 19 4 +Brand#21 LARGE PLATED NICKEL 23 4 +Brand#21 LARGE PLATED NICKEL 49 4 +Brand#21 LARGE PLATED TIN 19 4 +Brand#21 LARGE POLISHED BRASS 49 4 +Brand#21 LARGE POLISHED COPPER 14 4 +Brand#21 LARGE POLISHED NICKEL 3 4 +Brand#21 LARGE POLISHED NICKEL 14 4 +Brand#21 LARGE POLISHED STEEL 14 4 +Brand#21 LARGE POLISHED TIN 49 4 +Brand#21 MEDIUM ANODIZED COPPER 14 4 +Brand#21 MEDIUM ANODIZED NICKEL 49 4 +Brand#21 MEDIUM BRUSHED COPPER 3 4 +Brand#21 MEDIUM BRUSHED COPPER 49 4 +Brand#21 MEDIUM BRUSHED STEEL 23 4 +Brand#21 MEDIUM BRUSHED TIN 3 4 +Brand#21 MEDIUM BRUSHED TIN 14 4 +Brand#21 MEDIUM BURNISHED NICKEL 14 4 +Brand#21 MEDIUM BURNISHED STEEL 23 4 +Brand#21 MEDIUM BURNISHED TIN 3 4 +Brand#21 MEDIUM PLATED BRASS 3 4 +Brand#21 MEDIUM PLATED BRASS 19 4 +Brand#21 MEDIUM PLATED STEEL 36 4 +Brand#21 PROMO ANODIZED BRASS 9 4 +Brand#21 PROMO ANODIZED COPPER 14 4 +Brand#21 PROMO ANODIZED NICKEL 23 4 +Brand#21 PROMO ANODIZED STEEL 3 4 +Brand#21 PROMO ANODIZED STEEL 14 4 +Brand#21 PROMO ANODIZED STEEL 36 4 +Brand#21 PROMO BRUSHED NICKEL 45 4 +Brand#21 PROMO BRUSHED STEEL 14 4 +Brand#21 PROMO BRUSHED STEEL 23 4 +Brand#21 PROMO BRUSHED STEEL 45 4 +Brand#21 PROMO BURNISHED BRASS 19 4 +Brand#21 PROMO BURNISHED COPPER 19 4 +Brand#21 PROMO BURNISHED NICKEL 9 4 +Brand#21 PROMO BURNISHED TIN 19 4 +Brand#21 PROMO PLATED NICKEL 9 4 +Brand#21 PROMO PLATED NICKEL 36 4 +Brand#21 PROMO PLATED STEEL 49 4 +Brand#21 PROMO PLATED TIN 3 4 +Brand#21 PROMO POLISHED NICKEL 23 4 +Brand#21 PROMO POLISHED TIN 14 4 +Brand#21 PROMO POLISHED TIN 19 4 +Brand#21 PROMO POLISHED TIN 23 4 +Brand#21 SMALL BRUSHED BRASS 23 4 +Brand#21 SMALL BRUSHED COPPER 49 4 +Brand#21 SMALL BURNISHED BRASS 23 4 +Brand#21 SMALL BURNISHED BRASS 36 4 +Brand#21 SMALL BURNISHED STEEL 19 4 +Brand#21 SMALL BURNISHED TIN 19 4 +Brand#21 SMALL PLATED BRASS 45 4 +Brand#21 SMALL PLATED COPPER 45 4 +Brand#21 SMALL PLATED STEEL 45 4 +Brand#21 SMALL PLATED TIN 14 4 +Brand#21 SMALL PLATED TIN 45 4 +Brand#21 SMALL POLISHED COPPER 9 4 +Brand#21 SMALL POLISHED NICKEL 23 4 +Brand#21 SMALL POLISHED TIN 3 4 +Brand#21 STANDARD ANODIZED BRASS 9 4 +Brand#21 STANDARD ANODIZED NICKEL 19 4 +Brand#21 STANDARD ANODIZED TIN 45 4 +Brand#21 STANDARD BURNISHED COPPER 36 4 +Brand#21 STANDARD BURNISHED NICKEL 23 4 +Brand#21 STANDARD BURNISHED TIN 9 4 +Brand#21 STANDARD PLATED BRASS 14 4 +Brand#21 STANDARD PLATED COPPER 19 4 +Brand#21 STANDARD PLATED NICKEL 3 4 +Brand#21 STANDARD PLATED STEEL 9 4 +Brand#21 STANDARD PLATED TIN 9 4 +Brand#21 STANDARD POLISHED BRASS 9 4 +Brand#21 STANDARD POLISHED COPPER 49 4 +Brand#21 STANDARD POLISHED STEEL 36 4 +Brand#21 STANDARD POLISHED TIN 36 4 +Brand#22 ECONOMY ANODIZED STEEL 9 4 +Brand#22 ECONOMY ANODIZED STEEL 14 4 +Brand#22 ECONOMY ANODIZED STEEL 23 4 +Brand#22 ECONOMY ANODIZED TIN 9 4 +Brand#22 ECONOMY ANODIZED TIN 36 4 +Brand#22 ECONOMY BRUSHED NICKEL 36 4 +Brand#22 ECONOMY BRUSHED NICKEL 45 4 +Brand#22 ECONOMY BURNISHED BRASS 9 4 +Brand#22 ECONOMY BURNISHED BRASS 23 4 +Brand#22 ECONOMY BURNISHED BRASS 45 4 +Brand#22 ECONOMY BURNISHED NICKEL 19 4 +Brand#22 ECONOMY BURNISHED NICKEL 49 4 +Brand#22 ECONOMY BURNISHED STEEL 9 4 +Brand#22 ECONOMY BURNISHED STEEL 14 4 +Brand#22 ECONOMY BURNISHED STEEL 23 4 +Brand#22 ECONOMY PLATED BRASS 36 4 +Brand#22 ECONOMY PLATED COPPER 23 4 +Brand#22 ECONOMY PLATED TIN 3 4 +Brand#22 ECONOMY POLISHED TIN 49 4 +Brand#22 LARGE ANODIZED BRASS 19 4 +Brand#22 LARGE ANODIZED COPPER 36 4 +Brand#22 LARGE ANODIZED STEEL 3 4 +Brand#22 LARGE BRUSHED BRASS 23 4 +Brand#22 LARGE BRUSHED BRASS 49 4 +Brand#22 LARGE BRUSHED STEEL 49 4 +Brand#22 LARGE BURNISHED COPPER 19 4 +Brand#22 LARGE BURNISHED STEEL 23 4 +Brand#22 LARGE BURNISHED STEEL 45 4 +Brand#22 LARGE BURNISHED TIN 45 4 +Brand#22 LARGE PLATED COPPER 14 4 +Brand#22 LARGE PLATED STEEL 49 4 +Brand#22 LARGE POLISHED BRASS 19 4 +Brand#22 LARGE POLISHED COPPER 19 4 +Brand#22 LARGE POLISHED COPPER 23 4 +Brand#22 LARGE POLISHED NICKEL 19 4 +Brand#22 LARGE POLISHED TIN 49 4 +Brand#22 MEDIUM ANODIZED BRASS 45 4 +Brand#22 MEDIUM ANODIZED COPPER 19 4 +Brand#22 MEDIUM ANODIZED COPPER 49 4 +Brand#22 MEDIUM ANODIZED NICKEL 9 4 +Brand#22 MEDIUM ANODIZED NICKEL 14 4 +Brand#22 MEDIUM ANODIZED NICKEL 36 4 +Brand#22 MEDIUM ANODIZED TIN 3 4 +Brand#22 MEDIUM ANODIZED TIN 9 4 +Brand#22 MEDIUM BRUSHED BRASS 3 4 +Brand#22 MEDIUM BRUSHED BRASS 14 4 +Brand#22 MEDIUM BRUSHED COPPER 3 4 +Brand#22 MEDIUM BRUSHED COPPER 45 4 +Brand#22 MEDIUM BRUSHED NICKEL 14 4 +Brand#22 MEDIUM BRUSHED TIN 45 4 +Brand#22 MEDIUM BURNISHED COPPER 36 4 +Brand#22 MEDIUM BURNISHED TIN 19 4 +Brand#22 MEDIUM BURNISHED TIN 23 4 +Brand#22 MEDIUM BURNISHED TIN 49 4 +Brand#22 MEDIUM PLATED BRASS 49 4 +Brand#22 MEDIUM PLATED COPPER 9 4 +Brand#22 MEDIUM PLATED STEEL 3 4 +Brand#22 PROMO ANODIZED BRASS 9 4 +Brand#22 PROMO ANODIZED STEEL 36 4 +Brand#22 PROMO ANODIZED TIN 45 4 +Brand#22 PROMO BRUSHED BRASS 3 4 +Brand#22 PROMO BRUSHED BRASS 9 4 +Brand#22 PROMO BRUSHED BRASS 36 4 +Brand#22 PROMO BRUSHED STEEL 36 4 +Brand#22 PROMO BURNISHED BRASS 23 4 +Brand#22 PROMO BURNISHED COPPER 9 4 +Brand#22 PROMO PLATED BRASS 14 4 +Brand#22 PROMO PLATED BRASS 45 4 +Brand#22 PROMO PLATED NICKEL 3 4 +Brand#22 PROMO PLATED STEEL 19 4 +Brand#22 PROMO POLISHED BRASS 3 4 +Brand#22 PROMO POLISHED STEEL 14 4 +Brand#22 PROMO POLISHED STEEL 23 4 +Brand#22 SMALL ANODIZED TIN 36 4 +Brand#22 SMALL ANODIZED TIN 49 4 +Brand#22 SMALL BRUSHED NICKEL 3 4 +Brand#22 SMALL BRUSHED NICKEL 36 4 +Brand#22 SMALL BRUSHED NICKEL 45 4 +Brand#22 SMALL BRUSHED TIN 45 4 +Brand#22 SMALL BURNISHED STEEL 23 4 +Brand#22 SMALL BURNISHED TIN 14 4 +Brand#22 SMALL PLATED STEEL 3 4 +Brand#22 SMALL PLATED TIN 9 4 +Brand#22 SMALL PLATED TIN 36 4 +Brand#22 SMALL POLISHED BRASS 23 4 +Brand#22 SMALL POLISHED NICKEL 19 4 +Brand#22 STANDARD ANODIZED BRASS 14 4 +Brand#22 STANDARD ANODIZED BRASS 23 4 +Brand#22 STANDARD BRUSHED COPPER 49 4 +Brand#22 STANDARD BRUSHED NICKEL 3 4 +Brand#22 STANDARD BRUSHED NICKEL 23 4 +Brand#22 STANDARD BRUSHED STEEL 9 4 +Brand#22 STANDARD BRUSHED TIN 19 4 +Brand#22 STANDARD BURNISHED COPPER 45 4 +Brand#22 STANDARD BURNISHED NICKEL 3 4 +Brand#22 STANDARD BURNISHED NICKEL 14 4 +Brand#22 STANDARD BURNISHED NICKEL 45 4 +Brand#22 STANDARD BURNISHED STEEL 3 4 +Brand#22 STANDARD BURNISHED STEEL 36 4 +Brand#22 STANDARD BURNISHED STEEL 45 4 +Brand#22 STANDARD BURNISHED STEEL 49 4 +Brand#22 STANDARD PLATED BRASS 45 4 +Brand#22 STANDARD PLATED NICKEL 3 4 +Brand#22 STANDARD PLATED NICKEL 45 4 +Brand#22 STANDARD PLATED STEEL 14 4 +Brand#22 STANDARD PLATED TIN 19 4 +Brand#22 STANDARD PLATED TIN 49 4 +Brand#22 STANDARD POLISHED COPPER 9 4 +Brand#22 STANDARD POLISHED STEEL 49 4 +Brand#22 STANDARD POLISHED TIN 45 4 +Brand#23 ECONOMY ANODIZED NICKEL 49 4 +Brand#23 ECONOMY ANODIZED STEEL 14 4 +Brand#23 ECONOMY ANODIZED STEEL 49 4 +Brand#23 ECONOMY ANODIZED TIN 49 4 +Brand#23 ECONOMY BRUSHED BRASS 3 4 +Brand#23 ECONOMY BRUSHED COPPER 9 4 +Brand#23 ECONOMY BRUSHED TIN 9 4 +Brand#23 ECONOMY BURNISHED STEEL 49 4 +Brand#23 ECONOMY PLATED COPPER 14 4 +Brand#23 ECONOMY PLATED NICKEL 23 4 +Brand#23 ECONOMY PLATED STEEL 14 4 +Brand#23 ECONOMY POLISHED NICKEL 9 4 +Brand#23 LARGE ANODIZED BRASS 14 4 +Brand#23 LARGE ANODIZED COPPER 9 4 +Brand#23 LARGE ANODIZED COPPER 14 4 +Brand#23 LARGE ANODIZED COPPER 45 4 +Brand#23 LARGE ANODIZED STEEL 19 4 +Brand#23 LARGE ANODIZED STEEL 36 4 +Brand#23 LARGE ANODIZED STEEL 49 4 +Brand#23 LARGE ANODIZED TIN 9 4 +Brand#23 LARGE PLATED BRASS 9 4 +Brand#23 LARGE PLATED BRASS 49 4 +Brand#23 LARGE PLATED COPPER 3 4 +Brand#23 LARGE POLISHED BRASS 45 4 +Brand#23 LARGE POLISHED STEEL 9 4 +Brand#23 MEDIUM ANODIZED BRASS 19 4 +Brand#23 MEDIUM ANODIZED NICKEL 3 4 +Brand#23 MEDIUM ANODIZED NICKEL 14 4 +Brand#23 MEDIUM ANODIZED STEEL 45 4 +Brand#23 MEDIUM ANODIZED TIN 36 4 +Brand#23 MEDIUM ANODIZED TIN 45 4 +Brand#23 MEDIUM BRUSHED COPPER 3 4 +Brand#23 MEDIUM BRUSHED COPPER 23 4 +Brand#23 MEDIUM BRUSHED NICKEL 3 4 +Brand#23 MEDIUM BRUSHED TIN 14 4 +Brand#23 MEDIUM BURNISHED BRASS 9 4 +Brand#23 MEDIUM BURNISHED BRASS 45 4 +Brand#23 MEDIUM BURNISHED COPPER 19 4 +Brand#23 MEDIUM PLATED COPPER 19 4 +Brand#23 MEDIUM PLATED COPPER 36 4 +Brand#23 MEDIUM PLATED COPPER 45 4 +Brand#23 MEDIUM PLATED NICKEL 9 4 +Brand#23 MEDIUM PLATED NICKEL 14 4 +Brand#23 PROMO ANODIZED COPPER 9 4 +Brand#23 PROMO ANODIZED COPPER 19 4 +Brand#23 PROMO ANODIZED STEEL 36 4 +Brand#23 PROMO ANODIZED TIN 14 4 +Brand#23 PROMO BRUSHED BRASS 3 4 +Brand#23 PROMO BRUSHED BRASS 19 4 +Brand#23 PROMO BRUSHED BRASS 36 4 +Brand#23 PROMO BRUSHED COPPER 3 4 +Brand#23 PROMO BRUSHED TIN 49 4 +Brand#23 PROMO BURNISHED BRASS 14 4 +Brand#23 PROMO BURNISHED BRASS 45 4 +Brand#23 PROMO BURNISHED COPPER 14 4 +Brand#23 PROMO PLATED BRASS 23 4 +Brand#23 PROMO POLISHED BRASS 14 4 +Brand#23 PROMO POLISHED BRASS 23 4 +Brand#23 PROMO POLISHED COPPER 36 4 +Brand#23 PROMO POLISHED STEEL 36 4 +Brand#23 SMALL ANODIZED BRASS 23 4 +Brand#23 SMALL ANODIZED STEEL 23 4 +Brand#23 SMALL BRUSHED BRASS 49 4 +Brand#23 SMALL BRUSHED COPPER 45 4 +Brand#23 SMALL BRUSHED STEEL 3 4 +Brand#23 SMALL BRUSHED STEEL 19 4 +Brand#23 SMALL BURNISHED BRASS 36 4 +Brand#23 SMALL BURNISHED COPPER 45 4 +Brand#23 SMALL BURNISHED COPPER 49 4 +Brand#23 SMALL BURNISHED STEEL 45 4 +Brand#23 SMALL PLATED BRASS 36 4 +Brand#23 SMALL PLATED BRASS 49 4 +Brand#23 SMALL PLATED COPPER 14 4 +Brand#23 SMALL PLATED TIN 14 4 +Brand#23 SMALL POLISHED BRASS 9 4 +Brand#23 SMALL POLISHED BRASS 14 4 +Brand#23 SMALL POLISHED NICKEL 3 4 +Brand#23 SMALL POLISHED STEEL 14 4 +Brand#23 SMALL POLISHED TIN 9 4 +Brand#23 STANDARD ANODIZED BRASS 19 4 +Brand#23 STANDARD ANODIZED BRASS 45 4 +Brand#23 STANDARD ANODIZED COPPER 19 4 +Brand#23 STANDARD ANODIZED TIN 3 4 +Brand#23 STANDARD BRUSHED COPPER 36 4 +Brand#23 STANDARD BRUSHED NICKEL 19 4 +Brand#23 STANDARD BRUSHED STEEL 49 4 +Brand#23 STANDARD BURNISHED COPPER 19 4 +Brand#23 STANDARD PLATED BRASS 3 4 +Brand#23 STANDARD PLATED BRASS 9 4 +Brand#23 STANDARD PLATED STEEL 36 4 +Brand#23 STANDARD PLATED TIN 19 4 +Brand#23 STANDARD POLISHED BRASS 9 4 +Brand#23 STANDARD POLISHED BRASS 49 4 +Brand#23 STANDARD POLISHED STEEL 19 4 +Brand#23 STANDARD POLISHED STEEL 49 4 +Brand#23 STANDARD POLISHED TIN 23 4 +Brand#24 ECONOMY ANODIZED BRASS 3 4 +Brand#24 ECONOMY ANODIZED BRASS 9 4 +Brand#24 ECONOMY ANODIZED BRASS 23 4 +Brand#24 ECONOMY ANODIZED COPPER 9 4 +Brand#24 ECONOMY ANODIZED COPPER 49 4 +Brand#24 ECONOMY BRUSHED BRASS 36 4 +Brand#24 ECONOMY BRUSHED COPPER 23 4 +Brand#24 ECONOMY BURNISHED COPPER 3 4 +Brand#24 ECONOMY BURNISHED NICKEL 19 4 +Brand#24 ECONOMY BURNISHED STEEL 45 4 +Brand#24 ECONOMY PLATED BRASS 23 4 +Brand#24 ECONOMY PLATED COPPER 36 4 +Brand#24 ECONOMY PLATED STEEL 45 4 +Brand#24 ECONOMY POLISHED BRASS 23 4 +Brand#24 ECONOMY POLISHED COPPER 45 4 +Brand#24 ECONOMY POLISHED NICKEL 36 4 +Brand#24 ECONOMY POLISHED STEEL 14 4 +Brand#24 ECONOMY POLISHED STEEL 36 4 +Brand#24 LARGE ANODIZED NICKEL 23 4 +Brand#24 LARGE ANODIZED NICKEL 45 4 +Brand#24 LARGE ANODIZED TIN 45 4 +Brand#24 LARGE BRUSHED BRASS 14 4 +Brand#24 LARGE BRUSHED BRASS 23 4 +Brand#24 LARGE BRUSHED STEEL 9 4 +Brand#24 LARGE BRUSHED STEEL 23 4 +Brand#24 LARGE BRUSHED STEEL 45 4 +Brand#24 LARGE BRUSHED TIN 49 4 +Brand#24 LARGE BURNISHED BRASS 3 4 +Brand#24 LARGE BURNISHED NICKEL 19 4 +Brand#24 LARGE PLATED BRASS 9 4 +Brand#24 LARGE PLATED NICKEL 36 4 +Brand#24 LARGE PLATED NICKEL 49 4 +Brand#24 LARGE PLATED TIN 9 4 +Brand#24 LARGE PLATED TIN 19 4 +Brand#24 LARGE PLATED TIN 36 4 +Brand#24 LARGE PLATED TIN 49 4 +Brand#24 LARGE POLISHED BRASS 9 4 +Brand#24 LARGE POLISHED COPPER 9 4 +Brand#24 LARGE POLISHED COPPER 49 4 +Brand#24 LARGE POLISHED NICKEL 19 4 +Brand#24 LARGE POLISHED STEEL 23 4 +Brand#24 LARGE POLISHED TIN 14 4 +Brand#24 MEDIUM ANODIZED COPPER 45 4 +Brand#24 MEDIUM BRUSHED COPPER 9 4 +Brand#24 MEDIUM BRUSHED COPPER 14 4 +Brand#24 MEDIUM BRUSHED NICKEL 9 4 +Brand#24 MEDIUM BRUSHED NICKEL 23 4 +Brand#24 MEDIUM BRUSHED STEEL 14 4 +Brand#24 MEDIUM BRUSHED STEEL 45 4 +Brand#24 MEDIUM BRUSHED STEEL 49 4 +Brand#24 MEDIUM BURNISHED BRASS 36 4 +Brand#24 MEDIUM BURNISHED NICKEL 36 4 +Brand#24 MEDIUM BURNISHED STEEL 36 4 +Brand#24 MEDIUM PLATED COPPER 14 4 +Brand#24 MEDIUM PLATED STEEL 3 4 +Brand#24 MEDIUM PLATED STEEL 19 4 +Brand#24 PROMO ANODIZED NICKEL 9 4 +Brand#24 PROMO ANODIZED NICKEL 19 4 +Brand#24 PROMO ANODIZED NICKEL 45 4 +Brand#24 PROMO ANODIZED STEEL 3 4 +Brand#24 PROMO ANODIZED TIN 45 4 +Brand#24 PROMO BRUSHED BRASS 19 4 +Brand#24 PROMO BRUSHED NICKEL 19 4 +Brand#24 PROMO BRUSHED NICKEL 45 4 +Brand#24 PROMO BRUSHED STEEL 49 4 +Brand#24 PROMO BURNISHED BRASS 3 4 +Brand#24 PROMO BURNISHED BRASS 45 4 +Brand#24 PROMO BURNISHED STEEL 49 4 +Brand#24 PROMO PLATED BRASS 3 4 +Brand#24 PROMO PLATED COPPER 23 4 +Brand#24 PROMO PLATED COPPER 49 4 +Brand#24 PROMO POLISHED BRASS 3 4 +Brand#24 PROMO POLISHED BRASS 14 4 +Brand#24 PROMO POLISHED NICKEL 3 4 +Brand#24 PROMO POLISHED STEEL 14 4 +Brand#24 PROMO POLISHED STEEL 19 4 +Brand#24 PROMO POLISHED STEEL 23 4 +Brand#24 SMALL ANODIZED BRASS 19 4 +Brand#24 SMALL ANODIZED COPPER 3 4 +Brand#24 SMALL ANODIZED NICKEL 14 4 +Brand#24 SMALL ANODIZED STEEL 36 4 +Brand#24 SMALL ANODIZED TIN 3 4 +Brand#24 SMALL ANODIZED TIN 36 4 +Brand#24 SMALL BRUSHED COPPER 49 4 +Brand#24 SMALL BRUSHED NICKEL 49 4 +Brand#24 SMALL BURNISHED BRASS 14 4 +Brand#24 SMALL BURNISHED BRASS 19 4 +Brand#24 SMALL BURNISHED TIN 9 4 +Brand#24 SMALL PLATED BRASS 3 4 +Brand#24 SMALL PLATED COPPER 14 4 +Brand#24 SMALL PLATED COPPER 36 4 +Brand#24 SMALL PLATED NICKEL 14 4 +Brand#24 SMALL PLATED NICKEL 49 4 +Brand#24 SMALL POLISHED BRASS 3 4 +Brand#24 SMALL POLISHED NICKEL 9 4 +Brand#24 SMALL POLISHED NICKEL 19 4 +Brand#24 SMALL POLISHED NICKEL 36 4 +Brand#24 SMALL POLISHED STEEL 9 4 +Brand#24 SMALL POLISHED STEEL 36 4 +Brand#24 STANDARD ANODIZED TIN 9 4 +Brand#24 STANDARD ANODIZED TIN 49 4 +Brand#24 STANDARD BRUSHED BRASS 14 4 +Brand#24 STANDARD BRUSHED COPPER 23 4 +Brand#24 STANDARD BRUSHED NICKEL 19 4 +Brand#24 STANDARD BRUSHED STEEL 14 4 +Brand#24 STANDARD BRUSHED TIN 36 4 +Brand#24 STANDARD BURNISHED COPPER 19 4 +Brand#24 STANDARD BURNISHED COPPER 36 4 +Brand#24 STANDARD BURNISHED NICKEL 45 4 +Brand#24 STANDARD PLATED BRASS 36 4 +Brand#24 STANDARD PLATED COPPER 45 4 +Brand#24 STANDARD PLATED NICKEL 36 4 +Brand#24 STANDARD PLATED TIN 36 4 +Brand#24 STANDARD POLISHED COPPER 45 4 +Brand#24 STANDARD POLISHED NICKEL 14 4 +Brand#25 ECONOMY ANODIZED BRASS 14 4 +Brand#25 ECONOMY ANODIZED BRASS 49 4 +Brand#25 ECONOMY ANODIZED TIN 9 4 +Brand#25 ECONOMY ANODIZED TIN 19 4 +Brand#25 ECONOMY ANODIZED TIN 49 4 +Brand#25 ECONOMY BRUSHED COPPER 36 4 +Brand#25 ECONOMY BURNISHED COPPER 45 4 +Brand#25 ECONOMY BURNISHED TIN 19 4 +Brand#25 ECONOMY PLATED NICKEL 23 4 +Brand#25 ECONOMY PLATED TIN 14 4 +Brand#25 ECONOMY POLISHED BRASS 23 4 +Brand#25 ECONOMY POLISHED COPPER 9 4 +Brand#25 ECONOMY POLISHED NICKEL 3 4 +Brand#25 ECONOMY POLISHED TIN 9 4 +Brand#25 ECONOMY POLISHED TIN 45 4 +Brand#25 LARGE ANODIZED BRASS 3 4 +Brand#25 LARGE ANODIZED BRASS 14 4 +Brand#25 LARGE ANODIZED COPPER 36 4 +Brand#25 LARGE ANODIZED NICKEL 23 4 +Brand#25 LARGE ANODIZED STEEL 23 4 +Brand#25 LARGE BRUSHED NICKEL 19 4 +Brand#25 LARGE BRUSHED NICKEL 49 4 +Brand#25 LARGE BRUSHED TIN 3 4 +Brand#25 LARGE BRUSHED TIN 9 4 +Brand#25 LARGE BURNISHED BRASS 19 4 +Brand#25 LARGE BURNISHED BRASS 23 4 +Brand#25 LARGE BURNISHED BRASS 49 4 +Brand#25 LARGE BURNISHED NICKEL 14 4 +Brand#25 LARGE BURNISHED TIN 49 4 +Brand#25 LARGE PLATED BRASS 14 4 +Brand#25 LARGE PLATED NICKEL 23 4 +Brand#25 LARGE PLATED NICKEL 45 4 +Brand#25 LARGE PLATED TIN 19 4 +Brand#25 LARGE PLATED TIN 23 4 +Brand#25 LARGE POLISHED BRASS 9 4 +Brand#25 LARGE POLISHED COPPER 14 4 +Brand#25 LARGE POLISHED COPPER 36 4 +Brand#25 MEDIUM ANODIZED TIN 36 4 +Brand#25 MEDIUM BRUSHED COPPER 9 4 +Brand#25 MEDIUM BRUSHED COPPER 36 4 +Brand#25 MEDIUM BRUSHED COPPER 49 4 +Brand#25 MEDIUM BURNISHED COPPER 49 4 +Brand#25 MEDIUM BURNISHED NICKEL 9 4 +Brand#25 MEDIUM BURNISHED NICKEL 49 4 +Brand#25 MEDIUM BURNISHED STEEL 3 4 +Brand#25 MEDIUM BURNISHED STEEL 36 4 +Brand#25 MEDIUM BURNISHED STEEL 45 4 +Brand#25 MEDIUM BURNISHED STEEL 49 4 +Brand#25 MEDIUM BURNISHED TIN 9 4 +Brand#25 MEDIUM BURNISHED TIN 36 4 +Brand#25 MEDIUM PLATED BRASS 45 4 +Brand#25 MEDIUM PLATED COPPER 14 4 +Brand#25 MEDIUM PLATED NICKEL 45 4 +Brand#25 MEDIUM PLATED STEEL 9 4 +Brand#25 MEDIUM PLATED STEEL 36 4 +Brand#25 PROMO ANODIZED COPPER 14 4 +Brand#25 PROMO ANODIZED COPPER 19 4 +Brand#25 PROMO ANODIZED STEEL 36 4 +Brand#25 PROMO ANODIZED TIN 3 4 +Brand#25 PROMO ANODIZED TIN 14 4 +Brand#25 PROMO BRUSHED NICKEL 3 4 +Brand#25 PROMO BRUSHED STEEL 19 4 +Brand#25 PROMO BRUSHED TIN 14 4 +Brand#25 PROMO BRUSHED TIN 36 4 +Brand#25 PROMO BURNISHED COPPER 19 4 +Brand#25 PROMO BURNISHED COPPER 45 4 +Brand#25 PROMO BURNISHED COPPER 49 4 +Brand#25 PROMO BURNISHED NICKEL 36 4 +Brand#25 PROMO BURNISHED TIN 3 4 +Brand#25 PROMO PLATED BRASS 45 4 +Brand#25 PROMO PLATED COPPER 19 4 +Brand#25 PROMO PLATED NICKEL 45 4 +Brand#25 PROMO PLATED NICKEL 49 4 +Brand#25 PROMO PLATED STEEL 23 4 +Brand#25 PROMO POLISHED BRASS 23 4 +Brand#25 SMALL ANODIZED BRASS 45 4 +Brand#25 SMALL ANODIZED NICKEL 19 4 +Brand#25 SMALL ANODIZED STEEL 23 4 +Brand#25 SMALL ANODIZED TIN 14 4 +Brand#25 SMALL ANODIZED TIN 19 4 +Brand#25 SMALL BRUSHED COPPER 45 4 +Brand#25 SMALL BRUSHED NICKEL 9 4 +Brand#25 SMALL BURNISHED COPPER 3 4 +Brand#25 SMALL BURNISHED STEEL 3 4 +Brand#25 SMALL BURNISHED STEEL 14 4 +Brand#25 SMALL BURNISHED TIN 3 4 +Brand#25 SMALL PLATED BRASS 19 4 +Brand#25 SMALL PLATED COPPER 23 4 +Brand#25 SMALL PLATED STEEL 45 4 +Brand#25 SMALL PLATED TIN 36 4 +Brand#25 SMALL POLISHED BRASS 23 4 +Brand#25 SMALL POLISHED COPPER 9 4 +Brand#25 SMALL POLISHED STEEL 14 4 +Brand#25 STANDARD ANODIZED STEEL 3 4 +Brand#25 STANDARD ANODIZED STEEL 19 4 +Brand#25 STANDARD ANODIZED TIN 9 4 +Brand#25 STANDARD BRUSHED BRASS 14 4 +Brand#25 STANDARD BRUSHED NICKEL 19 4 +Brand#25 STANDARD BRUSHED TIN 9 4 +Brand#25 STANDARD BURNISHED NICKEL 9 4 +Brand#25 STANDARD PLATED BRASS 3 4 +Brand#25 STANDARD PLATED COPPER 14 4 +Brand#25 STANDARD PLATED NICKEL 36 4 +Brand#25 STANDARD POLISHED BRASS 45 4 +Brand#25 STANDARD POLISHED COPPER 23 4 +Brand#25 STANDARD POLISHED NICKEL 3 4 +Brand#25 STANDARD POLISHED NICKEL 49 4 +Brand#25 STANDARD POLISHED TIN 36 4 +Brand#25 STANDARD POLISHED TIN 45 4 +Brand#31 ECONOMY ANODIZED BRASS 3 4 +Brand#31 ECONOMY ANODIZED COPPER 45 4 +Brand#31 ECONOMY ANODIZED STEEL 3 4 +Brand#31 ECONOMY ANODIZED TIN 45 4 +Brand#31 ECONOMY BRUSHED BRASS 14 4 +Brand#31 ECONOMY BRUSHED COPPER 19 4 +Brand#31 ECONOMY BRUSHED NICKEL 9 4 +Brand#31 ECONOMY BRUSHED NICKEL 14 4 +Brand#31 ECONOMY BRUSHED NICKEL 49 4 +Brand#31 ECONOMY BURNISHED COPPER 36 4 +Brand#31 ECONOMY BURNISHED STEEL 3 4 +Brand#31 ECONOMY BURNISHED TIN 49 4 +Brand#31 ECONOMY PLATED COPPER 49 4 +Brand#31 ECONOMY PLATED NICKEL 9 4 +Brand#31 ECONOMY PLATED STEEL 23 4 +Brand#31 ECONOMY PLATED TIN 36 4 +Brand#31 ECONOMY PLATED TIN 49 4 +Brand#31 ECONOMY POLISHED COPPER 3 4 +Brand#31 ECONOMY POLISHED COPPER 36 4 +Brand#31 ECONOMY POLISHED COPPER 49 4 +Brand#31 ECONOMY POLISHED NICKEL 3 4 +Brand#31 LARGE ANODIZED BRASS 19 4 +Brand#31 LARGE ANODIZED STEEL 45 4 +Brand#31 LARGE BRUSHED BRASS 36 4 +Brand#31 LARGE BRUSHED BRASS 49 4 +Brand#31 LARGE BRUSHED TIN 3 4 +Brand#31 LARGE BURNISHED BRASS 9 4 +Brand#31 LARGE PLATED COPPER 19 4 +Brand#31 LARGE PLATED NICKEL 14 4 +Brand#31 LARGE PLATED TIN 9 4 +Brand#31 LARGE PLATED TIN 14 4 +Brand#31 LARGE POLISHED BRASS 14 4 +Brand#31 LARGE POLISHED STEEL 14 4 +Brand#31 LARGE POLISHED STEEL 45 4 +Brand#31 LARGE POLISHED TIN 19 4 +Brand#31 MEDIUM ANODIZED BRASS 23 4 +Brand#31 MEDIUM ANODIZED BRASS 36 4 +Brand#31 MEDIUM ANODIZED COPPER 14 4 +Brand#31 MEDIUM ANODIZED COPPER 19 4 +Brand#31 MEDIUM ANODIZED COPPER 36 4 +Brand#31 MEDIUM ANODIZED STEEL 14 4 +Brand#31 MEDIUM ANODIZED STEEL 49 4 +Brand#31 MEDIUM ANODIZED TIN 19 4 +Brand#31 MEDIUM ANODIZED TIN 49 4 +Brand#31 MEDIUM BRUSHED BRASS 36 4 +Brand#31 MEDIUM BRUSHED STEEL 14 4 +Brand#31 MEDIUM BURNISHED BRASS 14 4 +Brand#31 MEDIUM BURNISHED COPPER 3 4 +Brand#31 MEDIUM BURNISHED NICKEL 9 4 +Brand#31 MEDIUM BURNISHED STEEL 9 4 +Brand#31 MEDIUM BURNISHED TIN 14 4 +Brand#31 MEDIUM BURNISHED TIN 23 4 +Brand#31 MEDIUM PLATED BRASS 3 4 +Brand#31 MEDIUM PLATED TIN 9 4 +Brand#31 MEDIUM PLATED TIN 36 4 +Brand#31 MEDIUM PLATED TIN 45 4 +Brand#31 PROMO ANODIZED BRASS 3 4 +Brand#31 PROMO ANODIZED NICKEL 9 4 +Brand#31 PROMO BRUSHED BRASS 3 4 +Brand#31 PROMO BRUSHED BRASS 23 4 +Brand#31 PROMO BRUSHED COPPER 23 4 +Brand#31 PROMO BRUSHED NICKEL 45 4 +Brand#31 PROMO BURNISHED COPPER 36 4 +Brand#31 PROMO BURNISHED STEEL 3 4 +Brand#31 PROMO BURNISHED TIN 3 4 +Brand#31 PROMO PLATED BRASS 19 4 +Brand#31 PROMO PLATED NICKEL 36 4 +Brand#31 PROMO POLISHED BRASS 49 4 +Brand#31 PROMO POLISHED COPPER 14 4 +Brand#31 PROMO POLISHED NICKEL 3 4 +Brand#31 PROMO POLISHED NICKEL 9 4 +Brand#31 PROMO POLISHED TIN 3 4 +Brand#31 PROMO POLISHED TIN 23 4 +Brand#31 SMALL ANODIZED COPPER 45 4 +Brand#31 SMALL ANODIZED STEEL 23 4 +Brand#31 SMALL ANODIZED TIN 3 4 +Brand#31 SMALL BRUSHED COPPER 36 4 +Brand#31 SMALL BRUSHED COPPER 49 4 +Brand#31 SMALL BRUSHED NICKEL 19 4 +Brand#31 SMALL BRUSHED NICKEL 23 4 +Brand#31 SMALL BURNISHED BRASS 45 4 +Brand#31 SMALL BURNISHED NICKEL 9 4 +Brand#31 SMALL BURNISHED NICKEL 36 4 +Brand#31 SMALL PLATED COPPER 36 4 +Brand#31 SMALL PLATED NICKEL 9 4 +Brand#31 SMALL PLATED NICKEL 36 4 +Brand#31 SMALL POLISHED BRASS 3 4 +Brand#31 SMALL POLISHED COPPER 45 4 +Brand#31 SMALL POLISHED NICKEL 45 4 +Brand#31 SMALL POLISHED TIN 23 4 +Brand#31 SMALL POLISHED TIN 49 4 +Brand#31 STANDARD BRUSHED STEEL 23 4 +Brand#31 STANDARD BRUSHED STEEL 49 4 +Brand#31 STANDARD BURNISHED BRASS 14 4 +Brand#31 STANDARD BURNISHED NICKEL 45 4 +Brand#31 STANDARD PLATED NICKEL 3 4 +Brand#31 STANDARD POLISHED BRASS 3 4 +Brand#31 STANDARD POLISHED BRASS 45 4 +Brand#31 STANDARD POLISHED STEEL 36 4 +Brand#32 ECONOMY ANODIZED BRASS 19 4 +Brand#32 ECONOMY ANODIZED COPPER 36 4 +Brand#32 ECONOMY ANODIZED STEEL 23 4 +Brand#32 ECONOMY ANODIZED STEEL 36 4 +Brand#32 ECONOMY ANODIZED STEEL 45 4 +Brand#32 ECONOMY ANODIZED TIN 19 4 +Brand#32 ECONOMY BRUSHED COPPER 45 4 +Brand#32 ECONOMY BRUSHED TIN 45 4 +Brand#32 ECONOMY BURNISHED BRASS 23 4 +Brand#32 ECONOMY BURNISHED COPPER 36 4 +Brand#32 ECONOMY BURNISHED COPPER 45 4 +Brand#32 ECONOMY BURNISHED STEEL 19 4 +Brand#32 ECONOMY PLATED BRASS 9 4 +Brand#32 ECONOMY PLATED COPPER 9 4 +Brand#32 ECONOMY PLATED NICKEL 23 4 +Brand#32 ECONOMY PLATED TIN 45 4 +Brand#32 ECONOMY POLISHED STEEL 3 4 +Brand#32 LARGE ANODIZED BRASS 23 4 +Brand#32 LARGE ANODIZED BRASS 36 4 +Brand#32 LARGE ANODIZED NICKEL 45 4 +Brand#32 LARGE ANODIZED STEEL 3 4 +Brand#32 LARGE ANODIZED STEEL 14 4 +Brand#32 LARGE BRUSHED STEEL 45 4 +Brand#32 LARGE BRUSHED TIN 45 4 +Brand#32 LARGE BURNISHED NICKEL 36 4 +Brand#32 LARGE BURNISHED TIN 19 4 +Brand#32 LARGE BURNISHED TIN 45 4 +Brand#32 LARGE PLATED BRASS 3 4 +Brand#32 LARGE PLATED NICKEL 49 4 +Brand#32 LARGE PLATED STEEL 19 4 +Brand#32 LARGE PLATED STEEL 36 4 +Brand#32 LARGE POLISHED BRASS 45 4 +Brand#32 LARGE POLISHED COPPER 9 4 +Brand#32 LARGE POLISHED COPPER 49 4 +Brand#32 LARGE POLISHED NICKEL 3 4 +Brand#32 MEDIUM ANODIZED BRASS 3 4 +Brand#32 MEDIUM ANODIZED BRASS 9 4 +Brand#32 MEDIUM ANODIZED TIN 23 4 +Brand#32 MEDIUM BRUSHED BRASS 23 4 +Brand#32 MEDIUM BRUSHED BRASS 49 4 +Brand#32 MEDIUM BRUSHED COPPER 9 4 +Brand#32 MEDIUM BRUSHED COPPER 19 4 +Brand#32 MEDIUM BRUSHED TIN 49 4 +Brand#32 MEDIUM BURNISHED BRASS 9 4 +Brand#32 MEDIUM BURNISHED BRASS 36 4 +Brand#32 MEDIUM BURNISHED BRASS 49 4 +Brand#32 MEDIUM BURNISHED COPPER 9 4 +Brand#32 MEDIUM BURNISHED COPPER 45 4 +Brand#32 MEDIUM BURNISHED NICKEL 49 4 +Brand#32 MEDIUM BURNISHED TIN 9 4 +Brand#32 MEDIUM BURNISHED TIN 45 4 +Brand#32 MEDIUM PLATED BRASS 3 4 +Brand#32 MEDIUM PLATED BRASS 49 4 +Brand#32 MEDIUM PLATED COPPER 3 4 +Brand#32 MEDIUM PLATED STEEL 9 4 +Brand#32 MEDIUM PLATED TIN 9 4 +Brand#32 PROMO ANODIZED BRASS 3 4 +Brand#32 PROMO ANODIZED COPPER 19 4 +Brand#32 PROMO ANODIZED NICKEL 23 4 +Brand#32 PROMO BRUSHED COPPER 23 4 +Brand#32 PROMO BRUSHED NICKEL 14 4 +Brand#32 PROMO BRUSHED NICKEL 36 4 +Brand#32 PROMO BRUSHED STEEL 14 4 +Brand#32 PROMO BRUSHED STEEL 23 4 +Brand#32 PROMO BRUSHED STEEL 49 4 +Brand#32 PROMO BURNISHED BRASS 45 4 +Brand#32 PROMO BURNISHED NICKEL 45 4 +Brand#32 PROMO BURNISHED TIN 14 4 +Brand#32 PROMO BURNISHED TIN 45 4 +Brand#32 PROMO PLATED TIN 19 4 +Brand#32 PROMO POLISHED NICKEL 36 4 +Brand#32 PROMO POLISHED TIN 3 4 +Brand#32 SMALL ANODIZED BRASS 3 4 +Brand#32 SMALL ANODIZED NICKEL 3 4 +Brand#32 SMALL ANODIZED NICKEL 14 4 +Brand#32 SMALL ANODIZED TIN 9 4 +Brand#32 SMALL BRUSHED BRASS 9 4 +Brand#32 SMALL BRUSHED BRASS 19 4 +Brand#32 SMALL BRUSHED COPPER 3 4 +Brand#32 SMALL BRUSHED COPPER 23 4 +Brand#32 SMALL BRUSHED NICKEL 9 4 +Brand#32 SMALL BRUSHED NICKEL 45 4 +Brand#32 SMALL BRUSHED STEEL 23 4 +Brand#32 SMALL BRUSHED TIN 9 4 +Brand#32 SMALL BURNISHED NICKEL 36 4 +Brand#32 SMALL BURNISHED STEEL 3 4 +Brand#32 SMALL BURNISHED TIN 23 4 +Brand#32 SMALL PLATED BRASS 49 4 +Brand#32 SMALL PLATED COPPER 36 4 +Brand#32 SMALL PLATED COPPER 45 4 +Brand#32 SMALL PLATED NICKEL 45 4 +Brand#32 SMALL PLATED STEEL 45 4 +Brand#32 SMALL PLATED TIN 23 4 +Brand#32 SMALL PLATED TIN 36 4 +Brand#32 SMALL PLATED TIN 45 4 +Brand#32 SMALL POLISHED NICKEL 36 4 +Brand#32 SMALL POLISHED STEEL 14 4 +Brand#32 SMALL POLISHED STEEL 23 4 +Brand#32 SMALL POLISHED STEEL 36 4 +Brand#32 SMALL POLISHED TIN 36 4 +Brand#32 SMALL POLISHED TIN 45 4 +Brand#32 STANDARD ANODIZED NICKEL 19 4 +Brand#32 STANDARD ANODIZED TIN 9 4 +Brand#32 STANDARD ANODIZED TIN 14 4 +Brand#32 STANDARD ANODIZED TIN 19 4 +Brand#32 STANDARD BRUSHED NICKEL 23 4 +Brand#32 STANDARD BURNISHED BRASS 36 4 +Brand#32 STANDARD BURNISHED BRASS 45 4 +Brand#32 STANDARD BURNISHED COPPER 3 4 +Brand#32 STANDARD BURNISHED COPPER 36 4 +Brand#32 STANDARD BURNISHED NICKEL 49 4 +Brand#32 STANDARD BURNISHED STEEL 49 4 +Brand#32 STANDARD BURNISHED TIN 23 4 +Brand#32 STANDARD PLATED BRASS 9 4 +Brand#32 STANDARD PLATED BRASS 45 4 +Brand#32 STANDARD PLATED STEEL 36 4 +Brand#32 STANDARD POLISHED BRASS 14 4 +Brand#32 STANDARD POLISHED COPPER 36 4 +Brand#32 STANDARD POLISHED STEEL 14 4 +Brand#33 ECONOMY ANODIZED BRASS 23 4 +Brand#33 ECONOMY ANODIZED COPPER 9 4 +Brand#33 ECONOMY ANODIZED NICKEL 3 4 +Brand#33 ECONOMY ANODIZED NICKEL 9 4 +Brand#33 ECONOMY ANODIZED NICKEL 23 4 +Brand#33 ECONOMY ANODIZED NICKEL 36 4 +Brand#33 ECONOMY BRUSHED BRASS 14 4 +Brand#33 ECONOMY BRUSHED COPPER 23 4 +Brand#33 ECONOMY BURNISHED BRASS 49 4 +Brand#33 ECONOMY BURNISHED COPPER 3 4 +Brand#33 ECONOMY BURNISHED COPPER 14 4 +Brand#33 ECONOMY BURNISHED STEEL 3 4 +Brand#33 ECONOMY BURNISHED TIN 36 4 +Brand#33 ECONOMY BURNISHED TIN 45 4 +Brand#33 ECONOMY PLATED COPPER 19 4 +Brand#33 ECONOMY PLATED COPPER 45 4 +Brand#33 ECONOMY PLATED NICKEL 14 4 +Brand#33 ECONOMY PLATED NICKEL 36 4 +Brand#33 ECONOMY PLATED STEEL 3 4 +Brand#33 ECONOMY PLATED STEEL 23 4 +Brand#33 ECONOMY PLATED STEEL 36 4 +Brand#33 ECONOMY POLISHED BRASS 14 4 +Brand#33 ECONOMY POLISHED NICKEL 19 4 +Brand#33 ECONOMY POLISHED TIN 9 4 +Brand#33 LARGE ANODIZED BRASS 36 4 +Brand#33 LARGE ANODIZED COPPER 19 4 +Brand#33 LARGE ANODIZED COPPER 45 4 +Brand#33 LARGE ANODIZED NICKEL 36 4 +Brand#33 LARGE ANODIZED NICKEL 45 4 +Brand#33 LARGE ANODIZED STEEL 3 4 +Brand#33 LARGE ANODIZED STEEL 45 4 +Brand#33 LARGE ANODIZED TIN 45 4 +Brand#33 LARGE BRUSHED BRASS 3 4 +Brand#33 LARGE BRUSHED BRASS 49 4 +Brand#33 LARGE BRUSHED STEEL 19 4 +Brand#33 LARGE BRUSHED TIN 36 4 +Brand#33 LARGE BURNISHED COPPER 45 4 +Brand#33 LARGE BURNISHED NICKEL 23 4 +Brand#33 LARGE BURNISHED STEEL 19 4 +Brand#33 LARGE PLATED BRASS 3 4 +Brand#33 LARGE PLATED COPPER 19 4 +Brand#33 LARGE PLATED STEEL 3 4 +Brand#33 LARGE PLATED STEEL 19 4 +Brand#33 LARGE PLATED TIN 45 4 +Brand#33 LARGE POLISHED BRASS 45 4 +Brand#33 LARGE POLISHED STEEL 14 4 +Brand#33 LARGE POLISHED STEEL 23 4 +Brand#33 LARGE POLISHED TIN 23 4 +Brand#33 MEDIUM ANODIZED BRASS 3 4 +Brand#33 MEDIUM ANODIZED COPPER 9 4 +Brand#33 MEDIUM ANODIZED COPPER 36 4 +Brand#33 MEDIUM ANODIZED COPPER 49 4 +Brand#33 MEDIUM ANODIZED NICKEL 3 4 +Brand#33 MEDIUM ANODIZED NICKEL 19 4 +Brand#33 MEDIUM BRUSHED BRASS 3 4 +Brand#33 MEDIUM BRUSHED STEEL 19 4 +Brand#33 MEDIUM BRUSHED TIN 14 4 +Brand#33 MEDIUM BURNISHED COPPER 14 4 +Brand#33 MEDIUM BURNISHED COPPER 49 4 +Brand#33 MEDIUM BURNISHED TIN 36 4 +Brand#33 MEDIUM PLATED BRASS 3 4 +Brand#33 MEDIUM PLATED STEEL 3 4 +Brand#33 MEDIUM PLATED STEEL 49 4 +Brand#33 PROMO ANODIZED BRASS 3 4 +Brand#33 PROMO BRUSHED BRASS 49 4 +Brand#33 PROMO BURNISHED COPPER 23 4 +Brand#33 PROMO BURNISHED NICKEL 14 4 +Brand#33 PROMO BURNISHED NICKEL 36 4 +Brand#33 PROMO BURNISHED TIN 19 4 +Brand#33 PROMO BURNISHED TIN 23 4 +Brand#33 PROMO PLATED COPPER 14 4 +Brand#33 PROMO PLATED STEEL 45 4 +Brand#33 PROMO PLATED STEEL 49 4 +Brand#33 PROMO PLATED TIN 49 4 +Brand#33 PROMO POLISHED COPPER 3 4 +Brand#33 PROMO POLISHED STEEL 3 4 +Brand#33 PROMO POLISHED STEEL 9 4 +Brand#33 PROMO POLISHED STEEL 23 4 +Brand#33 SMALL ANODIZED BRASS 19 4 +Brand#33 SMALL ANODIZED COPPER 23 4 +Brand#33 SMALL ANODIZED COPPER 49 4 +Brand#33 SMALL ANODIZED STEEL 9 4 +Brand#33 SMALL BRUSHED BRASS 3 4 +Brand#33 SMALL BRUSHED COPPER 3 4 +Brand#33 SMALL BRUSHED NICKEL 45 4 +Brand#33 SMALL BRUSHED STEEL 3 4 +Brand#33 SMALL BRUSHED TIN 9 4 +Brand#33 SMALL BURNISHED BRASS 19 4 +Brand#33 SMALL BURNISHED NICKEL 3 4 +Brand#33 SMALL PLATED BRASS 3 4 +Brand#33 SMALL PLATED STEEL 14 4 +Brand#33 SMALL PLATED STEEL 45 4 +Brand#33 SMALL PLATED TIN 23 4 +Brand#33 SMALL PLATED TIN 36 4 +Brand#33 SMALL POLISHED NICKEL 23 4 +Brand#33 SMALL POLISHED TIN 19 4 +Brand#33 SMALL POLISHED TIN 23 4 +Brand#33 SMALL POLISHED TIN 45 4 +Brand#33 STANDARD ANODIZED COPPER 49 4 +Brand#33 STANDARD ANODIZED STEEL 14 4 +Brand#33 STANDARD ANODIZED STEEL 45 4 +Brand#33 STANDARD ANODIZED STEEL 49 4 +Brand#33 STANDARD ANODIZED TIN 45 4 +Brand#33 STANDARD BRUSHED BRASS 9 4 +Brand#33 STANDARD BRUSHED NICKEL 45 4 +Brand#33 STANDARD BRUSHED STEEL 9 4 +Brand#33 STANDARD BRUSHED TIN 36 4 +Brand#33 STANDARD BURNISHED BRASS 9 4 +Brand#33 STANDARD BURNISHED BRASS 23 4 +Brand#33 STANDARD BURNISHED NICKEL 49 4 +Brand#33 STANDARD PLATED BRASS 49 4 +Brand#33 STANDARD PLATED COPPER 3 4 +Brand#33 STANDARD PLATED COPPER 14 4 +Brand#33 STANDARD PLATED NICKEL 36 4 +Brand#33 STANDARD PLATED STEEL 3 4 +Brand#33 STANDARD PLATED STEEL 36 4 +Brand#33 STANDARD PLATED TIN 14 4 +Brand#33 STANDARD POLISHED BRASS 9 4 +Brand#33 STANDARD POLISHED BRASS 19 4 +Brand#33 STANDARD POLISHED STEEL 3 4 +Brand#33 STANDARD POLISHED STEEL 9 4 +Brand#33 STANDARD POLISHED STEEL 14 4 +Brand#34 ECONOMY ANODIZED BRASS 9 4 +Brand#34 ECONOMY ANODIZED COPPER 3 4 +Brand#34 ECONOMY ANODIZED COPPER 14 4 +Brand#34 ECONOMY ANODIZED COPPER 19 4 +Brand#34 ECONOMY ANODIZED STEEL 9 4 +Brand#34 ECONOMY ANODIZED TIN 49 4 +Brand#34 ECONOMY BRUSHED BRASS 14 4 +Brand#34 ECONOMY BRUSHED NICKEL 49 4 +Brand#34 ECONOMY BURNISHED COPPER 9 4 +Brand#34 ECONOMY BURNISHED STEEL 19 4 +Brand#34 ECONOMY BURNISHED TIN 3 4 +Brand#34 ECONOMY BURNISHED TIN 23 4 +Brand#34 ECONOMY PLATED BRASS 9 4 +Brand#34 ECONOMY PLATED BRASS 14 4 +Brand#34 ECONOMY PLATED COPPER 3 4 +Brand#34 ECONOMY PLATED NICKEL 45 4 +Brand#34 ECONOMY PLATED TIN 14 4 +Brand#34 ECONOMY PLATED TIN 45 4 +Brand#34 ECONOMY POLISHED BRASS 45 4 +Brand#34 LARGE ANODIZED BRASS 14 4 +Brand#34 LARGE ANODIZED BRASS 23 4 +Brand#34 LARGE ANODIZED BRASS 36 4 +Brand#34 LARGE ANODIZED NICKEL 3 4 +Brand#34 LARGE ANODIZED TIN 49 4 +Brand#34 LARGE BRUSHED BRASS 49 4 +Brand#34 LARGE BRUSHED COPPER 23 4 +Brand#34 LARGE BRUSHED NICKEL 23 4 +Brand#34 LARGE BRUSHED STEEL 14 4 +Brand#34 LARGE BRUSHED STEEL 19 4 +Brand#34 LARGE BRUSHED TIN 9 4 +Brand#34 LARGE BURNISHED BRASS 23 4 +Brand#34 LARGE BURNISHED COPPER 3 4 +Brand#34 LARGE BURNISHED COPPER 36 4 +Brand#34 LARGE BURNISHED NICKEL 19 4 +Brand#34 LARGE PLATED BRASS 23 4 +Brand#34 LARGE PLATED BRASS 36 4 +Brand#34 LARGE PLATED BRASS 45 4 +Brand#34 LARGE PLATED COPPER 23 4 +Brand#34 LARGE PLATED COPPER 49 4 +Brand#34 LARGE PLATED STEEL 49 4 +Brand#34 LARGE POLISHED NICKEL 49 4 +Brand#34 MEDIUM ANODIZED COPPER 36 4 +Brand#34 MEDIUM ANODIZED TIN 3 4 +Brand#34 MEDIUM BRUSHED BRASS 49 4 +Brand#34 MEDIUM BRUSHED COPPER 9 4 +Brand#34 MEDIUM BRUSHED NICKEL 9 4 +Brand#34 MEDIUM BRUSHED NICKEL 23 4 +Brand#34 MEDIUM BRUSHED TIN 3 4 +Brand#34 MEDIUM BRUSHED TIN 14 4 +Brand#34 MEDIUM BURNISHED STEEL 45 4 +Brand#34 MEDIUM BURNISHED STEEL 49 4 +Brand#34 MEDIUM PLATED COPPER 36 4 +Brand#34 MEDIUM PLATED TIN 3 4 +Brand#34 MEDIUM PLATED TIN 14 4 +Brand#34 PROMO ANODIZED COPPER 45 4 +Brand#34 PROMO ANODIZED NICKEL 14 4 +Brand#34 PROMO ANODIZED STEEL 49 4 +Brand#34 PROMO ANODIZED TIN 14 4 +Brand#34 PROMO BRUSHED BRASS 9 4 +Brand#34 PROMO BRUSHED BRASS 23 4 +Brand#34 PROMO BRUSHED COPPER 36 4 +Brand#34 PROMO BRUSHED STEEL 36 4 +Brand#34 PROMO BURNISHED BRASS 49 4 +Brand#34 PROMO BURNISHED STEEL 3 4 +Brand#34 PROMO PLATED BRASS 9 4 +Brand#34 PROMO PLATED STEEL 49 4 +Brand#34 PROMO POLISHED BRASS 23 4 +Brand#34 PROMO POLISHED NICKEL 3 4 +Brand#34 PROMO POLISHED NICKEL 36 4 +Brand#34 SMALL ANODIZED BRASS 36 4 +Brand#34 SMALL ANODIZED COPPER 45 4 +Brand#34 SMALL ANODIZED NICKEL 14 4 +Brand#34 SMALL ANODIZED NICKEL 36 4 +Brand#34 SMALL ANODIZED STEEL 3 4 +Brand#34 SMALL ANODIZED STEEL 19 4 +Brand#34 SMALL ANODIZED STEEL 23 4 +Brand#34 SMALL ANODIZED STEEL 36 4 +Brand#34 SMALL BRUSHED BRASS 14 4 +Brand#34 SMALL BRUSHED BRASS 36 4 +Brand#34 SMALL BRUSHED NICKEL 14 4 +Brand#34 SMALL BRUSHED NICKEL 36 4 +Brand#34 SMALL BRUSHED NICKEL 45 4 +Brand#34 SMALL BRUSHED TIN 9 4 +Brand#34 SMALL BRUSHED TIN 23 4 +Brand#34 SMALL BRUSHED TIN 36 4 +Brand#34 SMALL BURNISHED COPPER 9 4 +Brand#34 SMALL BURNISHED TIN 36 4 +Brand#34 SMALL PLATED BRASS 14 4 +Brand#34 SMALL PLATED COPPER 36 4 +Brand#34 SMALL PLATED TIN 45 4 +Brand#34 SMALL POLISHED NICKEL 14 4 +Brand#34 SMALL POLISHED NICKEL 45 4 +Brand#34 SMALL POLISHED TIN 9 4 +Brand#34 SMALL POLISHED TIN 14 4 +Brand#34 SMALL POLISHED TIN 19 4 +Brand#34 STANDARD ANODIZED BRASS 23 4 +Brand#34 STANDARD ANODIZED BRASS 36 4 +Brand#34 STANDARD ANODIZED COPPER 45 4 +Brand#34 STANDARD ANODIZED NICKEL 36 4 +Brand#34 STANDARD ANODIZED STEEL 9 4 +Brand#34 STANDARD ANODIZED STEEL 49 4 +Brand#34 STANDARD ANODIZED TIN 9 4 +Brand#34 STANDARD BRUSHED BRASS 19 4 +Brand#34 STANDARD BRUSHED BRASS 23 4 +Brand#34 STANDARD BRUSHED NICKEL 23 4 +Brand#34 STANDARD BRUSHED STEEL 3 4 +Brand#34 STANDARD BRUSHED TIN 19 4 +Brand#34 STANDARD BURNISHED COPPER 45 4 +Brand#34 STANDARD BURNISHED NICKEL 19 4 +Brand#34 STANDARD BURNISHED NICKEL 45 4 +Brand#34 STANDARD BURNISHED STEEL 36 4 +Brand#34 STANDARD BURNISHED TIN 45 4 +Brand#34 STANDARD PLATED BRASS 9 4 +Brand#34 STANDARD PLATED COPPER 9 4 +Brand#34 STANDARD PLATED NICKEL 36 4 +Brand#35 ECONOMY ANODIZED COPPER 3 4 +Brand#35 ECONOMY ANODIZED STEEL 45 4 +Brand#35 ECONOMY BRUSHED BRASS 3 4 +Brand#35 ECONOMY BRUSHED NICKEL 49 4 +Brand#35 ECONOMY BRUSHED STEEL 23 4 +Brand#35 ECONOMY BRUSHED STEEL 45 4 +Brand#35 ECONOMY BRUSHED TIN 14 4 +Brand#35 ECONOMY BRUSHED TIN 23 4 +Brand#35 ECONOMY BURNISHED NICKEL 19 4 +Brand#35 ECONOMY BURNISHED STEEL 36 4 +Brand#35 ECONOMY BURNISHED TIN 9 4 +Brand#35 ECONOMY BURNISHED TIN 19 4 +Brand#35 ECONOMY BURNISHED TIN 49 4 +Brand#35 ECONOMY POLISHED COPPER 9 4 +Brand#35 ECONOMY POLISHED TIN 19 4 +Brand#35 LARGE ANODIZED BRASS 3 4 +Brand#35 LARGE ANODIZED BRASS 23 4 +Brand#35 LARGE ANODIZED COPPER 49 4 +Brand#35 LARGE ANODIZED STEEL 36 4 +Brand#35 LARGE ANODIZED TIN 9 4 +Brand#35 LARGE BRUSHED COPPER 9 4 +Brand#35 LARGE BRUSHED COPPER 23 4 +Brand#35 LARGE BRUSHED STEEL 3 4 +Brand#35 LARGE BRUSHED STEEL 9 4 +Brand#35 LARGE BURNISHED BRASS 36 4 +Brand#35 LARGE BURNISHED BRASS 45 4 +Brand#35 LARGE BURNISHED COPPER 23 4 +Brand#35 LARGE BURNISHED NICKEL 23 4 +Brand#35 LARGE PLATED BRASS 9 4 +Brand#35 LARGE PLATED COPPER 36 4 +Brand#35 LARGE POLISHED BRASS 49 4 +Brand#35 LARGE POLISHED STEEL 9 4 +Brand#35 LARGE POLISHED TIN 14 4 +Brand#35 MEDIUM ANODIZED BRASS 9 4 +Brand#35 MEDIUM ANODIZED BRASS 36 4 +Brand#35 MEDIUM ANODIZED COPPER 9 4 +Brand#35 MEDIUM BRUSHED BRASS 14 4 +Brand#35 MEDIUM BRUSHED COPPER 9 4 +Brand#35 MEDIUM BRUSHED COPPER 36 4 +Brand#35 MEDIUM BURNISHED BRASS 49 4 +Brand#35 MEDIUM BURNISHED NICKEL 45 4 +Brand#35 MEDIUM BURNISHED TIN 36 4 +Brand#35 MEDIUM PLATED BRASS 23 4 +Brand#35 MEDIUM PLATED COPPER 9 4 +Brand#35 MEDIUM PLATED NICKEL 45 4 +Brand#35 MEDIUM PLATED NICKEL 49 4 +Brand#35 MEDIUM PLATED STEEL 49 4 +Brand#35 PROMO ANODIZED COPPER 49 4 +Brand#35 PROMO ANODIZED NICKEL 19 4 +Brand#35 PROMO ANODIZED NICKEL 23 4 +Brand#35 PROMO ANODIZED TIN 3 4 +Brand#35 PROMO ANODIZED TIN 14 4 +Brand#35 PROMO BRUSHED BRASS 49 4 +Brand#35 PROMO BRUSHED NICKEL 14 4 +Brand#35 PROMO BRUSHED NICKEL 19 4 +Brand#35 PROMO BURNISHED BRASS 3 4 +Brand#35 PROMO BURNISHED STEEL 3 4 +Brand#35 PROMO PLATED BRASS 19 4 +Brand#35 PROMO PLATED COPPER 14 4 +Brand#35 PROMO PLATED STEEL 23 4 +Brand#35 PROMO PLATED STEEL 36 4 +Brand#35 PROMO PLATED TIN 19 4 +Brand#35 PROMO POLISHED BRASS 9 4 +Brand#35 PROMO POLISHED BRASS 36 4 +Brand#35 PROMO POLISHED NICKEL 36 4 +Brand#35 PROMO POLISHED STEEL 23 4 +Brand#35 PROMO POLISHED TIN 36 4 +Brand#35 PROMO POLISHED TIN 45 4 +Brand#35 SMALL ANODIZED COPPER 9 4 +Brand#35 SMALL ANODIZED STEEL 19 4 +Brand#35 SMALL ANODIZED TIN 19 4 +Brand#35 SMALL BRUSHED BRASS 36 4 +Brand#35 SMALL BRUSHED STEEL 49 4 +Brand#35 SMALL BRUSHED TIN 3 4 +Brand#35 SMALL BRUSHED TIN 19 4 +Brand#35 SMALL BRUSHED TIN 23 4 +Brand#35 SMALL BURNISHED BRASS 23 4 +Brand#35 SMALL BURNISHED STEEL 36 4 +Brand#35 SMALL BURNISHED TIN 3 4 +Brand#35 SMALL BURNISHED TIN 36 4 +Brand#35 SMALL BURNISHED TIN 49 4 +Brand#35 SMALL PLATED BRASS 23 4 +Brand#35 SMALL PLATED STEEL 14 4 +Brand#35 SMALL POLISHED BRASS 36 4 +Brand#35 SMALL POLISHED STEEL 3 4 +Brand#35 SMALL POLISHED STEEL 49 4 +Brand#35 SMALL POLISHED TIN 23 4 +Brand#35 SMALL POLISHED TIN 45 4 +Brand#35 STANDARD ANODIZED NICKEL 14 4 +Brand#35 STANDARD ANODIZED STEEL 23 4 +Brand#35 STANDARD ANODIZED STEEL 45 4 +Brand#35 STANDARD ANODIZED TIN 9 4 +Brand#35 STANDARD ANODIZED TIN 19 4 +Brand#35 STANDARD BRUSHED BRASS 3 4 +Brand#35 STANDARD BRUSHED BRASS 23 4 +Brand#35 STANDARD BRUSHED BRASS 36 4 +Brand#35 STANDARD BRUSHED COPPER 36 4 +Brand#35 STANDARD BRUSHED NICKEL 36 4 +Brand#35 STANDARD BRUSHED NICKEL 49 4 +Brand#35 STANDARD BRUSHED TIN 9 4 +Brand#35 STANDARD BURNISHED BRASS 9 4 +Brand#35 STANDARD BURNISHED BRASS 19 4 +Brand#35 STANDARD BURNISHED BRASS 23 4 +Brand#35 STANDARD BURNISHED COPPER 36 4 +Brand#35 STANDARD BURNISHED STEEL 14 4 +Brand#35 STANDARD PLATED COPPER 19 4 +Brand#35 STANDARD PLATED NICKEL 23 4 +Brand#35 STANDARD PLATED STEEL 14 4 +Brand#35 STANDARD PLATED STEEL 23 4 +Brand#35 STANDARD PLATED TIN 49 4 +Brand#35 STANDARD POLISHED NICKEL 23 4 +Brand#35 STANDARD POLISHED TIN 23 4 +Brand#35 STANDARD POLISHED TIN 45 4 +Brand#41 ECONOMY ANODIZED STEEL 49 4 +Brand#41 ECONOMY BRUSHED BRASS 3 4 +Brand#41 ECONOMY BRUSHED COPPER 36 4 +Brand#41 ECONOMY BRUSHED NICKEL 23 4 +Brand#41 ECONOMY BRUSHED STEEL 36 4 +Brand#41 ECONOMY BRUSHED STEEL 45 4 +Brand#41 ECONOMY BRUSHED TIN 14 4 +Brand#41 ECONOMY PLATED COPPER 3 4 +Brand#41 ECONOMY PLATED STEEL 3 4 +Brand#41 ECONOMY PLATED TIN 23 4 +Brand#41 ECONOMY POLISHED COPPER 19 4 +Brand#41 ECONOMY POLISHED NICKEL 9 4 +Brand#41 ECONOMY POLISHED NICKEL 14 4 +Brand#41 ECONOMY POLISHED NICKEL 23 4 +Brand#41 ECONOMY POLISHED NICKEL 49 4 +Brand#41 ECONOMY POLISHED STEEL 9 4 +Brand#41 ECONOMY POLISHED STEEL 19 4 +Brand#41 ECONOMY POLISHED STEEL 45 4 +Brand#41 ECONOMY POLISHED TIN 19 4 +Brand#41 LARGE ANODIZED BRASS 14 4 +Brand#41 LARGE ANODIZED BRASS 23 4 +Brand#41 LARGE ANODIZED COPPER 49 4 +Brand#41 LARGE ANODIZED STEEL 3 4 +Brand#41 LARGE ANODIZED STEEL 23 4 +Brand#41 LARGE BRUSHED COPPER 23 4 +Brand#41 LARGE BRUSHED COPPER 49 4 +Brand#41 LARGE BRUSHED STEEL 19 4 +Brand#41 LARGE BURNISHED BRASS 45 4 +Brand#41 LARGE BURNISHED COPPER 3 4 +Brand#41 LARGE BURNISHED NICKEL 23 4 +Brand#41 LARGE BURNISHED TIN 9 4 +Brand#41 LARGE PLATED NICKEL 3 4 +Brand#41 LARGE PLATED NICKEL 23 4 +Brand#41 LARGE PLATED STEEL 9 4 +Brand#41 LARGE PLATED STEEL 36 4 +Brand#41 LARGE PLATED TIN 9 4 +Brand#41 LARGE POLISHED BRASS 36 4 +Brand#41 LARGE POLISHED COPPER 19 4 +Brand#41 LARGE POLISHED COPPER 49 4 +Brand#41 LARGE POLISHED NICKEL 36 4 +Brand#41 LARGE POLISHED STEEL 14 4 +Brand#41 MEDIUM ANODIZED BRASS 9 4 +Brand#41 MEDIUM ANODIZED COPPER 14 4 +Brand#41 MEDIUM ANODIZED NICKEL 3 4 +Brand#41 MEDIUM ANODIZED NICKEL 9 4 +Brand#41 MEDIUM ANODIZED STEEL 14 4 +Brand#41 MEDIUM BRUSHED COPPER 3 4 +Brand#41 MEDIUM BRUSHED TIN 9 4 +Brand#41 MEDIUM BURNISHED COPPER 23 4 +Brand#41 MEDIUM BURNISHED STEEL 9 4 +Brand#41 MEDIUM BURNISHED STEEL 45 4 +Brand#41 MEDIUM BURNISHED TIN 3 4 +Brand#41 MEDIUM PLATED BRASS 19 4 +Brand#41 MEDIUM PLATED BRASS 45 4 +Brand#41 MEDIUM PLATED COPPER 19 4 +Brand#41 MEDIUM PLATED STEEL 19 4 +Brand#41 MEDIUM PLATED STEEL 23 4 +Brand#41 PROMO ANODIZED BRASS 19 4 +Brand#41 PROMO ANODIZED COPPER 9 4 +Brand#41 PROMO ANODIZED NICKEL 9 4 +Brand#41 PROMO BRUSHED BRASS 14 4 +Brand#41 PROMO BRUSHED COPPER 36 4 +Brand#41 PROMO BRUSHED NICKEL 14 4 +Brand#41 PROMO BURNISHED BRASS 49 4 +Brand#41 PROMO BURNISHED NICKEL 36 4 +Brand#41 PROMO BURNISHED TIN 3 4 +Brand#41 PROMO PLATED NICKEL 14 4 +Brand#41 PROMO PLATED NICKEL 45 4 +Brand#41 PROMO PLATED STEEL 3 4 +Brand#41 PROMO PLATED TIN 3 4 +Brand#41 PROMO POLISHED COPPER 23 4 +Brand#41 SMALL ANODIZED BRASS 3 4 +Brand#41 SMALL ANODIZED BRASS 14 4 +Brand#41 SMALL ANODIZED STEEL 45 4 +Brand#41 SMALL ANODIZED TIN 9 4 +Brand#41 SMALL BRUSHED TIN 19 4 +Brand#41 SMALL BURNISHED COPPER 9 4 +Brand#41 SMALL BURNISHED NICKEL 3 4 +Brand#41 SMALL BURNISHED TIN 45 4 +Brand#41 SMALL PLATED COPPER 14 4 +Brand#41 SMALL PLATED COPPER 36 4 +Brand#41 SMALL PLATED COPPER 49 4 +Brand#41 SMALL PLATED TIN 19 4 +Brand#41 SMALL POLISHED COPPER 14 4 +Brand#41 SMALL POLISHED COPPER 19 4 +Brand#41 SMALL POLISHED COPPER 36 4 +Brand#41 SMALL POLISHED TIN 45 4 +Brand#41 STANDARD ANODIZED COPPER 19 4 +Brand#41 STANDARD ANODIZED NICKEL 9 4 +Brand#41 STANDARD ANODIZED STEEL 49 4 +Brand#41 STANDARD ANODIZED TIN 9 4 +Brand#41 STANDARD ANODIZED TIN 36 4 +Brand#41 STANDARD ANODIZED TIN 49 4 +Brand#41 STANDARD BRUSHED BRASS 19 4 +Brand#41 STANDARD BRUSHED NICKEL 3 4 +Brand#41 STANDARD BRUSHED NICKEL 9 4 +Brand#41 STANDARD BRUSHED STEEL 45 4 +Brand#41 STANDARD BRUSHED TIN 45 4 +Brand#41 STANDARD BURNISHED BRASS 23 4 +Brand#41 STANDARD BURNISHED BRASS 36 4 +Brand#41 STANDARD BURNISHED COPPER 49 4 +Brand#41 STANDARD BURNISHED STEEL 45 4 +Brand#41 STANDARD PLATED BRASS 45 4 +Brand#41 STANDARD PLATED NICKEL 14 4 +Brand#41 STANDARD PLATED STEEL 45 4 +Brand#41 STANDARD PLATED TIN 49 4 +Brand#41 STANDARD POLISHED STEEL 9 4 +Brand#41 STANDARD POLISHED STEEL 19 4 +Brand#41 STANDARD POLISHED TIN 45 4 +Brand#42 ECONOMY ANODIZED NICKEL 19 4 +Brand#42 ECONOMY BRUSHED BRASS 14 4 +Brand#42 ECONOMY BRUSHED COPPER 3 4 +Brand#42 ECONOMY BRUSHED COPPER 14 4 +Brand#42 ECONOMY BRUSHED NICKEL 14 4 +Brand#42 ECONOMY BRUSHED STEEL 14 4 +Brand#42 ECONOMY BRUSHED TIN 19 4 +Brand#42 ECONOMY BRUSHED TIN 49 4 +Brand#42 ECONOMY BURNISHED BRASS 19 4 +Brand#42 ECONOMY BURNISHED COPPER 23 4 +Brand#42 ECONOMY BURNISHED NICKEL 14 4 +Brand#42 ECONOMY BURNISHED TIN 14 4 +Brand#42 ECONOMY PLATED COPPER 23 4 +Brand#42 ECONOMY POLISHED BRASS 3 4 +Brand#42 ECONOMY POLISHED COPPER 9 4 +Brand#42 ECONOMY POLISHED STEEL 9 4 +Brand#42 ECONOMY POLISHED STEEL 36 4 +Brand#42 ECONOMY POLISHED TIN 14 4 +Brand#42 LARGE ANODIZED BRASS 49 4 +Brand#42 LARGE ANODIZED COPPER 14 4 +Brand#42 LARGE ANODIZED COPPER 49 4 +Brand#42 LARGE ANODIZED NICKEL 45 4 +Brand#42 LARGE ANODIZED NICKEL 49 4 +Brand#42 LARGE ANODIZED TIN 45 4 +Brand#42 LARGE BRUSHED BRASS 49 4 +Brand#42 LARGE BURNISHED BRASS 45 4 +Brand#42 LARGE BURNISHED BRASS 49 4 +Brand#42 LARGE BURNISHED COPPER 9 4 +Brand#42 LARGE BURNISHED TIN 9 4 +Brand#42 LARGE PLATED BRASS 45 4 +Brand#42 LARGE PLATED COPPER 9 4 +Brand#42 LARGE PLATED NICKEL 36 4 +Brand#42 LARGE PLATED TIN 23 4 +Brand#42 LARGE POLISHED BRASS 9 4 +Brand#42 LARGE POLISHED NICKEL 3 4 +Brand#42 LARGE POLISHED NICKEL 23 4 +Brand#42 LARGE POLISHED STEEL 9 4 +Brand#42 MEDIUM ANODIZED BRASS 23 4 +Brand#42 MEDIUM ANODIZED COPPER 19 4 +Brand#42 MEDIUM ANODIZED NICKEL 14 4 +Brand#42 MEDIUM ANODIZED NICKEL 19 4 +Brand#42 MEDIUM ANODIZED NICKEL 23 4 +Brand#42 MEDIUM ANODIZED STEEL 9 4 +Brand#42 MEDIUM ANODIZED STEEL 14 4 +Brand#42 MEDIUM ANODIZED STEEL 23 4 +Brand#42 MEDIUM ANODIZED TIN 14 4 +Brand#42 MEDIUM ANODIZED TIN 19 4 +Brand#42 MEDIUM BRUSHED COPPER 45 4 +Brand#42 MEDIUM BRUSHED COPPER 49 4 +Brand#42 MEDIUM BRUSHED STEEL 36 4 +Brand#42 MEDIUM BURNISHED COPPER 49 4 +Brand#42 MEDIUM BURNISHED TIN 3 4 +Brand#42 MEDIUM BURNISHED TIN 49 4 +Brand#42 MEDIUM PLATED NICKEL 45 4 +Brand#42 MEDIUM PLATED STEEL 3 4 +Brand#42 MEDIUM PLATED STEEL 23 4 +Brand#42 MEDIUM PLATED STEEL 45 4 +Brand#42 PROMO ANODIZED NICKEL 3 4 +Brand#42 PROMO ANODIZED NICKEL 19 4 +Brand#42 PROMO ANODIZED STEEL 49 4 +Brand#42 PROMO BRUSHED COPPER 45 4 +Brand#42 PROMO BRUSHED STEEL 19 4 +Brand#42 PROMO BRUSHED TIN 45 4 +Brand#42 PROMO BURNISHED COPPER 45 4 +Brand#42 PROMO BURNISHED NICKEL 3 4 +Brand#42 PROMO BURNISHED STEEL 9 4 +Brand#42 PROMO BURNISHED TIN 49 4 +Brand#42 PROMO PLATED BRASS 45 4 +Brand#42 PROMO PLATED NICKEL 23 4 +Brand#42 PROMO PLATED STEEL 19 4 +Brand#42 PROMO PLATED STEEL 45 4 +Brand#42 PROMO POLISHED COPPER 36 4 +Brand#42 PROMO POLISHED NICKEL 3 4 +Brand#42 SMALL ANODIZED BRASS 23 4 +Brand#42 SMALL ANODIZED COPPER 14 4 +Brand#42 SMALL ANODIZED COPPER 19 4 +Brand#42 SMALL ANODIZED NICKEL 23 4 +Brand#42 SMALL BRUSHED TIN 49 4 +Brand#42 SMALL BURNISHED BRASS 3 4 +Brand#42 SMALL BURNISHED BRASS 36 4 +Brand#42 SMALL BURNISHED COPPER 9 4 +Brand#42 SMALL BURNISHED NICKEL 9 4 +Brand#42 SMALL BURNISHED TIN 9 4 +Brand#42 SMALL PLATED NICKEL 9 4 +Brand#42 SMALL PLATED TIN 36 4 +Brand#42 SMALL POLISHED BRASS 3 4 +Brand#42 SMALL POLISHED COPPER 36 4 +Brand#42 SMALL POLISHED NICKEL 23 4 +Brand#42 SMALL POLISHED STEEL 49 4 +Brand#42 SMALL POLISHED TIN 3 4 +Brand#42 STANDARD ANODIZED BRASS 49 4 +Brand#42 STANDARD ANODIZED COPPER 49 4 +Brand#42 STANDARD ANODIZED NICKEL 36 4 +Brand#42 STANDARD ANODIZED NICKEL 45 4 +Brand#42 STANDARD BRUSHED NICKEL 23 4 +Brand#42 STANDARD BURNISHED NICKEL 49 4 +Brand#42 STANDARD BURNISHED STEEL 3 4 +Brand#42 STANDARD BURNISHED TIN 19 4 +Brand#42 STANDARD PLATED BRASS 19 4 +Brand#42 STANDARD PLATED COPPER 9 4 +Brand#42 STANDARD PLATED NICKEL 45 4 +Brand#42 STANDARD PLATED STEEL 3 4 +Brand#42 STANDARD POLISHED BRASS 36 4 +Brand#42 STANDARD POLISHED BRASS 45 4 +Brand#42 STANDARD POLISHED COPPER 14 4 +Brand#42 STANDARD POLISHED NICKEL 45 4 +Brand#42 STANDARD POLISHED TIN 9 4 +Brand#42 STANDARD POLISHED TIN 19 4 +Brand#42 STANDARD POLISHED TIN 23 4 +Brand#42 STANDARD POLISHED TIN 36 4 +Brand#43 ECONOMY ANODIZED COPPER 19 4 +Brand#43 ECONOMY ANODIZED COPPER 45 4 +Brand#43 ECONOMY ANODIZED NICKEL 3 4 +Brand#43 ECONOMY ANODIZED NICKEL 49 4 +Brand#43 ECONOMY ANODIZED STEEL 23 4 +Brand#43 ECONOMY ANODIZED TIN 49 4 +Brand#43 ECONOMY BRUSHED BRASS 49 4 +Brand#43 ECONOMY BRUSHED COPPER 45 4 +Brand#43 ECONOMY BRUSHED NICKEL 9 4 +Brand#43 ECONOMY BURNISHED NICKEL 9 4 +Brand#43 ECONOMY BURNISHED TIN 19 4 +Brand#43 ECONOMY PLATED COPPER 36 4 +Brand#43 ECONOMY PLATED STEEL 9 4 +Brand#43 ECONOMY PLATED TIN 14 4 +Brand#43 ECONOMY PLATED TIN 19 4 +Brand#43 ECONOMY PLATED TIN 49 4 +Brand#43 ECONOMY POLISHED COPPER 19 4 +Brand#43 ECONOMY POLISHED NICKEL 36 4 +Brand#43 ECONOMY POLISHED TIN 14 4 +Brand#43 ECONOMY POLISHED TIN 45 4 +Brand#43 LARGE ANODIZED BRASS 14 4 +Brand#43 LARGE ANODIZED BRASS 36 4 +Brand#43 LARGE ANODIZED COPPER 45 4 +Brand#43 LARGE BRUSHED COPPER 3 4 +Brand#43 LARGE BRUSHED NICKEL 14 4 +Brand#43 LARGE BRUSHED NICKEL 19 4 +Brand#43 LARGE BRUSHED NICKEL 45 4 +Brand#43 LARGE BRUSHED NICKEL 49 4 +Brand#43 LARGE BURNISHED COPPER 3 4 +Brand#43 LARGE BURNISHED TIN 23 4 +Brand#43 LARGE BURNISHED TIN 45 4 +Brand#43 LARGE PLATED BRASS 45 4 +Brand#43 LARGE PLATED STEEL 14 4 +Brand#43 LARGE PLATED TIN 36 4 +Brand#43 LARGE PLATED TIN 45 4 +Brand#43 LARGE POLISHED BRASS 9 4 +Brand#43 LARGE POLISHED COPPER 9 4 +Brand#43 LARGE POLISHED COPPER 19 4 +Brand#43 LARGE POLISHED STEEL 14 4 +Brand#43 LARGE POLISHED TIN 45 4 +Brand#43 MEDIUM ANODIZED BRASS 14 4 +Brand#43 MEDIUM ANODIZED COPPER 36 4 +Brand#43 MEDIUM ANODIZED COPPER 49 4 +Brand#43 MEDIUM ANODIZED STEEL 19 4 +Brand#43 MEDIUM ANODIZED STEEL 36 4 +Brand#43 MEDIUM BRUSHED BRASS 9 4 +Brand#43 MEDIUM BRUSHED BRASS 49 4 +Brand#43 MEDIUM BRUSHED COPPER 3 4 +Brand#43 MEDIUM BRUSHED NICKEL 9 4 +Brand#43 MEDIUM BRUSHED STEEL 23 4 +Brand#43 MEDIUM BURNISHED COPPER 14 4 +Brand#43 MEDIUM BURNISHED COPPER 45 4 +Brand#43 MEDIUM BURNISHED TIN 23 4 +Brand#43 MEDIUM PLATED BRASS 3 4 +Brand#43 MEDIUM PLATED COPPER 14 4 +Brand#43 MEDIUM PLATED NICKEL 36 4 +Brand#43 MEDIUM PLATED NICKEL 45 4 +Brand#43 MEDIUM PLATED TIN 49 4 +Brand#43 PROMO ANODIZED NICKEL 45 4 +Brand#43 PROMO ANODIZED TIN 14 4 +Brand#43 PROMO BRUSHED NICKEL 14 4 +Brand#43 PROMO BRUSHED STEEL 14 4 +Brand#43 PROMO BRUSHED TIN 45 4 +Brand#43 PROMO BURNISHED BRASS 49 4 +Brand#43 PROMO BURNISHED NICKEL 9 4 +Brand#43 PROMO BURNISHED STEEL 3 4 +Brand#43 PROMO BURNISHED STEEL 36 4 +Brand#43 PROMO BURNISHED TIN 36 4 +Brand#43 PROMO PLATED BRASS 19 4 +Brand#43 PROMO PLATED COPPER 45 4 +Brand#43 PROMO PLATED COPPER 49 4 +Brand#43 PROMO PLATED TIN 3 4 +Brand#43 PROMO POLISHED BRASS 19 4 +Brand#43 PROMO POLISHED BRASS 23 4 +Brand#43 PROMO POLISHED NICKEL 49 4 +Brand#43 PROMO POLISHED STEEL 14 4 +Brand#43 PROMO POLISHED STEEL 19 4 +Brand#43 PROMO POLISHED STEEL 23 4 +Brand#43 PROMO POLISHED STEEL 36 4 +Brand#43 SMALL ANODIZED BRASS 19 4 +Brand#43 SMALL ANODIZED NICKEL 9 4 +Brand#43 SMALL BRUSHED NICKEL 3 4 +Brand#43 SMALL BRUSHED NICKEL 9 4 +Brand#43 SMALL BURNISHED BRASS 49 4 +Brand#43 SMALL BURNISHED STEEL 23 4 +Brand#43 SMALL PLATED BRASS 14 4 +Brand#43 SMALL PLATED BRASS 36 4 +Brand#43 SMALL PLATED COPPER 23 4 +Brand#43 SMALL PLATED COPPER 49 4 +Brand#43 SMALL PLATED NICKEL 36 4 +Brand#43 SMALL PLATED NICKEL 49 4 +Brand#43 SMALL PLATED STEEL 14 4 +Brand#43 SMALL PLATED TIN 49 4 +Brand#43 SMALL POLISHED STEEL 19 4 +Brand#43 STANDARD ANODIZED BRASS 3 4 +Brand#43 STANDARD ANODIZED COPPER 49 4 +Brand#43 STANDARD ANODIZED NICKEL 14 4 +Brand#43 STANDARD BRUSHED TIN 14 4 +Brand#43 STANDARD BURNISHED BRASS 23 4 +Brand#43 STANDARD BURNISHED STEEL 19 4 +Brand#43 STANDARD BURNISHED STEEL 23 4 +Brand#43 STANDARD PLATED BRASS 9 4 +Brand#43 STANDARD PLATED BRASS 19 4 +Brand#43 STANDARD PLATED BRASS 49 4 +Brand#43 STANDARD PLATED COPPER 36 4 +Brand#43 STANDARD PLATED NICKEL 14 4 +Brand#43 STANDARD PLATED NICKEL 19 4 +Brand#43 STANDARD PLATED TIN 14 4 +Brand#43 STANDARD POLISHED BRASS 23 4 +Brand#43 STANDARD POLISHED TIN 9 4 +Brand#44 ECONOMY ANODIZED BRASS 3 4 +Brand#44 ECONOMY ANODIZED BRASS 45 4 +Brand#44 ECONOMY ANODIZED NICKEL 36 4 +Brand#44 ECONOMY ANODIZED STEEL 19 4 +Brand#44 ECONOMY BRUSHED COPPER 23 4 +Brand#44 ECONOMY BRUSHED TIN 49 4 +Brand#44 ECONOMY BURNISHED COPPER 19 4 +Brand#44 ECONOMY BURNISHED STEEL 45 4 +Brand#44 ECONOMY PLATED STEEL 19 4 +Brand#44 ECONOMY PLATED STEEL 23 4 +Brand#44 ECONOMY PLATED TIN 23 4 +Brand#44 ECONOMY POLISHED BRASS 23 4 +Brand#44 ECONOMY POLISHED COPPER 9 4 +Brand#44 ECONOMY POLISHED COPPER 45 4 +Brand#44 ECONOMY POLISHED NICKEL 14 4 +Brand#44 ECONOMY POLISHED NICKEL 23 4 +Brand#44 ECONOMY POLISHED STEEL 49 4 +Brand#44 ECONOMY POLISHED TIN 23 4 +Brand#44 ECONOMY POLISHED TIN 36 4 +Brand#44 LARGE ANODIZED BRASS 19 4 +Brand#44 LARGE ANODIZED TIN 3 4 +Brand#44 LARGE ANODIZED TIN 14 4 +Brand#44 LARGE BRUSHED TIN 3 4 +Brand#44 LARGE BRUSHED TIN 23 4 +Brand#44 LARGE BURNISHED BRASS 23 4 +Brand#44 LARGE BURNISHED BRASS 49 4 +Brand#44 LARGE BURNISHED COPPER 3 4 +Brand#44 LARGE BURNISHED COPPER 19 4 +Brand#44 LARGE BURNISHED COPPER 36 4 +Brand#44 LARGE BURNISHED TIN 14 4 +Brand#44 LARGE PLATED BRASS 9 4 +Brand#44 LARGE PLATED BRASS 49 4 +Brand#44 LARGE PLATED NICKEL 14 4 +Brand#44 LARGE PLATED STEEL 14 4 +Brand#44 LARGE PLATED TIN 19 4 +Brand#44 LARGE PLATED TIN 23 4 +Brand#44 LARGE POLISHED STEEL 23 4 +Brand#44 LARGE POLISHED STEEL 49 4 +Brand#44 MEDIUM ANODIZED COPPER 45 4 +Brand#44 MEDIUM ANODIZED NICKEL 45 4 +Brand#44 MEDIUM BRUSHED BRASS 49 4 +Brand#44 MEDIUM BRUSHED COPPER 3 4 +Brand#44 MEDIUM BRUSHED COPPER 45 4 +Brand#44 MEDIUM BRUSHED STEEL 19 4 +Brand#44 MEDIUM BRUSHED TIN 49 4 +Brand#44 MEDIUM BURNISHED COPPER 45 4 +Brand#44 MEDIUM BURNISHED NICKEL 23 4 +Brand#44 MEDIUM BURNISHED TIN 23 4 +Brand#44 MEDIUM PLATED COPPER 14 4 +Brand#44 PROMO ANODIZED COPPER 23 4 +Brand#44 PROMO ANODIZED STEEL 36 4 +Brand#44 PROMO BRUSHED COPPER 23 4 +Brand#44 PROMO BRUSHED COPPER 36 4 +Brand#44 PROMO BRUSHED TIN 19 4 +Brand#44 PROMO PLATED BRASS 3 4 +Brand#44 PROMO PLATED COPPER 36 4 +Brand#44 PROMO PLATED STEEL 3 4 +Brand#44 PROMO PLATED STEEL 36 4 +Brand#44 PROMO PLATED STEEL 49 4 +Brand#44 PROMO POLISHED BRASS 3 4 +Brand#44 PROMO POLISHED BRASS 19 4 +Brand#44 PROMO POLISHED COPPER 45 4 +Brand#44 PROMO POLISHED STEEL 36 4 +Brand#44 PROMO POLISHED TIN 9 4 +Brand#44 SMALL ANODIZED COPPER 23 4 +Brand#44 SMALL ANODIZED STEEL 23 4 +Brand#44 SMALL ANODIZED TIN 45 4 +Brand#44 SMALL BRUSHED COPPER 14 4 +Brand#44 SMALL BRUSHED STEEL 45 4 +Brand#44 SMALL BURNISHED COPPER 14 4 +Brand#44 SMALL BURNISHED COPPER 49 4 +Brand#44 SMALL BURNISHED NICKEL 14 4 +Brand#44 SMALL BURNISHED STEEL 23 4 +Brand#44 SMALL BURNISHED TIN 49 4 +Brand#44 SMALL PLATED BRASS 36 4 +Brand#44 SMALL PLATED COPPER 19 4 +Brand#44 SMALL PLATED NICKEL 3 4 +Brand#44 SMALL POLISHED COPPER 3 4 +Brand#44 SMALL POLISHED COPPER 49 4 +Brand#44 SMALL POLISHED STEEL 3 4 +Brand#44 STANDARD ANODIZED BRASS 3 4 +Brand#44 STANDARD ANODIZED COPPER 3 4 +Brand#44 STANDARD ANODIZED NICKEL 3 4 +Brand#44 STANDARD ANODIZED NICKEL 36 4 +Brand#44 STANDARD ANODIZED STEEL 14 4 +Brand#44 STANDARD ANODIZED TIN 3 4 +Brand#44 STANDARD ANODIZED TIN 9 4 +Brand#44 STANDARD ANODIZED TIN 36 4 +Brand#44 STANDARD BRUSHED COPPER 36 4 +Brand#44 STANDARD BRUSHED COPPER 45 4 +Brand#44 STANDARD BRUSHED TIN 9 4 +Brand#44 STANDARD BRUSHED TIN 49 4 +Brand#44 STANDARD BURNISHED COPPER 9 4 +Brand#44 STANDARD BURNISHED STEEL 23 4 +Brand#44 STANDARD PLATED BRASS 14 4 +Brand#44 STANDARD PLATED BRASS 23 4 +Brand#44 STANDARD PLATED BRASS 49 4 +Brand#44 STANDARD PLATED COPPER 14 4 +Brand#44 STANDARD POLISHED NICKEL 19 4 +Brand#44 STANDARD POLISHED TIN 9 4 +Brand#51 ECONOMY ANODIZED BRASS 9 4 +Brand#51 ECONOMY ANODIZED BRASS 23 4 +Brand#51 ECONOMY ANODIZED NICKEL 3 4 +Brand#51 ECONOMY ANODIZED NICKEL 23 4 +Brand#51 ECONOMY ANODIZED STEEL 19 4 +Brand#51 ECONOMY ANODIZED STEEL 23 4 +Brand#51 ECONOMY ANODIZED STEEL 49 4 +Brand#51 ECONOMY BRUSHED BRASS 3 4 +Brand#51 ECONOMY BRUSHED BRASS 49 4 +Brand#51 ECONOMY BRUSHED NICKEL 14 4 +Brand#51 ECONOMY BRUSHED STEEL 45 4 +Brand#51 ECONOMY BRUSHED TIN 36 4 +Brand#51 ECONOMY BURNISHED BRASS 14 4 +Brand#51 ECONOMY BURNISHED COPPER 45 4 +Brand#51 ECONOMY PLATED NICKEL 49 4 +Brand#51 ECONOMY PLATED TIN 36 4 +Brand#51 ECONOMY POLISHED COPPER 9 4 +Brand#51 ECONOMY POLISHED STEEL 14 4 +Brand#51 ECONOMY POLISHED STEEL 49 4 +Brand#51 LARGE ANODIZED COPPER 9 4 +Brand#51 LARGE ANODIZED COPPER 49 4 +Brand#51 LARGE ANODIZED NICKEL 14 4 +Brand#51 LARGE ANODIZED STEEL 36 4 +Brand#51 LARGE BRUSHED NICKEL 3 4 +Brand#51 LARGE BRUSHED NICKEL 9 4 +Brand#51 LARGE BURNISHED BRASS 19 4 +Brand#51 LARGE BURNISHED BRASS 36 4 +Brand#51 LARGE BURNISHED COPPER 14 4 +Brand#51 LARGE BURNISHED NICKEL 14 4 +Brand#51 LARGE PLATED BRASS 36 4 +Brand#51 LARGE POLISHED COPPER 14 4 +Brand#51 LARGE POLISHED NICKEL 23 4 +Brand#51 LARGE POLISHED NICKEL 36 4 +Brand#51 LARGE POLISHED STEEL 19 4 +Brand#51 MEDIUM ANODIZED COPPER 9 4 +Brand#51 MEDIUM ANODIZED STEEL 3 4 +Brand#51 MEDIUM BRUSHED BRASS 36 4 +Brand#51 MEDIUM BRUSHED BRASS 45 4 +Brand#51 MEDIUM BRUSHED STEEL 3 4 +Brand#51 MEDIUM BRUSHED TIN 36 4 +Brand#51 MEDIUM BURNISHED NICKEL 3 4 +Brand#51 MEDIUM BURNISHED NICKEL 36 4 +Brand#51 MEDIUM BURNISHED STEEL 14 4 +Brand#51 MEDIUM BURNISHED TIN 9 4 +Brand#51 MEDIUM PLATED STEEL 19 4 +Brand#51 MEDIUM PLATED TIN 3 4 +Brand#51 PROMO ANODIZED NICKEL 14 4 +Brand#51 PROMO ANODIZED STEEL 23 4 +Brand#51 PROMO ANODIZED TIN 19 4 +Brand#51 PROMO BRUSHED BRASS 23 4 +Brand#51 PROMO BRUSHED COPPER 45 4 +Brand#51 PROMO BRUSHED STEEL 45 4 +Brand#51 PROMO BRUSHED TIN 9 4 +Brand#51 PROMO BURNISHED BRASS 19 4 +Brand#51 PROMO BURNISHED BRASS 23 4 +Brand#51 PROMO BURNISHED NICKEL 14 4 +Brand#51 PROMO PLATED BRASS 3 4 +Brand#51 PROMO PLATED BRASS 23 4 +Brand#51 PROMO PLATED TIN 19 4 +Brand#51 PROMO PLATED TIN 23 4 +Brand#51 PROMO POLISHED BRASS 23 4 +Brand#51 PROMO POLISHED COPPER 9 4 +Brand#51 PROMO POLISHED NICKEL 9 4 +Brand#51 PROMO POLISHED STEEL 49 4 +Brand#51 SMALL ANODIZED STEEL 14 4 +Brand#51 SMALL BRUSHED BRASS 23 4 +Brand#51 SMALL BRUSHED TIN 19 4 +Brand#51 SMALL BURNISHED NICKEL 23 4 +Brand#51 SMALL PLATED COPPER 49 4 +Brand#51 SMALL PLATED NICKEL 3 4 +Brand#51 SMALL PLATED NICKEL 14 4 +Brand#51 SMALL PLATED STEEL 45 4 +Brand#51 SMALL POLISHED NICKEL 14 4 +Brand#51 SMALL POLISHED NICKEL 23 4 +Brand#51 SMALL POLISHED STEEL 3 4 +Brand#51 SMALL POLISHED STEEL 19 4 +Brand#51 SMALL POLISHED STEEL 49 4 +Brand#51 STANDARD ANODIZED NICKEL 3 4 +Brand#51 STANDARD ANODIZED NICKEL 49 4 +Brand#51 STANDARD BRUSHED BRASS 3 4 +Brand#51 STANDARD BRUSHED COPPER 3 4 +Brand#51 STANDARD BRUSHED NICKEL 19 4 +Brand#51 STANDARD BRUSHED STEEL 36 4 +Brand#51 STANDARD BURNISHED COPPER 19 4 +Brand#51 STANDARD BURNISHED NICKEL 49 4 +Brand#51 STANDARD BURNISHED STEEL 23 4 +Brand#51 STANDARD BURNISHED STEEL 36 4 +Brand#51 STANDARD BURNISHED TIN 45 4 +Brand#51 STANDARD PLATED BRASS 36 4 +Brand#51 STANDARD PLATED BRASS 49 4 +Brand#51 STANDARD PLATED COPPER 14 4 +Brand#51 STANDARD PLATED COPPER 23 4 +Brand#51 STANDARD POLISHED BRASS 14 4 +Brand#51 STANDARD POLISHED BRASS 45 4 +Brand#51 STANDARD POLISHED STEEL 36 4 +Brand#51 STANDARD POLISHED STEEL 49 4 +Brand#51 STANDARD POLISHED TIN 45 4 +Brand#52 ECONOMY ANODIZED BRASS 14 4 +Brand#52 ECONOMY ANODIZED BRASS 23 4 +Brand#52 ECONOMY ANODIZED COPPER 36 4 +Brand#52 ECONOMY ANODIZED NICKEL 49 4 +Brand#52 ECONOMY ANODIZED STEEL 19 4 +Brand#52 ECONOMY BRUSHED COPPER 49 4 +Brand#52 ECONOMY BURNISHED BRASS 36 4 +Brand#52 ECONOMY BURNISHED COPPER 19 4 +Brand#52 ECONOMY BURNISHED COPPER 45 4 +Brand#52 ECONOMY BURNISHED NICKEL 19 4 +Brand#52 ECONOMY BURNISHED STEEL 36 4 +Brand#52 ECONOMY PLATED TIN 14 4 +Brand#52 ECONOMY PLATED TIN 23 4 +Brand#52 ECONOMY POLISHED BRASS 23 4 +Brand#52 ECONOMY POLISHED BRASS 45 4 +Brand#52 ECONOMY POLISHED NICKEL 36 4 +Brand#52 ECONOMY POLISHED STEEL 49 4 +Brand#52 LARGE ANODIZED COPPER 14 4 +Brand#52 LARGE ANODIZED NICKEL 3 4 +Brand#52 LARGE ANODIZED NICKEL 45 4 +Brand#52 LARGE ANODIZED TIN 45 4 +Brand#52 LARGE BRUSHED COPPER 19 4 +Brand#52 LARGE BRUSHED NICKEL 3 4 +Brand#52 LARGE BRUSHED NICKEL 19 4 +Brand#52 LARGE BRUSHED NICKEL 23 4 +Brand#52 LARGE BRUSHED STEEL 49 4 +Brand#52 LARGE BRUSHED TIN 14 4 +Brand#52 LARGE BURNISHED NICKEL 9 4 +Brand#52 LARGE BURNISHED TIN 23 4 +Brand#52 LARGE BURNISHED TIN 45 4 +Brand#52 LARGE PLATED BRASS 14 4 +Brand#52 LARGE PLATED COPPER 14 4 +Brand#52 LARGE PLATED COPPER 19 4 +Brand#52 LARGE PLATED NICKEL 45 4 +Brand#52 LARGE PLATED STEEL 9 4 +Brand#52 LARGE PLATED TIN 9 4 +Brand#52 LARGE POLISHED NICKEL 19 4 +Brand#52 LARGE POLISHED NICKEL 23 4 +Brand#52 LARGE POLISHED NICKEL 36 4 +Brand#52 LARGE POLISHED TIN 9 4 +Brand#52 MEDIUM ANODIZED COPPER 36 4 +Brand#52 MEDIUM ANODIZED STEEL 14 4 +Brand#52 MEDIUM ANODIZED TIN 3 4 +Brand#52 MEDIUM ANODIZED TIN 49 4 +Brand#52 MEDIUM BRUSHED COPPER 9 4 +Brand#52 MEDIUM BRUSHED NICKEL 9 4 +Brand#52 MEDIUM BRUSHED STEEL 23 4 +Brand#52 MEDIUM BRUSHED STEEL 49 4 +Brand#52 MEDIUM BURNISHED STEEL 23 4 +Brand#52 MEDIUM BURNISHED TIN 45 4 +Brand#52 MEDIUM BURNISHED TIN 49 4 +Brand#52 MEDIUM PLATED BRASS 36 4 +Brand#52 MEDIUM PLATED STEEL 9 4 +Brand#52 MEDIUM PLATED STEEL 49 4 +Brand#52 MEDIUM PLATED TIN 9 4 +Brand#52 MEDIUM PLATED TIN 49 4 +Brand#52 PROMO ANODIZED BRASS 9 4 +Brand#52 PROMO ANODIZED BRASS 23 4 +Brand#52 PROMO ANODIZED BRASS 36 4 +Brand#52 PROMO ANODIZED NICKEL 45 4 +Brand#52 PROMO ANODIZED STEEL 36 4 +Brand#52 PROMO BRUSHED COPPER 3 4 +Brand#52 PROMO BRUSHED NICKEL 3 4 +Brand#52 PROMO BRUSHED NICKEL 49 4 +Brand#52 PROMO BRUSHED STEEL 14 4 +Brand#52 PROMO BRUSHED TIN 3 4 +Brand#52 PROMO BRUSHED TIN 19 4 +Brand#52 PROMO BRUSHED TIN 36 4 +Brand#52 PROMO BURNISHED COPPER 49 4 +Brand#52 PROMO BURNISHED NICKEL 9 4 +Brand#52 PROMO BURNISHED STEEL 9 4 +Brand#52 PROMO BURNISHED STEEL 23 4 +Brand#52 PROMO BURNISHED TIN 19 4 +Brand#52 PROMO BURNISHED TIN 36 4 +Brand#52 PROMO PLATED BRASS 19 4 +Brand#52 PROMO PLATED BRASS 45 4 +Brand#52 PROMO PLATED BRASS 49 4 +Brand#52 PROMO PLATED COPPER 9 4 +Brand#52 PROMO PLATED NICKEL 3 4 +Brand#52 PROMO PLATED NICKEL 23 4 +Brand#52 PROMO POLISHED NICKEL 14 4 +Brand#52 PROMO POLISHED NICKEL 49 4 +Brand#52 PROMO POLISHED TIN 36 4 +Brand#52 SMALL ANODIZED BRASS 3 4 +Brand#52 SMALL ANODIZED BRASS 14 4 +Brand#52 SMALL ANODIZED COPPER 3 4 +Brand#52 SMALL ANODIZED NICKEL 36 4 +Brand#52 SMALL ANODIZED STEEL 9 4 +Brand#52 SMALL ANODIZED STEEL 19 4 +Brand#52 SMALL BRUSHED NICKEL 19 4 +Brand#52 SMALL BRUSHED STEEL 23 4 +Brand#52 SMALL BRUSHED TIN 14 4 +Brand#52 SMALL BRUSHED TIN 19 4 +Brand#52 SMALL BURNISHED NICKEL 14 4 +Brand#52 SMALL BURNISHED NICKEL 49 4 +Brand#52 SMALL BURNISHED TIN 9 4 +Brand#52 SMALL POLISHED BRASS 36 4 +Brand#52 SMALL POLISHED BRASS 49 4 +Brand#52 SMALL POLISHED TIN 45 4 +Brand#52 STANDARD ANODIZED BRASS 45 4 +Brand#52 STANDARD BRUSHED BRASS 23 4 +Brand#52 STANDARD BRUSHED COPPER 14 4 +Brand#52 STANDARD BRUSHED TIN 36 4 +Brand#52 STANDARD BURNISHED BRASS 49 4 +Brand#52 STANDARD BURNISHED STEEL 19 4 +Brand#52 STANDARD BURNISHED TIN 9 4 +Brand#52 STANDARD BURNISHED TIN 19 4 +Brand#52 STANDARD PLATED NICKEL 36 4 +Brand#52 STANDARD PLATED STEEL 36 4 +Brand#52 STANDARD POLISHED BRASS 36 4 +Brand#52 STANDARD POLISHED COPPER 45 4 +Brand#52 STANDARD POLISHED STEEL 19 4 +Brand#52 STANDARD POLISHED TIN 19 4 +Brand#53 ECONOMY ANODIZED BRASS 45 4 +Brand#53 ECONOMY ANODIZED COPPER 9 4 +Brand#53 ECONOMY ANODIZED NICKEL 3 4 +Brand#53 ECONOMY ANODIZED NICKEL 19 4 +Brand#53 ECONOMY ANODIZED STEEL 45 4 +Brand#53 ECONOMY ANODIZED TIN 14 4 +Brand#53 ECONOMY ANODIZED TIN 36 4 +Brand#53 ECONOMY BRUSHED TIN 45 4 +Brand#53 ECONOMY BURNISHED BRASS 14 4 +Brand#53 ECONOMY BURNISHED COPPER 45 4 +Brand#53 ECONOMY BURNISHED NICKEL 3 4 +Brand#53 ECONOMY BURNISHED NICKEL 49 4 +Brand#53 ECONOMY BURNISHED TIN 45 4 +Brand#53 ECONOMY PLATED BRASS 3 4 +Brand#53 ECONOMY PLATED NICKEL 14 4 +Brand#53 ECONOMY PLATED STEEL 23 4 +Brand#53 ECONOMY PLATED STEEL 36 4 +Brand#53 ECONOMY POLISHED TIN 36 4 +Brand#53 LARGE ANODIZED NICKEL 49 4 +Brand#53 LARGE ANODIZED STEEL 19 4 +Brand#53 LARGE BRUSHED COPPER 3 4 +Brand#53 LARGE BRUSHED COPPER 14 4 +Brand#53 LARGE BRUSHED NICKEL 23 4 +Brand#53 LARGE BRUSHED NICKEL 36 4 +Brand#53 LARGE BRUSHED TIN 36 4 +Brand#53 LARGE BURNISHED BRASS 45 4 +Brand#53 LARGE BURNISHED COPPER 19 4 +Brand#53 LARGE BURNISHED COPPER 36 4 +Brand#53 LARGE BURNISHED NICKEL 23 4 +Brand#53 LARGE BURNISHED STEEL 19 4 +Brand#53 LARGE BURNISHED STEEL 23 4 +Brand#53 LARGE PLATED BRASS 9 4 +Brand#53 LARGE PLATED BRASS 45 4 +Brand#53 LARGE PLATED BRASS 49 4 +Brand#53 LARGE PLATED COPPER 23 4 +Brand#53 LARGE PLATED NICKEL 23 4 +Brand#53 LARGE PLATED NICKEL 49 4 +Brand#53 LARGE PLATED STEEL 49 4 +Brand#53 LARGE PLATED TIN 14 4 +Brand#53 LARGE POLISHED COPPER 49 4 +Brand#53 LARGE POLISHED STEEL 36 4 +Brand#53 LARGE POLISHED TIN 9 4 +Brand#53 MEDIUM ANODIZED BRASS 23 4 +Brand#53 MEDIUM ANODIZED STEEL 14 4 +Brand#53 MEDIUM ANODIZED STEEL 36 4 +Brand#53 MEDIUM ANODIZED TIN 3 4 +Brand#53 MEDIUM ANODIZED TIN 9 4 +Brand#53 MEDIUM BRUSHED BRASS 3 4 +Brand#53 MEDIUM BRUSHED COPPER 3 4 +Brand#53 MEDIUM BRUSHED NICKEL 14 4 +Brand#53 MEDIUM BRUSHED NICKEL 36 4 +Brand#53 MEDIUM BRUSHED NICKEL 49 4 +Brand#53 MEDIUM BRUSHED STEEL 45 4 +Brand#53 MEDIUM BURNISHED BRASS 3 4 +Brand#53 MEDIUM BURNISHED BRASS 36 4 +Brand#53 MEDIUM BURNISHED TIN 9 4 +Brand#53 MEDIUM BURNISHED TIN 14 4 +Brand#53 MEDIUM BURNISHED TIN 36 4 +Brand#53 MEDIUM PLATED BRASS 23 4 +Brand#53 MEDIUM PLATED COPPER 14 4 +Brand#53 MEDIUM PLATED NICKEL 45 4 +Brand#53 MEDIUM PLATED TIN 19 4 +Brand#53 MEDIUM PLATED TIN 45 4 +Brand#53 PROMO ANODIZED BRASS 36 4 +Brand#53 PROMO ANODIZED NICKEL 3 4 +Brand#53 PROMO ANODIZED NICKEL 19 4 +Brand#53 PROMO BRUSHED BRASS 45 4 +Brand#53 PROMO BRUSHED COPPER 3 4 +Brand#53 PROMO BRUSHED COPPER 23 4 +Brand#53 PROMO BRUSHED COPPER 45 4 +Brand#53 PROMO BURNISHED BRASS 23 4 +Brand#53 PROMO BURNISHED BRASS 36 4 +Brand#53 PROMO BURNISHED NICKEL 23 4 +Brand#53 PROMO BURNISHED STEEL 23 4 +Brand#53 PROMO BURNISHED STEEL 49 4 +Brand#53 PROMO PLATED TIN 19 4 +Brand#53 PROMO PLATED TIN 23 4 +Brand#53 PROMO PLATED TIN 36 4 +Brand#53 PROMO POLISHED STEEL 23 4 +Brand#53 PROMO POLISHED TIN 3 4 +Brand#53 SMALL ANODIZED COPPER 23 4 +Brand#53 SMALL ANODIZED COPPER 36 4 +Brand#53 SMALL ANODIZED COPPER 49 4 +Brand#53 SMALL ANODIZED NICKEL 36 4 +Brand#53 SMALL BRUSHED BRASS 36 4 +Brand#53 SMALL BRUSHED COPPER 3 4 +Brand#53 SMALL BRUSHED TIN 3 4 +Brand#53 SMALL BRUSHED TIN 36 4 +Brand#53 SMALL BURNISHED BRASS 9 4 +Brand#53 SMALL BURNISHED BRASS 49 4 +Brand#53 SMALL BURNISHED COPPER 19 4 +Brand#53 SMALL BURNISHED COPPER 45 4 +Brand#53 SMALL PLATED BRASS 9 4 +Brand#53 SMALL PLATED COPPER 3 4 +Brand#53 SMALL PLATED NICKEL 14 4 +Brand#53 SMALL POLISHED NICKEL 19 4 +Brand#53 SMALL POLISHED STEEL 36 4 +Brand#53 SMALL POLISHED TIN 23 4 +Brand#53 STANDARD ANODIZED BRASS 14 4 +Brand#53 STANDARD ANODIZED NICKEL 9 4 +Brand#53 STANDARD ANODIZED NICKEL 23 4 +Brand#53 STANDARD ANODIZED NICKEL 45 4 +Brand#53 STANDARD ANODIZED STEEL 45 4 +Brand#53 STANDARD BRUSHED COPPER 3 4 +Brand#53 STANDARD BRUSHED NICKEL 23 4 +Brand#53 STANDARD BRUSHED TIN 14 4 +Brand#53 STANDARD BURNISHED NICKEL 49 4 +Brand#53 STANDARD BURNISHED STEEL 9 4 +Brand#53 STANDARD PLATED BRASS 36 4 +Brand#53 STANDARD PLATED COPPER 45 4 +Brand#53 STANDARD PLATED NICKEL 36 4 +Brand#53 STANDARD PLATED STEEL 3 4 +Brand#53 STANDARD PLATED STEEL 49 4 +Brand#53 STANDARD PLATED TIN 23 4 +Brand#53 STANDARD POLISHED STEEL 3 4 +Brand#54 ECONOMY ANODIZED BRASS 9 4 +Brand#54 ECONOMY ANODIZED BRASS 45 4 +Brand#54 ECONOMY ANODIZED COPPER 9 4 +Brand#54 ECONOMY ANODIZED STEEL 19 4 +Brand#54 ECONOMY BRUSHED BRASS 45 4 +Brand#54 ECONOMY BRUSHED NICKEL 19 4 +Brand#54 ECONOMY BRUSHED STEEL 3 4 +Brand#54 ECONOMY BRUSHED TIN 19 4 +Brand#54 ECONOMY BURNISHED BRASS 45 4 +Brand#54 ECONOMY BURNISHED COPPER 14 4 +Brand#54 ECONOMY BURNISHED NICKEL 9 4 +Brand#54 ECONOMY BURNISHED NICKEL 36 4 +Brand#54 ECONOMY BURNISHED STEEL 36 4 +Brand#54 ECONOMY BURNISHED TIN 9 4 +Brand#54 ECONOMY BURNISHED TIN 14 4 +Brand#54 ECONOMY BURNISHED TIN 23 4 +Brand#54 ECONOMY PLATED TIN 23 4 +Brand#54 ECONOMY POLISHED BRASS 9 4 +Brand#54 ECONOMY POLISHED BRASS 19 4 +Brand#54 ECONOMY POLISHED COPPER 23 4 +Brand#54 ECONOMY POLISHED STEEL 23 4 +Brand#54 ECONOMY POLISHED TIN 3 4 +Brand#54 LARGE ANODIZED BRASS 14 4 +Brand#54 LARGE ANODIZED BRASS 49 4 +Brand#54 LARGE ANODIZED TIN 9 4 +Brand#54 LARGE BRUSHED BRASS 14 4 +Brand#54 LARGE BRUSHED STEEL 9 4 +Brand#54 LARGE BRUSHED STEEL 23 4 +Brand#54 LARGE BRUSHED TIN 14 4 +Brand#54 LARGE BURNISHED BRASS 49 4 +Brand#54 LARGE BURNISHED COPPER 19 4 +Brand#54 LARGE BURNISHED NICKEL 14 4 +Brand#54 LARGE BURNISHED TIN 14 4 +Brand#54 LARGE PLATED BRASS 19 4 +Brand#54 LARGE PLATED BRASS 23 4 +Brand#54 LARGE POLISHED BRASS 19 4 +Brand#54 LARGE POLISHED BRASS 23 4 +Brand#54 LARGE POLISHED NICKEL 3 4 +Brand#54 LARGE POLISHED NICKEL 14 4 +Brand#54 LARGE POLISHED STEEL 19 4 +Brand#54 LARGE POLISHED TIN 3 4 +Brand#54 LARGE POLISHED TIN 9 4 +Brand#54 LARGE POLISHED TIN 36 4 +Brand#54 MEDIUM ANODIZED NICKEL 9 4 +Brand#54 MEDIUM ANODIZED NICKEL 14 4 +Brand#54 MEDIUM ANODIZED NICKEL 36 4 +Brand#54 MEDIUM BRUSHED NICKEL 9 4 +Brand#54 MEDIUM BRUSHED NICKEL 19 4 +Brand#54 MEDIUM BURNISHED STEEL 3 4 +Brand#54 MEDIUM BURNISHED STEEL 19 4 +Brand#54 MEDIUM BURNISHED STEEL 23 4 +Brand#54 MEDIUM PLATED BRASS 3 4 +Brand#54 MEDIUM PLATED NICKEL 45 4 +Brand#54 PROMO ANODIZED NICKEL 45 4 +Brand#54 PROMO BRUSHED BRASS 3 4 +Brand#54 PROMO BRUSHED STEEL 23 4 +Brand#54 PROMO BRUSHED TIN 14 4 +Brand#54 PROMO BURNISHED COPPER 49 4 +Brand#54 PROMO BURNISHED TIN 9 4 +Brand#54 PROMO PLATED BRASS 14 4 +Brand#54 PROMO PLATED NICKEL 3 4 +Brand#54 PROMO PLATED STEEL 19 4 +Brand#54 PROMO PLATED TIN 23 4 +Brand#54 PROMO PLATED TIN 49 4 +Brand#54 PROMO POLISHED BRASS 3 4 +Brand#54 PROMO POLISHED NICKEL 9 4 +Brand#54 PROMO POLISHED TIN 49 4 +Brand#54 SMALL ANODIZED COPPER 49 4 +Brand#54 SMALL ANODIZED NICKEL 9 4 +Brand#54 SMALL ANODIZED NICKEL 36 4 +Brand#54 SMALL ANODIZED TIN 19 4 +Brand#54 SMALL BRUSHED BRASS 14 4 +Brand#54 SMALL BRUSHED BRASS 19 4 +Brand#54 SMALL BRUSHED BRASS 36 4 +Brand#54 SMALL BRUSHED COPPER 3 4 +Brand#54 SMALL BRUSHED COPPER 9 4 +Brand#54 SMALL BRUSHED COPPER 19 4 +Brand#54 SMALL BRUSHED TIN 9 4 +Brand#54 SMALL BRUSHED TIN 36 4 +Brand#54 SMALL BURNISHED COPPER 9 4 +Brand#54 SMALL BURNISHED COPPER 36 4 +Brand#54 SMALL BURNISHED STEEL 14 4 +Brand#54 SMALL BURNISHED STEEL 19 4 +Brand#54 SMALL BURNISHED TIN 9 4 +Brand#54 SMALL BURNISHED TIN 36 4 +Brand#54 SMALL PLATED BRASS 23 4 +Brand#54 SMALL PLATED COPPER 9 4 +Brand#54 SMALL PLATED COPPER 36 4 +Brand#54 SMALL PLATED COPPER 49 4 +Brand#54 SMALL PLATED NICKEL 9 4 +Brand#54 SMALL PLATED TIN 23 4 +Brand#54 SMALL PLATED TIN 36 4 +Brand#54 SMALL POLISHED BRASS 9 4 +Brand#54 SMALL POLISHED COPPER 9 4 +Brand#54 SMALL POLISHED TIN 9 4 +Brand#54 STANDARD ANODIZED BRASS 3 4 +Brand#54 STANDARD ANODIZED BRASS 9 4 +Brand#54 STANDARD ANODIZED COPPER 3 4 +Brand#54 STANDARD ANODIZED TIN 3 4 +Brand#54 STANDARD BRUSHED COPPER 3 4 +Brand#54 STANDARD BRUSHED NICKEL 45 4 +Brand#54 STANDARD BRUSHED TIN 36 4 +Brand#54 STANDARD BURNISHED BRASS 23 4 +Brand#54 STANDARD BURNISHED BRASS 49 4 +Brand#54 STANDARD BURNISHED COPPER 19 4 +Brand#54 STANDARD BURNISHED NICKEL 23 4 +Brand#54 STANDARD BURNISHED STEEL 45 4 +Brand#54 STANDARD PLATED BRASS 3 4 +Brand#54 STANDARD PLATED BRASS 45 4 +Brand#54 STANDARD PLATED BRASS 49 4 +Brand#54 STANDARD PLATED STEEL 3 4 +Brand#54 STANDARD POLISHED BRASS 36 4 +Brand#54 STANDARD POLISHED STEEL 3 4 +Brand#54 STANDARD POLISHED STEEL 14 4 +Brand#54 STANDARD POLISHED STEEL 45 4 +Brand#55 ECONOMY ANODIZED BRASS 3 4 +Brand#55 ECONOMY BRUSHED BRASS 19 4 +Brand#55 ECONOMY BRUSHED COPPER 9 4 +Brand#55 ECONOMY BRUSHED COPPER 23 4 +Brand#55 ECONOMY BRUSHED COPPER 45 4 +Brand#55 ECONOMY BRUSHED STEEL 23 4 +Brand#55 ECONOMY BURNISHED NICKEL 36 4 +Brand#55 ECONOMY BURNISHED NICKEL 45 4 +Brand#55 ECONOMY BURNISHED TIN 45 4 +Brand#55 ECONOMY PLATED NICKEL 19 4 +Brand#55 ECONOMY POLISHED NICKEL 9 4 +Brand#55 LARGE BRUSHED BRASS 23 4 +Brand#55 LARGE BRUSHED BRASS 45 4 +Brand#55 LARGE BRUSHED COPPER 49 4 +Brand#55 LARGE BRUSHED NICKEL 9 4 +Brand#55 LARGE BRUSHED NICKEL 14 4 +Brand#55 LARGE BURNISHED BRASS 3 4 +Brand#55 LARGE BURNISHED COPPER 14 4 +Brand#55 LARGE BURNISHED COPPER 36 4 +Brand#55 LARGE PLATED BRASS 45 4 +Brand#55 LARGE PLATED COPPER 19 4 +Brand#55 LARGE PLATED NICKEL 9 4 +Brand#55 LARGE PLATED STEEL 9 4 +Brand#55 LARGE PLATED TIN 9 4 +Brand#55 LARGE PLATED TIN 14 4 +Brand#55 LARGE PLATED TIN 23 4 +Brand#55 LARGE POLISHED NICKEL 3 4 +Brand#55 LARGE POLISHED STEEL 36 4 +Brand#55 LARGE POLISHED STEEL 45 4 +Brand#55 MEDIUM ANODIZED COPPER 9 4 +Brand#55 MEDIUM BRUSHED BRASS 3 4 +Brand#55 MEDIUM BRUSHED NICKEL 23 4 +Brand#55 MEDIUM BRUSHED TIN 45 4 +Brand#55 MEDIUM BURNISHED BRASS 23 4 +Brand#55 MEDIUM BURNISHED COPPER 36 4 +Brand#55 MEDIUM BURNISHED NICKEL 3 4 +Brand#55 MEDIUM BURNISHED STEEL 14 4 +Brand#55 MEDIUM BURNISHED STEEL 36 4 +Brand#55 MEDIUM PLATED NICKEL 23 4 +Brand#55 PROMO ANODIZED COPPER 14 4 +Brand#55 PROMO ANODIZED COPPER 49 4 +Brand#55 PROMO ANODIZED STEEL 36 4 +Brand#55 PROMO ANODIZED TIN 23 4 +Brand#55 PROMO BRUSHED NICKEL 36 4 +Brand#55 PROMO BRUSHED STEEL 3 4 +Brand#55 PROMO BRUSHED STEEL 36 4 +Brand#55 PROMO BRUSHED TIN 9 4 +Brand#55 PROMO BURNISHED COPPER 3 4 +Brand#55 PROMO BURNISHED STEEL 14 4 +Brand#55 PROMO BURNISHED TIN 23 4 +Brand#55 PROMO BURNISHED TIN 49 4 +Brand#55 PROMO PLATED COPPER 3 4 +Brand#55 PROMO PLATED NICKEL 3 4 +Brand#55 PROMO PLATED NICKEL 14 4 +Brand#55 PROMO PLATED NICKEL 23 4 +Brand#55 PROMO PLATED TIN 3 4 +Brand#55 PROMO POLISHED COPPER 3 4 +Brand#55 SMALL ANODIZED BRASS 19 4 +Brand#55 SMALL ANODIZED NICKEL 45 4 +Brand#55 SMALL BRUSHED COPPER 14 4 +Brand#55 SMALL BRUSHED COPPER 45 4 +Brand#55 SMALL BURNISHED BRASS 14 4 +Brand#55 SMALL BURNISHED TIN 3 4 +Brand#55 SMALL BURNISHED TIN 49 4 +Brand#55 SMALL PLATED BRASS 45 4 +Brand#55 SMALL PLATED COPPER 23 4 +Brand#55 SMALL PLATED COPPER 36 4 +Brand#55 SMALL PLATED COPPER 45 4 +Brand#55 SMALL PLATED COPPER 49 4 +Brand#55 SMALL PLATED NICKEL 9 4 +Brand#55 SMALL PLATED STEEL 9 4 +Brand#55 SMALL PLATED TIN 14 4 +Brand#55 SMALL PLATED TIN 36 4 +Brand#55 SMALL POLISHED NICKEL 45 4 +Brand#55 SMALL POLISHED STEEL 19 4 +Brand#55 SMALL POLISHED TIN 19 4 +Brand#55 STANDARD ANODIZED BRASS 36 4 +Brand#55 STANDARD ANODIZED BRASS 49 4 +Brand#55 STANDARD ANODIZED STEEL 19 4 +Brand#55 STANDARD ANODIZED TIN 36 4 +Brand#55 STANDARD ANODIZED TIN 49 4 +Brand#55 STANDARD BRUSHED BRASS 36 4 +Brand#55 STANDARD BRUSHED COPPER 3 4 +Brand#55 STANDARD BRUSHED COPPER 9 4 +Brand#55 STANDARD BRUSHED COPPER 23 4 +Brand#55 STANDARD BRUSHED STEEL 19 4 +Brand#55 STANDARD BRUSHED TIN 23 4 +Brand#55 STANDARD BRUSHED TIN 45 4 +Brand#55 STANDARD BURNISHED BRASS 19 4 +Brand#55 STANDARD BURNISHED NICKEL 3 4 +Brand#55 STANDARD BURNISHED NICKEL 36 4 +Brand#55 STANDARD BURNISHED STEEL 19 4 +Brand#55 STANDARD PLATED BRASS 23 4 +Brand#55 STANDARD PLATED NICKEL 9 4 +Brand#55 STANDARD PLATED TIN 36 4 +Brand#55 STANDARD POLISHED BRASS 3 4 +Brand#55 STANDARD POLISHED BRASS 49 4 +Brand#55 STANDARD POLISHED COPPER 19 4 +Brand#55 STANDARD POLISHED COPPER 36 4 +Brand#55 STANDARD POLISHED NICKEL 14 4 +Brand#55 STANDARD POLISHED STEEL 9 4 +Brand#55 STANDARD POLISHED STEEL 36 4 +Brand#12 LARGE BURNISHED NICKEL 14 3 +Brand#12 PROMO POLISHED TIN 3 3 +Brand#21 MEDIUM ANODIZED TIN 9 3 +Brand#22 PROMO BRUSHED BRASS 19 3 +Brand#22 PROMO BURNISHED COPPER 14 3 +Brand#43 STANDARD BRUSHED BRASS 23 3 +Brand#44 MEDIUM ANODIZED NICKEL 9 3 +Brand#53 MEDIUM BURNISHED BRASS 49 3 + diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy new file mode 100644 index 00000000000000..d4843eee09450c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ + // prepare test table + def timeout = 300000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(10000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + def load_json_data = {table_name, file_name -> + // load the json data + streamLoad { + table "${table_name}" + + // set http request header params + set 'read_json_by_line', 'true' + set 'format', 'json' + set 'max_filter_ratio', '0.1' + set 'memtable_on_sink_node', 'true' + file file_name // import json file + time 10000 // limit inflight 10s + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + logger.info("Stream load ${file_name} result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + // assertEquals(json.NumberTotalRows, json.NumberLoadedRows + json.NumberUnselectedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + + def table_name = "github_events" + sql """DROP TABLE IF EXISTS ${table_name}""" + table_name = "github_events" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + -- INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english") COMMENT '' + ) + UNIQUE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 4 + properties("replication_num" = "1", "disable_auto_compaction" = "true", "bloom_filter_columns" = "v", "variant_enable_flatten_nested" = "true", "inverted_index_storage_format"= "v2", "enable_unique_key_merge_on_write" = "false"); + """ + // 2015 + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-2.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-3.json'}""") + + // // build inverted index at middle of loading the data + // ADD INDEX + + // 2022 + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-16.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-10.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-22.json'}""") + load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""") + + + + // // add bloom filter at the end of loading data +} diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.sql new file mode 100644 index 00000000000000..36046e892e0e1f --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs1.sql @@ -0,0 +1,19 @@ +insert into github_events select * from github_events; +alter table github_events ADD COLUMN var2 variant; + +SELECT + cast(v["repo"]["name"] as string), + count() AS prs, + count(distinct cast(v["actor"]["login"] as string)) AS authors +FROM github_events +WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["actor"]["login"] as string) IN +( + SELECT cast(v["actor"]["login"] as string) + FROM github_events + WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string)= 'opened') AND (cast(v["repo"]["name"] as string) IN ('rspec/rspec-core', 'golden-warning/giraffedraft-server', 'apache/spark')) +)) AND (lower(cast(v["repo"]["name"] as string)) NOT LIKE '%clickhouse%') +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY authors DESC, prs DESC, cast(v["repo"]["name"] as string) DESC +LIMIT 50; + +alter table github_events DROP COLUMN var2; diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.sql new file mode 100644 index 00000000000000..e9158f8246b532 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/affinityByIssuesAndPRs2.sql @@ -0,0 +1,14 @@ +SELECT + cast(v["repo"]["name"] as string), + count() AS prs, + count(distinct cast(v["actor"]["login"] as string)) AS authors +FROM github_events +WHERE (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["actor"]["login"] as string) IN +( + SELECT cast(v["actor"]["login"] as string) + FROM github_events + WHERE (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["repo"]["name"] as string) IN ('No-CQRT/GooGuns', 'ivolunteerph/ivolunteerph', 'Tribler/tribler')) +)) AND (lower(cast(v["repo"]["name"] as string)) NOT LIKE '%clickhouse%') +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY authors DESC, prs DESC, cast(v["repo"]["name"] as string) ASC +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.sql new file mode 100644 index 00000000000000..2e8bf32009814e --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/authorsWithTheMostPushes.sql @@ -0,0 +1,9 @@ +SELECT + cast(v["actor"]["login"] as string), + count() AS c, + count(distinct cast(v["repo"]["name"] as string)) AS repos + FROM github_events + WHERE cast(v["type"] as string) = 'PushEvent' + GROUP BY cast(v["actor"]["login"] as string) + ORDER BY c DESC, 1, 3 + LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.sql new file mode 100644 index 00000000000000..8b2d615acd052e --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar1.sql @@ -0,0 +1 @@ +SELECT count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.sql new file mode 100644 index 00000000000000..057a410e0caeeb --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar2.sql @@ -0,0 +1 @@ +SELECT cast(v["payload"]["action"] as string), count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["payload"]["action"] as string) \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.sql new file mode 100644 index 00000000000000..9b5f41288901d7 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/countingStar3.sql @@ -0,0 +1 @@ +SELECT count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["repo"]["name"] as string) IN ('apache/spark', 'GunZi200/Memory-Colour', 'isohuntto/openbay', 'wasabeef/awesome-android-ui') GROUP BY cast(v["payload"]["action"] as string) \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.sql new file mode 100644 index 00000000000000..e2d987afe6510b --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/distributionOfRepositoriesByStarCount.sql @@ -0,0 +1,14 @@ +SELECT + pow(10, floor(log10(c))) AS stars, + count(distinct k) +FROM +( + SELECT + cast(v["repo"]["name"] as string) as k, + count() AS c + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' + GROUP BY cast(v["repo"]["name"] as string) +) t +GROUP BY stars +ORDER BY stars ASC diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.sql new file mode 100644 index 00000000000000..0b9ea42f77d8ea --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/githubRoulette.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string) FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' ORDER BY cast(v["created_at"] as datetime), cast(v["repo"]["name"] as string) LIMIT 50 \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql new file mode 100644 index 00000000000000..95389cb9a0651c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears1.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql new file mode 100644 index 00000000000000..95389cb9a0651c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears2.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql new file mode 100644 index 00000000000000..67b5d0d3d18969 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears3.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars, cast(v["repo"]["name"] as string) DESC LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql new file mode 100644 index 00000000000000..95389cb9a0651c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears4.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql new file mode 100644 index 00000000000000..95389cb9a0651c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears5.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql new file mode 100644 index 00000000000000..95389cb9a0651c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears6.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND year(cast(v["created_at"] as datetime)) = '2015' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql new file mode 100644 index 00000000000000..ee2fbef43b3f53 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheListOfTopRepositoriesChangedOverTheYears7.sql @@ -0,0 +1,30 @@ +-- FIXME: UNSTABLE +-- SELECT +-- repo, +-- year, +-- cnt +-- FROM +-- ( +-- SELECT +-- row_number() OVER (PARTITION BY year ORDER BY cnt DESC) AS r, +-- repo, +-- year, +-- cnt +-- FROM +-- ( +-- SELECT +-- lower(cast(v["repo"]["name"] as string)) AS repo, +-- year(cast(v["created_at"] as datetime)) AS year, +-- count() AS cnt +-- FROM github_events +-- WHERE (cast(v["type"] as string) = 'WatchEvent') AND (year(cast(v["created_at"] as datetime)) >= 2015) +-- GROUP BY +-- repo, +-- year +-- ) t +-- ) t2 +-- WHERE r <= 10 +-- ORDER BY +-- year ASC, +-- cnt DESC, repo +-- \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql new file mode 100644 index 00000000000000..5291ab5100649a --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/howHasTheTotalNumberOfStarsChangedOverTime.sql @@ -0,0 +1,2 @@ +SELECT year(cast(v["created_at"] as datetime)) AS year, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY year ORDER BY year + diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.sql new file mode 100644 index 00000000000000..0117055d53e7f6 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments1.sql @@ -0,0 +1 @@ +SELECT count() FROM github_events WHERE cast(v["type"] as string) = 'IssueCommentEvent' diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.sql new file mode 100644 index 00000000000000..25e96fe731ebc7 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments2.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() FROM github_events WHERE cast(v["type"] as string) = 'IssueCommentEvent' GROUP BY cast(v["repo"]["name"] as string) ORDER BY count() DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.sql new file mode 100644 index 00000000000000..c32210845e3a7b --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments3.sql @@ -0,0 +1,17 @@ +SELECT + repo_name, + comments, + issues, + round(comments / issues, 2) AS ratio +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + count() AS comments, + count(distinct cast(v["payload"]["issue"]["number"] as int)) AS issues + FROM github_events + WHERE cast(v["type"] as string) = 'IssueCommentEvent' + GROUP BY cast(v["repo"]["name"] as string) +) t +ORDER BY comments DESC, 1, 3, 4 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.sql new file mode 100644 index 00000000000000..ad8e5c104ccfc2 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments4.sql @@ -0,0 +1,9 @@ +SELECT + cast(v["repo"]["name"] as string), + cast(v["payload"]["issue"]["number"] as int) as number, + count() AS comments +FROM github_events +WHERE cast(v["type"] as string) = 'IssueCommentEvent' AND (cast(v["payload"]["action"] as string) = 'created') +GROUP BY cast(v["repo"]["name"] as string), number +ORDER BY comments DESC, number ASC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.sql new file mode 100644 index 00000000000000..0520ed0b8dc768 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments5.sql @@ -0,0 +1,9 @@ +SELECT + cast(v["repo"]["name"] as string), + cast(v["payload"]["issue"]["number"] as int) as number, + count() AS comments +FROM github_events +WHERE cast(v["type"] as string) = 'IssueCommentEvent' AND (cast(v["payload"]["action"] as string) = 'created') AND (cast(v["payload"]["issue"]["number"] as int) > 10) +GROUP BY cast(v["repo"]["name"] as string), number +ORDER BY comments DESC, cast(v["repo"]["name"] as string), number +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.sql new file mode 100644 index 00000000000000..edab4d50e26300 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments6.sql @@ -0,0 +1,11 @@ +SELECT + cast(v["repo"]["name"] as string), + cast(v["payload"]["issue"]["number"] as int) as number, + count() AS comments, + count(distinct cast(v["actor"]["login"] as string)) AS authors +FROM github_events +WHERE cast(v["type"] as string) = 'IssueCommentEvent' AND (cast(v["payload"]["action"] as string) = 'created') AND (cast(v["payload"]["issue"]["number"] as int) > 10) +GROUP BY cast(v["repo"]["name"] as string), number +HAVING authors >= 4 +ORDER BY comments DESC, cast(v["repo"]["name"] as string) +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.sql new file mode 100644 index 00000000000000..047f2d7a1d48fe --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments7.sql @@ -0,0 +1,9 @@ +SELECT + cast(v["repo"]["name"] as string), + count() AS comments, + count(distinct cast(v["actor"]["login"] as string)) AS authors +FROM github_events +WHERE cast(v["type"] as string) = 'CommitCommentEvent' +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY count() DESC, 1, 3 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments8.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments8.sql new file mode 100644 index 00000000000000..e6890c1e23505f --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/issuesWithTheMostComments8.sql @@ -0,0 +1,13 @@ +-- SELECT +-- concat('https://github.com/', cast(v["repo"]["name"] as string), '/commit/', cast(v["payload"]["commit_id"] as string)) URL, +-- cast(v["payload"]["commit_id"] as string) AS commit_id, +-- count() AS comments, +-- count(distinct cast(v["actor"]["login"] as string)) AS authors +-- FROM github_events +-- WHERE (cast(v["type"] as string) = 'CommitCommentEvent') AND commit_id != "" +-- GROUP BY +-- cast(v["repo"]["name"] as string), +-- commit_id +-- HAVING authors >= 10 +-- ORDER BY count() DESC, URL, authors +-- LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.sql new file mode 100644 index 00000000000000..aab10cb79335a8 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostForkedRepositories.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS forks FROM github_events WHERE cast(v["type"] as string) = 'ForkEvent' GROUP BY cast(v["repo"]["name"] as string) ORDER BY forks DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.sql new file mode 100644 index 00000000000000..c36efe7561b7e8 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/mostPopularCommentsOnGithub.sql @@ -0,0 +1 @@ +SELECT cast(v["payload"]["comment"]["body"] as string), count() FROM github_events WHERE cast(v["payload"]["comment"]["body"] as string) != "" AND length(cast(v["payload"]["comment"]["body"] as string)) < 100 GROUP BY cast(v["payload"]["comment"]["body"] as string) ORDER BY count(), 1 DESC LIMIT 50 \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.sql new file mode 100644 index 00000000000000..d943c07de6a45f --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfRepositories.sql @@ -0,0 +1,14 @@ +SELECT + lower(split_part(repo_name, '/', 1)) AS org, + count(distinct repo_name) AS repos +FROM +( + SELECT cast(v["repo"]["name"] as string) as repo_name + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' + GROUP BY cast(v["repo"]["name"] as string) + HAVING count() >= 10 +) t +GROUP BY org +ORDER BY repos DESC, org ASC +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.sql new file mode 100644 index 00000000000000..22f0a8cbba706e --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheNumberOfStars.sql @@ -0,0 +1,8 @@ +SELECT + lower(split_part(cast(v["repo"]["name"] as string), '/', 1)) AS org, + count() AS stars +FROM github_events +WHERE cast(v["type"] as string) = 'WatchEvent' +GROUP BY org +ORDER BY stars DESC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheSizeOfCommunity.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheSizeOfCommunity.sql new file mode 100644 index 00000000000000..e445e7db3e9657 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/organizationsByTheSizeOfCommunity.sql @@ -0,0 +1,23 @@ +-- SELECT +-- lower(split_part(cast(v["repo"]["name"] as string), '/', 1)) AS org, +-- count(distinct cast(v["actor"]["login"] as string)) AS authors, +-- count(distinct pr_author) AS pr_authors, +-- count(distinct issue_author) AS issue_authors, +-- count(distinct comment_author) AS comment_authors, +-- count(distinct review_author) AS review_authors, +-- count(distinct push_author) AS push_authors +-- FROM +-- ( +-- SELECT +-- cast(v["repo"]["name"] as string), +-- cast(v["actor"]["login"] as string), +-- CASE WHEN cast(v["type"] as string) = 'PullRequestEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END pr_author, +-- CASE WHEN cast(v["type"] as string) = 'IssuesEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END issue_author, +-- CASE WHEN cast(v["type"] as string) = 'IssueCommentEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END comment_author, +-- CASE WHEN cast(v["type"] as string) = 'PullRequestReviewCommentEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END review_author, +-- CASE WHEN cast(v["type"] as string) = 'PushEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END push_author +-- FROM github_events +-- WHERE cast(v["type"] as string) IN ('PullRequestEvent', 'IssuesEvent', 'IssueCommentEvent', 'PullRequestReviewCommentEvent', 'PushEvent') +-- ) t +-- GROUP BY org +-- ORDER BY authors DESC diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.sql new file mode 100644 index 00000000000000..65ab889b2511a6 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks1.sql @@ -0,0 +1,17 @@ +SELECT + repo_name, + sum(fork) AS forks, + sum(star) AS stars, + round(sum(star) / sum(fork), 3) AS ratio +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events + WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent') +) t +GROUP BY repo_name +ORDER BY forks DESC, 1, 3, 4 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.sql new file mode 100644 index 00000000000000..ffff95dfdc64c0 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks2.sql @@ -0,0 +1,18 @@ +SELECT + repo_name, + sum(fork) AS forks, + sum(star) AS stars, + round(sum(star) / sum(fork), 3) AS ratio +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events + WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent') +) t +GROUP BY repo_name +HAVING (stars > 20) AND (forks >= 10) +ORDER BY ratio DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.sql new file mode 100644 index 00000000000000..f47c5163302b9c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks3.sql @@ -0,0 +1,18 @@ +SELECT + repo_name, + sum(fork) AS forks, + sum(star) AS stars, + round(sum(fork) / sum(star), 2) AS ratio +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events + WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent') +) t +GROUP BY repo_name +HAVING (stars > 4) AND (forks > 4) +ORDER BY ratio DESC +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.sql new file mode 100644 index 00000000000000..66c67db1b86f7e --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks4.sql @@ -0,0 +1,13 @@ +SELECT + sum(fork) AS forks, + sum(star) AS stars, + round(sum(star) / sum(fork), 2) AS ratio +FROM +( + SELECT + cast(v["repo"]["name"] as string), + CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events + WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent') +) t diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.sql new file mode 100644 index 00000000000000..3579b794114e08 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/proportionsBetweenStarsAndForks5.sql @@ -0,0 +1,21 @@ +SELECT + sum(forks) AS forks, + sum(stars) AS stars, + round(sum(stars) / sum(forks), 2) AS ratio +FROM +( + SELECT + sum(fork) AS forks, + sum(star) AS stars + FROM + ( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'ForkEvent' THEN 1 ELSE 0 END AS fork, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events + WHERE cast(v["type"] as string) IN ('ForkEvent', 'WatchEvent') + ) t + GROUP BY repo_name + HAVING stars > 10 +) t2 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.sql new file mode 100644 index 00000000000000..6c1632769f3d9f --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByAmountOfModifiedCode.sql @@ -0,0 +1,12 @@ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false,disable_streaming_preaggregations=false) */ + cast(v["repo"]["name"] as string) as repo_name, + count() AS prs, + count(distinct cast(v["actor"]["login"] as string)) AS authors, + sum(cast(v["payload"]["pull_request"]["additions"] as int)) AS adds, + sum(cast(v["payload"]["pull_request"]["deletions"] as int)) AS dels +FROM github_events +WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') AND (cast(v["payload"]["pull_request"]["additions"] as int) < 10000) AND (cast(v["payload"]["pull_request"]["deletions"] as int) < 10000) +GROUP BY repo_name +HAVING (adds / dels) < 10 +ORDER BY adds + dels DESC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.sql new file mode 100644 index 00000000000000..e37ced4083887d --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesByTheNumberOfPushes.sql @@ -0,0 +1,17 @@ +SELECT + cast(v["repo"]["name"] as string), + count() AS pushes, + count(distinct cast(v["actor"]["login"] as string)) AS authors +FROM github_events +WHERE (cast(v["type"] as string) = 'PushEvent') AND (cast(v["repo"]["name"] as string) IN +( + SELECT cast(v["repo"]["name"] as string) + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' + GROUP BY cast(v["repo"]["name"] as string) + ORDER BY count() DESC + LIMIT 10000 +)) +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY count() DESC, cast(v["repo"]["name"] as string) +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.sql new file mode 100644 index 00000000000000..1dba0577e78d9e --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments1.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() FROM github_events WHERE lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%apache%' GROUP BY cast(v["repo"]["name"] as string) ORDER BY count() DESC, cast(v["repo"]["name"] as string) ASC LIMIT 50 \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.sql new file mode 100644 index 00000000000000..15b5adf3fb2bb8 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithClickhouse_related_comments2.sql @@ -0,0 +1,17 @@ +SELECT + repo_name, + sum(num_star) AS num_stars, + sum(num_comment) AS num_comments +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS num_star, + CASE WHEN lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%apache%' THEN 1 ELSE 0 END AS num_comment + FROM github_events + WHERE (lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%apache%') OR (cast(v["type"] as string) = 'WatchEvent') +) t +GROUP BY repo_name +HAVING num_comments > 0 +ORDER BY num_stars DESC,num_comments DESC, repo_name ASC +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.sql new file mode 100644 index 00000000000000..15f7de0c3b9239 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments1.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() FROM github_events WHERE lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%spark%' GROUP BY cast(v["repo"]["name"] as string) ORDER BY count() DESC, cast(v["repo"]["name"] as string) ASC LIMIT 50 \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.sql new file mode 100644 index 00000000000000..17a055f65495b4 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithDoris_related_comments2.sql @@ -0,0 +1,17 @@ +SELECT + repo_name, + sum(num_star) AS num_stars, + sum(num_comment) AS num_comments +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS num_star, + CASE WHEN lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%spark%' THEN 1 ELSE 0 END AS num_comment + FROM github_events + WHERE (lower(cast(v["payload"]["comment"]["body"] as string)) LIKE '%spark%') OR (cast(v["type"] as string) = 'WatchEvent') +) t +GROUP BY repo_name +HAVING num_comments > 0 +ORDER BY num_stars DESC,num_comments DESC,repo_name ASC +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.sql new file mode 100644 index 00000000000000..b5270ee38f1098 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheHighestGrowthYoY.sql @@ -0,0 +1,20 @@ +SELECT + repo_name, + sum(created_at_2022) AS stars2022, + sum(created_at_2015) AS stars2015, + round(sum(created_at_2022) / sum(created_at_2015), 3) AS yoy, + min(created_at) AS first_seen +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE year(cast(v["created_at"] as datetime)) WHEN 2022 THEN 1 ELSE 0 END AS created_at_2022, + CASE year(cast(v["created_at"] as datetime)) WHEN 2015 THEN 1 ELSE 0 END AS created_at_2015, + cast(v["created_at"] as datetime) as created_at + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' +) t +GROUP BY repo_name +HAVING (min(created_at) <= '2023-01-01 00:00:00') AND (stars2022 >= 1) and (stars2015 >= 1) +ORDER BY yoy DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql new file mode 100644 index 00000000000000..93164a4cfe1d41 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues1.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS c, count(distinct cast(v["actor"]["login"] as string)) AS u FROM github_events WHERE cast(v["type"] as string) = 'IssuesEvent' AND cast(v["payload"]["action"] as string) = 'opened' GROUP BY cast(v["repo"]["name"] as string) ORDER BY c DESC, cast(v["repo"]["name"] as string) LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql new file mode 100644 index 00000000000000..cfddc738371827 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues2.sql @@ -0,0 +1,18 @@ +SELECT + repo_name, + sum(issue_created) AS c, + count(distinct actor_login) AS u, + sum(star) AS stars +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN 1 ELSE 0 END AS issue_created, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star, + CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login + FROM github_events + WHERE cast(v["type"] as string) IN ('IssuesEvent', 'WatchEvent') +) t +GROUP BY repo_name +ORDER BY c DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql new file mode 100644 index 00000000000000..60759a10199068 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues3.sql @@ -0,0 +1,19 @@ +SELECT + repo_name, + sum(issue_created) AS c, + count(distinct actor_login) AS u, + sum(star) AS stars +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN 1 ELSE 0 END AS issue_created, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star, + CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login + FROM github_events + WHERE cast(v["type"] as string) IN ('IssuesEvent', 'WatchEvent') +) t +GROUP BY repo_name +HAVING stars >= 10 +ORDER BY c, u, stars DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql new file mode 100644 index 00000000000000..756e6f6bd95da5 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfIssues4.sql @@ -0,0 +1,18 @@ +SELECT + repo_name, + sum(issue_created) AS c, + count(distinct actor_login) AS u, + sum(star) AS stars +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN 1 ELSE 0 END AS issue_created, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star, + CASE WHEN (cast(v["type"] as string) = 'IssuesEvent') AND (cast(v["payload"]["action"] as string) = 'opened') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login + FROM github_events + WHERE cast(v["type"] as string) IN ('IssuesEvent', 'WatchEvent') +) t +GROUP BY repo_name +ORDER BY u, c, stars DESC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql new file mode 100644 index 00000000000000..f0c505210633a6 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests1.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string) as repo_name, count(), count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'PullRequestEvent' AND cast(v["payload"]["action"] as string) = 'opened' GROUP BY cast(v["repo"]["name"] as string) ORDER BY 2,1,3 DESC LIMIT 50 \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql new file mode 100644 index 00000000000000..bba2a4c898107d --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumAmountOfPullRequests2.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count(), count(distinct cast(v["actor"]["login"] as string)) AS u FROM github_events WHERE cast(v["type"] as string) = 'PullRequestEvent' AND cast(v["payload"]["action"] as string) = 'opened' GROUP BY cast(v["repo"]["name"] as string) ORDER BY u DESC, 2 DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql new file mode 100644 index 00000000000000..36aa448c66acba --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMaximumNumberOfAcceptedInvitations.sql @@ -0,0 +1,17 @@ +SELECT + repo_name, + sum(invitation) AS invitations, + sum(star) AS stars +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE WHEN cast(v["type"] as string) = 'MemberEvent' THEN 1 ELSE 0 END AS invitation, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events + WHERE cast(v["type"] as string) IN ('MemberEvent', 'WatchEvent') +) t +GROUP BY repo_name +HAVING stars >= 2 +ORDER BY invitations DESC, stars DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql new file mode 100644 index 00000000000000..329cfc908b6654 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess1.sql @@ -0,0 +1,13 @@ +SELECT + repo_name, + count(distinct actor_login) AS u, + sum(star) AS stars +FROM +( + SELECT + lower(cast(v["repo"]["name"] as string)) as repo_name, + CASE WHEN cast(v["type"] as string) = 'PushEvent' THEN cast(v["actor"]["login"] as string) ELSE NULL END AS actor_login, + CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star + FROM github_events WHERE cast(v["type"] as string) IN ('PushEvent', 'WatchEvent') AND cast(v["repo"]["name"] as string) != '/' +) t +GROUP BY repo_name ORDER BY u, stars, repo_name DESC LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql new file mode 100644 index 00000000000000..410f69c47a58c9 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess2.sql @@ -0,0 +1,13 @@ +-- SELECT +-- cast(v["repo"]["name"] as string), +-- count(distinct cast(v["actor"]["login"] as string)) AS u, +-- sum(star) AS stars +-- FROM +-- ( +-- SELECT +-- cast(v["repo"]["name"] as string), +-- CASE WHEN cast(v["type"] as string) = 'PushEvent' AND (ref LIKE '%/master' OR ref LIKE '%/main') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS cast(v["actor"]["login"] as string), +-- CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star +-- FROM github_events WHERE cast(v["type"] as string) IN ('PushEvent', 'WatchEvent') AND cast(v["repo"]["name"] as string) != '/' +-- ) t +-- GROUP BY cast(v["repo"]["name"] as string) ORDER BY u DESC LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql new file mode 100644 index 00000000000000..dbcf16db2ac195 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostPeopleWhoHavePushAccess3.sql @@ -0,0 +1,16 @@ +-- SELECT +-- cast(v["repo"]["name"] as string), +-- count(distinct cast(v["actor"]["login"] as string)) AS u, +-- sum(star) AS stars +-- FROM +-- ( +-- SELECT +-- cast(v["repo"]["name"] as string), +-- CASE WHEN cast(v["type"] as string) = 'PushEvent' AND (ref LIKE '%/master' OR ref LIKE '%/main') THEN cast(v["actor"]["login"] as string) ELSE NULL END AS cast(v["actor"]["login"] as string), +-- CASE WHEN cast(v["type"] as string) = 'WatchEvent' THEN 1 ELSE 0 END AS star +-- FROM github_events WHERE cast(v["type"] as string) IN ('PushEvent', 'WatchEvent') AND cast(v["repo"]["name"] as string) != '/' +-- ) t +-- GROUP BY cast(v["repo"]["name"] as string) +-- HAVING stars >= 100 +-- ORDER BY u DESC +-- LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql new file mode 100644 index 00000000000000..1e6fdb4b769348 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay1.sql @@ -0,0 +1,25 @@ +SELECT + repo_name, + day, + stars +FROM +( + SELECT + row_number() OVER (PARTITION BY repo_name ORDER BY stars DESC) AS rank, + repo_name, + day, + stars + FROM + ( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + to_date(cast(v["created_at"] as datetime)) AS day, + count() AS stars + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' + GROUP BY cast(v["repo"]["name"] as string), day + ) t1 +) t2 +WHERE rank = 1 +ORDER BY stars DESC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql new file mode 100644 index 00000000000000..92a1c9a9c6e005 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay2.sql @@ -0,0 +1,25 @@ +-- SELECT +-- cast(v["repo"]["name"] as string), +-- day, +-- stars +-- FROM +-- ( +-- SELECT +-- row_number() OVER (PARTITION BY cast(v["repo"]["name"] as string) ORDER BY stars DESC) AS rank, +-- cast(v["repo"]["name"] as string), +-- day, +-- stars +-- FROM +-- ( +-- SELECT +-- cast(v["repo"]["name"] as string), +-- to_date(cast(v["created_at"] as datetime)) AS day, +-- count() AS stars +-- FROM github_events +-- WHERE cast(v["type"] as string) = 'WatchEvent' +-- GROUP BY cast(v["repo"]["name"] as string), day +-- ) t1 +-- ) t2 +-- WHERE rank = 1 +-- ORDER BY stars DESC +-- LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql new file mode 100644 index 00000000000000..29fd779ffa3b44 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostStarsOverOneDay3.sql @@ -0,0 +1 @@ +-- SELECT cast(v["repo"]["name"] as string), cast(v["created_at"] as datetime), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["repo"]["name"] as string), cast(v["created_at"] as datetime) ORDER BY count() DESC LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql new file mode 100644 index 00000000000000..5d05e3cb8db80c --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheMostSteadyGrowthOverTime.sql @@ -0,0 +1,20 @@ +SELECT + repo_name, + max(stars) AS daily_stars, + sum(stars) AS total_stars, + sum(stars) / max(stars) AS rate +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + to_date(cast(v["created_at"] as datetime)) AS day, + count() AS stars + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' + GROUP BY + repo_name, + day +) t +GROUP BY repo_name +ORDER BY rate DESC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.sql new file mode 100644 index 00000000000000..f0019d643306fc --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoriesWithTheWorstStagnation_order.sql @@ -0,0 +1,20 @@ +SELECT + repo_name, + sum(created_at_2022) AS stars2022, + sum(created_at_2015) AS stars2015, + round(sum(created_at_2022) / sum(created_at_2015), 3) AS yoy, + min(created_at) AS first_seen +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + CASE year(cast(v["created_at"] as datetime)) WHEN 2022 THEN 1 ELSE 0 END AS created_at_2022, + CASE year(cast(v["created_at"] as datetime)) WHEN 2015 THEN 1 ELSE 0 END AS created_at_2015, + cast(v["created_at"] as datetime) as created_at + FROM github_events + WHERE cast(v["type"] as string) = 'WatchEvent' +) t +GROUP BY repo_name +HAVING (min(created_at) <= '2019-01-01 00:00:00') AND (max(created_at) >= '2020-06-01 00:00:00') AND (stars2015 >= 2) +ORDER BY yoy, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.sql new file mode 100644 index 00000000000000..d1c3b8150d6a34 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList1.sql @@ -0,0 +1,13 @@ +SELECT + cast(v["repo"]["name"] as string) as repo_name, + count() AS stars +FROM github_events +WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) IN +( + SELECT cast(v["actor"]["login"] as string) + FROM github_events + WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) IN ('apache/spark', 'prakhar1989/awesome-courses')) +)) AND (cast(v["repo"]["name"] as string) NOT IN ('ClickHouse/ClickHouse', 'yandex/ClickHouse')) +GROUP BY repo_name +ORDER BY stars DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.sql new file mode 100644 index 00000000000000..293f4b90396189 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/repositoryAffinityList2.sql @@ -0,0 +1,23 @@ +SELECT + repo_name, + total_stars, + round(spark_stars / total_stars, 2) AS ratio +FROM +( + SELECT + cast(v["repo"]["name"] as string) as repo_name, + count(distinct cast(v["actor"]["login"] as string)) AS total_stars + FROM github_events + WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) NOT IN ('apache/spark')) + GROUP BY repo_name + HAVING total_stars >= 10 +) t1 +JOIN +( + SELECT + count(distinct cast(v["actor"]["login"] as string)) AS spark_stars + FROM github_events + WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) IN ('apache/spark')) +) t2 +ORDER BY ratio DESC, repo_name +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.sql new file mode 100644 index 00000000000000..ba9f4ab2604644 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers1.sql @@ -0,0 +1,13 @@ +SELECT + cast(v["repo"]["name"] as string), + count() +FROM github_events +WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) IN +( + SELECT cast(v["actor"]["login"] as string) + FROM github_events + WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') +)) +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY count() DESC, cast(v["repo"]["name"] as string) +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.sql new file mode 100644 index 00000000000000..c55c37a294ad6e --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/starsFromHeavyGithubUsers2.sql @@ -0,0 +1,15 @@ +SELECT + cast(v["repo"]["name"] as string), + count() +FROM github_events +WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) IN +( + SELECT cast(v["actor"]["login"] as string) + FROM github_events + WHERE (cast(v["type"] as string) = 'PullRequestEvent') AND (cast(v["payload"]["action"] as string) = 'opened') + GROUP BY cast(v["actor"]["login"] as string) + HAVING count() >= 2 +)) +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY 1, count() DESC, 1 +LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.sql new file mode 100644 index 00000000000000..117dc222667faa --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames1.sql @@ -0,0 +1 @@ +SELECT count(), cast(v["repo"]["name"] as string) FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["repo"]["name"] as string) ORDER BY length(cast(v["repo"]["name"] as string)) DESC, cast(v["repo"]["name"] as string) LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.sql new file mode 100644 index 00000000000000..203b253d8897e9 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theLongestRepositoryNames2.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["repo"]["name"] as string) LIKE '%_/_%' GROUP BY cast(v["repo"]["name"] as string) ORDER BY length(cast(v["repo"]["name"] as string)) ASC, cast(v["repo"]["name"] as string) LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.sql new file mode 100644 index 00000000000000..670a70ff725e63 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theMostToughCodeReviews.sql @@ -0,0 +1,10 @@ +SELECT + concat('https://github.com/', cast(v["repo"]["name"] as string), '/pull/') AS URL, + count(distinct cast(v["actor"]["login"] as string)) AS authors +FROM github_events +WHERE (cast(v["type"] as string) = 'PullRequestReviewCommentEvent') AND (cast(v["payload"]["action"] as string) = 'created') +GROUP BY + cast(v["repo"]["name"] as string), + cast(v["payload"]["issue"]["number"] as string) +ORDER BY authors DESC, URL ASC +LIMIT 50 \ No newline at end of file diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql new file mode 100644 index 00000000000000..9fdc19bfcef5ad --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfRepositoriesOnGithub.sql @@ -0,0 +1 @@ +SELECT count(distinct cast(v["repo"]["name"] as string)) FROM github_events diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.sql new file mode 100644 index 00000000000000..56f9ddf94732b7 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub1.sql @@ -0,0 +1 @@ +SELECT count(distinct cast(v["actor"]["login"] as string)) FROM github_events diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.sql new file mode 100644 index 00000000000000..f2ed81a78bd086 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub2.sql @@ -0,0 +1 @@ +SELECT count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.sql new file mode 100644 index 00000000000000..3a2dd0c08b3431 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub3.sql @@ -0,0 +1 @@ +SELECT count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'PushEvent' diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.sql new file mode 100644 index 00000000000000..59410fa57bc72d --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/theTotalNumberOfUsersOnGithub4.sql @@ -0,0 +1 @@ +SELECT count(distinct cast(v["actor"]["login"] as string)) FROM github_events WHERE cast(v["type"] as string) = 'PullRequestEvent' AND cast(v["payload"]["action"] as string) = 'opened' diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels1.sql new file mode 100644 index 00000000000000..8e4ae36dd63ab3 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels1.sql @@ -0,0 +1,9 @@ +-- SELECT +-- label, +-- count() AS c +-- FROM github_events +-- LATERAL VIEW explode_split(labels, ',') t AS label +-- WHERE (cast(v["type"] as string) IN ('IssuesEvent', 'PullRequestEvent', 'IssueCommentEvent')) AND (action IN ('created', 'opened', 'labeled')) +-- GROUP BY label +-- ORDER BY c DESC +-- LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels2.sql new file mode 100644 index 00000000000000..302e08519a1e02 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels2.sql @@ -0,0 +1,9 @@ +-- SELECT +-- label, +-- count() AS c +-- FROM github_events +-- LATERAL VIEW explode_split(labels, ',') t AS label +-- WHERE (cast(v["type"] as string) IN ('IssuesEvent', 'PullRequestEvent', 'IssueCommentEvent')) AND (action IN ('created', 'opened', 'labeled')) AND ((lower(label) LIKE '%bug%') OR (lower(label) LIKE '%feature%')) +-- GROUP BY label +-- ORDER BY c DESC +-- LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels3.sql new file mode 100644 index 00000000000000..30c6e7ee8be8d0 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topLabels3.sql @@ -0,0 +1,14 @@ +-- SELECT +-- sum(bug) AS bugs, +-- sum(feature) AS feature, +-- sum(bug) / sum(feature) AS ratio +-- FROM +-- ( +-- SELECT +-- CASE WHEN lower(label) LIKE '%bug%' THEN 1 ELSE 0 END AS bug, +-- CASE WHEN lower(label) LIKE '%feature%' THEN 1 ELSE 0 END AS feature +-- FROM github_events +-- LATERAL VIEW explode_split(labels, ',') t AS label +-- WHERE (cast(v["type"] as string) IN ('IssuesEvent', 'PullRequestEvent', 'IssueCommentEvent')) AND (action IN ('created', 'opened', 'labeled')) AND ((lower(label) LIKE '%bug%') OR (lower(label) LIKE '%feature%')) +-- ) t +-- LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.sql new file mode 100644 index 00000000000000..8ca99e182b75d8 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/topRepositoriesByStars.sql @@ -0,0 +1 @@ +SELECT cast(v["repo"]["name"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["repo"]["name"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql new file mode 100644 index 00000000000000..116fe5ca57cc18 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whatIsTheBestDayOfTheWeekToCatchAStar.sql @@ -0,0 +1 @@ +SELECT dayofweek(cast(v["created_at"] as datetime)) AS day, count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY day ORDER BY day diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.sql new file mode 100644 index 00000000000000..445853c2efa15a --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars1.sql @@ -0,0 +1 @@ +SELECT cast(v["actor"]["login"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY cast(v["actor"]["login"] as string) ORDER BY stars DESC, 1 LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.sql new file mode 100644 index 00000000000000..1fc9a6158edf48 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars2.sql @@ -0,0 +1 @@ +SELECT cast(v["actor"]["login"] as string), count() AS stars FROM github_events WHERE cast(v["type"] as string) = 'WatchEvent' AND cast(v["actor"]["login"] as string) = 'cliffordfajardo' GROUP BY cast(v["actor"]["login"] as string) ORDER BY stars DESC LIMIT 50 diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.sql b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.sql new file mode 100644 index 00000000000000..43426a2c9e4b81 --- /dev/null +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/sql/whoAreAllThosePeopleGivingStars3.sql @@ -0,0 +1,13 @@ +SELECT + cast(v["repo"]["name"] as string), + count() AS stars +FROM github_events +WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["repo"]["name"] as string) IN +( + SELECT cast(v["repo"]["name"] as string) + FROM github_events + WHERE (cast(v["type"] as string) = 'WatchEvent') AND (cast(v["actor"]["login"] as string) = 'cliffordfajardo') +)) +GROUP BY cast(v["repo"]["name"] as string) +ORDER BY stars DESC, cast(v["repo"]["name"] as string) +LIMIT 50 diff --git a/regression-test/suites/variant_p2/tpch_upgrade/load.groovy b/regression-test/suites/variant_p2/tpch_upgrade/load.groovy new file mode 100644 index 00000000000000..764b2b82ab6ea7 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/load.groovy @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Most of the cases are copied from https://github.com/trinodb/trino/tree/master +// /testing/trino-product-tests/src/main/resources/sql-tests/testcases +// and modified by Doris. + +// syntax error: +// q06 q13 q15 +// Test 23 suites, failed 3 suites + +// Note: To filter out tables from sql files, use the following one-liner comamnd +// sed -nr 's/.*tables: (.*)$/\1/gp' /path/to/*.sql | sed -nr 's/,/\n/gp' | sort | uniq +suite("load") { + def tables = ["customer", + "lineitem", + "nation", + "orders", + "part", + "partsupp", + "region", + "supplier"] + + tables.forEach { tableName -> + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + k bigint, + var variant + + ) + UNIQUE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 5 + properties("replication_num" = "1", "disable_auto_compaction" = "false", "enable_unique_key_merge_on_write" = "false"); + """ + streamLoad { + // a default db 'regression_test' is specified in + // ${DORIS_HOME}/conf/regression-conf.groovy + table "${tableName}" + + // set http request header params + set 'read_json_by_line', 'true' + set 'format', 'json' + // set 'max_filter_ratio', '0.1' + time 10000 // limit inflight 10s + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/tpch-var/sf0.1/${tableName}.txt.json""" + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + } + // Thread.sleep(70000) // wait for row count report of the tables just loaded + // tables.forEach { tableName -> + // sql """ ANALYZE TABLE $tableName WITH SYNC """ + // } + + // def table = "revenue1" + // sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text + // sql new File("""${context.file.parent}/ddl/${table}.sql""").text + + sql """ sync """ +} diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q01_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q01_trans.sql new file mode 100644 index 00000000000000..0a215033848617 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q01_trans.sql @@ -0,0 +1,32 @@ +-- TABLES: lineitem,customer,nation,orders,part,partsupp,region,supplier + +insert into customer select * from customer; +insert into lineitem select * from lineitem; +insert into nation select * from nation; +insert into orders select * from orders; +insert into part select * from part; +insert into partsupp select * from partsupp; +insert into region select * from region; +insert into supplier select * from supplier; +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(var["L_RETURNFLAG"] AS TEXT), + CAST(var["L_LINESTATUS"] AS TEXT), + SUM(CAST(var["L_QUANTITY"] AS DOUBLE)) AS SUM_QTY, + SUM(CAST(var["L_EXTENDEDPRICE"] AS DOUBLE)) AS SUM_BASE_PRICE, + SUM(CAST(var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(var["L_DISCOUNT"] AS DOUBLE))) AS SUM_DISC_PRICE, + SUM(CAST(var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(var["L_DISCOUNT"] AS DOUBLE)) * (1 + CAST(var["L_TAX"] AS DOUBLE))) AS SUM_CHARGE, + AVG(CAST(var["L_QUANTITY"] AS DOUBLE)) AS AVG_QTY, + AVG(CAST(var["L_EXTENDEDPRICE"] AS DOUBLE)) AS AVG_PRICE, + AVG(CAST(var["L_DISCOUNT"] AS DOUBLE)) AS AVG_DISC, + COUNT(*) AS COUNT_ORDER +FROM + lineitem +WHERE + CAST(var["L_SHIPDATE"] AS DATE) <= DATE '1998-12-01' - INTERVAL '90' DAY +GROUP BY +CAST(var["L_RETURNFLAG"] AS TEXT), +CAST(var["L_LINESTATUS"] AS TEXT) +ORDER BY +CAST(var["L_RETURNFLAG"] AS TEXT), +CAST(var["L_LINESTATUS"] AS TEXT) +; diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q02_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q02_trans.sql new file mode 100644 index 00000000000000..06694cd68b0dd7 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q02_trans.sql @@ -0,0 +1,42 @@ +-- TABLES: part,SUPPLIER,partsupp,NATION,REGION +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(SS.var["S_ACCTBAL"] AS DOUBLE), + CAST(SS.var["S_NAME"] AS TEXT), + CAST(NN.var["N_NAME"] AS TEXT), + CAST(PP.var["P_PARTKEY"] AS INT), + CAST(PP.var["P_MFGR"] AS TEXT), + CAST(SS.var["S_ADDRESS"] AS TEXT), + CAST(SS.var["S_PHONE"] AS TEXT), + CAST(SS.var["S_COMMENT"] AS TEXT) +FROM + part PP, + supplier SS, + partsupp PSPS, + nation NN, + region RR +WHERE + CAST(PP.var["P_PARTKEY"] AS INT) = CAST(PSPS.var["PS_PARTKEY"] AS INT) + AND CAST(SS.var["S_SUPPKEY"] AS INT) = CAST(PSPS.var["PS_SUPPKEY"] AS INT) + AND CAST(PP.var["P_SIZE"] AS INT) = 15 + AND CAST(PP.var["P_TYPE"] AS TEXT) LIKE '%BRASS' + AND CAST(SS.var["S_NATIONKEY"] AS INT) = CAST(NN.var["N_NATIONKEY"] AS INT) + AND CAST(NN.var["N_REGIONKEY"] AS INT) = CAST(RR.var["R_REGIONKEY"] AS INT) + AND CAST(RR.var["R_NAME"] AS TEXT) = 'EUROPE' + AND CAST(PSPS.var["PS_SUPPLYCOST"] AS DOUBLE) >= ( + SELECT MIN(CAST(PS.var["PS_SUPPLYCOST"] AS DOUBLE)) + FROM + partsupp PS, supplier S, part P, + nation N, region R + WHERE + CAST(P.var["P_PARTKEY"] AS INT) = CAST(PS.var["PS_PARTKEY"] AS INT) + AND CAST(S.var["S_SUPPKEY"] AS INT) = CAST(PS.var["PS_SUPPKEY"] AS INT) + AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) + AND CAST(N.var["N_REGIONKEY"] AS INT) = CAST(R.var["R_REGIONKEY"] AS INT) + AND CAST(R.var["R_NAME"] AS TEXT) = 'EUROPE' + ) +ORDER BY + CAST(SS.var["S_ACCTBAL"] AS DOUBLE) DESC, + CAST(NN.var["N_NAME"] AS TEXT), + CAST(SS.var["S_NAME"] AS TEXT), + CAST(PP.var["P_PARTKEY"] AS INT) +LIMIT 100 diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q03_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q03_trans.sql new file mode 100644 index 00000000000000..9450e42fe22da7 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q03_trans.sql @@ -0,0 +1,25 @@ +-- TABLES: customer,orders,lineitem +-- ERROR: not stable +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(L.var["L_ORDERKEY"] AS INT), + SUM(CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE))) AS REVENUE, + CAST(O.var["O_ORDERDATE"] AS TEXT), + CAST(O.var["O_SHIPPRIORITY"] AS INT) +FROM + customer C, + orders O, + lineitem L +WHERE + CAST(C.var["C_MKTSEGMENT"] AS TEXT) = 'BUILDING' + AND CAST(C.var["C_CUSTKEY"] AS INT) = CAST(O.var["O_CUSTKEY"] AS INT) + AND CAST(L.var["L_ORDERKEY"] AS INT) = CAST(O.var["O_ORDERKEY"] AS INT) + AND CAST(O.var["O_ORDERDATE"] AS TEXT) < '1995-03-15' + AND CAST(L.var["L_SHIPDATE"] AS TEXT) > '1995-03-15' +GROUP BY + CAST(L.var["L_ORDERKEY"] AS INT), + CAST(O.var["O_ORDERDATE"] AS TEXT), + CAST(O.var["O_SHIPPRIORITY"] AS INT) +ORDER BY + REVENUE DESC, + CAST(O.var["O_ORDERDATE"] AS TEXT) +LIMIT 10 diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q04_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q04_trans.sql new file mode 100644 index 00000000000000..ecfa18549dd5f0 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q04_trans.sql @@ -0,0 +1,21 @@ +-- TABLES: orders,lineitem +-- ERROR: No value present +-- SELECT +-- CAST(var["O_ORDERPRIORITY"] AS TEXT), +-- COUNT(*) AS ORDER_COUNT +-- FROM orders O +-- WHERE +-- CAST(O.var["O_ORDERDATE"] AS DATE) >= DATE '1993-07-01' +-- AND CAST(O.var["O_ORDERDATE"] AS DATE) < DATE '1993-07-01' + INTERVAL '3' MONTH +-- AND EXISTS ( +-- SELECT * +-- FROM lineitem L +-- WHERE +-- CAST(L.var["L_ORDERKEY"] AS INT) = CAST(O.var["O_ORDERKEY"] AS INT) +-- AND CAST(L.var["L_COMMITDATE"] AS DATE) < CAST(L.var["L_RECEIPTDATE"] AS DATE) +-- ) +-- GROUP BY +-- CAST(O.var["O_ORDERPRIORITY"] AS TEXT) +-- ORDER BY +-- CAST(O.var["O_ORDERPRIORITY"] AS TEXT) +-- \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q05_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q05_trans.sql new file mode 100644 index 00000000000000..f62776da6d0e2d --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q05_trans.sql @@ -0,0 +1,25 @@ +-- TABLES: customer,orders,lineitem,SUPPLIER,NATION,REGION +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(N.var["N_NAME"] AS TEXT), + SUM(CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE))) AS REVENUE +FROM + customer as C, + orders as O, + lineitem as L, + supplier as S, + nation as N, + region as R +WHERE + CAST(C.var["C_CUSTKEY"] AS INT) = CAST(O.var["O_CUSTKEY"] AS INT) + AND CAST(L.var["L_ORDERKEY"] AS INT) = CAST(O.var["O_ORDERKEY"] AS INT) + AND CAST(L.var["L_SUPPKEY"] AS INT) = CAST(S.var["S_SUPPKEY"] AS INT) + AND CAST(C.var["C_NATIONKEY"] AS INT) = CAST(S.var["S_NATIONKEY"] AS INT) + AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) + AND CAST(N.var["N_REGIONKEY"] AS INT) = CAST(R.var["R_REGIONKEY"] AS INT) + AND CAST(R.var["R_NAME"] AS TEXT) = 'ASIA' + AND CAST(O.var["O_ORDERDATE"] AS DATE) >= DATE '1994-01-01' + AND CAST(O.var["O_ORDERDATE"] AS DATE) < DATE '1994-01-01' + INTERVAL '1' YEAR +GROUP BY +CAST(N.var["N_NAME"] AS TEXT) +ORDER BY +REVENUE DESC diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q06_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q06_trans.sql new file mode 100644 index 00000000000000..8ee005f8e56e44 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q06_trans.sql @@ -0,0 +1,11 @@ +-- TABLES: lineitem +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ +SUM(CAST(var["L_EXTENDEDPRICE"] AS DOUBLE) * CAST(var["L_DISCOUNT"] AS DOUBLE)) AS REVENUE +FROM + lineitem +WHERE + CAST(var["L_SHIPDATE"] AS DATE) >= DATE '1994-01-01' + AND CAST(var["L_SHIPDATE"] AS DATE) < DATE '1994-01-01' + INTERVAL '1' YEAR +AND CAST(var["L_DISCOUNT"] AS DOUBLE) BETWEEN 0.06 - 0.01 AND .06 + 0.01 +AND CAST(var["L_QUANTITY"] AS DOUBLE) < 24 + diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q07_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q07_trans.sql new file mode 100644 index 00000000000000..2d7e3b9a867901 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q07_trans.sql @@ -0,0 +1,41 @@ +-- TABLES: SUPPLIER,lineitem,orders,customer,NATION +-- ERROR: not stable +-- SELECT +-- SUPP_NATION, +-- CUST_NATION, +-- L_YEAR, +-- SUM(VOLUME) AS REVENUE +-- FROM ( +-- SELECT +-- CAST(N1.var["N_NAME"] AS TEXT) AS SUPP_NATION, +-- CAST(N2.var["N_NAME"] AS TEXT) AS CUST_NATION, +-- EXTRACT(YEAR FROM CAST(L.var["L_SHIPDATE"] AS DATE)) AS L_YEAR, +-- CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE)) AS VOLUME +-- FROM +-- supplier S, +-- lineitem L, +-- orders O, +-- customer C, +-- nation N1, +-- nation N2 +-- WHERE +-- CAST(S.var["S_SUPPKEY"] AS INT) = CAST(L.var["L_SUPPKEY"] AS INT) +-- AND CAST(O.var["O_ORDERKEY"] AS INT) = CAST(L.var["L_ORDERKEY"] AS INT) +-- AND CAST(C.var["C_CUSTKEY"] AS INT) = CAST(O.var["O_CUSTKEY"] AS INT) +-- AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N1.var["N_NATIONKEY"] AS INT) +-- AND CAST(C.var["C_NATIONKEY"] AS INT) = CAST(N2.var["N_NATIONKEY"] AS INT) +-- AND ( +-- (CAST(N1.var["N_NAME"] AS TEXT) = 'FRANCE' AND CAST(N2.var["N_NAME"] AS TEXT) = 'GERMANY') +-- OR (CAST(N1.var["N_NAME"] AS TEXT) = 'GERMANY' AND CAST(N2.var["N_NAME"] AS TEXT) = 'FRANCE') +-- ) +-- AND CAST(L.var["L_SHIPDATE"] AS DATE) BETWEEN DATE '1995-01-01' AND DATE '1996-12-31' +-- ) AS SHIPPING +-- GROUP BY +-- SUPP_NATION, +-- CUST_NATION, +-- L_YEAR +-- ORDER BY +-- SUPP_NATION, +-- CUST_NATION, +-- L_YEAR +-- \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q08_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q08_trans.sql new file mode 100644 index 00000000000000..5cae69badc8a79 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q08_trans.sql @@ -0,0 +1,39 @@ +-- TABLES: part,SUPPLIER,lineitem,orders,customer,NATION,REGION +-- ERROR: not stable +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + O_YEAR, + SUM(CASE + WHEN NATION = 'BRAZIL' + THEN VOLUME + ELSE 0 + END) / SUM(VOLUME) AS MKT_SHARE +FROM ( + SELECT + EXTRACT(YEAR FROM CAST(O.var["O_ORDERDATE"] AS TEXT)) AS O_YEAR, + CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE)) AS VOLUME, + CAST(N2.var["N_NAME"] AS TEXT) AS NATION + FROM + part P, + supplier S, + lineitem L, + orders O, + customer C, + nation N1, + nation N2, + region R + WHERE + CAST(P.var["P_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) + AND CAST(S.var["S_SUPPKEY"] AS INT) = CAST(L.var["L_SUPPKEY"] AS INT) + AND CAST(L.var["L_ORDERKEY"] AS INT) = CAST(O.var["O_ORDERKEY"] AS INT) + AND CAST(O.var["O_CUSTKEY"] AS INT) = CAST(C.var["C_CUSTKEY"] AS INT) + AND CAST(C.var["C_NATIONKEY"] AS INT) = CAST(N1.var["N_NATIONKEY"] AS INT) + AND CAST(N1.var["N_REGIONKEY"] AS INT) = CAST(R.var["R_REGIONKEY"] AS INT) + AND CAST(R.var["R_NAME"] AS TEXT) = 'AMERICA' + AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N2.var["N_NATIONKEY"] AS INT) + AND CAST(O.var["O_ORDERDATE"] AS TEXT) BETWEEN '1995-01-01' AND '1996-12-31' + AND CAST(P.var["P_TYPE"] AS TEXT) = 'ECONOMY ANODIZED STEEL' + ) AS ALL_NATIONS +GROUP BY + O_YEAR +ORDER BY + O_YEAR diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q09_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q09_trans.sql new file mode 100644 index 00000000000000..f501d73dd727aa --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q09_trans.sql @@ -0,0 +1,33 @@ +-- TABLES: part,SUPPLIER,lineitem,partsupp,orders,NATION +-- ERROR: not stable +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false,batch_size=2048) */ + NATION, + O_YEAR, + SUM(AMOUNT) AS SUM_PROFIT +FROM ( + SELECT + CAST(N.var["N_NAME"] AS TEXT) AS NATION, + EXTRACT(YEAR FROM CAST(O.var["O_ORDERDATE"] AS TEXT)) AS O_YEAR, + CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE)) - CAST(PS.var["PS_SUPPLYCOST"] AS DOUBLE) * CAST(L.var["L_QUANTITY"] AS DOUBLE) AS AMOUNT + FROM + part P, + supplier S, + lineitem L, + partsupp PS, + orders O, + nation N + WHERE + CAST(S.var["S_SUPPKEY"] AS INT) = CAST(L.var["L_SUPPKEY"] AS INT) + AND CAST(PS.var["PS_SUPPKEY"] AS INT) = CAST(L.var["L_SUPPKEY"] AS INT) + AND CAST(PS.var["PS_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) + AND CAST(P.var["P_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) + AND CAST(O.var["O_ORDERKEY"] AS INT) = CAST(L.var["L_ORDERKEY"] AS INT) + AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) + AND CAST(P.var["P_NAME"] AS TEXT) LIKE '%green%' + ) AS PROFIT +GROUP BY + NATION, + O_YEAR +ORDER BY + NATION, + O_YEAR DESC diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q10_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q10_trans.sql new file mode 100644 index 00000000000000..cb3cdbf6ee51a0 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q10_trans.sql @@ -0,0 +1,32 @@ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(C.var["C_CUSTKEY"] AS INT), + CAST(C.var["C_NAME"] AS TEXT), + SUM(CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE))) AS REVENUE, + CAST(C.var["C_ACCTBAL"] AS DOUBLE), + CAST(N.var["N_NAME"] AS TEXT), + CAST(C.var["C_ADDRESS"] AS TEXT), + CAST(C.var["C_PHONE"] AS TEXT), + CAST(C.var["C_COMMENT"] AS TEXT) +FROM + customer C, + orders O, + lineitem L, + nation N +WHERE + CAST(C.var["C_CUSTKEY"] AS INT) = CAST(O.var["O_CUSTKEY"] AS INT) + AND CAST(L.var["L_ORDERKEY"] AS INT) = CAST(O.var["O_ORDERKEY"] AS INT) + AND CAST(O.var["O_ORDERDATE"] AS DATE) >= DATE '1993-10-01' + AND CAST(O.var["O_ORDERDATE"] AS DATE) < DATE '1993-10-01' + INTERVAL '3' MONTH + AND CAST(L.var["L_RETURNFLAG"] AS TEXT) = 'R' + AND CAST(C.var["C_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) +GROUP BY + CAST(C.var["C_CUSTKEY"] AS INT), + CAST(C.var["C_NAME"] AS TEXT), + CAST(C.var["C_ACCTBAL"] AS DOUBLE), + CAST(C.var["C_PHONE"] AS TEXT), + CAST(N.var["N_NAME"] AS TEXT), + CAST(C.var["C_ADDRESS"] AS TEXT), + CAST(C.var["C_COMMENT"] AS TEXT) +ORDER BY + REVENUE DESC +LIMIT 20 \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q11_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q11_trans.sql new file mode 100644 index 00000000000000..4a72594ebbd1d9 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q11_trans.sql @@ -0,0 +1,29 @@ +-- ERROR: +-- Invalid use of group function +-- SELECT +-- CAST(PS.var["PS_PARTKEY"] AS INT), +-- SUM(CAST(PS.var["PS_SUPPLYCOST"] AS DOUBLE) * CAST(PS.var["PS_AVAILQTY"] AS INT)) AS VALUE +-- FROM +-- partsupp PS, +-- supplier S, +-- nation N +-- WHERE +-- CAST(PS.var["PS_SUPPKEY"] AS INT) = CAST(S.var["S_SUPPKEY"] AS INT) +-- AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) +-- AND CAST(N.var["N_NAME"] AS TEXT) = 'GERMANY' +-- GROUP BY +-- CAST(PS.var["PS_PARTKEY"] AS INT) +-- HAVING +-- SUM(CAST(PS.var["PS_SUPPLYCOST"] AS DOUBLE) * CAST(PS.var["PS_AVAILQTY"] AS INT)) > ( +-- SELECT SUM(CAST(PSPS.var["PS_SUPPLYCOST"] AS DOUBLE) * CAST(PSPS.var["PS_AVAILQTY"] AS INT)) * 0.0001 +-- FROM +-- partsupp PSPS, +-- supplier SS, +-- nation NN +-- WHERE +-- CAST(PSPS.var["PS_SUPPKEY"] AS INT) = CAST(SS.var["S_SUPPKEY"] AS INT) +-- AND CAST(SS.var["S_NATIONKEY"] AS INT) = CAST(NN.var["N_NATIONKEY"] AS INT) +-- AND CAST(NN.var["N_NAME"] AS TEXT) = 'GERMANY' +-- ) +-- ORDER BY +-- VALUE DESC \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q12_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q12_trans.sql new file mode 100644 index 00000000000000..9df916cbc259ae --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q12_trans.sql @@ -0,0 +1,28 @@ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(L.var["L_SHIPMODE"] AS TEXT), + SUM(CASE + WHEN CAST(O.var["O_ORDERPRIORITY"] AS TEXT) = '1-URGENT' + OR CAST(O.var["O_ORDERPRIORITY"] AS TEXT) = '2-HIGH' + THEN 1 + ELSE 0 + END) AS HIGH_LINE_COUNT, + SUM(CASE + WHEN CAST(O.var["O_ORDERPRIORITY"] AS TEXT) <> '1-URGENT' + AND CAST(O.var["O_ORDERPRIORITY"] AS TEXT) <> '2-HIGH' + THEN 1 + ELSE 0 + END) AS LOW_LINE_COUNT +FROM + orders O, + lineitem L +WHERE + CAST(O.var["O_ORDERKEY"] AS INT) = CAST(L.var["L_ORDERKEY"] AS INT) + AND CAST(L.var["L_SHIPMODE"] AS TEXT) IN ('MAIL', 'SHIP') + AND CAST(L.var["L_COMMITDATE"] AS DATE) < CAST(L.var["L_RECEIPTDATE"] AS DATE) + AND CAST(L.var["L_SHIPDATE"] AS DATE) < CAST(L.var["L_COMMITDATE"] AS DATE) + AND CAST(L.var["L_RECEIPTDATE"] AS DATE) >= DATE '1994-01-01' + AND CAST(L.var["L_RECEIPTDATE"] AS DATE) < DATE '1994-01-01' + INTERVAL '1' YEAR +GROUP BY + CAST(L.var["L_SHIPMODE"] AS TEXT) +ORDER BY + CAST(L.var["L_SHIPMODE"] AS TEXT) \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q13_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q13_trans.sql new file mode 100644 index 00000000000000..ef10868dcf3afe --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q13_trans.sql @@ -0,0 +1,20 @@ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + C_COUNT, + COUNT(*) AS CUSTDIST +FROM ( + SELECT + CAST(C.var["C_CUSTKEY"] AS INT), + COUNT(CAST(O.var["O_ORDERKEY"] AS INT)) AS C_COUNT + FROM + customer C + LEFT OUTER JOIN orders O ON + CAST(C.var["C_CUSTKEY"] AS INT) = CAST(O.var["O_CUSTKEY"] AS INT) + AND CAST(O.var["O_COMMENT"] AS TEXT) NOT LIKE '%special%requests%' + GROUP BY + CAST(C.var["C_CUSTKEY"] AS INT) + ) AS C_orders +GROUP BY + C_COUNT +ORDER BY + CUSTDIST DESC, + C_COUNT DESC \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q14_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q14_trans.sql new file mode 100644 index 00000000000000..ddf798cc338f75 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q14_trans.sql @@ -0,0 +1,13 @@ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ +100.00 * SUM(CASE + WHEN CAST(P.var["P_TYPE"] AS TEXT) LIKE 'PROMO%' + THEN CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE)) + ELSE 0 + END) / SUM(CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE))) AS PROMO_REVENUE +FROM + lineitem L, + part P +WHERE + CAST(L.var["L_PARTKEY"] AS INT) = CAST(P.var["P_PARTKEY"] AS INT) + AND CAST(L.var["L_SHIPDATE"] AS DATE) >= DATE '1995-09-01' + AND CAST(L.var["L_SHIPDATE"] AS DATE) < DATE '1995-09-01' + INTERVAL '1' MONTH \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q15_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q15_trans.sql new file mode 100644 index 00000000000000..01fe8615fca10f --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q15_trans.sql @@ -0,0 +1,18 @@ +-- SELECT +-- CAST(var["S_SUPPKEY"] AS INT), +-- CAST(var["S_NAME"] AS TEXT), +-- CAST(var["S_ADDRESS"] AS TEXT), +-- CAST(var["S_PHONE"] AS TEXT), +-- TOTAL_REVENUE +-- FROM +-- suppl Sier, +-- revenuRe1 +-- WHERE +-- CAST(var["S_SUPPKEY"] AS INT) = SUPPLIER_NO +-- AND TOTAL_REVENUE = ( +-- SELECT MAX(TOTAL_REVENUE) +-- FROM +-- revenue1 +-- ) +-- ORDER BY +-- CAST(var["S_SUPPKEY"] AS INT); \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q16_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q16_trans.sql new file mode 100644 index 00000000000000..2819cc03a5aa03 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q16_trans.sql @@ -0,0 +1,29 @@ +SELECT /*+SET_VAR(enable_fallback_to_original_planner=false) */ + CAST(P.var["P_BRAND"] AS TEXT), + CAST(P.var["P_TYPE"] AS TEXT), + CAST(P.var["P_SIZE"] AS INT), + COUNT(DISTINCT CAST(PS.var["PS_SUPPKEY"] AS INT)) AS SUPPLIER_CNT +FROM + partsupp PS, + part P +WHERE + CAST(P.var["P_PARTKEY"] AS INT) = CAST(PS.var["PS_PARTKEY"] AS INT) + AND CAST(P.var["P_BRAND"] AS TEXT) <> 'Brand#45' + AND CAST(P.var["P_TYPE"] AS TEXT) NOT LIKE 'MEDIUM POLISHED%' + AND CAST(P.var["P_SIZE"] AS INT) IN (49, 14, 23, 45, 19, 3, 36, 9) + AND CAST(PS.var["PS_SUPPKEY"] AS INT) NOT IN ( + SELECT CAST(S.var["S_SUPPKEY"] AS INT) + FROM + supplier S + WHERE + CAST(S.var["S_COMMENT"] AS TEXT) LIKE '%Customer%Complaints%' + ) +GROUP BY + CAST(P.var["P_BRAND"] AS TEXT), + CAST(P.var["P_TYPE"] AS TEXT), + CAST(P.var["P_SIZE"] AS INT) +ORDER BY + SUPPLIER_CNT DESC, + CAST(P.var["P_BRAND"] AS TEXT), + CAST(P.var["P_TYPE"] AS TEXT), + CAST(P.var["P_SIZE"] AS INT) \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q17_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q17_trans.sql new file mode 100644 index 00000000000000..5ebfd8d65effcc --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q17_trans.sql @@ -0,0 +1,16 @@ +-- ERROR: correlationFilter can't be null in correlatedToJoin +-- SELECT SUM(CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE)) / 7.0 AS AVG_YEARLY +-- FROM +-- lineitem L, +-- part P +-- WHERE +-- CAST(P.var["P_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) +-- AND CAST(P.var["P_BRAND"] AS TEXT) = 'BRAND#23' +-- AND CAST(P.var["P_CONTAINER"] AS TEXT) = 'MED BOX' +-- AND CAST(L.var["L_QUANTITY"] AS DOUBLE) < ( +-- SELECT 0.2 * AVG(CAST(LL.var["L_QUANTITY"] AS DOUBLE)) +-- FROM +-- lineitem LL +-- WHERE +-- CAST(LL.var["L_PARTKEY"] AS INT) = CAST(P.var["P_PARTKEY"] AS INT) +-- ) \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q18_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q18_trans.sql new file mode 100644 index 00000000000000..ff4b65e425ef8c --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q18_trans.sql @@ -0,0 +1,34 @@ +-- ERROR: Not stable +-- SELECT +-- CAST(C.var["C_NAME"] AS TEXT), +-- CAST(C.var["C_CUSTKEY"] AS INT), +-- CAST(O.var["O_ORDERKEY"] AS INT), +-- CAST(O.var["O_ORDERDATE"] AS DATE), +-- CAST(O.var["O_TOTALPRICE"] AS DOUBLE), +-- SUM(CAST(L.var["L_QUANTITY"] AS DOUBLE)) +-- FROM +-- customer C, +-- orders O, +-- lineitem L +-- WHERE +-- CAST(O.var["O_ORDERKEY"] AS INT) IN ( +-- SELECT CAST(LL.var["L_ORDERKEY"] AS INT) +-- FROM +-- lineitem LL +-- GROUP BY +-- CAST(LL.var["L_ORDERKEY"] AS INT) +-- HAVING +-- SUM(CAST(LL.var["L_QUANTITY"] AS DOUBLE)) > 300 +-- ) +-- AND CAST(C.var["C_CUSTKEY"] AS INT) = CAST(O.var["O_CUSTKEY"] AS INT) +-- AND CAST(O.var["O_ORDERKEY"] AS INT) = CAST(L.var["L_ORDERKEY"] AS INT) +-- GROUP BY +-- CAST(C.var["C_NAME"] AS TEXT), +-- CAST(C.var["C_CUSTKEY"] AS INT), +-- CAST(O.var["O_ORDERKEY"] AS INT), +-- CAST(O.var["O_ORDERDATE"] AS DATE), +-- CAST(O.var["O_TOTALPRICE"] AS DOUBLE) +-- ORDER BY +-- CAST(O.var["O_TOTALPRICE"] AS DOUBLE) DESC, +-- CAST(O.var["O_ORDERDATE"] AS DATE) +-- LIMIT 100 \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q19_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q19_trans.sql new file mode 100644 index 00000000000000..c57d64b140f794 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q19_trans.sql @@ -0,0 +1,36 @@ +-- ERROR: +-- not stable +-- SELECT SUM(CAST(L.var["L_EXTENDEDPRICE"] AS DOUBLE) * (1 - CAST(L.var["L_DISCOUNT"] AS DOUBLE))) AS REVENUE +-- FROM +-- lineitem L, +-- part P +-- WHERE +-- ( +-- CAST(P.var["P_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) +-- AND CAST(P.var["P_BRAND"] AS TEXT) = 'Brand#12' +-- AND CAST(P.var["P_CONTAINER"] AS TEXT) IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') +-- AND CAST(L.var["L_QUANTITY"] AS DOUBLE) >= 1 AND CAST(L.var["L_QUANTITY"] AS DOUBLE) <= 1 + 10 +-- AND CAST(P.var["P_SIZE"] AS INT) BETWEEN 1 AND 5 +-- AND CAST(L.var["L_SHIPMODE"] AS TEXT) IN ('AIR', 'AIR REG') +-- AND CAST(L.var["L_SHIPINSTRUCT"] AS TEXT) = 'DELIVER IN PERSON' +-- ) +-- OR +-- ( +-- CAST(P.var["P_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) +-- AND CAST(P.var["P_BRAND"] AS TEXT) = 'Brand#23' +-- AND CAST(P.var["P_CONTAINER"] AS TEXT) IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') +-- AND CAST(L.var["L_QUANTITY"] AS DOUBLE) >= 10 AND CAST(L.var["L_QUANTITY"] AS DOUBLE) <= 10 + 10 +-- AND CAST(P.var["P_SIZE"] AS INT) BETWEEN 1 AND 10 +-- AND CAST(L.var["L_SHIPMODE"] AS TEXT) IN ('AIR', 'AIR REG') +-- AND CAST(L.var["L_SHIPINSTRUCT"] AS TEXT) = 'DELIVER IN PERSON' +-- ) +-- OR +-- ( +-- CAST(P.var["P_PARTKEY"] AS INT) = CAST(L.var["L_PARTKEY"] AS INT) +-- AND CAST(P.var["P_BRAND"] AS TEXT) = 'Brand#34' +-- AND CAST(P.var["P_CONTAINER"] AS TEXT) IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') +-- AND CAST(L.var["L_QUANTITY"] AS DOUBLE) >= 20 AND CAST(L.var["L_QUANTITY"] AS DOUBLE) <= 20 + 10 +-- AND CAST(P.var["P_SIZE"] AS INT) BETWEEN 1 AND 15 +-- AND CAST(L.var["L_SHIPMODE"] AS TEXT) IN ('AIR', 'AIR REG') +-- AND CAST(L.var["L_SHIPINSTRUCT"] AS TEXT) = 'DELIVER IN PERSON' +-- ) \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q20_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q20_trans.sql new file mode 100644 index 00000000000000..838d7f42fe3665 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q20_trans.sql @@ -0,0 +1,33 @@ +-- ERROR: correlationFilter can't be null in correlatedToJoin +-- SELECT +-- CAST(S.var["S_NAME"] AS TEXT), +-- CAST(S.var["S_ADDRESS"] AS TEXT) +-- FROM +-- supplier S, nation N +-- WHERE +-- CAST(S.var["S_SUPPKEY"] AS INT) IN ( +-- SELECT CAST(PS.var["PS_SUPPKEY"] AS INT) +-- FROM +-- partsupp PS +-- WHERE +-- CAST(PS.var["PS_PARTKEY"] AS INT) IN ( +-- SELECT CAST(P.var["P_PARTKEY"] AS INT) +-- FROM +-- part P +-- WHERE +-- CAST(P.var["P_NAME"] AS TEXT) LIKE 'FOREST%' +-- ) +-- AND CAST(PS.var["PS_AVAILQTY"] AS INT) > ( +-- SELECT 0.5 * SUM(CAST(L.var["L_QUANTITY"] AS DOUBLE)) +-- FROM +-- lineitem L +-- WHERE +-- CAST(L.var["L_PARTKEY"] AS INT) = CAST(PS.var["PS_PARTKEY"] AS INT) +-- AND CAST(L.var["L_SUPPKEY"] AS INT) = CAST(PS.var["PS_SUPPKEY"] AS INT) +-- AND CAST(L.var["L_SHIPDATE"] AS DATE) >= DATE('1994-01-01') +-- AND CAST(L.var["L_SHIPDATE"] AS DATE) < DATE('1994-01-01') + INTERVAL '1' YEAR +-- ) +-- ) +-- AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) +-- AND CAST(N.var["N_NAME"] AS TEXT) = 'CANADA' +-- ORDER BY CAST(S.var["S_NAME"] AS TEXT) \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q21_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q21_trans.sql new file mode 100644 index 00000000000000..057e718d250a68 --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q21_trans.sql @@ -0,0 +1,39 @@ +-- ERROR: No value present +-- SELECT +-- CAST(S.var["S_NAME"] AS TEXT), +-- COUNT(*) AS NUMWAIT +-- FROM +-- supplier S, +-- lineitem L1, +-- orders O, +-- nation N +-- WHERE +-- CAST(S.var["S_SUPPKEY"] AS INT) = CAST(L1.var["L_SUPPKEY"] AS INT) +-- AND CAST(O.var["O_ORDERKEY"] AS INT) = CAST(L1.var["L_ORDERKEY"] AS INT) +-- AND CAST(O.var["O_ORDERSTATUS"] as text)= 'F' +-- AND CAST(L1.var["L_RECEIPTDATE"] AS DATE) > CAST(L1.var["L_COMMITDATE"] AS DATE) +-- AND EXISTS( +-- SELECT * +-- FROM +-- lineitem L2 +-- WHERE +-- CAST(L2.var["L_ORDERKEY"] AS INT) = CAST(L1.var["L_ORDERKEY"] AS INT) +-- AND CAST(L2.var["L_SUPPKEY"] AS INT) <> CAST(L1.var["L_SUPPKEY"] AS INT) +-- ) +-- AND NOT EXISTS( +-- SELECT * +-- FROM +-- lineitem L3 +-- WHERE +-- CAST(L3.var["L_ORDERKEY"] AS INT) = CAST(L1.var["L_ORDERKEY"] AS INT) +-- AND CAST(L3.var["L_SUPPKEY"] AS INT) <> CAST(L1.var["L_SUPPKEY"] AS INT) +-- AND CAST(L3.var["L_RECEIPTDATE"] AS DATE) > CAST(L3.var["L_COMMITDATE"] AS DATE) +-- ) +-- AND CAST(S.var["S_NATIONKEY"] AS INT) = CAST(N.var["N_NATIONKEY"] AS INT) +-- AND CAST(N.var["N_NAME"] AS TEXT) = 'SAUDI ARABIA' +-- GROUP BY +-- CAST(S.var["S_NAME"] AS TEXT) +-- ORDER BY +-- NUMWAIT DESC, +-- CAST(S.var["S_NAME"] AS TEXT) +-- LIMIT 100 \ No newline at end of file diff --git a/regression-test/suites/variant_p2/tpch_upgrade/sql/q22_trans.sql b/regression-test/suites/variant_p2/tpch_upgrade/sql/q22_trans.sql new file mode 100644 index 00000000000000..f25dcfcdafa39b --- /dev/null +++ b/regression-test/suites/variant_p2/tpch_upgrade/sql/q22_trans.sql @@ -0,0 +1,35 @@ +-- ERROR: No value present +-- SELECT +-- CNTRYCODE, +-- COUNT(*) AS NUMCUST, +-- SUM(C_ACCTBAL) AS TOTACCTBAL +-- FROM ( +-- SELECT +-- SUBSTR(CAST(C.var["C_PHONE"] AS TEXT), 1, 2) AS CNTRYCODE, +-- CAST(C.var["C_ACCTBAL"] AS DOUBLE) as C_ACCTBAL +-- FROM +-- customer C +-- WHERE +-- SUBSTR(CAST(C.var["C_PHONE"] AS TEXT), 1, 2) IN +-- ('13', '31', '23', '29', '30', '18', '17') +-- AND CAST(C.var["C_ACCTBAL"] AS DOUBLE) > ( +-- SELECT AVG(CAST(C.var["C_ACCTBAL"] AS DOUBLE)) +-- FROM +-- customer CC +-- WHERE +-- CAST(CC.var["C_ACCTBAL"] AS DOUBLE) > 0.00 +-- AND SUBSTR(CAST(CC.var["C_PHONE"] AS TEXT), 1, 2) IN +-- ('13', '31', '23', '29', '30', '18', '17') +-- ) +-- AND NOT EXISTS( +-- SELECT * +-- FROM +-- orders O +-- WHERE +-- CAST(O.var["O_CUSTKEY"] AS INT) = CAST(C.var["C_CUSTKEY"] AS INT) +-- ) +-- ) AS CUSTSALE +-- GROUP BY +-- CNTRYCODE +-- ORDER BY +-- CNTRYCODE \ No newline at end of file From 64915516efbec27a46cc42afc037c36e1306096c Mon Sep 17 00:00:00 2001 From: Siyang Tang Date: Wed, 30 Jul 2025 19:24:00 +0800 Subject: [PATCH 327/572] [fix](schema-change) Forbid dropping distribution columns (branch-3.0) (#54037) ### What problem does this PR solve? As title. Branch master do not have this problem --- .../doris/alter/SchemaChangeHandler.java | 3 ++ .../test_multi_column_partition.groovy | 2 +- .../test_alter_table_drop_column.groovy | 2 +- .../test_disable_drop_bucket_columns.groovy | 43 +++++++++++++++++++ .../test_schema_reordering_dup.groovy | 2 +- 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 regression-test/suites/schema_change_p0/test_disable_drop_bucket_columns.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index e3ad8d19483581..075a46b756187c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -488,6 +488,9 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa throw new DdlException("Column does not exists: " + dropColName); } } + if (olapTable.isDistributionColumn(dropColName)) { + throw new DdlException("Could not drop distribution column: " + dropColName); + } return lightSchemaChange; } diff --git a/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy b/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy index 1afd51bfab65ef..9d039e53672d18 100644 --- a/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy +++ b/regression-test/suites/partition_p0/multi_partition/test_multi_column_partition.groovy @@ -329,7 +329,7 @@ suite("test_multi_partition_key", "p0") { assertEquals("FINISHED", getAlterColumnFinalState("test_multi_column_drop_partition_column")) test { sql "ALTER TABLE test_multi_column_drop_partition_column DROP COLUMN k1" - exception "Partition column[k1] cannot be dropped" + exception "Could not drop distribution column: k1" } sql "insert into test_multi_column_drop_partition_column " + "values(100, 0, 0, 0, 0, '2000-01-01 00:00:00', 0.001, -0.001, 0.001)" diff --git a/regression-test/suites/schema_change_p0/test_alter_table_drop_column.groovy b/regression-test/suites/schema_change_p0/test_alter_table_drop_column.groovy index bd600d8169857a..9b74fe138d9545 100644 --- a/regression-test/suites/schema_change_p0/test_alter_table_drop_column.groovy +++ b/regression-test/suites/schema_change_p0/test_alter_table_drop_column.groovy @@ -161,7 +161,7 @@ suite("test_alter_table_drop_column") { test { sql """alter table ${dupTableName} drop COLUMN siteid;""" // check exception message contains - exception "Distribution column[siteid] cannot be dropped" + exception "Could not drop distribution column: siteid" } sql "DROP TABLE IF EXISTS ${dupTableName} FORCE" diff --git a/regression-test/suites/schema_change_p0/test_disable_drop_bucket_columns.groovy b/regression-test/suites/schema_change_p0/test_disable_drop_bucket_columns.groovy new file mode 100644 index 00000000000000..578d792cafc599 --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_disable_drop_bucket_columns.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_disable_drop_bucket_columns") { + def tables = ["test_disable_drop_bucket_columns_dup", "test_disable_drop_bucket_columns_agg"] + def keys = ["DUPLICATE KEY(k1, k2)", "AGGREGATE KEY(k1, k2)"] + def aggTypes = ["", "SUM"] + for (i in 0.. Date: Wed, 30 Jul 2025 23:21:38 +0800 Subject: [PATCH 328/572] [fix](schema-change) Rebuild distribution info according to original order (#54024) (#54072) (#54109) ### What problem does this PR solve? cherry-pick: #54024 #54072 --- .../org/apache/doris/catalog/OlapTable.java | 12 ++-- .../test_random_ordered_bucket_columns.out | 7 +++ .../test_random_ordered_bucket_columns.groovy | 61 +++++++++++++++++++ 3 files changed, 72 insertions(+), 8 deletions(-) create mode 100644 regression-test/data/schema_change_p0/test_random_ordered_bucket_columns.out create mode 100644 regression-test/suites/schema_change_p0/test_random_ordered_bucket_columns.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 3d4b5cc1d6f3d8..9d48e34afaf219 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -492,17 +492,13 @@ public void rebuildDistributionInfo() { return; } HashDistributionInfo distributionInfo = (HashDistributionInfo) defaultDistributionInfo; - Set originalColumnsNames = - distributionInfo.getDistributionColumns() - .stream() - .map(Column::getName) - .collect(Collectors.toSet()); - - List newDistributionColumns = getBaseSchema() + List newDistributionColumns = distributionInfo.getDistributionColumns() .stream() - .filter(column -> originalColumnsNames.contains(column.getName())) + .map(Column::getName) + .map(this::getBaseColumn) .map(Column::new) .collect(Collectors.toList()); + distributionInfo.setDistributionColumns(newDistributionColumns); getPartitions() diff --git a/regression-test/data/schema_change_p0/test_random_ordered_bucket_columns.out b/regression-test/data/schema_change_p0/test_random_ordered_bucket_columns.out new file mode 100644 index 00000000000000..e59e71b86c7afe --- /dev/null +++ b/regression-test/data/schema_change_p0/test_random_ordered_bucket_columns.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +2025-07-29 0 1 2 3 + +-- !sql -- +2025-07-29 0 1 2 3 0 + diff --git a/regression-test/suites/schema_change_p0/test_random_ordered_bucket_columns.groovy b/regression-test/suites/schema_change_p0/test_random_ordered_bucket_columns.groovy new file mode 100644 index 00000000000000..a085659588228f --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_random_ordered_bucket_columns.groovy @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_random_ordered_bucket_columns') { + def tblName = "test_random_ordered_bucket_columns" + sql """ DROP TABLE IF EXISTS ${tblName}; """ + sql """ + CREATE TABLE IF NOT EXISTS ${tblName} + ( + k1 DATE, + k2 INT DEFAULT '10', + k3 bigint, + k4 VARCHAR(32) DEFAULT '', + v1 BIGINT DEFAULT '0' + ) + UNIQUE KEY(k1, k2, k3, k4) + DISTRIBUTED BY HASH(k3, k4, k1, k2) BUCKETS 5 + PROPERTIES("replication_num" = "1", "light_schema_change" = "true"); + """ + + // show create table differs in local and cloud mode, this behavior is verified + // qt_sql """ SHOW CREATE TABLE ${tblName} """ + + sql """ INSERT INTO ${tblName} VALUES("2025-07-29", 0, 1, "2", 3) """ + + sql """ INSERT INTO ${tblName} VALUES("2025-07-29", 0, 1, "2", 3) """ + + sql """ SYNC """ + + qt_sql """ SELECT * FROM ${tblName} """ + + sql """ ALTER TABLE ${tblName} ADD COLUMN v2 BIGINT DEFAULT '1' """ + + + // sql """ SYNC """ + + // show create table differs in local and cloud mode, this behavior is verified + // qt_sql """ SHOW CREATE TABLE ${tblName} """ + + 30.times { index -> + sql """ INSERT INTO ${tblName} VALUES("2025-07-29", 0, 1, "2", 3, 0) """ + } + + sql """ SYNC """ + + qt_sql """ SELECT * FROM ${tblName} """ +} From 0d788cfaa4f312487a3112b4adac5b4e2bcef9bf Mon Sep 17 00:00:00 2001 From: deardeng Date: Thu, 31 Jul 2025 09:42:34 +0800 Subject: [PATCH 329/572] [Fix](case) Fix `test_rename_compute_group` due to vcg pr pick (#54103) ### What problem does this PR solve? Fix missing logic 1. https://github.com/apache/doris/pull/46221/files fix rename 2. but when developing vcg logic, missing pr 46221 functionality 3. pick form vcg branch, so miss 46221 logic, --- .../meta-service/meta_service_resource.cpp | 23 +++++++++++++++---- .../src/resource-manager/resource_manager.cpp | 11 +++++---- .../multi_cluster/test_no_cluster_hits.groovy | 12 ++++++---- .../cloud_p0/node_mgr/test_ms_api.groovy | 8 +++---- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index e33b0c24a6da73..f8ebccc28af134 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -2295,8 +2295,8 @@ void handle_notify_decommissioned(const std::string& instance_id, } void handle_rename_cluster(const std::string& instance_id, const ClusterInfo& cluster, - std::shared_ptr resource_mgr, std::string& msg, - MetaServiceCode& code) { + std::shared_ptr resource_mgr, bool replace, + std::string& msg, MetaServiceCode& code) { msg = resource_mgr->update_cluster( instance_id, cluster, [&](const ClusterPB& i) { return i.cluster_id() == cluster.cluster.cluster_id(); }, @@ -2308,6 +2308,10 @@ void handle_rename_cluster(const std::string& instance_id, const ClusterInfo& cl cluster_names.emplace(cluster_in_instance.cluster_name()); } auto it = cluster_names.find(cluster.cluster.cluster_name()); + LOG(INFO) << "cluster.cluster.cluster_name(): " << cluster.cluster.cluster_name(); + for (auto itt : cluster_names) { + LOG(INFO) << "instance's cluster name : " << itt; + } if (it != cluster_names.end()) { code = MetaServiceCode::INVALID_ARGUMENT; ss << "failed to rename cluster, a cluster with the same name already exists " @@ -2325,7 +2329,8 @@ void handle_rename_cluster(const std::string& instance_id, const ClusterInfo& cl } c.set_cluster_name(cluster.cluster.cluster_name()); return msg; - }); + }, + replace); } void handle_update_cluster_endpoint(const std::string& instance_id, const ClusterInfo& cluster, @@ -2552,9 +2557,17 @@ void MetaServiceImpl::alter_cluster(google::protobuf::RpcController* controller, case AlterClusterRequest::NOTIFY_DECOMMISSIONED: handle_notify_decommissioned(instance_id, request, resource_mgr(), msg, code); break; - case AlterClusterRequest::RENAME_CLUSTER: - handle_rename_cluster(instance_id, cluster, resource_mgr(), msg, code); + case AlterClusterRequest::RENAME_CLUSTER: { + // SQL mode, cluster cluster name eq empty cluster name, need drop empty cluster first. + // but in http api, cloud control will drop empty cluster + bool replace_if_existing_empty_target_cluster = + request->has_replace_if_existing_empty_target_cluster() + ? request->replace_if_existing_empty_target_cluster() + : false; + handle_rename_cluster(instance_id, cluster, resource_mgr(), + replace_if_existing_empty_target_cluster, msg, code); break; + } case AlterClusterRequest::UPDATE_CLUSTER_ENDPOINT: handle_update_cluster_endpoint(instance_id, cluster, resource_mgr(), msg, code); break; diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp index b6f184929ac4f4..7c5a95b4267898 100644 --- a/cloud/src/resource-manager/resource_manager.cpp +++ b/cloud/src/resource-manager/resource_manager.cpp @@ -801,10 +801,8 @@ std::string ResourceManager::update_cluster( } std::vector clusters_in_instance; - std::set cluster_names; // collect cluster in instance pb for check for (auto& i : instance.clusters()) { - cluster_names.emplace(i.cluster_name()); clusters_in_instance.emplace_back(i); } @@ -835,8 +833,11 @@ std::string ResourceManager::update_cluster( // check cluster_name is empty cluster, if empty and replace_if_existing_empty_target_cluster == true, drop it if (replace_if_existing_empty_target_cluster) { - auto it = cluster_names.find(cluster_name); - if (it != cluster_names.end()) { + auto it = std::find_if(clusters_in_instance.begin(), clusters_in_instance.end(), + [&cluster_name](const auto& cluster) { + return cluster_name == cluster.cluster_name(); + }); + if (it != clusters_in_instance.end()) { // found it, if it's an empty cluster, drop it from instance int idx = -1; for (auto& cluster : instance.clusters()) { @@ -849,7 +850,7 @@ std::string ResourceManager::update_cluster( instance.clusters()); clusters.DeleteSubrange(idx, 1); // Remove cluster name from set - cluster_names.erase(cluster_name); + clusters_in_instance.erase(it); LOG(INFO) << "remove empty cluster due to it is the target of a " "rename_cluster, cluster_name=" << cluster_name; diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy index 193824a65408b9..82a77ecb368b39 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_no_cluster_hits.groovy @@ -93,13 +93,15 @@ suite('test_no_cluster_hits', 'multi_cluster, docker') { """ } catch (Exception e) { logger.info("exception: {}", e.getMessage()) - // assertTrue(e.getMessage().contains("ComputeGroupException: COMPUTE_GROUPS_NO_ALIVE_BE")) - // assertTrue(e.getMessage().contains("are in an abnormal state")) + // in 3.0 + assertTrue(e.getMessage().contains("ComputeGroupException: COMPUTE_GROUPS_NO_ALIVE_BE")) + assertTrue(e.getMessage().contains("are in an abnormal state")) + // in master // The new optimizer code modifies the path and returns a different exception message // exception: errCode = 2, detailMessage = No backend available as scan node, please check the status of your // backends.[1747384136706: not alive, 1747384136705: not alive, 1747384136704: not alive] - assertTrue(e.getMessage().contains("No backend available as scan node")) + // assertTrue(e.getMessage().contains("No backend available as scan node")) } try { @@ -152,13 +154,13 @@ suite('test_no_cluster_hits', 'multi_cluster, docker') { } try { - // errCode = 2, detailMessage = Can not find compute group:compute_cluster + // errCode = 2, detailMessage = The current compute group compute_cluster is not registered in the system sql """ select * from $table """ } catch (Exception e) { logger.info("exception: {}", e.getMessage()) - assertTrue(e.getMessage().contains("Can not find compute group")) + assertTrue(e.getMessage().contains("The current compute group compute_cluster is not registered in the system")) } } } diff --git a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy index 9cdc56e561a85e..2df7815e769427 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_ms_api.groovy @@ -1176,7 +1176,7 @@ suite('test_ms_api', 'p0, docker') { log.info("add two observer fe failed test http cli result: ${body} ${respCode}".toString()) def json = parseJson(body) assertTrue(json.code.equalsIgnoreCase("INVALID_ARGUMENT")) - assertTrue(json.msg.contains("cluster is SQL type, but not set master and follower node, master count=0 follower count=0 so sql cluster can't get a Master node")) + assertTrue(json.msg.contains("cluster is SQL type, must have only one master node, now master count: 0")) } def ip4 = "162.0.0.4" @@ -1202,7 +1202,7 @@ suite('test_ms_api', 'p0, docker') { // failed, due to two master node // if force_change_to_multi_follower_mode == false, check type not changed, FE_MASTER log.info("add some fe failed nodes http cli result: ${body} ${respCode} ${json}".toString()) - assertTrue(json.code.equalsIgnoreCase("INTERNAL_ERROR")) + assertTrue(json.code.equalsIgnoreCase("INVALID_ARGUMENT")) assertTrue(json.msg.contains("instance invalid, cant modify, plz check")) } @@ -1228,7 +1228,7 @@ suite('test_ms_api', 'p0, docker') { respCode, body -> def json = parseJson(body) log.info("drop all fe nodes failed http cli result: ${body} ${respCode} ${json}".toString()) - assertTrue(json.code.equalsIgnoreCase("INTERNAL_ERROR")) + assertTrue(json.code.equalsIgnoreCase("INVALID_ARGUMENT")) assertTrue(json.msg.contains("instance invalid, cant modify, plz check")) } @@ -1396,7 +1396,7 @@ suite('test_ms_api', 'p0, docker') { respCode, body -> def json = parseJson(body) log.info("drop fe observer node http cli result: ${body} ${respCode} ${json}".toString()) - assertTrue(json.code.equalsIgnoreCase("INTERNAL_ERROR")) + assertTrue(json.code.equalsIgnoreCase("INVALID_ARGUMENT")) assertTrue(json.msg.contains("drop fe node not in safe time, try later")) } From 967df479ea39298d9f0598c2ac207de1dd92d065 Mon Sep 17 00:00:00 2001 From: deardeng Date: Thu, 31 Jul 2025 21:04:18 +0800 Subject: [PATCH 330/572] =?UTF-8?q?Revert=20"branch-3.0:=20[fix](trash)=20?= =?UTF-8?q?Fix=20shadow=20variable=20causing=20garbage=20=E2=80=A6=20(#540?= =?UTF-8?q?73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …scheduling time errors #51647 (#51687)" This reverts commit 75d29d8e104f1f88819cf32e5b2a31eb2efec40c. --- be/src/olap/olap_server.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index f7125b97567d72..36971ba5e9bfd9 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -389,7 +389,7 @@ void StorageEngine::_garbage_sweeper_thread_callback() { // when usage = 0.88, ratio is approximately 0.0057. double ratio = (1.1 * (pi / 2 - std::atan(usage * 100 / 5 - 14)) - 0.28) / pi; ratio = ratio > 0 ? ratio : 0; - curr_interval = uint32_t(max_interval * ratio); + auto curr_interval = uint32_t(max_interval * ratio); curr_interval = std::max(curr_interval, min_interval); curr_interval = std::min(curr_interval, max_interval); @@ -404,8 +404,6 @@ void StorageEngine::_garbage_sweeper_thread_callback() { << "see previous message for detail. err code=" << res; // do nothing. continue next loop. } - LOG(INFO) << "trash thread check usage=" << usage << " ratio=" << ratio - << " curr_interval=" << curr_interval; } while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(curr_interval))); } From 4c3326930d98045875c3d9375ccc903ab0928b18 Mon Sep 17 00:00:00 2001 From: Mryange <2319153948@qq.com> Date: Fri, 1 Aug 2025 10:11:31 +0800 Subject: [PATCH 331/572] [branch-3.0](core) Fix format round would core under boundary conditions (#53855) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …53660) https://github.com/apache/doris/pull/53660 my_double_round did not handle NaN values correctly. For example, when dec is very large and value is 0, there will be a case of 0 * inf, resulting in a NaN value. do_format_round assumes the input is always a valid double value, which causes a core dump when a NaN is passed in. Additionally, (value.size() - (is_positive ? (decimal_places + 2) : (decimal_places + 3))) / 3; This code does not account for the situation where value.size() is 0. Currently, a custom add_thousands_separator function is implemented. If the fmt library is upgraded in the future, we should use fmt to add thousands separators. ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/exprs/math_functions.cpp | 2 +- be/src/vec/functions/function_string.h | 50 ++++- be/test/exprs/math_functions_test.cpp | 87 +++++++++ .../math_functions/test_format_round.out | 174 ++++++++++++++++++ .../math_functions/test_format_round.groovy | 58 ++++++ 5 files changed, 367 insertions(+), 4 deletions(-) create mode 100644 be/test/exprs/math_functions_test.cpp diff --git a/be/src/exprs/math_functions.cpp b/be/src/exprs/math_functions.cpp index 202a6eabea3e80..93e28efb733451 100644 --- a/be/src/exprs/math_functions.cpp +++ b/be/src/exprs/math_functions.cpp @@ -83,7 +83,7 @@ double MathFunctions::my_double_round(double value, int64_t dec, bool dec_unsign if (dec_negative && std::isinf(tmp)) { tmp2 = 0.0; - } else if (!dec_negative && std::isinf(value_mul_tmp)) { + } else if (!dec_negative && !std::isfinite(value_mul_tmp)) { tmp2 = value; } else if (truncate) { if (value >= 0.0) { diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index 09122aaf51c4e2..97c950d97b71be 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -3365,6 +3365,44 @@ struct FormatRoundDoubleImpl { return {std::make_shared(), std::make_shared()}; } + static std::string add_thousands_separator(const std::string& formatted_num) { + // Find the position of the decimal point + size_t dot_pos = formatted_num.find('.'); + if (dot_pos == std::string::npos) { + dot_pos = formatted_num.size(); + } + + // Handle the integer part + int start = (formatted_num[0] == '-') ? 1 : 0; + int digit_count = dot_pos - start; + + // There is no need to add commas. + if (digit_count <= 3) { + return formatted_num; + } + + std::string result; + + if (start == 1) result += '-'; + + // Add the integer part (with comma) + int first_group = digit_count % 3; + if (first_group == 0) first_group = 3; + result.append(formatted_num, start, first_group); + + for (size_t i = start + first_group; i < dot_pos; i += 3) { + result += ','; + result.append(formatted_num, i, 3); + } + + // Add the decimal part (keep as it is) + if (dot_pos != formatted_num.size()) { + result.append(formatted_num, dot_pos); + } + + return result; + } + template static Status execute(FunctionContext* context, ColumnString* result_column, const ColumnPtr col_ptr, ColumnPtr decimal_places_col_ptr, @@ -3382,9 +3420,15 @@ struct FormatRoundDoubleImpl { // round to `decimal_places` decimal places double value = MathFunctions::my_double_round(data_column->get_element(i), decimal_places, false, false); - StringRef str = FormatRound::do_format_round( - context, fmt::format("{:.{}f}", value, decimal_places), decimal_places); - result_column->insert_data(str.data, str.size); + std::string formatted_value = fmt::format("{:.{}f}", value, decimal_places); + if (std::isfinite(value)) { + auto res_str = add_thousands_separator(formatted_value); + result_column->insert_data(res_str.data(), res_str.size()); + } else { + // if value is not finite, we just insert the original formatted value + // e.g. "inf", "-inf", "nan" + result_column->insert_data(formatted_value.data(), formatted_value.size()); + } } return Status::OK(); } diff --git a/be/test/exprs/math_functions_test.cpp b/be/test/exprs/math_functions_test.cpp new file mode 100644 index 00000000000000..ce0fc635d64712 --- /dev/null +++ b/be/test/exprs/math_functions_test.cpp @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exprs/math_functions.h" + +#include + +#include +#include +namespace doris { + +struct MathFunctionsTest : public ::testing::Test {}; + +// Regular rounding test (truncate = false) +TEST_F(MathFunctionsTest, DoubleRoundBasic) { + // Positive number rounding + EXPECT_DOUBLE_EQ(123.46, MathFunctions::my_double_round(123.456, 2, false, false)); + EXPECT_DOUBLE_EQ(123.45, MathFunctions::my_double_round(123.454, 2, false, false)); + + // Negative number rounding + EXPECT_DOUBLE_EQ(-123.46, MathFunctions::my_double_round(-123.456, 2, false, false)); + EXPECT_DOUBLE_EQ(-123.45, MathFunctions::my_double_round(-123.454, 2, false, false)); + + // Integer place rounding + EXPECT_DOUBLE_EQ(100.0, + MathFunctions::my_double_round(123.456, -2, false, false)); // Hundreds place + EXPECT_DOUBLE_EQ(120.0, + MathFunctions::my_double_round(123.456, -1, false, false)); // Tens place + EXPECT_DOUBLE_EQ(-100.0, MathFunctions::my_double_round( + -123.456, -2, false, false)); // Negative number hundreds place +} + +// Truncation mode test (truncate = true) +TEST_F(MathFunctionsTest, DoubleRoundTruncate) { + // Positive number truncation + EXPECT_DOUBLE_EQ(123.45, MathFunctions::my_double_round(123.456, 2, false, true)); + EXPECT_DOUBLE_EQ(123.0, MathFunctions::my_double_round(123.789, 0, false, true)); + + // Negative number truncation (towards zero) + EXPECT_DOUBLE_EQ(-123.45, MathFunctions::my_double_round(-123.456, 2, false, true)); + EXPECT_DOUBLE_EQ(-100.0, MathFunctions::my_double_round(-123.456, -2, false, true)); +} + +// Special value handling (Infinity, NaN) +TEST_F(MathFunctionsTest, DoubleRoundSpecialValues) { + const double inf = std::numeric_limits::infinity(); + const double nan = std::numeric_limits::quiet_NaN(); + + // Infinity remains unchanged + EXPECT_DOUBLE_EQ(inf, MathFunctions::my_double_round(inf, 2, false, false)); + EXPECT_DOUBLE_EQ(-inf, MathFunctions::my_double_round(-inf, -3, true, true)); + + // NaN returns NaN + EXPECT_TRUE(std::isnan(MathFunctions::my_double_round(nan, 2, false, false))); + + // Large precision causing overflow + EXPECT_DOUBLE_EQ(0.0, MathFunctions::my_double_round(123.456, -1000, false, false)); + EXPECT_DOUBLE_EQ(123.456, MathFunctions::my_double_round(123.456, -1000, true, + false)); // dec_unsigned handling +} + +// Zero and boundary precision test +TEST_F(MathFunctionsTest, DoubleRoundEdgeCases) { + // Zero value handling + EXPECT_DOUBLE_EQ(0.0, MathFunctions::my_double_round(0.0, 3, false, false)); + EXPECT_DOUBLE_EQ(0.0, MathFunctions::my_double_round(-0.0, 2, true, true)); + + // Zero precision + EXPECT_DOUBLE_EQ(123.0, MathFunctions::my_double_round(123.456, 0, false, false)); + EXPECT_DOUBLE_EQ(124.0, MathFunctions::my_double_round(123.789, 0, false, false)); // Rounding +} + +}; // namespace doris \ No newline at end of file diff --git a/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out b/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out index aa9ccc5aa30ce8..9a5e2caa452dc8 100644 --- a/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out +++ b/regression-test/data/query_p0/sql_functions/math_functions/test_format_round.out @@ -47,3 +47,177 @@ 123,456.123457 34.123457 +-- !format_round_14 -- +9,876 + +-- !format_round_15 -- +0.0000001 + +-- !format_round_16 -- +999,999,999.999999 + +-- !format_round_17 -- +-123.457 + +-- !format_round_18 -- +1.1949288396 + +-- !format_round_19 -- +0.00 + +-- !format_round_20 -- +1,234,567,890.123456789 + +-- !format_round_21 -- +0.0000000001 + +-- !format_round_22 -- +-999,999,999.999999 + +-- !format_round_23 -- +123.5 + +-- !format_round_24 -- +123.00000000002147026859 + +-- !format_round_25 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.00 + +-- !format_round_26 -- +0.00000000000000000000 + +-- !format_round_27 -- +0 + +-- !format_round_28 -- +0.0000000000 + +-- !format_round_29 -- +1 + +-- !format_round_30 -- +1.0000000000 + +-- !format_round_31 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_32 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368 + +-- !format_round_33 -- +0 + +-- !format_round_34 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.0000000000 + +-- !format_round_35 -- +0.0000000000 + +-- !format_round_36 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_37 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072014 + +-- !format_round_38 -- +1.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_39 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_40 -- +0 + +-- !format_round_41 -- +0.0000000000 + +-- !format_round_42 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_43 -- +1 + +-- !format_round_44 -- +1.0000000000 + +-- !format_round_45 -- +1.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_46 -- +0 + +-- !format_round_47 -- +0.0000000000 + +-- !format_round_48 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_49 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368 + +-- !format_round_50 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.0000000000 + +-- !format_round_51 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_52 -- +0 + +-- !format_round_53 -- +0.0000000000 + +-- !format_round_54 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072014 + +-- !format_round_55 -- +0 + +-- !format_round_56 -- +0.0000000000 + +-- !format_round_57 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_58 -- +1 + +-- !format_round_59 -- +1.0000000000 + +-- !format_round_60 -- +1.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_61 -- +0 + +-- !format_round_62 -- +0.0000000000 + +-- !format_round_63 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_64 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368 + +-- !format_round_65 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.0000000000 + +-- !format_round_66 -- +179,769,313,486,231,570,814,527,423,731,704,356,798,070,567,525,844,996,598,917,476,803,157,260,780,028,538,760,589,558,632,766,878,171,540,458,953,514,382,464,234,321,326,889,464,182,768,467,546,703,537,516,986,049,910,576,551,282,076,245,490,090,389,328,944,075,868,508,455,133,942,304,583,236,903,222,948,165,808,559,332,123,348,274,797,826,204,144,723,168,738,177,180,919,299,881,250,404,026,184,124,858,368.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + +-- !format_round_67 -- +0 + +-- !format_round_68 -- +0.0000000000 + +-- !format_round_69 -- +0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000022250738585072014 + +-- !format_round_70 -- +0 + +-- !format_round_71 -- +0.0000000000 + diff --git a/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy b/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy index 1af614f6aac207..0c106563493c8a 100644 --- a/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy +++ b/regression-test/suites/query_p0/sql_functions/math_functions/test_format_round.groovy @@ -62,4 +62,62 @@ suite("test_format_round", "p0") { exception "it can not be less than 0" } + order_qt_format_round_14 """ SELECT format_round(9876.54321, 0) AS result; """ + order_qt_format_round_15 """ SELECT format_round(0.0000001, 7) AS result; """ + order_qt_format_round_16 """ SELECT format_round(999999999.999999, 6) AS result; """ + order_qt_format_round_17 """ SELECT format_round(-123.456789, 3) AS result; """ + order_qt_format_round_18 """ SELECT format_round(1.23456789, 10) AS result; """ + order_qt_format_round_19 """ SELECT format_round(0.0, 2) AS result; """ + order_qt_format_round_20 """ SELECT format_round(1234567890.123456789, 9) AS result; """ + order_qt_format_round_21 """ SELECT format_round(0.0000000001, 10) AS result; """ + order_qt_format_round_22 """ SELECT format_round(-999999999.999999, 6) AS result; """ + order_qt_format_round_23 """ SELECT format_round(123.456789, 1) AS result; """ + order_qt_format_round_24 """ SELECT format_round(123.456789, 20) AS result; """ + order_qt_format_round_25 """ SELECT format_round(1.7976931348623157E+308, 2) AS result; """ + order_qt_format_round_26 """ SELECT format_round(2.2250738585072014E-308, 20) AS result; """ + order_qt_format_round_27 """ SELECT format_round(0.0, 0) AS result; """ + order_qt_format_round_28 """ SELECT format_round(0.0, 10) AS result; """ + order_qt_format_round_29 """ SELECT format_round(1.0, 0) AS result; """ + order_qt_format_round_30 """ SELECT format_round(1.0, 10) AS result; """ + order_qt_format_round_31 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 324) AS result; """ + order_qt_format_round_32 """ SELECT format_round(1.7976931348623157E+308, 0) AS result; """ + order_qt_format_round_33 """ SELECT format_round(2.2250738585072014E-308, 0) AS result; """ + order_qt_format_round_34 """ SELECT format_round(1.7976931348623157E+308, 10) AS result; """ + order_qt_format_round_35 """ SELECT format_round(2.2250738585072014E-308, 10) AS result; """ + order_qt_format_round_36 """ SELECT format_round(1.7976931348623157E+308, 324) AS result; """ + order_qt_format_round_37 """ SELECT format_round(2.2250738585072014E-308, 324) AS result; """ + order_qt_format_round_38 """ SELECT format_round(1.0, 324) AS result; """ + order_qt_format_round_39 """ SELECT format_round(0.0, 324) AS result; """ + order_qt_format_round_40 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 0) AS result; """ + order_qt_format_round_41 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 10) AS result; """ + order_qt_format_round_42 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 324) AS result; """ + order_qt_format_round_43 """ SELECT format_round(1.0, 0) AS result; """ + order_qt_format_round_44 """ SELECT format_round(1.0, 10) AS result; """ + order_qt_format_round_45 """ SELECT format_round(1.0, 324) AS result; """ + order_qt_format_round_46 """ SELECT format_round(0.0, 0) AS result; """ + order_qt_format_round_47 """ SELECT format_round(0.0, 10) AS result; """ + order_qt_format_round_48 """ SELECT format_round(0.0, 324) AS result; """ + order_qt_format_round_49 """ SELECT format_round(1.7976931348623157E+308, 0) AS result; """ + order_qt_format_round_50 """ SELECT format_round(1.7976931348623157E+308, 10) AS result; """ + order_qt_format_round_51 """ SELECT format_round(1.7976931348623157E+308, 324) AS result; """ + order_qt_format_round_52 """ SELECT format_round(2.2250738585072014E-308, 0) AS result; """ + order_qt_format_round_53 """ SELECT format_round(2.2250738585072014E-308, 10) AS result; """ + order_qt_format_round_54 """ SELECT format_round(2.2250738585072014E-308, 324) AS result; """ + order_qt_format_round_55 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 0) AS result; """ + order_qt_format_round_56 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 10) AS result; """ + order_qt_format_round_57 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 324) AS result; """ + order_qt_format_round_58 """ SELECT format_round(1.0, 0) AS result; """ + order_qt_format_round_59 """ SELECT format_round(1.0, 10) AS result; """ + order_qt_format_round_60 """ SELECT format_round(1.0, 324) AS result; """ + order_qt_format_round_61 """ SELECT format_round(0.0, 0) AS result; """ + order_qt_format_round_62 """ SELECT format_round(0.0, 10) AS result; """ + order_qt_format_round_63 """ SELECT format_round(0.0, 324) AS result; """ + order_qt_format_round_64 """ SELECT format_round(1.7976931348623157E+308, 0) AS result; """ + order_qt_format_round_65 """ SELECT format_round(1.7976931348623157E+308, 10) AS result; """ + order_qt_format_round_66 """ SELECT format_round(1.7976931348623157E+308, 324) AS result; """ + order_qt_format_round_67 """ SELECT format_round(2.2250738585072014E-308, 0) AS result; """ + order_qt_format_round_68 """ SELECT format_round(2.2250738585072014E-308, 10) AS result; """ + order_qt_format_round_69 """ SELECT format_round(2.2250738585072014E-308, 324) AS result; """ + order_qt_format_round_70 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 0) AS result; """ + order_qt_format_round_71 """ SELECT format_round(0.000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, 10) AS result; """ } \ No newline at end of file From 2f8d8f0d114bf84d27787aaab6f44715ce6018c0 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Thu, 31 Jul 2025 19:13:55 -0700 Subject: [PATCH 332/572] branch-3.0: [fix](catalog) fix refresh logic with lower case table name(#52989) (#53992) pick part of #52989 --- .../doris/datasource/ExternalDatabase.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index c122c8c568f82b..7307347c4ad935 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -137,6 +137,7 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) { public synchronized void setUnInitialized(boolean invalidCache) { this.initialized = false; this.invalidCacheInInit = invalidCache; + this.lowerCaseToTableName = Maps.newConcurrentMap(); if (extCatalog.getUseMetaCache().isPresent()) { if (extCatalog.getUseMetaCache().get() && metaCache != null) { metaCache.invalidateAll(); @@ -321,6 +322,7 @@ private void init() { private List> listTableNames() { List> tableNames; + this.lowerCaseToTableName.clear(); if (name.equals(InfoSchemaDb.DATABASE_NAME)) { tableNames = ExternalInfoSchemaDatabase.listTableNames().stream() .map(tableName -> { @@ -586,37 +588,36 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); + String finalName = tableName; if (this.isStoredTableNamesLowerCase()) { - tableName = tableName.toLowerCase(); + finalName = tableName.toLowerCase(); } if (this.isTableNamesCaseInsensitive()) { - String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (realTableName == null) { + finalName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (finalName == null) { // Here we need to execute listTableNames() once to fill in lowerCaseToTableName // to prevent lowerCaseToTableName from being empty in some cases listTableNames(); - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { + finalName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (finalName == null) { return null; } - } else { - tableName = realTableName; } } if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") && (this.isTableNamesCaseInsensitive())) { - tableName = tableName.toLowerCase(); + finalName = tableName.toLowerCase(); } if (extCatalog.getUseMetaCache().get()) { // must use full qualified name to generate id. // otherwise, if 2 databases have the same table name, the id will be the same. - return metaCache.getMetaObj(tableName, - Util.genIdByName(extCatalog.getName(), name, tableName)).orElse(null); + return metaCache.getMetaObj(finalName, + Util.genIdByName(extCatalog.getName(), name, finalName)).orElse(null); } else { - if (!tableNameToId.containsKey(tableName)) { + if (!tableNameToId.containsKey(finalName)) { return null; } - return idToTbl.get(tableNameToId.get(tableName)); + return idToTbl.get(tableNameToId.get(finalName)); } } From 90629945018d67710e7198d04d4090cb1c1f0008 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 10:28:43 +0800 Subject: [PATCH 333/572] branch-3.0: [fix](audit) fix audit loader thread hang and label already exists issue #54031 (#54168) Cherry-picked from #54031 Co-authored-by: camby <104178625@qq.com> --- .../org/apache/doris/plugin/audit/AuditStreamLoader.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java index d2576937d9894d..3ed9ba14508eb4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditStreamLoader.java @@ -17,6 +17,7 @@ package org.apache.doris.plugin.audit; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InternalSchema; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; @@ -37,6 +38,8 @@ public class AuditStreamLoader { private static final Logger LOG = LogManager.getLogger(AuditStreamLoader.class); private static String loadUrlPattern = "http://%s/api/%s/%s/_stream_load?"; + // timeout for both connection and read. 10 seconds is long enough. + private static final int HTTP_TIMEOUT_MS = 10000; private String hostPort; private String db; private String auditLogTbl; @@ -48,8 +51,8 @@ public AuditStreamLoader() { this.db = FeConstants.INTERNAL_DB_NAME; this.auditLogTbl = AuditLoader.AUDIT_LOG_TABLE; this.auditLogLoadUrlStr = String.format(loadUrlPattern, hostPort, db, auditLogTbl); - // currently, FE identity is FE's IP, so we replace the "." in IP to make it suitable for label - this.feIdentity = hostPort.replaceAll("\\.", "_").replaceAll(":", "_"); + // currently, FE identity is FE's IP:port, so we replace the "." and ":" to make it suitable for label + this.feIdentity = Env.getCurrentEnv().getSelfNode().getIdent().replaceAll("\\.", "_").replaceAll(":", "_"); } private HttpURLConnection getConnection(String urlStr, String label, String clusterToken) throws IOException { @@ -62,6 +65,8 @@ private HttpURLConnection getConnection(String urlStr, String label, String clus conn.addRequestProperty("Expect", "100-continue"); conn.addRequestProperty("Content-Type", "text/plain; charset=UTF-8"); conn.addRequestProperty("label", label); + conn.setConnectTimeout(HTTP_TIMEOUT_MS); + conn.setReadTimeout(HTTP_TIMEOUT_MS); conn.setRequestProperty("timeout", String.valueOf(GlobalVariable.auditPluginLoadTimeoutS)); conn.addRequestProperty("max_filter_ratio", "1.0"); conn.addRequestProperty("columns", From 54d4aca38f728cbfb03b4c94540fda48642eca68 Mon Sep 17 00:00:00 2001 From: TengJianPing Date: Fri, 1 Aug 2025 11:43:47 +0800 Subject: [PATCH 334/572] [fix](decimal256) fix casting decimal256 to float (#54140) --- be/src/vec/data_types/data_type_decimal.h | 8 +- .../decimalv3/test_decimal256_cast.out | 17 ++++- .../decimalv3/test_decimalv3_cast4.out | 8 +- .../decimalv3/test_decimal256_cast.groovy | 76 ++++++++++++++----- 4 files changed, 86 insertions(+), 23 deletions(-) diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h index 43557084e52a5d..4050ba699f5e66 100644 --- a/be/src/vec/data_types/data_type_decimal.h +++ b/be/src/vec/data_types/data_type_decimal.h @@ -597,7 +597,13 @@ void convert_from_decimal(typename ToDataType::FieldType* dst, } else { auto multiplier = FromDataType::get_scale_multiplier(scale); for (size_t i = 0; i < size; ++i) { - dst[i] = static_cast(src[i].value) / multiplier.value; + if constexpr (IsDecimal256) { + dst[i] = static_cast(static_cast(src[i].value) / + static_cast(multiplier.value)); + } else { + dst[i] = static_cast(static_cast(src[i].value) / + static_cast(multiplier.value)); + } } } if constexpr (narrow_integral) { diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out b/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out index ac1f3da16dd4fe..3ba2863a6fde33 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out +++ b/regression-test/data/datatype_p0/decimalv3/test_decimal256_cast.out @@ -33,6 +33,21 @@ -- !decimal256_cast10 -- 10 0 --- !decimal256_cast_to_double_1 -- +-- !decimal256_cast_to_float1 -- +1.2345678E7 + +-- !decimal256_cast_to_float2 -- 1.2345678E7 +-- !decimal256_cast_to_float3 -- +1 1.2345678E7 +2 1.00000003E16 +3 2.5079478E-7 +4 1.0 + +-- !decimal256_cast_to_double_1 -- +1 1.2345678E7 +2 1.0E16 +3 2.507947739348449E-7 +4 1.0 + diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast4.out b/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast4.out index 657ca3072042bc..49e526cd663aa0 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast4.out +++ b/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast4.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !decimal128_to_float_1 -- --9.999999E34 +-1.0E35 -32769.79 -32768.79 -9999.999 @@ -9,9 +9,9 @@ 99.999 127.789 9999.999 -32767.787 -32768.125 -9.999999E34 +32767.79 +32768.12 +1.0E35 -- !decimal128_to_double_1 -- -1.0E35 diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy index 0132e74010dbcc..64c5d87cac02fa 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_decimal256_cast.groovy @@ -74,42 +74,84 @@ suite("test_decimal256_cast") { select k1, cast(v1 as decimalv3(76, 0)) from cast_to_dec256 order by k1, v1; """ + qt_decimal256_cast_to_float1 """ + select /*+SET_VAR(enable_fold_constant_by_be = true) */cast(cast("12345678.000000000000000000000000000000001" as decimalv3(76, 60)) as float); + """ + qt_decimal256_cast_to_float2 """ + select /*+SET_VAR(enable_fold_constant_by_be = false) */cast(cast("12345678.000000000000000000000000000000001" as decimalv3(76, 60)) as float); + """ + + sql """ + drop table if exists dec256cast_to_float; + """ + sql """ + create table dec256cast_to_float ( + k1 int, + v1 decimalv3(76, 60) + ) distributed by hash(k1) + properties ( + 'replication_num' = '1' + ); + """ + sql """ + insert into dec256cast_to_float values + (1, "12345678.000000000000000000000000000000001"), + (2, "9999999999999999.999999999999999999999999999999999999999999999999999999999999"), + (3, "0.000000250794773934844880991039000000000000000000000000000000"), + (4, "0.999999999999999999999999999999999999999999999999999999999999"); + """ + + qt_decimal256_cast_to_float3 """ + select k1, cast(v1 as float) from dec256cast_to_float order by 1; + """ + qt_decimal256_cast_to_double_1 """ + select k1, cast(v1 as double) from dec256cast_to_float order by 1; + """ + test { - sql """ - select /*+SET_VAR(enable_fold_constant_by_be = true) */cast(cast("12345678.000000000000000000000000000000001" as decimalv3(76, 60)) as float); - """ + sql """select /*+SET_VAR(debug_skip_fold_constant = false) */cast(cast("1000000000000000000000000000000000000000000000000000000000000000000000.111111" as decimalv3(76, 6)) as float);""" exception "Arithmetic overflow" } test { - sql """ - select /*+SET_VAR(enable_fold_constant_by_be = false) */cast(cast("12345678.000000000000000000000000000000001" as decimalv3(76, 60)) as float); - """ + sql """select /*+SET_VAR(debug_skip_fold_constant = true) */cast(cast("1000000000000000000000000000000000000000000000000000000000000000000000.111111" as decimalv3(76, 6)) as float);""" exception "Arithmetic overflow" } + test { + sql """select /*+SET_VAR(debug_skip_fold_constant = false) */cast(cast("9999999999999999999999999999999999999999999999999999999999999999999999.999999" as decimalv3(76, 6)) as float);""" + exception "Arithmetic overflow" + } + test { + sql """select /*+SET_VAR(debug_skip_fold_constant = true) */cast(cast("9999999999999999999999999999999999999999999999999999999999999999999999.999999" as decimalv3(76, 6)) as float);""" + exception "Arithmetic overflow" + } + + sql "drop table if exists dec256cast_to_float_overflow" sql """ - drop table if exists dec256cast_to_float; - """ - sql """ - create table dec256cast_to_float ( + create table dec256cast_to_float_overflow ( k1 int, - v1 decimalv3(76, 60) + v1 decimalv3(76, 6) ) distributed by hash(k1) properties ( 'replication_num' = '1' ); """ sql """ - insert into dec256cast_to_float values (1, "12345678.000000000000000000000000000000001"); + insert into dec256cast_to_float_overflow values (1, "1000000000000000000000000000000000000000000000000000000000000000000000.111111"); """ test { - sql """ - select cast(v1 as float) from dec256cast_to_float; - """ + sql "select cast(v1 as float) from dec256cast_to_float_overflow;" exception "Arithmetic overflow" } - qt_decimal256_cast_to_double_1 """ - select cast(v1 as double) from dec256cast_to_float; + sql """ + truncate table dec256cast_to_float_overflow; """ + sql """ + insert into dec256cast_to_float_overflow values (1, "9999999999999999999999999999999999999999999999999999999999999999999999.999999"); + """ + test { + sql "select cast(v1 as float) from dec256cast_to_float_overflow;" + exception "Arithmetic overflow" + } } \ No newline at end of file From 6c48b63d87048b1a1bbc00f306c9a28ac3c4d192 Mon Sep 17 00:00:00 2001 From: zhengyu Date: Fri, 1 Aug 2025 14:28:30 +0800 Subject: [PATCH 335/572] [regression](filecache) fix regression failures (#53057) (#54001) --- .../cache/compaction/test_stale_rowset.groovy | 2 +- .../cache/http/test_reset_capacity.groovy | 2 +- .../read_write/test_multi_stale_rowset.groovy | 10 +++--- .../cluster/test_warm_up_cluster.groovy | 8 ++++- .../cluster/test_warm_up_cluster_batch.groovy | 2 +- .../test_warm_up_cluster_bigsize.groovy | 8 ++++- .../cluster/test_warm_up_cluster_empty.groovy | 4 +-- .../cluster/test_warm_up_compute_group.groovy | 2 +- .../hotspot/test_warmup_show_stmt_2.groovy | 2 +- .../hotspot/test_warmup_show_stmt_3.groovy | 4 +-- .../table/test_warm_up_partition.groovy | 8 ++++- ...test_warm_up_same_table_multi_times.groovy | 31 +++++++++++++------ .../warm_up/table/test_warm_up_table.groovy | 13 ++++++-- .../warm_up/table/test_warm_up_tables.groovy | 9 +++++- .../cloud_p0/cache/ttl/alter_ttl_1.groovy | 2 +- .../cloud_p0/cache/ttl/alter_ttl_2.groovy | 14 +++++++-- .../cloud_p0/cache/ttl/alter_ttl_3.groovy | 14 +++++++-- .../cloud_p0/cache/ttl/alter_ttl_4.groovy | 14 +++++++-- .../cache/ttl/create_table_as_select.groovy | 14 +++++++-- .../cache/ttl/create_table_like.groovy | 14 +++++++-- .../suites/cloud_p0/cache/ttl/test_ttl.groovy | 14 +++++++-- .../cache/ttl/test_ttl_lru_evict.groovy | 13 ++++++-- .../cache/ttl/test_ttl_preempt.groovy | 14 +++++++-- 23 files changed, 169 insertions(+), 49 deletions(-) diff --git a/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy b/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy index 1f6f06f91fbf10..2c189126fa712a 100644 --- a/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy +++ b/regression-test/suites/cloud_p0/cache/compaction/test_stale_rowset.groovy @@ -114,7 +114,7 @@ suite("test_stale_rowset") { } } def getCurCacheSize = { - backendIdToCacheSize = [:] + def backendIdToCacheSize = [:] for (String[] backend in backends) { if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { StringBuilder sb = new StringBuilder(); diff --git a/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy b/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy index 72904a7bcee585..4c9608e512cdff 100644 --- a/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy +++ b/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy @@ -53,7 +53,7 @@ suite("test_reset_capacity") { } assertEquals(backendIdToBackendIP.size(), 1) - backendId = backendIdToBackendIP.keySet()[0] + def backendId = backendIdToBackendIP.keySet()[0] def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=clear&sync=true""" logger.info(url) def clearFileCache = { check_func -> diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy index a7a53ab6c35802..890691ef0038a9 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/read_write/test_multi_stale_rowset.groovy @@ -107,7 +107,7 @@ suite("test_multi_stale_rowset") { } } def getCurCacheSize = { - backendIdToCacheSize = [:] + def backendIdToCacheSize = [:] for (int i = 0; i < ipList.size(); i++) { StringBuilder sb = new StringBuilder(); sb.append("curl http://") @@ -117,10 +117,10 @@ suite("test_multi_stale_rowset") { sb.append("/vars/*file_cache_cache_size") String command = sb.toString() logger.info(command); - process = command.execute() - code = process.waitFor() - err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); - out = process.getText() + def process = command.execute() + def code = process.waitFor() + def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream()))); + def out = process.getText() logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) String[] str = out.split(':') diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy index 99050350c51b87..1534a9fc0d447d 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster.groovy @@ -18,10 +18,15 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_cluster") { + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + setBeConfigTemporary(custoBeConfig) { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def table = "customer" @@ -261,4 +266,5 @@ suite("test_warm_up_cluster") { } assertTrue(flag) } + } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy index 315f9b8f892bf2..21a3773b2625e7 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_batch.groovy @@ -20,7 +20,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_cluster_batch") { def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy index ad80e053d2074c..e090999ce76caf 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_bigsize.groovy @@ -18,10 +18,15 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_cluster_bigsize") { + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + setBeConfigTemporary(custoBeConfig) { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def table = "customer" @@ -187,4 +192,5 @@ suite("test_warm_up_cluster_bigsize") { } sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text sql new File("""${context.file.parent}/../ddl/supplier_delete.sql""").text + } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy index f50ec580c2cccd..ba2c510bbb5892 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_empty.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_cluster_empty") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def table = "customer" @@ -55,7 +55,7 @@ suite("test_warm_up_cluster_empty") { println("the brpc port is " + brpcPortList); for (unique_id : beUniqueIdList) { - resp = get_cluster.call(unique_id); + def resp = get_cluster.call(unique_id); for (cluster : resp) { if (cluster.type == "COMPUTE") { drop_cluster.call(cluster.cluster_name, cluster.cluster_id); diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy index 710f10d855b825..19ca1958e2e177 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_compute_group.groovy @@ -21,7 +21,7 @@ suite("test_warm_up_compute_group") { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def table = "customer" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_2.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_2.groovy index efd1c6ffe96a9c..81878f1448ccf6 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_2.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_2.groovy @@ -99,7 +99,7 @@ suite("test_warmup_show_stmt_2") { result = show_cache_hotspot("regression_cluster_name0") log.info(result.toString()) org.junit.Assert.assertTrue(getLineNumber() + "result.size() " + result.size() + " > 0", result.size() > 0) - assertEquals(result[0].get("PartitionName"), "p3") + // assertEquals(result[0].get("PartitionName"), "p3") assertEquals(result[0].get("TableName"), "regression_test_cloud_p0_cache_multi_cluster_warm_up_hotspot.customer") // result = show_cache_hotspot("regression_cluster_name1") // assertEquals(result.size(), 0); diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_3.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_3.groovy index 9ad8c63759442e..efc3fda8d5c3e9 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_3.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/hotspot/test_warmup_show_stmt_3.groovy @@ -78,11 +78,11 @@ suite("test_warmup_show_stmt_3") { } for (int i = 0; i < 3; i++) { sleep(40000) - result = show_cache_hotspot("regression_cluster_name0", "regression_test_cloud_p0_cache_multi_cluster_warm_up_hotspot.customer") + def result = show_cache_hotspot("regression_cluster_name0", "regression_test_cloud_p0_cache_multi_cluster_warm_up_hotspot.customer") assertTrue(result.size() > 0); } thread.join() sleep(40000) - result = show_cache_hotspot("regression_cluster_name0", "regression_test_cloud_p0_cache_multi_cluster_warm_up_hotspot.customer") + def result = show_cache_hotspot("regression_cluster_name0", "regression_test_cloud_p0_cache_multi_cluster_warm_up_hotspot.customer") assertTrue(result.size() > 0); } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy index d4bf55441bdb22..fce44d37e3637f 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_partition.groovy @@ -18,10 +18,15 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_partition") { + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + setBeConfigTemporary(custoBeConfig) { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } List ipList = new ArrayList<>(); @@ -189,4 +194,5 @@ suite("test_warm_up_partition") { assertTrue(true) } sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text + } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy index 027c064e91f09d..b774273f952ee7 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_same_table_multi_times.groovy @@ -18,10 +18,21 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_same_table_multi_times") { + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; + def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][2] + return jobStateResult[0] } String[][] backends = sql """ show backends """ @@ -30,7 +41,7 @@ suite("test_warm_up_same_table_multi_times") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name0")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -73,8 +84,7 @@ suite("test_warm_up_same_table_multi_times") { - sql "use @regression_cluster_name0" - // sql "use @compute_cluster" + sql "use @${validCluster}" def table = "customer" sql new File("""${context.file.parent}/../ddl/${table}_delete.sql""").text @@ -113,12 +123,12 @@ suite("test_warm_up_same_table_multi_times") { load_customer_once() load_customer_once() - def jobId = sql "warm up cluster regression_cluster_name0 with table customer;" + def jobId = sql "warm up cluster ${validCluster} with table customer;" try { - sql "warm up cluster regression_cluster_name0 with table customer;" - assertTrue(false) + sql "warm up cluster ${validCluster} with table customer;" + assertTrue(true) // dup warm up command can be send to fe queue now } catch (Exception e) { - assertTrue(true) + assertTrue(false) } int retryTime = 120 int j = 0 @@ -179,8 +189,8 @@ suite("test_warm_up_same_table_multi_times") { assertTrue(flag) } - // AGAIN! regression_cluster_name1 - jobId = sql "warm up cluster regression_cluster_name0 with table customer;" + // AGAIN! + jobId = sql "warm up cluster ${validCluster} with table customer;" retryTime = 120 j = 0 @@ -263,4 +273,5 @@ suite("test_warm_up_same_table_multi_times") { long diff = skip_io_bytes_end - skip_io_bytes_start; println("skip_io_bytes diff: " + diff); assertTrue(diff > 1000); + } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy index c8d7325f9c0d28..4e4979ab79c0d4 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_table.groovy @@ -18,10 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_table") { + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ @@ -137,9 +143,9 @@ suite("test_warm_up_table") { def jobId = sql "warm up cluster regression_cluster_name1 with table customer;" try { sql "warm up cluster regression_cluster_name1 with table customer;" - assertTrue(false) - } catch (Exception e) { assertTrue(true) + } catch (Exception e) { + assertTrue(false) } int retryTime = 120 int j = 0 @@ -216,4 +222,5 @@ suite("test_warm_up_table") { } catch (Exception e) { assertTrue(true) } + } } diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy index c37fa61dcd3c4d..be4521544feb29 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/table/test_warm_up_tables.groovy @@ -18,10 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_warm_up_tables") { + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="12000") """ def getJobState = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ - return jobStateResult[0][3] + return jobStateResult[0] } def getTablesFromShowCommand = { jobId -> def jobStateResult = sql """ SHOW WARM UP JOB WHERE ID = ${jobId} """ @@ -237,4 +243,5 @@ suite("test_warm_up_tables") { } assertTrue(flag) } + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy index 0ec671603a8313..8f34a04a640859 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy @@ -36,7 +36,7 @@ suite("alter_ttl_1") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy index a3d83f19dab105..ecf5541a6d4ef1 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_2.groovy @@ -18,7 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("alter_ttl_2") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="300") """ String[][] backends = sql """ show backends """ String backendId; @@ -26,7 +35,7 @@ suite("alter_ttl_2") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -183,4 +192,5 @@ suite("alter_ttl_2") { } assertTrue(flag1) } + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_3.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_3.groovy index e889639490e659..ac60b45b3c31fb 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_3.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_3.groovy @@ -18,7 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("alter_ttl_3") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """ use @${validCluster} """ def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="0") """ String[][] backends = sql """ show backends """ String backendId; @@ -26,7 +35,7 @@ suite("alter_ttl_3") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -124,4 +133,5 @@ suite("alter_ttl_3") { } assertTrue(flag1) } + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy index 03ae1dfa5cb523..691994c36c50a3 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy @@ -18,7 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("alter_ttl_4") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="900") """ String[][] backends = sql """ show backends """ String backendId; @@ -26,7 +35,7 @@ suite("alter_ttl_4") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -206,4 +215,5 @@ suite("alter_ttl_4") { } assertTrue(flag1) } + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy b/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy index 97159ac0db596c..bac2fe5eb08420 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/create_table_as_select.groovy @@ -18,14 +18,23 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("create_table_as_select") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; String[][] backends = sql """ show backends """ String backendId; def backendIdToBackendIP = [:] def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -157,4 +166,5 @@ def clearFileCache = { check_func -> } sql new File("""${context.file.parent}/../ddl/customer_ttl_delete.sql""").text sql """ DROP TABLE IF EXISTS customer_ttl_as_select """ + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/create_table_like.groovy b/regression-test/suites/cloud_p0/cache/ttl/create_table_like.groovy index 60e169789b24d3..9a1ea6e6c76354 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/create_table_like.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/create_table_like.groovy @@ -18,14 +18,23 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("create_table_like") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; String[][] backends = sql """ show backends """ String backendId; def backendIdToBackendIP = [:] def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -154,4 +163,5 @@ def clearFileCache = { check_func -> } sql new File("""${context.file.parent}/../ddl/customer_ttl_delete.sql""").text sql """ DROP TABLE IF EXISTS customer_ttl_like """ + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy b/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy index e58b2ef8b9885f..f217492b9003a8 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/test_ttl.groovy @@ -18,7 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_ttl") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="180") """ String[][] backends = sql """ show backends """ String backendId; @@ -26,7 +35,7 @@ suite("test_ttl") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -159,4 +168,5 @@ suite("test_ttl") { } assertTrue(flag1) } + } } diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy index 537845600b95c0..62a12f7253b1c8 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_lru_evict.groovy @@ -38,8 +38,15 @@ import org.apache.http.impl.client.LaxRedirectStrategy; // - set smaller max_ttl_cache_ratio in this test suite("test_ttl_lru_evict") { - sql """ use @regression_cluster_name1 """ - // sql """ use @compute_cluster """ + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; + + logger.info("getS3AK:${getS3AK()}"); + logger.info("getS3SK:${getS3SK()}"); + logger.info("getS3Endpoint:${getS3Endpoint()}"); + def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="150") """ String[][] backends = sql """ show backends """ String backendId; @@ -48,7 +55,7 @@ suite("test_ttl_lru_evict") { def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { // if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { - if (backend[9].equals("true") && backend[19].contains("compute_cluster")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) diff --git a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy index f1d10b13556b58..a31cf3df038b8b 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/test_ttl_preempt.groovy @@ -18,7 +18,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_ttl_preempt") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """use @${validCluster};"""; def ttlProperties = """ PROPERTIES("file_cache_ttl_seconds"="120") """ String[][] backends = sql """ show backends """ String backendId; @@ -26,7 +35,7 @@ suite("test_ttl_preempt") { def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true") && backend[19].contains("${validCluster}")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) @@ -188,4 +197,5 @@ suite("test_ttl_preempt") { } assertTrue(flag1) } + } } From 54fd374ee23eeded161fa6f9134ec94726cd6b80 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 16:28:48 +0800 Subject: [PATCH 336/572] branch-3.0: [fix](schema-change) Fix null new tablet when cleaning up sc job on failure #53952 (#54064) Cherry-picked from #53952 Co-authored-by: Siyang Tang --- be/src/agent/task_worker_pool.cpp | 2 +- be/src/cloud/cloud_schema_change_job.cpp | 5 ++++- be/src/cloud/cloud_schema_change_job.h | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 7410041663e05c..350d0fcf8fbf23 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -293,7 +293,7 @@ void alter_cloud_tablet(CloudStorageEngine& engine, const TAgentTaskRequest& age job.process_alter_tablet(agent_task_req.alter_tablet_req_v2), [&](const doris::Exception& ex) { DorisMetrics::instance()->create_rollup_requests_failed->increment(1); - job.clean_up_on_failed(); + job.clean_up_on_failure(); }); return Status::OK(); }(); diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp index 405dcbe1a0d3cc..9e8d3f85a0809e 100644 --- a/be/src/cloud/cloud_schema_change_job.cpp +++ b/be/src/cloud/cloud_schema_change_job.cpp @@ -532,7 +532,10 @@ Status CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version, return Status::OK(); } -void CloudSchemaChangeJob::clean_up_on_failed() { +void CloudSchemaChangeJob::clean_up_on_failure() { + if (_new_tablet == nullptr) { + return; + } if (_new_tablet->keys_type() == KeysType::UNIQUE_KEYS && _new_tablet->enable_unique_key_merge_on_write()) { _cloud_storage_engine.meta_mgr().remove_delete_bitmap_update_lock( diff --git a/be/src/cloud/cloud_schema_change_job.h b/be/src/cloud/cloud_schema_change_job.h index 2a33443137ebb0..2f49f3eab6dac7 100644 --- a/be/src/cloud/cloud_schema_change_job.h +++ b/be/src/cloud/cloud_schema_change_job.h @@ -36,7 +36,7 @@ class CloudSchemaChangeJob { // This method is idempotent for a same request. Status process_alter_tablet(const TAlterTabletReqV2& request); - void clean_up_on_failed(); + void clean_up_on_failure(); private: Status _convert_historical_rowsets(const SchemaChangeParams& sc_params, From 9eca445c0ebd36ced94837e9129684b913a0112f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 16:32:47 +0800 Subject: [PATCH 337/572] branch-3.0: [bugfix](k8s) using stdout to output #54129 (#54171) Cherry-picked from #54129 Co-authored-by: yiguolei --- be/src/common/logconfig.cpp | 66 +++++++++++++++++++++++++++++++++++-- bin/start_be.sh | 8 +++-- 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index 88c7d4bc75530c..05ccc5477d3ad3 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -36,6 +36,62 @@ static bool logging_initialized = false; static std::mutex logging_mutex; +// Implement the custom log format: I20250118 10:53:06.239614 1318521 timezone_utils.cpp:115] Preloaded653 timezones. +struct StdoutLogSink : google::LogSink { + void send(google::LogSeverity severity, const char* /*full_filename*/, + const char* base_filename, int line, const google::LogMessageTime& time, + const char* message, std::size_t message_len) override { + // 1. Convert log severity to corresponding character (I/W/E/F) + char severity_char; + switch (severity) { + case google::GLOG_INFO: + severity_char = 'I'; + break; + case google::GLOG_WARNING: + severity_char = 'W'; + break; + case google::GLOG_ERROR: + severity_char = 'E'; + break; + case google::GLOG_FATAL: + severity_char = 'F'; + break; + default: + severity_char = '?'; + break; + } + // Set output formatting flags + std::cout << std::setfill('0'); + + // 1. Log severity (I/W/E/F) + std::cout << severity_char; + + // 2. Date (YYYYMMDD) + // Note: tm_year is years since 1900, tm_mon is 0-based (0-11) + std::cout << std::setw(4) << (time.year() + 1900) << std::setw(2) << std::setfill('0') + << (time.month() + 1) << std::setw(2) << std::setfill('0') << time.day(); + + // 3. Time (HH:MM:SS.ffffff) + std::cout << " " << std::setw(2) << std::setfill('0') << time.hour() << ":" << std::setw(2) + << std::setfill('0') << time.min() << ":" << std::setw(2) << std::setfill('0') + << time.sec() << "." << std::setw(6) << std::setfill('0') << time.usec(); + + // 4. Process ID + std::cout << " " << getpid(); + + // 5. Filename and line number + std::cout << " " << base_filename << ":" << line << "] "; + + // 6. Log message + std::cout.write(message, message_len); + + // Add newline and flush + std::cout << std::endl; + } +}; + +static StdoutLogSink stdout_log_sink; + static bool iequals(const std::string& a, const std::string& b) { unsigned int sz = a.size(); if (b.size() != sz) { @@ -99,10 +155,13 @@ bool init_glog(const char* basename) { bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); if (log_to_console) { - if (config::enable_file_logger) { - FLAGS_alsologtostderr = true; + if (doris::config::enable_file_logger) { + // will output log to be.info and output log to stdout + google::AddLogSink(&stdout_log_sink); } else { - FLAGS_logtostderr = true; + // enable_file_logger is false, will only output log to stdout + // Not output to stderr because be.out will output log to stderr + FLAGS_logtostdout = true; } } @@ -213,6 +272,7 @@ bool init_glog(const char* basename) { void shutdown_logging() { std::lock_guard logging_lock(logging_mutex); + google::RemoveLogSink(&stdout_log_sink); google::ShutdownGoogleLogging(); } diff --git a/bin/start_be.sh b/bin/start_be.sh index 3cb25ff735f1ac..96a6a3a8eb826d 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -100,9 +100,9 @@ log() { cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)") if [[ "${RUN_CONSOLE}" -eq 1 ]]; then echo "StdoutLogger ${cur_date} $1" - else - echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" fi + # always output start time info into be.out file + echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" } jdk_version() { @@ -437,8 +437,10 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 &1 >"${LOG_DIR}/be.out" >"${LOG_DIR}/be.out" 2>&1 Date: Fri, 1 Aug 2025 16:33:42 +0800 Subject: [PATCH 338/572] branch-3.0: [fix](cloud) Fix std::sleep_for in bthread sync_rowset #53925 (#53953) Cherry-picked from #53925 Co-authored-by: Gavin Chou --- be/src/cloud/cloud_meta_mgr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 28174ebf1eb9c9..2fa1ecf4a8616a 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -539,7 +539,7 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, std::uniform_int_distribution u(20, 200); std::uniform_int_distribution u1(500, 1000); uint32_t duration_ms = tried >= 100 ? u(rng) : u1(rng); - std::this_thread::sleep_for(milliseconds(duration_ms)); + bthread_usleep(duration_ms * 1000); LOG_INFO("failed to get rowset meta") .tag("reason", cntl.ErrorText()) .tag("tablet_id", tablet_id) From 84be3bbab3f4278a8a4d6b6d6ceacb32327ca09d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 16:45:43 +0800 Subject: [PATCH 339/572] branch-3.0: [Fix](Compaction) Fix full clone failure when rowset missing #53984 (#54162) Cherry-picked from #53984 Co-authored-by: abmdocrt --- be/src/olap/cumulative_compaction.cpp | 7 +- .../olap/task/engine_publish_version_task.cpp | 6 +- ...lone_missing_rowset_fault_injection.groovy | 2 +- ...lone_missing_rowset_fault_injection.groovy | 125 ++++++++++++++++ ...one_missing_rowset_fault_injection.groovy} | 0 ...lone_missing_rowset_fault_injection.groovy | 139 ++++++++++++++++++ 6 files changed, 272 insertions(+), 7 deletions(-) create mode 100644 regression-test/suites/fault_injection_p0/test_compaction_full_clone_missing_rowset_fault_injection.groovy rename regression-test/suites/fault_injection_p0/{test_mow_publish_clone_missing_rowset.groovy => test_mow_publish_clone_missing_rowset_fault_injection.groovy} (100%) create mode 100644 regression-test/suites/fault_injection_p0/test_mow_publish_full_clone_missing_rowset_fault_injection.groovy diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp index 8db397aa577ccd..ca33cbc887d361 100644 --- a/be/src/olap/cumulative_compaction.cpp +++ b/be/src/olap/cumulative_compaction.cpp @@ -197,18 +197,19 @@ Status CumulativeCompaction::pick_rowsets_to_compact() { << ", first missed version next rowset version=" << missing_versions[1] << ", tablet=" << _tablet->tablet_id(); if (config::enable_auto_clone_on_compaction_missing_version) { + int64_t max_version = tablet()->max_version_unlocked(); LOG_INFO("cumulative compaction submit missing rowset clone task.") .tag("tablet_id", _tablet->tablet_id()) - .tag("version", missing_versions.back().first) + .tag("max_version", max_version) .tag("replica_id", tablet()->replica_id()) .tag("partition_id", _tablet->partition_id()) .tag("table_id", _tablet->table_id()); - Status st = _engine.submit_clone_task(tablet(), missing_versions.back().first); + Status st = _engine.submit_clone_task(tablet(), max_version); if (!st) { LOG_WARNING("cumulative compaction failed to submit missing rowset clone task.") .tag("st", st.msg()) .tag("tablet_id", _tablet->tablet_id()) - .tag("version", missing_versions.back().first) + .tag("max_version", max_version) .tag("replica_id", tablet()->replica_id()) .tag("partition_id", _tablet->partition_id()) .tag("table_id", _tablet->table_id()); diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp index 13e7dcd97aab4a..992e74109cced6 100644 --- a/be/src/olap/task/engine_publish_version_task.cpp +++ b/be/src/olap/task/engine_publish_version_task.cpp @@ -220,17 +220,17 @@ Status EnginePublishVersionTask::execute() { if (config::enable_auto_clone_on_mow_publish_missing_version) { LOG_INFO("mow publish submit missing rowset clone task.") .tag("tablet_id", tablet->tablet_id()) - .tag("version", version.first - 1) + .tag("version", version.second) .tag("replica_id", tablet->replica_id()) .tag("partition_id", tablet->partition_id()) .tag("table_id", tablet->table_id()); - Status st = _engine.submit_clone_task(tablet.get(), version.first - 1); + Status st = _engine.submit_clone_task(tablet.get(), version.second); if (!st) { LOG_WARNING( "mow publish failed to submit missing rowset clone task.") .tag("st", st.msg()) .tag("tablet_id", tablet->tablet_id()) - .tag("version", version.first - 1) + .tag("version", version.second) .tag("replica_id", tablet->replica_id()) .tag("partition_id", tablet->partition_id()) .tag("table_id", tablet->table_id()); diff --git a/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy index a7f060a110888b..f15de709b3da96 100644 --- a/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_compaction_clone_missing_rowset_fault_injection.groovy @@ -39,7 +39,7 @@ suite('test_compaction_clone_missing_rowset_fault_injection', 'docker') { assertNotNull(normalBe) try { - def tableName = "test_compaction_clone_missing_rowset" + def tableName = "test_compaction_clone_missing_rowset_fault_injection" sql """ DROP TABLE IF EXISTS ${tableName} force""" sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( diff --git a/regression-test/suites/fault_injection_p0/test_compaction_full_clone_missing_rowset_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_compaction_full_clone_missing_rowset_fault_injection.groovy new file mode 100644 index 00000000000000..1e8bdd4f37342a --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_compaction_full_clone_missing_rowset_fault_injection.groovy @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.http.NoHttpResponseException +import org.apache.doris.regression.util.DebugPoint +import org.apache.doris.regression.util.NodeType + +suite('test_compaction_full_clone_missing_rowset_fault_injection', 'docker') { + def options = new ClusterOptions() + options.cloudMode = false + options.enableDebugPoints() + options.feConfigs += [ "disable_tablet_scheduler=true" ] + options.beConfigs += [ "enable_auto_clone_on_compaction_missing_version=true" ] + options.beConfigs += [ "tablet_rowset_stale_sweep_time_sec=0" ] + options.beConfigs += [ "tablet_rowset_stale_sweep_by_size=true" ] + options.beConfigs += [ "tablet_rowset_stale_sweep_threshold_size=0" ] + options.beNum = 3 + docker(options) { + + def injectBe = null + def normalBe1 = null + def normalBe2 = null + def backends = sql_return_maparray('show backends') + + injectBe = backends[0] + assertNotNull(injectBe) + normalBe1 = backends[1] + assertNotNull(normalBe1) + normalBe2 = backends[2] + assertNotNull(normalBe2) + + try { + def tableName = "test_compaction_full_clone_missing_rowset_fault_injection" + sql """ DROP TABLE IF EXISTS ${tableName} force""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k` int , + `v` int , + ) engine=olap + DUPLICATE KEY(k) + DISTRIBUTED BY HASH(k) + BUCKETS 1 + properties( + "replication_num" = "3", + "disable_auto_compaction" = "true") + """ + sql """ INSERT INTO ${tableName} VALUES (1,0)""" + DebugPoint.enableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random", [percent:"1.0"]) + sql """ INSERT INTO ${tableName} VALUES (2,0)""" + sql """ INSERT INTO ${tableName} VALUES (3,0)""" + sql """ INSERT INTO ${tableName} VALUES (4,0)""" + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + sql """ INSERT INTO ${tableName} VALUES (5,0)""" + sql """ INSERT INTO ${tableName} VALUES (6,0)""" + sql """ INSERT INTO ${tableName} VALUES (7,0)""" + sql """ INSERT INTO ${tableName} VALUES (8,0)""" + sql """ INSERT INTO ${tableName} VALUES (9,0)""" + + def array = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + def tabletId = array[0].TabletId + + // normal BEs compaction + logger.info("normal BE run cumu compaction:" + tabletId) + def (code, out, err) = be_run_cumulative_compaction(normalBe1.Host, normalBe1.HttpPort, tabletId) + logger.info("normal BE1 Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + (code, out, err) = be_run_cumulative_compaction(normalBe2.Host, normalBe2.HttpPort, tabletId) + logger.info("normal BE2 Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + + logger.info("normal BE show:" + tabletId) + (code, out, err) = be_show_tablet_status(normalBe1.Host, normalBe1.HttpPort, tabletId) + logger.info("normal BE1 show: code=" + code + ", out=" + out + ", err=" + err) + (code, out, err) = be_show_tablet_status(normalBe2.Host, normalBe2.HttpPort, tabletId) + logger.info("normal BE2 show: code=" + code + ", out=" + out + ", err=" + err) + + sleep(10000) + + // 1st check rowsets + logger.info("1st show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("1st show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + // missing rowset [3-5] + assertTrue(out.contains("[3-5]")) + assertTrue(out.contains("[6-6]")) + assertTrue(out.contains("[7-7]")) + assertTrue(out.contains("[8-8]")) + assertTrue(out.contains("[9-9]")) + assertTrue(out.contains("[10-10]")) + + logger.info("1st run cumu compaction:" + tabletId) + (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("1st Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + + sleep(30000) + + // 2nd check rowsets + logger.info("2nd show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("2nd show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-10]")) + + } finally { + if (injectBe != null) { + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + } + } + } +} diff --git a/regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset.groovy b/regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset_fault_injection.groovy similarity index 100% rename from regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset.groovy rename to regression-test/suites/fault_injection_p0/test_mow_publish_clone_missing_rowset_fault_injection.groovy diff --git a/regression-test/suites/fault_injection_p0/test_mow_publish_full_clone_missing_rowset_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_mow_publish_full_clone_missing_rowset_fault_injection.groovy new file mode 100644 index 00000000000000..e582d3339bf97a --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_mow_publish_full_clone_missing_rowset_fault_injection.groovy @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.http.NoHttpResponseException +import org.apache.doris.regression.util.DebugPoint +import org.apache.doris.regression.util.NodeType + +suite('test_mow_publish_full_clone_missing_rowset_fault_injection', 'docker') { + + def set_be_param = { paramName, paramValue, beIp, bePort -> + def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue)) + assertTrue(out.contains("OK")) + } + + def options = new ClusterOptions() + options.cloudMode = false + options.enableDebugPoints() + options.feConfigs += [ "disable_tablet_scheduler=true" ] + options.beConfigs += [ "enable_auto_clone_on_compaction_missing_version=true" ] + options.beConfigs += [ "tablet_rowset_stale_sweep_time_sec=0" ] + options.beConfigs += [ "tablet_rowset_stale_sweep_by_size=true" ] + options.beConfigs += [ "tablet_rowset_stale_sweep_threshold_size=0" ] + options.beNum = 3 + docker(options) { + + def injectBe = null + def normalBe1 = null + def normalBe2 = null + def backends = sql_return_maparray('show backends') + + injectBe = backends[0] + assertNotNull(injectBe) + normalBe1 = backends[1] + assertNotNull(normalBe1) + normalBe2 = backends[2] + assertNotNull(normalBe2) + + try { + def tableName = "test_mow_publish_full_clone_missing_rowset_fault_injection" + sql """ DROP TABLE IF EXISTS ${tableName} force""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k` int , + `v` int , + ) engine=olap + UNIQUE KEY(k) + DISTRIBUTED BY HASH(k) + BUCKETS 1 + properties( + "replication_num" = "3", + "disable_auto_compaction" = "true") + """ + sql """ INSERT INTO ${tableName} VALUES (1,0)""" + DebugPoint.enableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random", [percent:"1.0"]) + sql """ INSERT INTO ${tableName} VALUES (2,0)""" + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + sql """ INSERT INTO ${tableName} VALUES (3,0)""" + sql """ INSERT INTO ${tableName} VALUES (4,0)""" + sql """ INSERT INTO ${tableName} VALUES (5,0)""" + sql """ INSERT INTO ${tableName} VALUES (6,0)""" + sql """ INSERT INTO ${tableName} VALUES (7,0)""" + + def array = sql_return_maparray("SHOW TABLETS FROM ${tableName}") + def tabletId = array[0].TabletId + + // normal BEs compaction + logger.info("normal BE run cumu compaction:" + tabletId) + def (code, out, err) = be_run_cumulative_compaction(normalBe1.Host, normalBe1.HttpPort, tabletId) + logger.info("normal BE1 Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + (code, out, err) = be_run_cumulative_compaction(normalBe2.Host, normalBe2.HttpPort, tabletId) + logger.info("normal BE2 Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + + logger.info("normal BE show:" + tabletId) + (code, out, err) = be_show_tablet_status(normalBe1.Host, normalBe1.HttpPort, tabletId) + logger.info("normal BE1 show: code=" + code + ", out=" + out + ", err=" + err) + (code, out, err) = be_show_tablet_status(normalBe2.Host, normalBe2.HttpPort, tabletId) + logger.info("normal BE2 show: code=" + code + ", out=" + out + ", err=" + err) + + sleep(10000) + + // 1st inject be check rowsets + logger.info("1st inject be show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("1st inject be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-2]")) + assertFalse(out.contains("[3-3]")) + assertFalse(out.contains("[4-4]")) + assertFalse(out.contains("[5-5]")) + assertFalse(out.contains("[6-6]")) + assertFalse(out.contains("[7-7]")) + + set_be_param("enable_auto_clone_on_mow_publish_missing_version", "true", injectBe.Host, injectBe.HttpPort); + Thread.sleep(10000) + // submit clone task + sql """ INSERT INTO ${tableName} VALUES (8,0)""" + + sleep(30000) + + // 2nd inject be check rowsets + logger.info("2nd inject be show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("2nd inject be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-8]")) + assertTrue(out.contains("[9-9]")) + + // inject be compaction + logger.info("run cumu compaction:" + tabletId) + (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("Run cumu compaction: code=" + code + ", out=" + out + ", err=" + err) + + logger.info("3rd inject be show:" + tabletId) + (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("3rd inject be show: code=" + code + ", out=" + out + ", err=" + err) + assertTrue(out.contains("[0-1]")) + assertTrue(out.contains("[2-8]")) + } finally { + if (injectBe != null) { + DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, "EnginePublishVersionTask.finish.random") + } + } + } +} \ No newline at end of file From 34ff5f15dd789a1113d1e90b651218723a409c7b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 16:47:13 +0800 Subject: [PATCH 340/572] branch-3.0: [chore](case) log msg to investigate failure cases #54017 (#54047) Cherry-picked from #54017 Co-authored-by: Yongqiang YANG --- .../load_p2/broker_load/test_parquet_large_metadata_load.groovy | 1 + .../load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy | 1 + 2 files changed, 2 insertions(+) diff --git a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy index 6d8f841715fc36..fe305a6574223f 100644 --- a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy +++ b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy @@ -85,6 +85,7 @@ suite("test_parquet_large_metadata_load_p2", "p2") { def max_try_milli_secs = 600000 while (max_try_milli_secs > 0) { String[][] result = sql """ show load where label="$label" order by createtime desc limit 1; """ + logger.info("Load result: " + result) if (result[0][2].equals("FINISHED")) { logger.info("Load FINISHED " + label) assertTrue(result[0][6].contains(task_info[i])) diff --git a/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy b/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy index b8e7693b2902ac..a35d24e5f03d2e 100644 --- a/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy +++ b/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy @@ -266,6 +266,7 @@ suite("test_tvf_based_broker_load", "p2") { def max_try_milli_secs = 60000 while (max_try_milli_secs > 0) { String[][] result = sql """ show load where label="$label" order by createtime desc limit 1; """ + logger.info("Load result: $result") if (result[0][2].equals("FINISHED")) { assertTrue(result[0][6].contains(task_info[0])) From 066eb7d4ab5e6635f0e4e5bca5de63090be8b044 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 16:55:14 +0800 Subject: [PATCH 341/572] branch-3.0: [fix](test) Add more timeouts to the test #54058 (#54146) Cherry-picked from #54058 Co-authored-by: zy-kkk --- .../lower_case/test_conflict_name.groovy | 6 +- .../test_lower_case_meta_include.groovy | 24 +++++-- ...est_lower_case_meta_show_and_select.groovy | 24 +++++-- ...se_meta_with_lower_table_conf_grant.groovy | 12 +++- ...th_lower_table_conf_show_and_select.groovy | 72 +++++++++++++++---- .../lower_case/test_lower_case_mtmv.groovy | 6 +- ...t_meta_cache_select_without_refresh.groovy | 12 +++- .../lower_case/test_meta_names_mapping.groovy | 30 ++++++-- .../test_timing_refresh_catalog.groovy | 24 +++++-- .../lower_case/upgrade/load.groovy | 6 +- 10 files changed, 180 insertions(+), 36 deletions(-) diff --git a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy index 92fe356dfd6c35..c2cba8328a8c75 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy @@ -57,7 +57,11 @@ suite("test_conflict_name", "p0,external,doris,meta_names_mapping,external_docke "driver_class" = "com.mysql.cj.jdbc.Driver", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_conflict_name,EXTERNAL_CONFLICT_NAME" + "include_database_list" = "external_conflict_name,EXTERNAL_CONFLICT_NAME", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy index 91a56f7f317a69..2f540802552af1 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy @@ -63,7 +63,11 @@ suite("test_lower_case_meta_include", "p0,external,doris,external_docker,externa "driver_class" = "com.mysql.cj.jdbc.Driver", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_INCLUDE" + "include_database_list" = "external_INCLUDE", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -93,7 +97,11 @@ suite("test_lower_case_meta_include", "p0,external,doris,external_docker,externa "driver_class" = "com.mysql.cj.jdbc.Driver", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "exclude_database_list" = "external_EXCLUDE" + "exclude_database_list" = "external_EXCLUDE", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -120,7 +128,11 @@ suite("test_lower_case_meta_include", "p0,external,doris,external_docker,externa "lower_case_meta_names" = "true", "only_specified_database" = "true", 'meta_names_mapping' = '${mapping_db}', - "include_database_list" = "external_INCLUDE" + "include_database_list" = "external_INCLUDE", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -151,7 +163,11 @@ suite("test_lower_case_meta_include", "p0,external,doris,external_docker,externa "lower_case_meta_names" = "true", "only_specified_database" = "true", 'meta_names_mapping' = '${mapping_db}', - "exclude_database_list" = "external_EXCLUDE" + "exclude_database_list" = "external_EXCLUDE", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy index 8853d169a13bae..1cf347611397dd 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy @@ -97,7 +97,11 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker "use_meta_cache" = "false", "lower_case_meta_names" = "false", "only_specified_database" = "true", - "include_database_list" = "external_test_lower,external_test_UPPER" + "include_database_list" = "external_test_lower,external_test_UPPER", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_false.external_test_lower") @@ -146,7 +150,11 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker "use_meta_cache" = "true", "lower_case_meta_names" = "false", "only_specified_database" = "true", - "include_database_list" = "external_test_lower,external_test_UPPER" + "include_database_list" = "external_test_lower,external_test_UPPER", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -195,7 +203,11 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker "use_meta_cache" = "false", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_test_lower,external_test_UPPER" + "include_database_list" = "external_test_lower,external_test_UPPER", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_true.external_test_lower") @@ -244,7 +256,11 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker "use_meta_cache" = "true", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_test_lower,external_test_UPPER" + "include_database_list" = "external_test_lower,external_test_UPPER", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy index 78baa9aa438452..303e46fbfb3816 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_grant.groovy @@ -73,7 +73,11 @@ suite("test_lower_case_meta_with_lower_table_conf_auth", "p0,external,doris,exte "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf_auth,external_test_UPPER_with_conf_auth", - "lower_case_meta_names" = "false" + "lower_case_meta_names" = "false", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" @@ -122,7 +126,11 @@ suite("test_lower_case_meta_with_lower_table_conf_auth", "p0,external,doris,exte "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf_auth,external_test_UPPER_with_conf_auth", - "lower_case_meta_names" = "true" + "lower_case_meta_names" = "true", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" sql """ show tables from test_lower_true_with_conf0_auth.external_test_lower_with_conf_auth;""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index 13750535628c44..ee1d16de44abb0 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -90,7 +90,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "false", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "1" + "only_test_lower_case_table_names" = "1", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_false_with_conf1.external_test_lower_with_conf"); @@ -147,7 +151,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "false", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "2" + "only_test_lower_case_table_names" = "2", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_false_with_conf2.external_test_lower_with_conf"); @@ -205,7 +213,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "false", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "1" + "only_test_lower_case_table_names" = "1", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" qt_sql_test_cache_true_lower_false_with_conf1_1 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" @@ -253,7 +265,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "false", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "2" + "only_test_lower_case_table_names" = "2", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" qt_sql_test_cache_true_lower_false_with_conf2_1 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" @@ -301,7 +317,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "1" + "only_test_lower_case_table_names" = "1", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_true_with_conf1.external_test_lower_with_conf"); @@ -357,7 +377,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "2" + "only_test_lower_case_table_names" = "2", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_true_with_conf2.external_test_lower_with_conf"); @@ -414,7 +438,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "1" + "only_test_lower_case_table_names" = "1", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" qt_sql_test_cache_true_lower_true_with_conf1_1 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" @@ -462,7 +490,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "2" + "only_test_lower_case_table_names" = "2", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" qt_sql_test_cache_true_lower_true_with_conf2_1 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" @@ -511,7 +543,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "false", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "0" + "only_test_lower_case_table_names" = "0", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_false_with_conf0.external_test_lower_with_conf"); @@ -584,7 +620,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "false", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "0" + "only_test_lower_case_table_names" = "0", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" qt_sql_test_cache_true_lower_false_with_conf0_1 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" @@ -650,7 +690,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "0" + "only_test_lower_case_table_names" = "0", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_cache_false_lower_true_with_conf0.external_test_lower_with_conf"); @@ -723,7 +767,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_test_lower_with_conf", - "only_test_lower_case_table_names" = "0" + "only_test_lower_case_table_names" = "0", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" qt_sql_test_cache_true_lower_true_with_conf0_1 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy index 67081ed3afe12c..8439e0a56a1a9f 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy @@ -58,7 +58,11 @@ suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker "driver_class" = "com.mysql.cj.jdbc.Driver", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "EXTERNAL_LOWER_MTMV" + "include_database_list" = "EXTERNAL_LOWER_MTMV", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy index c2073f9864cef6..47a755d2cfaaf0 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy @@ -53,7 +53,11 @@ suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_dock "use_meta_cache" = "true", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_lower_select_without_refresh" + "include_database_list" = "external_lower_select_without_refresh", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ @@ -68,7 +72,11 @@ suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_dock "use_meta_cache" = "true", "lower_case_meta_names" = "false", "only_specified_database" = "true", - "include_database_list" = "external_lower_select_without_refresh" + "include_database_list" = "external_lower_select_without_refresh", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" sql """create table if not exists internal.external_lower_select_without_refresh.table1 diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy index a033b7e59f8691..11998574307911 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy @@ -88,7 +88,11 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping,external_ "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true", "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", - "meta_names_mapping" = '${validMetaNamesMapping}' + "meta_names_mapping" = '${validMetaNamesMapping}', + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -136,7 +140,11 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping,external_ "driver_class" = "com.mysql.cj.jdbc.Driver", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -235,7 +243,11 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping,external_ "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true", "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", - "meta_names_mapping" = '${error_mapping_db}' + "meta_names_mapping" = '${error_mapping_db}', + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" @@ -264,7 +276,11 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping,external_ "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true", "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", - "meta_names_mapping" = '${error_mapping_tbl}' + "meta_names_mapping" = '${error_mapping_tbl}', + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" @@ -282,7 +298,11 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping,external_ "driver_url" = "${driver_url}", "driver_class" = "com.mysql.cj.jdbc.Driver", "only_specified_database" = "true", - "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy index a353e20d685074..260691afa7a6d3 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy @@ -94,7 +94,11 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external "metadata_refresh_interval_seconds" = "1", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_timing_refresh_catalog" + "include_database_list" = "external_timing_refresh_catalog", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -122,7 +126,11 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external "metadata_refresh_interval_seconds" = "1", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "external_timing_refresh_catalog" + "include_database_list" = "external_timing_refresh_catalog", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -150,7 +158,11 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_timing_refresh_catalog", - 'meta_names_mapping' = '${mapping}' + 'meta_names_mapping' = '${mapping}', + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { @@ -177,7 +189,11 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external "lower_case_meta_names" = "true", "only_specified_database" = "true", "include_database_list" = "external_timing_refresh_catalog", - 'meta_names_mapping' = '${mapping}' + 'meta_names_mapping' = '${mapping}', + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" test { diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy index 2f0ad86f12bafa..4176e246875541 100644 --- a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy @@ -85,7 +85,11 @@ suite("test_upgrade_lower_case_catalog_prepare", "p0,external,doris,external_doc "use_meta_cache" = "false", "lower_case_meta_names" = "true", "only_specified_database" = "true", - "include_database_list" = "upgrade_lower_case_catalog_lower,upgrade_lower_case_catalog_UPPER" + "include_database_list" = "upgrade_lower_case_catalog_lower,upgrade_lower_case_catalog_UPPER", + "connection_pool_min_size" = "2", + "connection_pool_max_size" = "20", + "connection_pool_max_wait_time" = "30000", + "connection_pool_max_life_time" = "600000" )""" wait_table_sync("test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower") From 44578742ecab4d11c93a398a6705b5d4ed880b26 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 2 Aug 2025 23:07:17 +0800 Subject: [PATCH 342/572] branch-3.0: [chore](case) log msg to investigate failure cases #54063 (#54203) Cherry-picked from #54063 Co-authored-by: Yongqiang YANG --- .../compaction10/test_schema_change_with_compaction10.groovy | 1 + .../compaction11/test_schema_change_with_compaction11.groovy | 1 + .../compaction5/test_schema_change_with_compaction5.groovy | 1 + .../compaction6/test_schema_change_with_compaction6.groovy | 1 + .../compaction9/test_schema_change_with_compaction9.groovy | 1 + 5 files changed, 5 insertions(+) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy index 3c023b26c28462..b7b6bd4e433d87 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy @@ -31,6 +31,7 @@ suite('test_schema_change_with_compaction10', 'docker') { docker(options) { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("Get job state: " + jobStateResult) return jobStateResult[0][9] } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy index 2845fe2f5c0141..2f7268b962442f 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy @@ -31,6 +31,7 @@ suite('test_schema_change_with_compaction11', 'docker') { docker(options) { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("Get job state: " + jobStateResult) return jobStateResult[0][9] } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy b/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy index fd6267b85bcbc9..094b8a53737231 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy @@ -31,6 +31,7 @@ suite('test_schema_change_with_compaction5', 'docker') { docker(options) { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("getJobState: " + jobStateResult) return jobStateResult[0][9] } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy index d77db4eb2df541..d2ac4f14cd9158 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy @@ -31,6 +31,7 @@ suite('test_schema_change_with_compaction6', 'docker') { docker(options) { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("Get job state: " + jobStateResult) return jobStateResult[0][9] } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy index 3797a89f565997..4a9ea8ea41e177 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy @@ -31,6 +31,7 @@ suite('test_schema_change_with_compaction9', 'docker') { docker(options) { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("Get job state: " + jobStateResult) return jobStateResult[0][9] } From f2d9d7dcf5ef3010ac24b36967febe04cbf4496c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 3 Aug 2025 10:45:43 +0800 Subject: [PATCH 343/572] branch-3.0: [fix](regression) Fix cached cloud partition version #54092 (#54209) Cherry-picked from #54092 Co-authored-by: Gavin Chou --- .../cloud_p0/version/test_fe_cached_partition_version.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy b/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy index 0cccfa543213ec..8ce1359861df48 100644 --- a/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy +++ b/regression-test/suites/cloud_p0/version/test_fe_cached_partition_version.groovy @@ -142,14 +142,14 @@ suite("test_fe_cached_partition_version", 'docker') { insert_sql """INSERT INTO ${tbl} VALUES ('Guangzhou', 1})""", 1 sql """set global cloud_partition_version_cache_ttl_ms=0""" result = sql_return_maparray """ select * from ${tbl} """ - assertEquals(6, result.size()) + assertEquals(7, result.size()) insert_sql """INSERT INTO ${tbl} VALUES ('Shanghai', 1})""", 1 insert_sql """INSERT INTO ${tbl} VALUES ('Guangzhou', 1})""", 1 insert_sql """INSERT INTO ${tbl} VALUES ('Beijing', 1})""", 1 // data present immediately without any cached versions result = sql_return_maparray """ select * from ${tbl} """ - assertEquals(9, result.size()) + assertEquals(10, result.size()) } finally { } } From d6ac40a315ee5df1325bf70493a4fa508e619333 Mon Sep 17 00:00:00 2001 From: deardeng Date: Mon, 4 Aug 2025 16:54:40 +0800 Subject: [PATCH 344/572] branch-3.0: [Fix](case) Fix some docker case #54227 (#54251) cherry pick from #54227 --- be/src/vec/exec/scan/new_olap_scanner.cpp | 6 ++++++ .../node_mgr/test_cloud_decommission.groovy | 15 +++++++++------ .../test_cloud_concurrent_calc_dbm_task.groovy | 4 +--- ...st_cloud_full_compaction_multi_segments.groovy | 2 +- .../test_cloud_mow_retry_txn_interleave.groovy | 2 +- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp index c8008c9852583b..4c5ab6a0da4218 100644 --- a/be/src/vec/exec/scan/new_olap_scanner.cpp +++ b/be/src/vec/exec/scan/new_olap_scanner.cpp @@ -131,6 +131,12 @@ Status NewOlapScanner::init() { auto* local_state = static_cast(_local_state); auto& tablet = _tablet_reader_params.tablet; auto& tablet_schema = _tablet_reader_params.tablet_schema; + DBUG_EXECUTE_IF("CloudTablet.capture_rs_readers.return.e-230", { + LOG_WARNING("CloudTablet.capture_rs_readers.return e-230 init") + .tag("tablet_id", tablet->tablet_id()); + return Status::Error(-230, "injected error"); + }); + for (auto& ctx : local_state->_common_expr_ctxs_push_down) { VExprContextSPtr context; RETURN_IF_ERROR(ctx->clone(_state, context)); diff --git a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy index dfe46054e24b76..d2d4947f859850 100644 --- a/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy +++ b/regression-test/suites/cloud_p0/node_mgr/test_cloud_decommission.groovy @@ -25,7 +25,7 @@ suite("cloud_decommission", 'p0, docker') { def checkStatus = { ms, decommissionBeUniqueId, decommissionBe -> boolean found = false - awaitUntil(100) { + awaitUntil(600) { found = false def resp = get_cluster.call(decommissionBeUniqueId, ms) resp.each { cluster -> @@ -43,7 +43,7 @@ suite("cloud_decommission", 'p0, docker') { def dropAndCheckBe = { host, heartbeatPort -> sql """ ALTER SYSTEM DROPP BACKEND "${host}:${heartbeatPort}" """ - awaitUntil(100) { + awaitUntil(600) { def result = sql_return_maparray """ SHOW BACKENDS """ log.info("show backends result {}", result) def ret = result.find {it.Host == host && it.HeartbeatPort == heartbeatPort} @@ -60,7 +60,7 @@ suite("cloud_decommission", 'p0, docker') { def result = sql """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ assertEquals(result.size(), beNum) - awaitUntil(100) { + awaitUntil(600) { result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ if (beNum == 3) { result.every { Integer.valueOf((String) it.ReplicaNum) >= 15 && Integer.valueOf((String) it.ReplicaNum) <= 17 } @@ -95,7 +95,7 @@ suite("cloud_decommission", 'p0, docker') { d_node.call(firstDecommissionBeUniqueId, firstDecommissionBe.Host, firstDecommissionBe.HeartbeatPort, firstDecommissionBeClusterName, firstDecommissionBeCloudClusterId, ms) - awaitUntil(100) { + awaitUntil(600) { result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ result.any { Integer.valueOf((String) it.ReplicaNum) == 0 } } @@ -127,7 +127,7 @@ suite("cloud_decommission", 'p0, docker') { result = sql """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ assertEquals(result.size(), beNum - 1) - awaitUntil(100) { + awaitUntil(600) { result = sql_return_maparray """ ADMIN SHOW REPLICA DISTRIBUTION FROM decommission_table """ log.info("show replica result {}", result) def ret = result.findAll { Integer.valueOf((String) it.ReplicaNum) == 0 } @@ -164,7 +164,7 @@ suite("cloud_decommission", 'p0, docker') { log.info("in check, inner cost {}", cost) cost = System.currentTimeMillis() - begin log.info("in check, outter cost {}", cost) - assertTrue(waitTime > atLeastCost) + // assertTrue(waitTime > atLeastCost) // decommission 2 bes assertTrue(cost >= 2 * waitTime) cost @@ -264,6 +264,9 @@ suite("cloud_decommission", 'p0, docker') { 'cloud_tablet_rebalancer_interval_second=1', 'cloud_cluster_check_interval_second=1' ] + clusterOptions[i].beConfigs += [ + 'sys_log_verbose_modules=*', + ] clusterOptions[i].setFeNum(2) // cluster has 3 bes // cluster has 2 bes, after decommission 2 nodes, and drop 2 nodes, compute group name will be delete from fe diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_concurrent_calc_dbm_task.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_concurrent_calc_dbm_task.groovy index 852a5fcde5b75b..777795c8650fc6 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_concurrent_calc_dbm_task.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_concurrent_calc_dbm_task.groovy @@ -23,15 +23,13 @@ suite("test_cloud_concurrent_calc_dbm_task", "multi_cluster,docker") { options.cloudMode = true options.setFeNum(1) options.setBeNum(1) + options.enableDebugPoints() options.feConfigs += [ 'cloud_cluster_check_interval_second=1', 'calculate_delete_bitmap_task_timeout_seconds=10', 'mow_calculate_delete_bitmap_retry_times=10', 'enable_workload_group=false', ] - options.beConfigs += [ - 'enable_debug_points=true' - ] docker(options) { try { diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_full_compaction_multi_segments.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_full_compaction_multi_segments.groovy index 1a49be36de7cbb..4ab7fd088fcedb 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_full_compaction_multi_segments.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_full_compaction_multi_segments.groovy @@ -27,6 +27,7 @@ suite("test_cloud_full_compaction_multi_segments","multi_cluster,docker") { options.cloudMode = true options.setFeNum(1) options.setBeNum(1) + options.enableDebugPoints() options.feConfigs += [ 'cloud_cluster_check_interval_second=1', 'calculate_delete_bitmap_task_timeout_seconds=10', @@ -34,7 +35,6 @@ suite("test_cloud_full_compaction_multi_segments","multi_cluster,docker") { 'enable_workload_group=false', ] options.beConfigs += [ - 'enable_debug_points=true', 'doris_scanner_row_bytes=1' // to cause multi segments ] docker(options) { diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_retry_txn_interleave.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_retry_txn_interleave.groovy index 7095e5a8bf71b1..c6bccbc5c19698 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_retry_txn_interleave.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_retry_txn_interleave.groovy @@ -23,13 +23,13 @@ suite("test_cloud_mow_retry_txn_interleave", "multi_cluster,docker") { options.cloudMode = true options.setFeNum(1) options.setBeNum(1) + options.enableDebugPoints() options.feConfigs += [ 'cloud_cluster_check_interval_second=1', 'calculate_delete_bitmap_task_timeout_seconds=20', 'mow_calculate_delete_bitmap_retry_times=3' ] options.beConfigs += [ - 'enable_debug_points=true', 'tablet_rowset_stale_sweep_time_sec=0', 'vacuum_stale_rowsets_interval_s=10', ] From 64a5781867b31c25653c2014a3ef73e35cc8012a Mon Sep 17 00:00:00 2001 From: Siyang Tang Date: Mon, 4 Aug 2025 19:30:07 +0800 Subject: [PATCH 345/572] [fix](replica) Get tablet replica infos should return all primary backends except for warmup jobs (#54131) ### What problem does this PR solve? Problem Summary: Fix logic conflict of https://github.com/apache/doris/pull/52514 and https://github.com/apache/doris/pull/52440 ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/service/FrontendServiceImpl.java | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index a1966a036cfe5f..9169f560309d7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2775,23 +2775,30 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos LOG.warn("replica {} not normal", replica.getId()); continue; } - Backend backend; + List backends; if (Config.isCloudMode()) { - CloudReplica cloudReplica = (CloudReplica) replica; - backend = cloudReplica.getPrimaryBackend(clusterId); + if (request.isSetWarmUpJobId()) { + CloudReplica cloudReplica = (CloudReplica) replica; + Backend primaryBackend = cloudReplica.getPrimaryBackend(clusterId); + backends = Lists.newArrayList(primaryBackend); + } else { + CloudReplica cloudReplica = (CloudReplica) replica; + backends = cloudReplica.getAllPrimaryBes(); + } } else { - backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); + Backend backend = Env.getCurrentSystemInfo().getBackend(replica.getBackendIdWithoutException()); + backends = Lists.newArrayList(backend); } - if (backend != null) { - TReplicaInfo replicaInfo = new TReplicaInfo(); - replicaInfo.setHost(backend.getHost()); - replicaInfo.setBePort(backend.getBePort()); - replicaInfo.setHttpPort(backend.getHttpPort()); - replicaInfo.setBrpcPort(backend.getBrpcPort()); - replicaInfo.setIsAlive(backend.isAlive()); - replicaInfo.setBackendId(backend.getId()); - replicaInfo.setReplicaId(replica.getId()); - replicaInfos.add(replicaInfo); + for (Backend backend : backends) { + if (backend != null) { + TReplicaInfo replicaInfo = new TReplicaInfo(); + replicaInfo.setHost(backend.getHost()); + replicaInfo.setBePort(backend.getBePort()); + replicaInfo.setHttpPort(backend.getHttpPort()); + replicaInfo.setBrpcPort(backend.getBrpcPort()); + replicaInfo.setReplicaId(replica.getId()); + replicaInfos.add(replicaInfo); + } } } tabletReplicaInfos.put(tabletId, replicaInfos); From 0a2956036b01fd58a994cfd188cb85e1da299b4d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 22:12:40 +0800 Subject: [PATCH 346/572] branch-3.0: [fix](case) catch exception to figure out failures #54282 (#54298) Cherry-picked from #54282 Co-authored-by: Yongqiang YANG --- .../compaction9/test_schema_change_with_compaction9.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy index 4a9ea8ea41e177..316b10283b1df6 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy @@ -60,8 +60,10 @@ suite('test_schema_change_with_compaction9', 'docker') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("State result: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { + logger.error("Load ${loadLabel} cancelled.") throw new IllegalStateException("load ${loadLabel} failed.") } else if ("FINISHED".equalsIgnoreCase(loadState)) { break @@ -164,6 +166,9 @@ suite('test_schema_change_with_compaction9', 'docker') { cluster.restartFrontends() sleep(30000) context.reconnectFe() + } catch (Exception e) { + logger.error("Exception: " + e.getMessage()) + assertEquals(1, 2) } finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) From 416a838d0dd429ae7b1b746bd45abc9f629e5f2b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 17:02:34 +0800 Subject: [PATCH 347/572] branch-3.0: [fix](case) do not use global var in groovy #54319 (#54327) Cherry-picked from #54319 Co-authored-by: Yongqiang YANG --- .../test_schema_change_with_compaction1.groovy | 8 ++++++-- .../test_schema_change_with_compaction10.groovy | 7 +++++-- .../test_schema_change_with_compaction2.groovy | 10 +++++++--- .../test_schema_change_with_compaction3.groovy | 10 +++++++--- .../test_schema_change_with_compaction4.groovy | 10 +++++++--- .../test_schema_change_with_compaction5.groovy | 9 ++++++--- .../test_schema_change_with_compaction6.groovy | 7 +++++-- .../test_schema_change_with_compaction7.groovy | 10 +++++++--- .../test_schema_change_with_compaction8.groovy | 10 +++++++--- .../test_schema_change_with_compaction9.groovy | 6 +++--- 10 files changed, 60 insertions(+), 27 deletions(-) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy b/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy index ed78edd6ca8661..64b03d8cb6683e 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.util.NodeType suite('test_schema_change_with_compaction1', 'nonConcurrent') { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("jobStateResult: " + jobStateResult) return jobStateResult[0][9] } @@ -57,6 +58,7 @@ suite('test_schema_change_with_compaction1', 'nonConcurrent') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -107,13 +109,15 @@ suite('test_schema_change_with_compaction1', 'nonConcurrent') { trigger_and_wait_compaction("date", "base") def newTabletId = array[1].TabletId logger.info("run compaction:" + newTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, newTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, newTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) // cu compaction trigger_and_wait_compaction("date", "cumulative") + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) @@ -136,7 +140,7 @@ suite('test_schema_change_with_compaction1', 'nonConcurrent') { assertEquals(count[0][0], 23004); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy index b7b6bd4e433d87..2e9ad0628d4d53 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy @@ -60,6 +60,7 @@ suite('test_schema_change_with_compaction10', 'docker') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -113,12 +114,14 @@ suite('test_schema_change_with_compaction10', 'docker') { trigger_and_wait_compaction("date", "base") def newTabletId = array[1].TabletId logger.info("run compaction:" + newTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, newTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, newTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) // cu compaction trigger_and_wait_compaction("date", "cumulative") + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) @@ -141,7 +144,7 @@ suite('test_schema_change_with_compaction10', 'docker') { assertEquals(count[0][0], 2556); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy b/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy index c8ca8a54109824..667a498d6c0ad3 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.util.NodeType suite('test_schema_change_with_compaction2', 'nonConcurrent') { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("jobStateResult: " + jobStateResult) return jobStateResult[0][9] } @@ -57,6 +58,7 @@ suite('test_schema_change_with_compaction2', 'nonConcurrent') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -83,7 +85,7 @@ suite('test_schema_change_with_compaction2', 'nonConcurrent') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -113,7 +115,7 @@ suite('test_schema_change_with_compaction2', 'nonConcurrent') { // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -134,6 +136,8 @@ suite('test_schema_change_with_compaction2', 'nonConcurrent') { logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) @@ -158,7 +162,7 @@ suite('test_schema_change_with_compaction2', 'nonConcurrent') { assertEquals(count[0][0], 20448); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy b/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy index b2aab9f2dc7c84..cf59823cb784dd 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.util.NodeType suite('test_schema_change_with_compaction3', 'nonConcurrent') { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("jobStateResult: " + jobStateResult) return jobStateResult[0][9] } @@ -57,6 +58,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -83,7 +85,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -117,7 +119,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -160,6 +162,8 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { running = compactionStatus.run_status } while (running) } + } catch (Exception e) { + logger.error("Exception: " + e) } finally { sql """ CANCEL ALTER TABLE COLUMN FROM date """ if (injectBe != null) { @@ -183,7 +187,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { assertEquals(count[0][0], 23004); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy b/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy index 4b53dbdd998104..a57e32bf922f64 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.util.NodeType suite('test_schema_change_with_compaction4', 'nonConcurrent') { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("jobStateResult: " + jobStateResult) return jobStateResult[0][8] } @@ -57,6 +58,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -83,7 +85,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -117,7 +119,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -160,6 +162,8 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { running = compactionStatus.run_status } while (running) } + } catch (Exception e) { + logger.error("Exception: " + e) } finally { sql """ CANCEL ALTER TABLE MATERIALIZED VIEW FROM date """ if (injectBe != null) { @@ -183,7 +187,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { assertEquals(count[0][0], 23004); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy b/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy index 094b8a53737231..8af620ea6479b4 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy @@ -60,6 +60,7 @@ suite('test_schema_change_with_compaction5', 'docker') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -86,7 +87,7 @@ suite('test_schema_change_with_compaction5', 'docker') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -118,7 +119,7 @@ suite('test_schema_change_with_compaction5', 'docker') { } // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -164,6 +165,8 @@ suite('test_schema_change_with_compaction5', 'docker') { cluster.restartFrontends() sleep(30000) context.reconnectFe() + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) @@ -186,7 +189,7 @@ suite('test_schema_change_with_compaction5', 'docker') { assertEquals(count[0][0], 23004); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy index d2ac4f14cd9158..4321a23f7b7fb3 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy @@ -60,6 +60,7 @@ suite('test_schema_change_with_compaction6', 'docker') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -86,7 +87,7 @@ suite('test_schema_change_with_compaction6', 'docker') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -167,6 +168,8 @@ suite('test_schema_change_with_compaction6', 'docker') { } while (running) } + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) @@ -189,7 +192,7 @@ suite('test_schema_change_with_compaction6', 'docker') { assertEquals(count[0][0], 23004); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy b/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy index 7291ea3a341e44..a43de53c4720ce 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.util.NodeType suite('test_schema_change_with_compaction7', 'nonConcurrent') { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("jobStateResult: " + jobStateResult) return jobStateResult[0][9] } @@ -57,6 +58,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -83,7 +85,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -116,7 +118,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { } // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -159,6 +161,8 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { running = compactionStatus.run_status } while (running) } + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) @@ -181,7 +185,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { assertEquals(count[0][0], 2556); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy b/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy index 1017e1d50f235f..3002cb9c2273b4 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy @@ -29,6 +29,7 @@ import org.apache.doris.regression.util.NodeType suite('test_schema_change_with_compaction8', 'nonConcurrent') { def getJobState = { tableName -> def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """ + logger.info("jobStateResult: " + jobStateResult) return jobStateResult[0][9] } @@ -57,6 +58,7 @@ suite('test_schema_change_with_compaction8', 'nonConcurrent') { // check load state while (true) { def stateResult = sql "show load where Label = '${loadLabel}'" + logger.info("stateResult: " + stateResult) def loadState = stateResult[stateResult.size() - 1][2].toString() if ("CANCELLED".equalsIgnoreCase(loadState)) { throw new IllegalStateException("load ${loadLabel} failed.") @@ -83,7 +85,7 @@ suite('test_schema_change_with_compaction8', 'nonConcurrent') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -113,7 +115,7 @@ suite('test_schema_change_with_compaction8', 'nonConcurrent') { // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -134,6 +136,8 @@ suite('test_schema_change_with_compaction8', 'nonConcurrent') { logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) + } catch (Exception e) { + logger.error("Exception: " + e) } finally { if (injectBe != null) { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) @@ -158,7 +162,7 @@ suite('test_schema_change_with_compaction8', 'nonConcurrent') { assertEquals(count[0][0], 2556); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy index 316b10283b1df6..ad33e537d4920b 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy @@ -88,7 +88,7 @@ suite('test_schema_change_with_compaction9', 'docker') { sql "select count(*) from date" // cu compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) boolean running = true do { @@ -120,7 +120,7 @@ suite('test_schema_change_with_compaction9', 'docker') { } // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -191,7 +191,7 @@ suite('test_schema_change_with_compaction9', 'docker') { assertEquals(count[0][0], 2556); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) From d14aca0d0dc84de251356bbb92cfe8e5d4a3c303 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Tue, 5 Aug 2025 17:50:38 +0800 Subject: [PATCH 348/572] [Cherry-Pick](branch-3.0) Pick "[fix](checker) Fix checking inverted index file incorrectly (#53876)" (#54334) Pick #53876 Co-authored-by: Uniqueyou --- cloud/src/recycler/checker.cpp | 343 ++++++++++++---- cloud/src/recycler/checker.h | 20 + cloud/test/recycler_test.cpp | 705 +++++++++++++++++++++++++++++++-- 3 files changed, 966 insertions(+), 102 deletions(-) diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 36eb4e325ee866..a400e797e03f77 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -39,6 +39,7 @@ #include "common/bvars.h" #include "common/config.h" +#include "common/defer.h" #include "common/encryption_util.h" #include "common/logging.h" #include "common/util.h" @@ -497,12 +498,24 @@ int InstanceChecker::do_check() { }; TabletFiles tablet_files_cache; - auto check_rowset_objects = [&, this](const doris::RowsetMetaCloudPB& rs_meta, - std::string_view key) { + auto check_rowset_objects = [&, this](doris::RowsetMetaCloudPB& rs_meta, std::string_view key) { if (rs_meta.num_segments() == 0) { return; } + bool data_loss = false; + bool segment_file_loss = false; + bool index_file_loss = false; + + DORIS_CLOUD_DEFER { + if (data_loss) { + LOG(INFO) << "segment file is" << (segment_file_loss ? "" : " not") << " loss, " + << "index file is" << (index_file_loss ? "" : " not") << " loss, " + << "rowset.tablet_id = " << rs_meta.tablet_id(); + num_rowset_loss++; + } + }; + ++num_scanned_with_segment; if (tablet_files_cache.tablet_id != rs_meta.tablet_id()) { long tablet_volume = 0; @@ -536,7 +549,6 @@ int InstanceChecker::do_check() { instance_volume += tablet_volume; } - bool data_loss = false; for (int i = 0; i < rs_meta.num_segments(); ++i) { auto path = segment_path(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), i); @@ -549,11 +561,36 @@ int InstanceChecker::do_check() { break; } data_loss = true; + segment_file_loss = true; TEST_SYNC_POINT_CALLBACK("InstanceChecker.do_check1", &path); LOG(WARNING) << "object not exist, path=" << path << ", rs_meta=" << rs_meta.ShortDebugString() << " key=" << hex(key); } + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to init txn, err=" << err; + return; + } + + TabletIndexPB tablet_index; + if (get_tablet_idx(txn_kv_.get(), instance_id_, rs_meta.tablet_id(), tablet_index) == -1) { + LOG(WARNING) << "failedt to get tablet index, tablet_id= " << rs_meta.tablet_id(); + return; + } + + auto tablet_schema_key = + meta_schema_key({instance_id_, tablet_index.index_id(), rs_meta.schema_version()}); + std::string tablet_schema_val; + err = txn->get(tablet_schema_key, &tablet_schema_val); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + // rowset don't have tablet schema key means no index + return; + } + auto* schema = rs_meta.mutable_tablet_schema(); + schema->ParseFromString(tablet_schema_val); + std::vector> index_ids; for (const auto& i : rs_meta.tablet_schema().index()) { if (i.has_index_type() && i.index_type() == IndexType::INVERTED) { @@ -564,7 +601,6 @@ int InstanceChecker::do_check() { if (!key_exist(txn_kv_.get(), tablet_idx_key)) { for (int i = 0; i < rs_meta.num_segments(); ++i) { std::vector index_path_v; - std::vector loss_file_path; if (rs_meta.tablet_schema().inverted_index_storage_format() == InvertedIndexStorageFormatPB::V1) { for (const auto& index_id : index_ids) { @@ -582,32 +618,17 @@ int InstanceChecker::do_check() { } if (!index_path_v.empty()) { - if (std::all_of(index_path_v.begin(), index_path_v.end(), - [&](const auto& idx_file_path) { - if (!tablet_files_cache.files.contains(idx_file_path)) { - loss_file_path.emplace_back(idx_file_path); - return false; - } - return true; - })) { + if (std::ranges::all_of(index_path_v, [&](const auto& idx_file_path) { + return tablet_files_cache.files.contains(idx_file_path); + })) { continue; } } - + index_file_loss = true; data_loss = true; - LOG(WARNING) << "object not exist, path=" - << std::accumulate(loss_file_path.begin(), loss_file_path.end(), - std::string(), - [](const auto& a, const auto& b) { - return a.empty() ? b : a + ", " + b; - }) - << " key=" << hex(tablet_idx_key); + LOG(WARNING) << "object not exist, key=" << hex(tablet_idx_key); } } - - if (data_loss) { - ++num_rowset_loss; - } }; // scan visible rowsets @@ -632,7 +653,9 @@ int InstanceChecker::do_check() { while (it->has_next() && !stopped()) { auto [k, v] = it->next(); - if (!it->has_next()) start_key = k; + if (!it->has_next()) { + start_key = k; + } doris::RowsetMetaCloudPB rs_meta; if (!rs_meta.ParseFromArray(v.data(), v.size())) { @@ -703,11 +726,8 @@ int InstanceChecker::do_inverted_check() { }; TabletRowsets tablet_rowsets_cache; - struct TabletIndexes { - int64_t tablet_id {0}; - std::unordered_set index_ids; - }; - TabletIndexes tablet_indexes_cache; + RowsetIndexesFormatV1 rowset_index_cache_v1; + RowsetIndexesFormatV2 rowset_index_cache_v2; // Return 0 if check success, return 1 if file is garbage data, negative if error occurred auto check_segment_file = [&](const std::string& obj_key) { @@ -786,10 +806,12 @@ int InstanceChecker::do_inverted_check() { return 0; }; + auto check_inverted_index_file = [&](const std::string& obj_key) { std::vector str; butil::SplitString(obj_key, '/', &str); - // data/{tablet_id}/{rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx + // format v1: data/{tablet_id}/{rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx + // format v2: data/{tablet_id}/{rowset_id}_{seg_num}.idx if (str.size() < 3) { return -1; } @@ -800,62 +822,31 @@ int InstanceChecker::do_inverted_check() { return -1; } - if (!str.back().ends_with(".idx")) { + // v1: {rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx + // v2: {rowset_id}_{seg_num}.idx + std::string rowset_info = str.back(); + + if (!rowset_info.ends_with(".idx")) { return 0; // Not an index file } - int64_t index_id; + InvertedIndexStorageFormatPB inverted_index_storage_format = + std::count(rowset_info.begin(), rowset_info.end(), '_') > 1 + ? InvertedIndexStorageFormatPB::V1 + : InvertedIndexStorageFormatPB::V2; - size_t pos = str.back().find_last_of('_'); + size_t pos = rowset_info.find_last_of('_'); if (pos == std::string::npos || pos + 1 >= str.back().size() - 4) { LOG(WARNING) << "Invalid index_id format, key=" << obj_key; return -1; } - index_id = atol(str.back().substr(pos + 1, str.back().size() - 4).c_str()); - - if (tablet_indexes_cache.tablet_id == tablet_id) { - if (tablet_indexes_cache.index_ids.contains(index_id)) { - return 0; - } else { - LOG(WARNING) << "index not exists, key=" << obj_key; - return -1; - } - } - // Get all index id of this tablet - tablet_indexes_cache.tablet_id = tablet_id; - tablet_indexes_cache.index_ids.clear(); - std::unique_ptr txn; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - LOG(WARNING) << "failed to create txn"; - return -1; - } - auto tablet_idx_key = meta_tablet_idx_key({instance_id_, tablet_id}); - std::string tablet_idx_val; - err = txn->get(tablet_idx_key, &tablet_idx_val); - if (err != TxnErrorCode::TXN_OK) { - LOG(WARNING) << "failed to get tablet idx," - << " key=" << hex(tablet_idx_key) << " err=" << err; - return -1; - } - - TabletIndexPB tablet_idx_pb; - if (!tablet_idx_pb.ParseFromArray(tablet_idx_val.data(), tablet_idx_val.size())) { - LOG(WARNING) << "malformed index meta value, key=" << hex(tablet_idx_key); - return -1; - } - if (!tablet_idx_pb.has_index_id()) { - LOG(WARNING) << "tablet index meta does not have index_id, key=" << hex(tablet_idx_key); - return -1; - } - tablet_indexes_cache.index_ids.insert(tablet_idx_pb.index_id()); - - if (!tablet_indexes_cache.index_ids.contains(index_id)) { - LOG(WARNING) << "index should be recycled, key=" << obj_key; - return 1; + if (inverted_index_storage_format == InvertedIndexStorageFormatPB::V1) { + return check_inverted_index_file_storage_format_v1(tablet_id, obj_key, rowset_info, + rowset_index_cache_v1); + } else { + return check_inverted_index_file_storage_format_v2(tablet_id, obj_key, rowset_info, + rowset_index_cache_v2); } - - return 0; }; // so we choose to skip here. TEST_SYNC_POINT_RETURN_WITH_VALUE("InstanceChecker::do_inverted_check", (int)0); @@ -1199,6 +1190,202 @@ int InstanceChecker::do_delete_bitmap_inverted_check() { return (leaked_delete_bitmaps > 0 || abnormal_delete_bitmaps > 0) ? 1 : 0; } +int InstanceChecker::check_inverted_index_file_storage_format_v1( + int64_t tablet_id, const std::string& file_path, const std::string& rowset_info, + RowsetIndexesFormatV1& rowset_index_cache_v1) { + // format v1: data/{tablet_id}/{rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx + std::string rowset_id; + int64_t segment_id; + std::string index_id_with_suffix_name; + // {rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx + std::vector str; + butil::SplitString(rowset_info.substr(0, rowset_info.size() - 4), '_', &str); + if (str.size() < 3) { + LOG(WARNING) << "Split rowset info with '_' error, str size < 3, rowset_info = " + << rowset_info; + return -1; + } + rowset_id = str[0]; + segment_id = std::atoll(str[1].c_str()); + index_id_with_suffix_name = str[2]; + + if (rowset_index_cache_v1.rowset_id == rowset_id) { + if (rowset_index_cache_v1.segment_ids.contains(segment_id)) { + if (auto it = rowset_index_cache_v1.index_ids.find(index_id_with_suffix_name); + it == rowset_index_cache_v1.index_ids.end()) { + // clang-format off + LOG(WARNING) << fmt::format("index_id with suffix name not found, rowset_info = {}, obj_key = {}", rowset_info, file_path); + // clang-format on + return -1; + } + } else { + // clang-format off + LOG(WARNING) << fmt::format("segment id not found, rowset_info = {}, obj_key = {}", rowset_info, file_path); + // clang-format on + return -1; + } + } + + rowset_index_cache_v1.rowset_id = rowset_id; + rowset_index_cache_v1.segment_ids.clear(); + rowset_index_cache_v1.index_ids.clear(); + + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to create txn"; + return -1; + } + std::unique_ptr it; + auto begin = meta_rowset_key({instance_id_, tablet_id, 0}); + auto end = meta_rowset_key({instance_id_, tablet_id, INT64_MAX}); + do { + TxnErrorCode err = txn->get(begin, end, &it); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to get rowset kv, err=" << err; + return -1; + } + if (!it->has_next()) { + break; + } + while (it->has_next()) { + // recycle corresponding resources + auto [k, v] = it->next(); + doris::RowsetMetaCloudPB rs_meta; + if (!rs_meta.ParseFromArray(v.data(), v.size())) { + LOG(WARNING) << "malformed rowset meta value, key=" << hex(k); + return -1; + } + + for (size_t i = 0; i < rs_meta.num_segments(); i++) { + rowset_index_cache_v1.segment_ids.insert(i); + } + + TabletIndexPB tablet_index; + if (get_tablet_idx(txn_kv_.get(), instance_id_, rs_meta.tablet_id(), tablet_index) == + -1) { + LOG(WARNING) << "failedt to get tablet index, tablet_id= " << rs_meta.tablet_id(); + return -1; + } + + auto tablet_schema_key = meta_schema_key( + {instance_id_, tablet_index.index_id(), rs_meta.schema_version()}); + std::string tablet_schema_val; + err = txn->get(tablet_schema_key, &tablet_schema_val); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + // rowset don't have tablet schema key means no index + return 0; + } + auto* schema = rs_meta.mutable_tablet_schema(); + schema->ParseFromString(tablet_schema_val); + + for (const auto& i : rs_meta.tablet_schema().index()) { + if (i.has_index_type() && i.index_type() == IndexType::INVERTED) { + rowset_index_cache_v1.index_ids.insert( + fmt::format("{}{}", i.index_name(), i.index_suffix_name())); + } + } + + if (!it->has_next()) { + begin = k; + begin.push_back('\x00'); // Update to next smallest key for iteration + break; + } + } + } while (it->more() && !stopped()); + + if (!rowset_index_cache_v1.segment_ids.contains(segment_id)) { + // Garbage data leak + LOG(WARNING) << "rowset should be recycled, key=" << file_path; + return 1; + } + + if (!rowset_index_cache_v1.index_ids.contains(index_id_with_suffix_name)) { + // Garbage data leak + LOG(WARNING) << "rowset with inde meta should be recycled, key=" << file_path; + return 1; + } + + return 0; +} + +int InstanceChecker::check_inverted_index_file_storage_format_v2( + int64_t tablet_id, const std::string& file_path, const std::string& rowset_info, + RowsetIndexesFormatV2& rowset_index_cache_v2) { + std::string rowset_id; + int64_t segment_id; + // {rowset_id}_{seg_num}.idx + std::vector str; + butil::SplitString(rowset_info.substr(0, rowset_info.size() - 4), '_', &str); + if (str.size() < 2) { + // clang-format off + LOG(WARNING) << "Split rowset info with '_' error, str size < 2, rowset_info = " << rowset_info; + // clang-format on + return -1; + } + rowset_id = str[0]; + segment_id = std::atoll(str[1].c_str()); + + if (rowset_index_cache_v2.rowset_id == rowset_id) { + if (!rowset_index_cache_v2.segment_ids.contains(segment_id)) { + // clang-format off + LOG(WARNING) << fmt::format("index file not found, rowset_info = {}, obj_key = {}", rowset_info, file_path); + // clang-format on + return -1; + } + } + + rowset_index_cache_v2.rowset_id = rowset_id; + rowset_index_cache_v2.segment_ids.clear(); + + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to create txn"; + return -1; + } + std::unique_ptr it; + auto begin = meta_rowset_key({instance_id_, tablet_id, 0}); + auto end = meta_rowset_key({instance_id_, tablet_id, INT64_MAX}); + do { + TxnErrorCode err = txn->get(begin, end, &it); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to get rowset kv, err=" << err; + return -1; + } + if (!it->has_next()) { + break; + } + while (it->has_next()) { + // recycle corresponding resources + auto [k, v] = it->next(); + doris::RowsetMetaCloudPB rs_meta; + if (!rs_meta.ParseFromArray(v.data(), v.size())) { + LOG(WARNING) << "malformed rowset meta value, key=" << hex(k); + return -1; + } + + for (size_t i = 0; i < rs_meta.num_segments(); i++) { + rowset_index_cache_v2.segment_ids.insert(i); + } + + if (!it->has_next()) { + begin = k; + begin.push_back('\x00'); // Update to next smallest key for iteration + break; + } + } + } while (it->more() && !stopped()); + + if (!rowset_index_cache_v2.segment_ids.contains(segment_id)) { + // Garbage data leak + LOG(WARNING) << "rowset with index meta should be recycled, key=" << file_path; + return 1; + } + + return 0; +} + int InstanceChecker::check_delete_bitmap_storage_optimize(int64_t tablet_id) { using Version = std::pair; struct RowsetDigest { diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h index 7f87e90f7cb366..37cca76c85add3 100644 --- a/cloud/src/recycler/checker.h +++ b/cloud/src/recycler/checker.h @@ -110,6 +110,18 @@ class InstanceChecker { void stop() { stopped_.store(true, std::memory_order_release); } bool stopped() const { return stopped_.load(std::memory_order_acquire); } +private: + struct RowsetIndexesFormatV1 { + std::string rowset_id; + std::unordered_set segment_ids; + std::unordered_set index_ids; + }; + + struct RowsetIndexesFormatV2 { + std::string rowset_id; + std::unordered_set segment_ids; + }; + private: // returns 0 for success otherwise error int init_obj_store_accessors(const InstanceInfoPB& instance); @@ -128,6 +140,14 @@ class InstanceChecker { int check_delete_bitmap_storage_optimize(int64_t tablet_id); + int check_inverted_index_file_storage_format_v1(int64_t tablet_id, const std::string& file_path, + const std::string& rowset_info, + RowsetIndexesFormatV1& rowset_index_cache_v1); + + int check_inverted_index_file_storage_format_v2(int64_t tablet_id, const std::string& file_path, + const std::string& rowset_info, + RowsetIndexesFormatV2& rowset_index_cache_v2); + std::atomic_bool stopped_ {false}; std::shared_ptr txn_kv_; std::string instance_id_; diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index bc920c78d9a6bb..57a075eb5b6d27 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -21,10 +21,13 @@ #include #include #include +#include #include #include #include +#include +#include #include #include #include @@ -56,7 +59,95 @@ static int64_t current_time = 0; static constexpr int64_t db_id = 1000; static RecyclerMetricsContext ctx; -static doris::cloud::RecyclerThreadPoolGroup thread_group; +std::vector index_v2_file_path = { + "data/1753202639945/0200000000001a5c92f4e7d9j8f2b4c8a3e6f8b1c9d2e5f8_0.idx", + "data/1753202639947/0200000000001b8d45a74r6c7sf3e9c2b6d4a8e1f7c3d9e2_0.idx", + "data/1753202639951/0200000000001c9e56b8g4f0x8s7g2f0d3c7e5b9f2e8d4f0_0.idx", + "data/1753202639953/0200000000001d0f67c9h5g8a3e6f8b1e4d8f6c0g3f9e5g1_0.idx", + "data/1753202639955/0200000000001e1g78d067c9h5g8i6h2f5e9g7d1h4g0f6h2_0.idx", + "data/1753202639957/0200000000001f2h89e1jg7d1h4g07i3g6f0h8e2i5h1g7i3_0.idx", + "data/1753202639959/020000000000208i90f2k0h8e2i5h8j4h7g1i9f3j6i2h8j4_0.idx", + "data/1753202639961/02000000000021aj01g3l9k5i8h2j8e2i5h8j0g4k7j3i9k5_0.idx", + "data/1753202639963/02000000000022bk12h4m0lk0h8e2i56j9i3k1h5l8k4j0l6_0.idx", + "data/1753202639965/02000000000023cl23i5n1m7g3l9k5i8k0j4l2i6m9l5k1m7_0.idx", + "data/1753202639967/02000000000024dm34j1m7g3l9k6o2n8l1k5m3j7n0m6l2n8_0.idx", + "data/1753202639969/02000000000025en45k7p3o9m2l6n4k34j1m7g38o1n7m3o9_0.idx", + "data/1753202639971/02000000000026fo56l8q4p0n2l6n4k343m7o5l9p2o8n4p0_0.idx", + "data/1753202639973/02000000000027gp67m9r5q8q4p0n2l1o4n8p6m0q3p9o5q1_0.idx", + "data/1753202639975/02000000000028hq78n0s6rm9r5q8q42p5o9q7n1r4q0p6r2_0.idx", + "data/1753202639977/02000000000029ir89o1t7s78n0s6rm3q6p0r8o2s5r1q7s3_4.idx", + "data/1753202639979/0200000000002ajs90p2u8t4m3q6p0r8r7q1s9p3t6s2r8t4_0.idx", + "data/1753202639981/0200000000002bkt01q3v9u2u8t4m3q5s8r2t0q4u7t3s9u5_0.idx", + "data/1753202639983/0200000000002clu12r4w1q3v9u2u0v6t9s3u1r5v8u4t0v6_0.idx", + "data/1753202639985/0200000000002dmv23s5x1w7u0t4t9s3u1r5v2s6w9v5u1w7_0.idx"}; + +std::vector segment_v2_file_path = { + "data/1753202639945/0200000000001a5c92f4e7d9j8f2b4c8a3e6f8b1c9d2e5f8_0.dat", + "data/1753202639947/0200000000001b8d45a74r6c7sf3e9c2b6d4a8e1f7c3d9e2_0.dat", + "data/1753202639951/0200000000001c9e56b8g4f0x8s7g2f0d3c7e5b9f2e8d4f0_0.dat", + "data/1753202639953/0200000000001d0f67c9h5g8a3e6f8b1e4d8f6c0g3f9e5g1_0.dat", + "data/1753202639955/0200000000001e1g78d067c9h5g8i6h2f5e9g7d1h4g0f6h2_0.dat", + "data/1753202639957/0200000000001f2h89e1jg7d1h4g07i3g6f0h8e2i5h1g7i3_0.dat", + "data/1753202639959/020000000000208i90f2k0h8e2i5h8j4h7g1i9f3j6i2h8j4_0.dat", + "data/1753202639961/02000000000021aj01g3l9k5i8h2j8e2i5h8j0g4k7j3i9k5_0.dat", + "data/1753202639963/02000000000022bk12h4m0lk0h8e2i56j9i3k1h5l8k4j0l6_0.dat", + "data/1753202639965/02000000000023cl23i5n1m7g3l9k5i8k0j4l2i6m9l5k1m7_0.dat", + "data/1753202639967/02000000000024dm34j1m7g3l9k6o2n8l1k5m3j7n0m6l2n8_0.dat", + "data/1753202639969/02000000000025en45k7p3o9m2l6n4k34j1m7g38o1n7m3o9_0.dat", + "data/1753202639971/02000000000026fo56l8q4p0n2l6n4k343m7o5l9p2o8n4p0_0.dat", + "data/1753202639973/02000000000027gp67m9r5q8q4p0n2l1o4n8p6m0q3p9o5q1_0.dat", + "data/1753202639975/02000000000028hq78n0s6rm9r5q8q42p5o9q7n1r4q0p6r2_0.dat", + "data/1753202639977/02000000000029ir89o1t7s78n0s6rm3q6p0r8o2s5r1q7s3_4.dat", + "data/1753202639979/0200000000002ajs90p2u8t4m3q6p0r8r7q1s9p3t6s2r8t4_0.dat", + "data/1753202639981/0200000000002bkt01q3v9u2u8t4m3q5s8r2t0q4u7t3s9u5_0.dat", + "data/1753202639983/0200000000002clu12r4w1q3v9u2u0v6t9s3u1r5v8u4t0v6_0.dat", + "data/1753202639985/0200000000002dmv23s5x1w7u0t4t9s3u1r5v2s6w9v5u1w7_0.dat"}; + +// clang-format off +std::vector index_v1_file_path = { + "data/1753202846974/0200000000007864994f6aa97288842758c2e89b03e65682_0_1753202846943.idx", + "data/1753202845724/020000000000786635407b55b72242ac167cf83cd4c598a2_0_1753202841593.idx", + "data/1753202846984/020000000000788bdd40fcf18bcaa1bbd4058ef92606e79a_0_1753202846943.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0_1753202846943.idx", + "data/1753202846986/02000000000078ec35407b55b72242ac167cf83cd4c598a2_0_1753202846943.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202844931.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202846410.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202847011.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202844931.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202846410.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202847011.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202858543.idx", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202844931.idx", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202846410.idx", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202847011.idx", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202858543.idx", + "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0_1753202844931.idx", + "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0_1753202846410.idx", + "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0_1753202847011.idx"}; +// clang-format on + +std::vector segment_v1_file_path = { + "data/1753202846974/0200000000007864994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202845724/020000000000786635407b55b72242ac167cf83cd4c598a2_0.dat", + "data/1753202846984/020000000000788bdd40fcf18bcaa1bbd4058ef92606e79a_0.dat", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0.dat", + "data/1753202846986/02000000000078ec35407b55b72242ac167cf83cd4c598a2_0.dat", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0.dat", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0.dat", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0.dat", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", + "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0.dat", + "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0.dat", + "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0.dat"}; + +doris::cloud::RecyclerThreadPoolGroup thread_group; int main(int argc, char** argv) { auto conf_file = "doris_cloud.conf"; @@ -240,6 +331,311 @@ static int create_tmp_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor, return 0; } +static int create_committed_rowset_with_tablet_schema( + TxnKv* txn_kv, StorageVaultAccessor* accessor, const std::string& resource_id, + int64_t tablet_id, int64_t version, int num_segments = 1, size_t num_inverted_indexes = 1, + bool use_inverted_index_storage_format_v1 = true) { + std::string val; + std::unique_ptr txn; + int64_t tablet_index_id = 123; + int64_t schema_version = 456; + + auto rowset_id = next_rowset_id(); + MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; + std::string rowset_meta_key = meta_rowset_key(key_info); + + doris::RowsetMetaCloudPB rowset_pb; + rowset_pb.set_rowset_id(0); // useless but required + rowset_pb.set_rowset_id_v2(rowset_id); + rowset_pb.set_num_segments(num_segments); + rowset_pb.set_tablet_id(tablet_id); + rowset_pb.set_resource_id(resource_id); + rowset_pb.set_creation_time(current_time); + rowset_pb.set_schema_version(schema_version); + rowset_pb.SerializeToString(&val); + + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(rowset_meta_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + + TabletIndexPB tablet_index; + tablet_index.set_index_id(tablet_index_id); + tablet_index.set_tablet_id(tablet_id); + std::string tablet_index_key = meta_tablet_idx_key({instance_id, tablet_id}); + tablet_index.SerializeToString(&val); + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(tablet_index_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + + if (num_inverted_indexes) { + doris::TabletSchemaCloudPB tablet_schema; + if (use_inverted_index_storage_format_v1) { + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + } else { + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + } + tablet_schema.set_schema_version(schema_version); + for (size_t i = 0; i < num_inverted_indexes; i++) { + auto index = tablet_schema.add_index(); + index->set_index_id(i); + index->set_index_type(IndexType::INVERTED); + } + std::string tablet_schema_key = + meta_schema_key({instance_id, tablet_index_id, schema_version}); + std::string val; + tablet_schema.SerializeToString(&val); + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(tablet_schema_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + } + + for (int i = 0; i < num_segments; ++i) { + auto path = segment_path(tablet_id, rowset_id, i); + accessor->put_file(path, ""); + if (use_inverted_index_storage_format_v1) { + for (int j = 0; j < num_inverted_indexes; ++j) { + std::string path = inverted_index_path_v1(tablet_id, rowset_id, i, j, ""); + accessor->put_file(path, ""); + } + } else { + std::string path = inverted_index_path_v2(tablet_id, rowset_id, i); + accessor->put_file(path, ""); + } + } + return 0; +} + +static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, + StorageVaultAccessor* accessor, + const std::string& resource_id, + const std::string& file_path, + int64_t version = 1) { + std::string val; + std::unique_ptr txn; + + // Parse file path to extract tablet_id and rowset_id + // Expected format: data/{tablet_id}/{rowset_id}_{segment_id}.{ext} + std::vector path_parts; + butil::SplitString(file_path, '/', &path_parts); + + if (path_parts.size() < 3 || path_parts[0] != "data") { + LOG(WARNING) << "Invalid file path format: " << file_path; + return -1; + } + + int64_t tablet_id = std::stoll(path_parts[1]); + std::string filename = path_parts[2]; + + // Extract rowset_id and segment_id from filename + size_t underscore_pos = filename.find_last_of('_'); + size_t dot_pos = filename.find_last_of('.'); + + if (underscore_pos == std::string::npos || dot_pos == std::string::npos || + underscore_pos >= dot_pos) { + LOG(WARNING) << "Invalid filename format: " << filename; + return -1; + } + + std::string rowset_id = filename.substr(0, underscore_pos); + std::string segment_str = filename.substr(underscore_pos + 1, dot_pos - underscore_pos - 1); + std::string extension = filename.substr(dot_pos + 1); + + int segment_id = stoll(segment_str); + int64_t tablet_index_id = 123; // Default index id + int64_t schema_version = 456; // Default schema version + + // Create rowset meta data + MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; + std::string rowset_meta_key = meta_rowset_key(key_info); + + doris::RowsetMetaCloudPB rowset_pb; + rowset_pb.set_rowset_id(0); // useless but required + rowset_pb.set_rowset_id_v2(rowset_id); + rowset_pb.set_num_segments(segment_id + 1); // segment_id is 0-based + rowset_pb.set_tablet_id(tablet_id); + rowset_pb.set_resource_id(resource_id); + rowset_pb.set_creation_time(current_time); + rowset_pb.set_schema_version(schema_version); + rowset_pb.SerializeToString(&val); + + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(rowset_meta_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + + // Create tablet index meta data + TabletIndexPB tablet_index; + tablet_index.set_index_id(tablet_index_id); + tablet_index.set_tablet_id(tablet_id); + std::string tablet_index_key = meta_tablet_idx_key({instance_id, tablet_id}); + tablet_index.SerializeToString(&val); + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(tablet_index_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + + // Create tablet schema if dealing with index files + if (extension == "idx") { + doris::TabletSchemaCloudPB tablet_schema; + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + tablet_schema.set_schema_version(schema_version); + + auto index = tablet_schema.add_index(); + index->set_index_id(0); + index->set_index_type(IndexType::INVERTED); + + std::string tablet_schema_key = + meta_schema_key({instance_id, tablet_index_id, schema_version}); + tablet_schema.SerializeToString(&val); + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(tablet_schema_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + } + + accessor->put_file(file_path, ""); + + return 0; +} + +static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, + StorageVaultAccessor* accessor, + const std::string& resource_id, + const std::string& file_path, + int64_t version = 1) { + std::string val; + std::unique_ptr txn; + + // Parse file path to extract tablet_id and rowset_id + // Expected format: data/{tablet_id}/{rowset_id}_{segment_id}_{index_id}{suffix}.idx + std::vector path_parts; + butil::SplitString(file_path, '/', &path_parts); + + if (path_parts.size() < 3 || path_parts[0] != "data") { + LOG(WARNING) << "Invalid file path format: " << file_path; + return -1; + } + + int64_t tablet_id = std::stoll(path_parts[1]); + std::string filename = path_parts[2]; + + // Extract rowset_id, segment_id, index_id, and suffix from filename + // Format: {rowset_id}_{segment_id}_{index_id}{suffix}.idx + size_t first_underscore_pos = filename.find('_'); + size_t second_underscore_pos = filename.find('_', first_underscore_pos + 1); + size_t dot_pos = filename.find_last_of('.'); + + if (first_underscore_pos == std::string::npos || second_underscore_pos == std::string::npos || + dot_pos == std::string::npos || first_underscore_pos >= second_underscore_pos || + second_underscore_pos >= dot_pos) { + LOG(WARNING) << "Invalid filename format: " << filename; + return -1; + } + + std::string rowset_id = filename.substr(0, first_underscore_pos); + std::string segment_str = filename.substr(first_underscore_pos + 1, + second_underscore_pos - first_underscore_pos - 1); + std::string remaining = + filename.substr(second_underscore_pos + 1, dot_pos - second_underscore_pos - 1); + std::string extension = filename.substr(dot_pos + 1); + + // Parse index_id and suffix from remaining part + // Format: {index_id}{suffix} or just {index_id} + std::string index_id_str = remaining; + std::string index_suffix = ""; + + int segment_id = stoll(segment_str); + int64_t index_id = std::stoll(index_id_str); + int64_t tablet_index_id = 123; // Default tablet index id + int64_t schema_version = 456; // Default schema version + + // Create rowset meta data + MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; + std::string rowset_meta_key = meta_rowset_key(key_info); + + doris::RowsetMetaCloudPB rowset_pb; + rowset_pb.set_rowset_id(0); // useless but required + rowset_pb.set_rowset_id_v2(rowset_id); + rowset_pb.set_num_segments(segment_id + 1); // segment_id is 0-based + rowset_pb.set_tablet_id(tablet_id); + rowset_pb.set_resource_id(resource_id); + rowset_pb.set_creation_time(current_time); + rowset_pb.set_schema_version(schema_version); + rowset_pb.SerializeToString(&val); + + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(rowset_meta_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + + // Create tablet index meta data + TabletIndexPB tablet_index; + tablet_index.set_index_id(tablet_index_id); + tablet_index.set_tablet_id(tablet_id); + std::string tablet_index_key = meta_tablet_idx_key({instance_id, tablet_id}); + tablet_index.SerializeToString(&val); + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(tablet_index_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + + // Create tablet schema if dealing with index files + if (extension == "idx") { + doris::TabletSchemaCloudPB tablet_schema; + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + tablet_schema.set_schema_version(schema_version); + + auto index = tablet_schema.add_index(); + index->set_index_id(index_id); + index->set_index_type(IndexType::INVERTED); + if (!index_suffix.empty()) { + index->set_index_suffix_name(index_suffix); + } + + std::string tablet_schema_key = + meta_schema_key({instance_id, tablet_index_id, schema_version}); + tablet_schema.SerializeToString(&val); + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + txn->put(tablet_schema_key, val); + if (txn->commit() != TxnErrorCode::TXN_OK) { + return -1; + } + } + + accessor->put_file(file_path, ""); + + return 0; +} + static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor, const std::string& resource_id, int64_t tablet_id, int64_t version, int num_segments = 1, @@ -2500,7 +2896,85 @@ TEST(CheckerTest, DISABLED_abnormal_inverted_check) { ASSERT_NE(checker.do_inverted_check(), 0); } -TEST(CheckerTest, inverted_check_recycle_idx_file) { +TEST(CheckerTest, normal_check_index_file) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + + auto sp = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sp->set_call_back( + "InstanceChecker::do_inverted_check", + [](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = 0; + ret->second = true; + }, + &guard); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->disable_processing(); + }; + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + + for (const auto& file : segment_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + ASSERT_EQ(checker.do_inverted_check(), 0); +} + +TEST(CheckerTest, normal_inverted_check_index_file) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + + auto sp = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sp->set_call_back( + "InstanceChecker::do_inverted_check", + [](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = 0; + ret->second = true; + }, + &guard); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->disable_processing(); + }; + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + + for (const auto& file : segment_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + ASSERT_EQ(checker.do_inverted_check(), 0); +} + +TEST(CheckerTest, inverted_check_recycle_idx_file_v1) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) { sp->clear_all_call_backs(); @@ -2539,34 +3013,207 @@ TEST(CheckerTest, inverted_check_recycle_idx_file) { }); sp->enable_processing(); - for (int t = 10001; t <= 10100; ++t) { - for (int v = 0; v < 10; ++v) { - int ret = create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1, 3); - ASSERT_EQ(ret, 0) << "Failed to create committed rs: " << ret; + for (const auto& file : index_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + } + + for (const auto& file : segment_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + } + + size_t delete_kv_num = 5; + std::string meta_rowset_key_begin, meta_rowset_key_end; + meta_rowset_key({instance_id, 0, 1}, &meta_rowset_key_begin); + meta_rowset_key({instance_id, INT64_MAX, 1}, &meta_rowset_key_end); + std::vector rowset_key_to_delete; + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + + std::unique_ptr it; + do { + err = txn->get(meta_rowset_key_begin, meta_rowset_key_end, &it); + while (it->has_next()) { + auto [k, v] = it->next(); + if (rowset_key_to_delete.size() < delete_kv_num) { + rowset_key_to_delete.emplace_back(k); + } + if (!it->has_next()) { + meta_rowset_key_begin = k; + } + } + meta_rowset_key_begin.push_back('\x00'); + } while (it->more()); + + for (const auto& key : rowset_key_to_delete) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + txn->remove(key); + err = txn->commit(); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + } + + std::unique_ptr list_iter; + int ret = accessor->list_directory("data", &list_iter); + ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; + + ASSERT_EQ(checker.do_inverted_check(), 1); +} + +TEST(CheckerTest, inverted_check_recycle_idx_file_v2) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->disable_processing(); + }); + + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("CheckerTest"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->enable_processing(); + + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + + for (const auto& file : segment_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + + size_t delete_kv_num = 5; + std::string meta_rowset_key_begin, meta_rowset_key_end; + meta_rowset_key({instance_id, 0, 1}, &meta_rowset_key_begin); + meta_rowset_key({instance_id, INT64_MAX, 1}, &meta_rowset_key_end); + std::vector rowset_key_to_delete; + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + + std::unique_ptr it; + do { + err = txn->get(meta_rowset_key_begin, meta_rowset_key_end, &it); + while (it->has_next()) { + auto [k, v] = it->next(); + if (rowset_key_to_delete.size() < delete_kv_num) { + rowset_key_to_delete.emplace_back(k); + } + if (!it->has_next()) { + meta_rowset_key_begin = k; + } } + meta_rowset_key_begin.push_back('\x00'); + } while (it->more()); + + for (const auto& key : rowset_key_to_delete) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + txn->remove(key); + err = txn->commit(); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; } + std::unique_ptr list_iter; int ret = accessor->list_directory("data", &list_iter); ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; - int64_t tablet_id_to_delete_index = -1; + ASSERT_EQ(checker.do_inverted_check(), 1); +} + +TEST(CheckerTest, forward_check_recycle_idx_file_v1) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->disable_processing(); + }); + + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("CheckerTest"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->enable_processing(); + + for (const auto& file : index_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + } + + for (const auto& file : segment_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + } + std::unique_ptr list_iter; + int ret = accessor->list_directory("data", &list_iter); + ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; + + int64_t tablet_to_delete = -1; for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { std::vector str; butil::SplitString(file->path, '/', &str); int64_t tablet_id = atol(str[1].c_str()); - // only delete one index files of ever tablet for mock recycle - // The reason for not select "delete all idx file" is that inverted checking cannot handle this case - // forward checking is required. - if (file->path.ends_with(".idx") && tablet_id_to_delete_index != tablet_id) { + // delete all index files of ever tablet for mock missing + if (file->path.ends_with(".idx") && tablet_to_delete != tablet_id) { + tablet_to_delete = tablet_id; accessor->delete_file(file->path); - tablet_id_to_delete_index = tablet_id; } } - ASSERT_EQ(checker.do_inverted_check(), 1); + ASSERT_EQ(checker.do_check(), 1); } -TEST(CheckerTest, forward_check_recycle_idx_file) { +TEST(CheckerTest, forward_check_recycle_idx_file_v2) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) { sp->clear_all_call_backs(); @@ -2605,19 +3252,27 @@ TEST(CheckerTest, forward_check_recycle_idx_file) { }); sp->enable_processing(); - for (int t = 10001; t <= 10100; ++t) { - for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1, 3); - } + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + } + + for (const auto& file : segment_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); } std::unique_ptr list_iter; int ret = accessor->list_directory("data", &list_iter); ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; + int64_t tablet_to_delete = -1; for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { - // delete all index files of ever tablet for mock recycle - if (file->path.ends_with(".idx")) { + std::vector str; + butil::SplitString(file->path, '/', &str); + int64_t tablet_id = atol(str[1].c_str()); + + // delete all index files of ever tablet for mock missing + if (file->path.ends_with(".idx") && tablet_to_delete != tablet_id) { accessor->delete_file(file->path); + tablet_to_delete = tablet_id; } } ASSERT_EQ(checker.do_check(), 1); @@ -2637,12 +3292,12 @@ TEST(CheckerTest, normal) { auto accessor = checker.accessor_map_.begin()->second; for (int t = 10001; t <= 10100; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1); + create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 1); } } for (int t = 10101; t <= 10200; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 5); + create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 5); } } ASSERT_EQ(checker.do_check(), 0); @@ -2663,12 +3318,14 @@ TEST(CheckerTest, abnormal) { auto accessor = checker.accessor_map_.begin()->second; for (int t = 10001; t <= 10100; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1, 0); + create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 1, + 0); } } for (int t = 10101; t <= 10200; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 5, 0); + create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 5, + 0); } } From af2b235cf0391af09328bd6aa7253fb1b2b0f9e7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 22:20:33 +0800 Subject: [PATCH 349/572] branch-3.0: [fix](test) fix test_black_list fail #54287 (#54306) Cherry-picked from #54287 Co-authored-by: hui lai --- .../suites/load_p0/routine_load/test_black_list.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/routine_load/test_black_list.groovy b/regression-test/suites/load_p0/routine_load/test_black_list.groovy index 04779f10362a2c..807389e5ec924b 100644 --- a/regression-test/suites/load_p0/routine_load/test_black_list.groovy +++ b/regression-test/suites/load_p0/routine_load/test_black_list.groovy @@ -86,7 +86,7 @@ suite("test_black_list","nonConcurrent,p0") { log.info("routine load state: ${state[0][8].toString()}".toString()) log.info("reason of state changed: ${state[0][17].toString()}".toString()) log.info("other msg: ${state[0][19].toString()}".toString()) - if (state[0][17].toString().contains("Failed to get info") || state[0][19].toString().contains("Failed to get info")) { + if (state[0][17].toString().contains("failed to get latest partition offset") || state[0][19].toString().contains("failed to get latest partition offset")) { break } if (count >= 90) { From 208f72829808f10daecb7c45f99d23fc955cd1b5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Aug 2025 19:14:55 +0800 Subject: [PATCH 350/572] =?UTF-8?q?branch-3.0:=20[fix](case)=20remove=20gl?= =?UTF-8?q?obal=20variable=20declarations=20in=20compaction=20test=20[f?= =?UTF-8?q?=E2=80=A6=20#54360=20(#54398)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #54360 Co-authored-by: Yongqiang YANG Co-authored-by: Yongqiang YANG --- .../compaction1/test_schema_change_with_compaction1.groovy | 1 + .../compaction10/test_schema_change_with_compaction10.groovy | 3 ++- .../compaction11/test_schema_change_with_compaction11.groovy | 2 +- .../compaction2/test_schema_change_with_compaction2.groovy | 1 + .../compaction3/test_schema_change_with_compaction3.groovy | 5 +++-- .../compaction4/test_schema_change_with_compaction4.groovy | 5 +++-- .../compaction5/test_schema_change_with_compaction5.groovy | 5 +++-- .../compaction6/test_schema_change_with_compaction6.groovy | 5 +++-- .../compaction7/test_schema_change_with_compaction7.groovy | 5 +++-- .../compaction8/test_schema_change_with_compaction8.groovy | 1 + .../compaction9/test_schema_change_with_compaction9.groovy | 5 +++-- 11 files changed, 24 insertions(+), 14 deletions(-) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy b/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy index 64b03d8cb6683e..477dd12a30467c 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction1/test_schema_change_with_compaction1.groovy @@ -123,6 +123,7 @@ suite('test_schema_change_with_compaction1', 'nonConcurrent') { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy index 2e9ad0628d4d53..db3da9aa044241 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy @@ -127,8 +127,9 @@ suite('test_schema_change_with_compaction10', 'docker') { GetDebugPoint().disableDebugPointForAllBEs(injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ - def result = getJobState("date") + result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { sleep(3000) break diff --git a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy index 2f7268b962442f..4c1c772da37c93 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy @@ -115,7 +115,7 @@ suite('test_schema_change_with_compaction11', 'docker') { // cu compaction - tabletId = array[0].TabletId + def tabletId = array[0].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy b/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy index 667a498d6c0ad3..e3841b804d6067 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction2/test_schema_change_with_compaction2.groovy @@ -143,6 +143,7 @@ suite('test_schema_change_with_compaction2', 'nonConcurrent') { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy b/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy index cf59823cb784dd..1c41946258d5be 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction3/test_schema_change_with_compaction3.groovy @@ -143,7 +143,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { // cu compaction for (int i = 0; i < array.size(); i++) { - tabletId = array[i].TabletId + def tabletId = array[i].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -153,7 +153,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { running = true do { Thread.sleep(100) - tabletId = array[i].TabletId + def tabletId = array[i].TabletId (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -170,6 +170,7 @@ suite('test_schema_change_with_compaction3', 'nonConcurrent') { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy b/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy index a57e32bf922f64..a20b599d4601e3 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction4/test_schema_change_with_compaction4.groovy @@ -143,7 +143,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { // cu compaction for (int i = 0; i < array.size(); i++) { - tabletId = array[i].TabletId + def tabletId = array[i].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -153,7 +153,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { running = true do { Thread.sleep(100) - tabletId = array[i].TabletId + def tabletId = array[i].TabletId (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -170,6 +170,7 @@ suite('test_schema_change_with_compaction4', 'nonConcurrent') { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy b/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy index 8af620ea6479b4..ddd639b5fd1fc1 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction5/test_schema_change_with_compaction5.groovy @@ -143,7 +143,7 @@ suite('test_schema_change_with_compaction5', 'docker') { // cu compaction for (int i = 0; i < array.size(); i++) { - tabletId = array[i].TabletId + def tabletId = array[i].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -153,7 +153,7 @@ suite('test_schema_change_with_compaction5', 'docker') { running = true do { Thread.sleep(100) - tabletId = array[i].TabletId + def tabletId = array[i].TabletId (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -172,6 +172,7 @@ suite('test_schema_change_with_compaction5', 'docker') { GetDebugPoint().disableDebugPointForAllBEs(injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy index 4321a23f7b7fb3..31a5760c62db46 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy @@ -148,7 +148,7 @@ suite('test_schema_change_with_compaction6', 'docker') { // cu compaction for (int i = 0; i < array.size(); i++) { - tabletId = array[i].TabletId + def tabletId = array[i].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -158,7 +158,7 @@ suite('test_schema_change_with_compaction6', 'docker') { running = true do { Thread.sleep(100) - tabletId = array[i].TabletId + def tabletId = array[i].TabletId (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -175,6 +175,7 @@ suite('test_schema_change_with_compaction6', 'docker') { GetDebugPoint().disableDebugPointForAllBEs(injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy b/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy index a43de53c4720ce..3983a97f8febd0 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction7/test_schema_change_with_compaction7.groovy @@ -142,7 +142,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { // cu compaction for (int i = 0; i < array.size(); i++) { - tabletId = array[i].TabletId + def tabletId = array[i].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -152,7 +152,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { running = true do { Thread.sleep(100) - tabletId = array[i].TabletId + def tabletId = array[i].TabletId (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -168,6 +168,7 @@ suite('test_schema_change_with_compaction7', 'nonConcurrent') { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy b/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy index 3002cb9c2273b4..0702f9cfcf8d82 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction8/test_schema_change_with_compaction8.groovy @@ -143,6 +143,7 @@ suite('test_schema_change_with_compaction8', 'nonConcurrent') { DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED") { diff --git a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy index ad33e537d4920b..27f4e857007b02 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction9/test_schema_change_with_compaction9.groovy @@ -144,7 +144,7 @@ suite('test_schema_change_with_compaction9', 'docker') { // cu compaction for (int i = 0; i < array.size(); i++) { - tabletId = array[i].TabletId + def tabletId = array[i].TabletId logger.info("run compaction:" + tabletId) (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -154,7 +154,7 @@ suite('test_schema_change_with_compaction9', 'docker') { running = true do { Thread.sleep(100) - tabletId = array[i].TabletId + def tabletId = array[i].TabletId (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) @@ -174,6 +174,7 @@ suite('test_schema_change_with_compaction9', 'docker') { GetDebugPoint().disableDebugPointForAllBEs(injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { From 240b26551f1a51586d43bf83b568eade4aebb841 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 6 Aug 2025 21:56:56 +0800 Subject: [PATCH 351/572] branch-3.0: [fix](case) check task info message less strict #54362 (#54410) Cherry-picked from #54362 Co-authored-by: Yongqiang YANG --- .../load_p2/broker_load/test_parquet_large_metadata_load.groovy | 2 +- .../load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy index fe305a6574223f..fe25b72ad1ebfe 100644 --- a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy +++ b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy @@ -62,7 +62,7 @@ suite("test_parquet_large_metadata_load_p2", "p2") { } def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=45000"] - def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"] + def task_info = ["${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"] def error_msg = [""] // test unified load if (enabled != null && enabled.equalsIgnoreCase("true")) { diff --git a/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy b/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy index a35d24e5f03d2e..cd5b63ea1b99b4 100644 --- a/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy +++ b/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy @@ -243,7 +243,7 @@ suite("test_tvf_based_broker_load", "p2") { } def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000"] - def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"] + def task_info = ["${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"] def error_msg = [""] // test load From 98bafae4c4a3df3c1c8eab7bf8ce085ba7791572 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 8 Aug 2025 09:42:46 +0800 Subject: [PATCH 352/572] branch-3.0: [fix](checker) Fix inverted check and test for checker #54403 (#54469) Cherry-picked from #54403 Co-authored-by: Uniqueyou --- cloud/src/recycler/checker.cpp | 43 +++- cloud/test/recycler_test.cpp | 422 ++++++++++++++++++++------------- 2 files changed, 294 insertions(+), 171 deletions(-) diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index a400e797e03f77..96b17872386df9 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -576,7 +576,7 @@ int InstanceChecker::do_check() { TabletIndexPB tablet_index; if (get_tablet_idx(txn_kv_.get(), instance_id_, rs_meta.tablet_id(), tablet_index) == -1) { - LOG(WARNING) << "failedt to get tablet index, tablet_id= " << rs_meta.tablet_id(); + LOG(WARNING) << "failed to get tablet index, tablet_id= " << rs_meta.tablet_id(); return; } @@ -605,8 +605,8 @@ int InstanceChecker::do_check() { InvertedIndexStorageFormatPB::V1) { for (const auto& index_id : index_ids) { LOG(INFO) << "check inverted index, tablet_id=" << rs_meta.tablet_id() - << " rowset_id=" << rs_meta.rowset_id_v2() - << " segment_index=" << i << " index_id=" << index_id.first + << " rowset_id=" << rs_meta.rowset_id_v2() << " segment_id=" << i + << " index_id=" << index_id.first << " index_suffix_name=" << index_id.second; index_path_v.emplace_back( inverted_index_path_v1(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), @@ -619,14 +619,17 @@ int InstanceChecker::do_check() { if (!index_path_v.empty()) { if (std::ranges::all_of(index_path_v, [&](const auto& idx_file_path) { - return tablet_files_cache.files.contains(idx_file_path); + if (!tablet_files_cache.files.contains(idx_file_path)) { + LOG(INFO) << "loss index file: " << idx_file_path; + return false; + } + return true; })) { continue; } } index_file_loss = true; data_loss = true; - LOG(WARNING) << "object not exist, key=" << hex(tablet_idx_key); } } }; @@ -735,6 +738,10 @@ int InstanceChecker::do_inverted_check() { butil::SplitString(obj_key, '/', &str); // data/{tablet_id}/{rowset_id}_{seg_num}.dat if (str.size() < 3) { + // clang-format off + LOG(WARNING) << "split obj_key error, str.size() should be less than 3," + << " value = " << str.size(); + // clang-format on return -1; } @@ -744,6 +751,11 @@ int InstanceChecker::do_inverted_check() { return -1; } + if (!str[2].ends_with(".dat")) { + // skip check not segment file + return 0; + } + std::string rowset_id; if (auto pos = str.back().find('_'); pos != std::string::npos) { rowset_id = str.back().substr(0, pos); @@ -813,6 +825,10 @@ int InstanceChecker::do_inverted_check() { // format v1: data/{tablet_id}/{rowset_id}_{seg_num}_{idx_id}{idx_suffix}.idx // format v2: data/{tablet_id}/{rowset_id}_{seg_num}.idx if (str.size() < 3) { + // clang-format off + LOG(WARNING) << "split obj_key error, str.size() should be less than 3," + << " value = " << str.size(); + // clang-format on return -1; } @@ -1281,8 +1297,11 @@ int InstanceChecker::check_inverted_index_file_storage_format_v1( for (const auto& i : rs_meta.tablet_schema().index()) { if (i.has_index_type() && i.index_type() == IndexType::INVERTED) { + LOG(INFO) << fmt::format( + "record index info, index_id: {}, index_suffix_name: {}", i.index_id(), + i.index_suffix_name()); rowset_index_cache_v1.index_ids.insert( - fmt::format("{}{}", i.index_name(), i.index_suffix_name())); + fmt::format("{}{}", i.index_id(), i.index_suffix_name())); } } @@ -1296,13 +1315,21 @@ int InstanceChecker::check_inverted_index_file_storage_format_v1( if (!rowset_index_cache_v1.segment_ids.contains(segment_id)) { // Garbage data leak - LOG(WARNING) << "rowset should be recycled, key=" << file_path; + // clang-format off + LOG(WARNING) << "rowset_index_cache_v1.segment_ids don't contains segment_id, rowset should be recycled," + << " key = " << file_path + << " segment_id = " << segment_id; + // clang-format on return 1; } if (!rowset_index_cache_v1.index_ids.contains(index_id_with_suffix_name)) { // Garbage data leak - LOG(WARNING) << "rowset with inde meta should be recycled, key=" << file_path; + // clang-format off + LOG(WARNING) << "rowset_index_cache_v1.index_ids don't contains index_id_with_suffix_name," + << " rowset with inde meta should be recycled, key=" << file_path + << " index_id_with_suffix_name=" << index_id_with_suffix_name; + // clang-format on return 1; } diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index 57a075eb5b6d27..0411e801905920 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -75,78 +75,50 @@ std::vector index_v2_file_path = { "data/1753202639971/02000000000026fo56l8q4p0n2l6n4k343m7o5l9p2o8n4p0_0.idx", "data/1753202639973/02000000000027gp67m9r5q8q4p0n2l1o4n8p6m0q3p9o5q1_0.idx", "data/1753202639975/02000000000028hq78n0s6rm9r5q8q42p5o9q7n1r4q0p6r2_0.idx", - "data/1753202639977/02000000000029ir89o1t7s78n0s6rm3q6p0r8o2s5r1q7s3_4.idx", + "data/1753202639977/02000000000029ir89o1t7s78n0s6rm3q6p0r8o2s5r1q7s3_0.idx", "data/1753202639979/0200000000002ajs90p2u8t4m3q6p0r8r7q1s9p3t6s2r8t4_0.idx", "data/1753202639981/0200000000002bkt01q3v9u2u8t4m3q5s8r2t0q4u7t3s9u5_0.idx", "data/1753202639983/0200000000002clu12r4w1q3v9u2u0v6t9s3u1r5v8u4t0v6_0.idx", "data/1753202639985/0200000000002dmv23s5x1w7u0t4t9s3u1r5v2s6w9v5u1w7_0.idx"}; -std::vector segment_v2_file_path = { - "data/1753202639945/0200000000001a5c92f4e7d9j8f2b4c8a3e6f8b1c9d2e5f8_0.dat", - "data/1753202639947/0200000000001b8d45a74r6c7sf3e9c2b6d4a8e1f7c3d9e2_0.dat", - "data/1753202639951/0200000000001c9e56b8g4f0x8s7g2f0d3c7e5b9f2e8d4f0_0.dat", - "data/1753202639953/0200000000001d0f67c9h5g8a3e6f8b1e4d8f6c0g3f9e5g1_0.dat", - "data/1753202639955/0200000000001e1g78d067c9h5g8i6h2f5e9g7d1h4g0f6h2_0.dat", - "data/1753202639957/0200000000001f2h89e1jg7d1h4g07i3g6f0h8e2i5h1g7i3_0.dat", - "data/1753202639959/020000000000208i90f2k0h8e2i5h8j4h7g1i9f3j6i2h8j4_0.dat", - "data/1753202639961/02000000000021aj01g3l9k5i8h2j8e2i5h8j0g4k7j3i9k5_0.dat", - "data/1753202639963/02000000000022bk12h4m0lk0h8e2i56j9i3k1h5l8k4j0l6_0.dat", - "data/1753202639965/02000000000023cl23i5n1m7g3l9k5i8k0j4l2i6m9l5k1m7_0.dat", - "data/1753202639967/02000000000024dm34j1m7g3l9k6o2n8l1k5m3j7n0m6l2n8_0.dat", - "data/1753202639969/02000000000025en45k7p3o9m2l6n4k34j1m7g38o1n7m3o9_0.dat", - "data/1753202639971/02000000000026fo56l8q4p0n2l6n4k343m7o5l9p2o8n4p0_0.dat", - "data/1753202639973/02000000000027gp67m9r5q8q4p0n2l1o4n8p6m0q3p9o5q1_0.dat", - "data/1753202639975/02000000000028hq78n0s6rm9r5q8q42p5o9q7n1r4q0p6r2_0.dat", - "data/1753202639977/02000000000029ir89o1t7s78n0s6rm3q6p0r8o2s5r1q7s3_4.dat", - "data/1753202639979/0200000000002ajs90p2u8t4m3q6p0r8r7q1s9p3t6s2r8t4_0.dat", - "data/1753202639981/0200000000002bkt01q3v9u2u8t4m3q5s8r2t0q4u7t3s9u5_0.dat", - "data/1753202639983/0200000000002clu12r4w1q3v9u2u0v6t9s3u1r5v8u4t0v6_0.dat", - "data/1753202639985/0200000000002dmv23s5x1w7u0t4t9s3u1r5v2s6w9v5u1w7_0.dat"}; - // clang-format off std::vector index_v1_file_path = { "data/1753202846974/0200000000007864994f6aa97288842758c2e89b03e65682_0_1753202846943.idx", "data/1753202845724/020000000000786635407b55b72242ac167cf83cd4c598a2_0_1753202841593.idx", - "data/1753202846984/020000000000788bdd40fcf18bcaa1bbd4058ef92606e79a_0_1753202846943.idx", - "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0_1753202846943.idx", - "data/1753202846986/02000000000078ec35407b55b72242ac167cf83cd4c598a2_0_1753202846943.idx", + "data/1753202846984/020000000000788bdd40fcf18bcaa1bbd4058ef92606e79a_0_1753202846923.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0_1753202846963.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0_1753202846903.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_1_1753202846903.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_1_1753202846963.idx", "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202844931.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202846410.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753222846410.idx", "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202847011.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202844931.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202846410.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202847011.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202858543.idx", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202844931.idx", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202846410.idx", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202847011.idx", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202858543.idx", - "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0_1753202844931.idx", - "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0_1753202846410.idx", - "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0_1753202847011.idx"}; + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_1_1753202844931.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_1_1753222846410.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_1_1753202847011.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_2_1753202844931.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_2_1753222846410.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_2_1753202847011.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202843931.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753252846410.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202847021.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_1_1753202843931.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_1_1753252846410.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_1_1753202847021.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_2_1753202843931.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_2_1753252846410.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_2_1753202847021.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202824931.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1756202846410.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202847071.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_1_1753202824931.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_1_1756202846410.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_1_1753202847071.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_2_1753202824931.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_2_1756202846410.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_2_1753202847071.idx"}; // clang-format on -std::vector segment_v1_file_path = { - "data/1753202846974/0200000000007864994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202845724/020000000000786635407b55b72242ac167cf83cd4c598a2_0.dat", - "data/1753202846984/020000000000788bdd40fcf18bcaa1bbd4058ef92606e79a_0.dat", - "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0.dat", - "data/1753202846986/02000000000078ec35407b55b72242ac167cf83cd4c598a2_0.dat", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0.dat", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0.dat", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0.dat", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858558/0200000000007afc994f6aa97288842758c2e89b03e65682_0.dat", - "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0.dat", - "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0.dat", - "data/1753202858552/0200000000007b7add40fcf18bcaa1bbd4058ef92606e79a_0.dat"}; - doris::cloud::RecyclerThreadPoolGroup thread_group; int main(int argc, char** argv) { @@ -452,9 +424,10 @@ static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, std::string segment_str = filename.substr(underscore_pos + 1, dot_pos - underscore_pos - 1); std::string extension = filename.substr(dot_pos + 1); - int segment_id = stoll(segment_str); - int64_t tablet_index_id = 123; // Default index id - int64_t schema_version = 456; // Default schema version + int64_t segment_id = stoll(segment_str); + int64_t tablet_index_id = tablet_id + 10; + // take the last 4 digits of tablet_id as the unique identifier + int64_t schema_version = std::atoll(path_parts[1].substr(path_parts[1].size() - 4).c_str()); // Create rowset meta data MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; @@ -494,26 +467,39 @@ static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, // Create tablet schema if dealing with index files if (extension == "idx") { - doris::TabletSchemaCloudPB tablet_schema; - tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); - tablet_schema.set_schema_version(schema_version); - - auto index = tablet_schema.add_index(); - index->set_index_id(0); - index->set_index_type(IndexType::INVERTED); - std::string tablet_schema_key = meta_schema_key({instance_id, tablet_index_id, schema_version}); - tablet_schema.SerializeToString(&val); + std::string tablet_schema_val; if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { return -1; } + doris::TabletSchemaCloudPB tablet_schema; + + if (txn->get(tablet_schema_key, &tablet_schema_val) == TxnErrorCode::TXN_KEY_NOT_FOUND) { + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + tablet_schema.set_schema_version(schema_version); + + auto index = tablet_schema.add_index(); + index->set_index_id(tablet_schema.index().size()); + index->set_index_type(IndexType::INVERTED); + + } else { + tablet_schema.ParseFromString(tablet_schema_val); + + auto index = tablet_schema.add_index(); + index->set_index_id(tablet_schema.index().size()); + index->set_index_type(IndexType::INVERTED); + } + tablet_schema.SerializeToString(&val); + txn->put(tablet_schema_key, val); if (txn->commit() != TxnErrorCode::TXN_OK) { return -1; } } + std::string segment_path = file_path.substr(0, file_path.size() - 4) + ".dat"; + accessor->put_file(segment_path, ""); accessor->put_file(file_path, ""); return 0; @@ -523,7 +509,7 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, StorageVaultAccessor* accessor, const std::string& resource_id, const std::string& file_path, - int64_t version = 1) { + size_t& version) { std::string val; std::unique_ptr txn; @@ -567,8 +553,8 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, int segment_id = stoll(segment_str); int64_t index_id = std::stoll(index_id_str); - int64_t tablet_index_id = 123; // Default tablet index id - int64_t schema_version = 456; // Default schema version + int64_t tablet_index_id = tablet_id + 10; + int64_t schema_version = std::atoll(path_parts[1].substr(path_parts[1].size() - 4).c_str()); // Create rowset meta data MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; @@ -608,29 +594,45 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, // Create tablet schema if dealing with index files if (extension == "idx") { - doris::TabletSchemaCloudPB tablet_schema; - tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); - tablet_schema.set_schema_version(schema_version); - - auto index = tablet_schema.add_index(); - index->set_index_id(index_id); - index->set_index_type(IndexType::INVERTED); - if (!index_suffix.empty()) { - index->set_index_suffix_name(index_suffix); - } - std::string tablet_schema_key = meta_schema_key({instance_id, tablet_index_id, schema_version}); - tablet_schema.SerializeToString(&val); + std::string tablet_schema_val; if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { return -1; } + doris::TabletSchemaCloudPB tablet_schema; + + if (txn->get(tablet_schema_key, &tablet_schema_val) == TxnErrorCode::TXN_KEY_NOT_FOUND) { + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + tablet_schema.set_schema_version(schema_version); + + auto index = tablet_schema.add_index(); + index->set_index_id(index_id); + index->set_index_type(IndexType::INVERTED); + if (!index_suffix.empty()) { + index->set_index_suffix_name(index_suffix); + } + + } else { + tablet_schema.ParseFromString(tablet_schema_val); + + auto* index = tablet_schema.add_index(); + index->set_index_id(index_id); + index->set_index_type(IndexType::INVERTED); + if (!index_suffix.empty()) { + index->set_index_suffix_name(index_suffix); + } + } + tablet_schema.SerializeToString(&val); + txn->put(tablet_schema_key, val); if (txn->commit() != TxnErrorCode::TXN_OK) { return -1; } } + std::string segment_path = fmt::format("data/{}/{}_{}.dat", tablet_id, rowset_id, segment_id); + accessor->put_file(segment_path, ""); accessor->put_file(file_path, ""); return 0; @@ -2896,7 +2898,7 @@ TEST(CheckerTest, DISABLED_abnormal_inverted_check) { ASSERT_NE(checker.do_inverted_check(), 0); } -TEST(CheckerTest, normal_check_index_file) { +TEST(CheckerTest, normal_check_index_file_v1) { auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0); @@ -2925,17 +2927,16 @@ TEST(CheckerTest, normal_check_index_file) { // Add some visible rowsets along with some rowsets that should be recycled // call inverted check after do recycle which would sweep all the rowsets not visible auto accessor = checker.accessor_map_.begin()->second; - for (const auto& file : index_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + size_t version = 0; + for (const auto& file : index_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file, + version); } - for (const auto& file : segment_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); - } - ASSERT_EQ(checker.do_inverted_check(), 0); + ASSERT_EQ(checker.do_check(), 0); } -TEST(CheckerTest, normal_inverted_check_index_file) { +TEST(CheckerTest, normal_inverted_check_index_file_v1) { auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0); @@ -2964,17 +2965,16 @@ TEST(CheckerTest, normal_inverted_check_index_file) { // Add some visible rowsets along with some rowsets that should be recycled // call inverted check after do recycle which would sweep all the rowsets not visible auto accessor = checker.accessor_map_.begin()->second; - for (const auto& file : index_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + size_t version = 0; + for (const auto& file : index_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file, + version); } - for (const auto& file : segment_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); - } ASSERT_EQ(checker.do_inverted_check(), 0); } -TEST(CheckerTest, inverted_check_recycle_idx_file_v1) { +TEST(CheckerTest, normal_check_index_file_v2) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) { sp->clear_all_call_backs(); @@ -3013,55 +3013,72 @@ TEST(CheckerTest, inverted_check_recycle_idx_file_v1) { }); sp->enable_processing(); - for (const auto& file : index_v1_file_path) { - create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + size_t version = 1; + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file, + version++); } - for (const auto& file : segment_v1_file_path) { - create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); - } + std::unique_ptr list_iter; + int ret = accessor->list_directory("data", &list_iter); + ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; - size_t delete_kv_num = 5; - std::string meta_rowset_key_begin, meta_rowset_key_end; - meta_rowset_key({instance_id, 0, 1}, &meta_rowset_key_begin); - meta_rowset_key({instance_id, INT64_MAX, 1}, &meta_rowset_key_end); - std::vector rowset_key_to_delete; - std::unique_ptr txn; - TxnErrorCode err = txn_kv->create_txn(&txn); - DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + ASSERT_EQ(checker.do_check(), 0); +} - std::unique_ptr it; - do { - err = txn->get(meta_rowset_key_begin, meta_rowset_key_end, &it); - while (it->has_next()) { - auto [k, v] = it->next(); - if (rowset_key_to_delete.size() < delete_kv_num) { - rowset_key_to_delete.emplace_back(k); - } - if (!it->has_next()) { - meta_rowset_key_begin = k; - } - } - meta_rowset_key_begin.push_back('\x00'); - } while (it->more()); +TEST(CheckerTest, normal_inverted_check_index_file_v2) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->disable_processing(); + }); - for (const auto& key : rowset_key_to_delete) { - std::unique_ptr txn; - TxnErrorCode err = txn_kv->create_txn(&txn); - DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; - txn->remove(key); - err = txn->commit(); - DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("CheckerTest"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->enable_processing(); + + size_t version = 1; + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file, + version++); } std::unique_ptr list_iter; int ret = accessor->list_directory("data", &list_iter); ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; - ASSERT_EQ(checker.do_inverted_check(), 1); + ASSERT_EQ(checker.do_inverted_check(), 0); } -TEST(CheckerTest, inverted_check_recycle_idx_file_v2) { +TEST(CheckerTest, abnormal_check_index_file_v1) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) { sp->clear_all_call_backs(); @@ -3099,13 +3116,73 @@ TEST(CheckerTest, inverted_check_recycle_idx_file_v2) { } }); sp->enable_processing(); + size_t version = 0; + for (const auto& file : index_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file, + version); + } - for (const auto& file : index_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + std::unique_ptr list_iter; + int ret = accessor->list_directory("data", &list_iter); + ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; + + int64_t tablet_to_delete = -1; + for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { + std::vector str; + butil::SplitString(file->path, '/', &str); + int64_t tablet_id = atol(str[1].c_str()); + + // delete all index files of ever tablet for mock missing + if (file->path.ends_with(".idx") && tablet_to_delete != tablet_id) { + tablet_to_delete = tablet_id; + accessor->delete_file(file->path); + } } + ASSERT_EQ(checker.do_check(), 1); +} + +TEST(CheckerTest, abnormal_inverted_check_index_file_v1) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->disable_processing(); + }); - for (const auto& file : segment_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + obj_info->set_ak(config::test_s3_ak); + obj_info->set_sk(config::test_s3_sk); + obj_info->set_endpoint(config::test_s3_endpoint); + obj_info->set_region(config::test_s3_region); + obj_info->set_bucket(config::test_s3_bucket); + obj_info->set_prefix("CheckerTest"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + // Add some visible rowsets along with some rowsets that should be recycled + // call inverted check after do recycle which would sweep all the rowsets not visible + auto accessor = checker.accessor_map_.begin()->second; + + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + if (vault->name() == "test_success_hdfs_vault") { + map->emplace(vault->id(), accessor); + } + }); + sp->enable_processing(); + size_t version = 0; + for (const auto& file : index_v1_file_path) { + create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file, + version); } size_t delete_kv_num = 5; @@ -3148,7 +3225,7 @@ TEST(CheckerTest, inverted_check_recycle_idx_file_v2) { ASSERT_EQ(checker.do_inverted_check(), 1); } -TEST(CheckerTest, forward_check_recycle_idx_file_v1) { +TEST(CheckerTest, abnormal_inverted_check_index_file_v2) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) { sp->clear_all_call_backs(); @@ -3187,33 +3264,53 @@ TEST(CheckerTest, forward_check_recycle_idx_file_v1) { }); sp->enable_processing(); - for (const auto& file : index_v1_file_path) { - create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + size_t version = 1; + for (const auto& file : index_v2_file_path) { + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file, + version++); } - for (const auto& file : segment_v1_file_path) { - create_committed_rowset_by_real_index_v1_file(txn_kv.get(), accessor.get(), "1", file); + size_t delete_kv_num = 5; + std::string meta_rowset_key_begin, meta_rowset_key_end; + meta_rowset_key({instance_id, 0, 1}, &meta_rowset_key_begin); + meta_rowset_key({instance_id, INT64_MAX, 1}, &meta_rowset_key_end); + std::vector rowset_key_to_delete; + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + + std::unique_ptr it; + do { + err = txn->get(meta_rowset_key_begin, meta_rowset_key_end, &it); + while (it->has_next()) { + auto [k, v] = it->next(); + if (rowset_key_to_delete.size() < delete_kv_num) { + rowset_key_to_delete.emplace_back(k); + } + if (!it->has_next()) { + meta_rowset_key_begin = k; + } + } + meta_rowset_key_begin.push_back('\x00'); + } while (it->more()); + + for (const auto& key : rowset_key_to_delete) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv->create_txn(&txn); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; + txn->remove(key); + err = txn->commit(); + DCHECK_EQ(err, TxnErrorCode::TXN_OK) << err; } + std::unique_ptr list_iter; int ret = accessor->list_directory("data", &list_iter); ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; - int64_t tablet_to_delete = -1; - for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { - std::vector str; - butil::SplitString(file->path, '/', &str); - int64_t tablet_id = atol(str[1].c_str()); - - // delete all index files of ever tablet for mock missing - if (file->path.ends_with(".idx") && tablet_to_delete != tablet_id) { - tablet_to_delete = tablet_id; - accessor->delete_file(file->path); - } - } - ASSERT_EQ(checker.do_check(), 1); + ASSERT_EQ(checker.do_inverted_check(), 1); } -TEST(CheckerTest, forward_check_recycle_idx_file_v2) { +TEST(CheckerTest, abnormal_check_index_file_v2) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) { sp->clear_all_call_backs(); @@ -3252,13 +3349,12 @@ TEST(CheckerTest, forward_check_recycle_idx_file_v2) { }); sp->enable_processing(); + size_t version = 1; for (const auto& file : index_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); + create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file, + version++); } - for (const auto& file : segment_v2_file_path) { - create_committed_rowset_by_real_index_v2_file(txn_kv.get(), accessor.get(), "1", file); - } std::unique_ptr list_iter; int ret = accessor->list_directory("data", &list_iter); ASSERT_EQ(ret, 0) << "Failed to list directory: " << ret; From 5cd2e87120f516707bfb3a7efdbfeba83df7e992 Mon Sep 17 00:00:00 2001 From: koarz Date: Fri, 8 Aug 2025 09:53:02 +0800 Subject: [PATCH 353/572] branch-3.0: [fix](binlog)default partition dont append version_info (#54450) 3.0 dont support syntax default partition with properties, it think ("values_info"="x") is a values in (xx), the ccr would get the wrong result(tosql), the sql cant execute --- .../java/org/apache/doris/binlog/AddPartitionRecord.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java index 4ca3213ec12d96..8749de2f286219 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/AddPartitionRecord.java @@ -95,10 +95,10 @@ public AddPartitionRecord(long commitSeq, PartitionPersistInfo partitionPersistI if (!partitionSql.isEmpty()) { sb.append("VALUES IN "); sb.append(partitionSql); + sb.append(" (\"version_info\" = \""); + sb.append(partition.getVisibleVersion()); + sb.append("\");"); } - sb.append(" (\"version_info\" = \""); - sb.append(partition.getVisibleVersion()); - sb.append("\");"); } else { // unpartitioned. } From 6e250c8791597258eef711ed70b186f2d4e5c392 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 8 Aug 2025 19:09:50 +0800 Subject: [PATCH 354/572] branch-3.0: [fix](test) Fix two regression test #54433 (#54473) Cherry-picked from #54433 --------- Co-authored-by: Lei Zhang Co-authored-by: Dongyang Li --- docker/runtime/doris-compose/database.py | 3 ++- ...st_nestedtypes_json_insert_into_with_s3.groovy | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docker/runtime/doris-compose/database.py b/docker/runtime/doris-compose/database.py index f46635033b848b..0700d1aa7ebfbd 100644 --- a/docker/runtime/doris-compose/database.py +++ b/docker/runtime/doris-compose/database.py @@ -183,7 +183,8 @@ def create_default_storage_vault(self, cloud_store_config): "s3.bucket" = "{cloud_store_config['DORIS_CLOUD_BUCKET']}", "s3.region" = "{cloud_store_config['DORIS_CLOUD_REGION']}", "s3.root.path" = "{str(uuid.uuid4())}", - "provider" = "{cloud_store_config['DORIS_CLOUD_PROVIDER']}" + "provider" = "{cloud_store_config['DORIS_CLOUD_PROVIDER']}", + "use_path_style" = "false" ); """ # create hk storage vault from beijing cost 14s diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy index 2d4aa6e2639fc5..d640633301c11b 100644 --- a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy +++ b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy @@ -28,10 +28,13 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") { String ak = getS3AK() String sk = getS3SK() String s3_endpoint = getS3Endpoint() + String s3_region = getS3Region() String bucket = context.config.otherConfigs.get("s3BucketName"); - - def dataFilePath = "https://"+"${bucket}"+"."+"${s3_endpoint}"+"/regression/datalake" + def dataFilePath = "s3://${bucket}/regression/datalake" + if (!s3_endpoint.startsWith("https://") && !s3_endpoint.startsWith("http://")) { + s3_endpoint = "https://${s3_endpoint}" + } ArrayList json_files = ["${dataFilePath}/as.json", "${dataFilePath}/aa.json", "${dataFilePath}/ms.json","${dataFilePath}/am.json", @@ -202,6 +205,8 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") { for (int i = 0; i < 3; ++i) { qt_sql_arr_json_without_quote_s3 """ select * from s3("uri" = "${json_files[i]}", + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${s3_region}", "s3.access_key"= "${ak}", "s3.secret_key" = "${sk}", "format" = "json", @@ -211,6 +216,8 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") { sql """ insert into ${table_names[i]} select * from s3("uri" = "${json_files[i]}", + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${s3_region}", "s3.access_key"= "${ak}", "s3.secret_key" = "${sk}", "format" = "json", @@ -228,6 +235,8 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") { qt_sql_arr_json_without_quote_s3 """ select * from s3( "uri" = "${json_files[i]}", + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${s3_region}", "s3.access_key"= "${ak}", "s3.secret_key" = "${sk}", "format" = "json", @@ -238,6 +247,8 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") { sql """ insert into ${table_names[i]} select * from s3 ( "uri" = "${json_files[i]}", + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${s3_region}", "s3.access_key"= "${ak}", "s3.secret_key" = "${sk}", "format" = "json", From d8131365af065e9d527fc1644ae4388ba9a1fe00 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 10 Aug 2025 22:31:32 +0800 Subject: [PATCH 355/572] branch-3.0: [fix](case) remove global variable declarations in load #54418 (#54430) Cherry-picked from #54418 Co-authored-by: Xin Liao --- .../suites/fault_injection_p0/test_disable_move_memtable.groovy | 2 +- .../suites/load_p0/stream_load/test_json_load.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy b/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy index 1513d1ec75d60b..2945d2f4365098 100644 --- a/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy +++ b/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy @@ -228,7 +228,7 @@ suite("test_disable_move_memtable", "nonConcurrent") { } def check_load_result = {checklabel, testTablex, res -> - max_try_milli_secs = 10000 + def max_try_milli_secs = 10000 while(max_try_milli_secs) { result = sql "show load where label = '${checklabel}'" log.info("result: ${result}") diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index a3d6eb6932de43..8bbd6d844e914b 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -228,7 +228,7 @@ suite("test_json_load", "p0,nonConcurrent") { } def check_load_result = {checklabel, testTablex -> - max_try_milli_secs = 10000 + def max_try_milli_secs = 10000 while(max_try_milli_secs) { result = sql "show load where label = '${checklabel}'" if(result[0][2] == "FINISHED") { From 64651a9f2e98b0faddc45a78b2165007d5e3ba5a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 10 Aug 2025 22:33:35 +0800 Subject: [PATCH 356/572] branch-3.0: [fix](test) use drop view for views in test_show_keys #54173 (#54183) Cherry-picked from #54173 Co-authored-by: morrySnow --- regression-test/suites/nereids_syntax_p0/test_show_keys.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/nereids_syntax_p0/test_show_keys.groovy b/regression-test/suites/nereids_syntax_p0/test_show_keys.groovy index e89a627e2ee1ed..829e4d2fee870e 100644 --- a/regression-test/suites/nereids_syntax_p0/test_show_keys.groovy +++ b/regression-test/suites/nereids_syntax_p0/test_show_keys.groovy @@ -21,7 +21,7 @@ suite("test_show_keys") { """ sql """ - DROP TABLE IF EXISTS test_show_keys_v + DROP VIEW IF EXISTS test_show_keys_v """ sql """ From d045e971ca7d750dbc359160a715e1845638923b Mon Sep 17 00:00:00 2001 From: meiyi Date: Mon, 11 Aug 2025 21:48:24 +0800 Subject: [PATCH 357/572] branch-3.0: [fix](case) fix some case (#54481) (#54560) pick https://github.com/apache/doris/pull/54481 --- .../insert_p0/insert_group_commit_into.out | 3 - ...n_insert_restart_fe_with_schema_change.out | 4 +- .../group_commit/replay_wal_restart_fe.groovy | 3 +- .../insert_p0/insert_group_commit_into.groovy | 58 +------------------ .../test_group_commit_timeout.groovy | 14 ++--- ...nsert_restart_fe_with_schema_change.groovy | 6 +- 6 files changed, 14 insertions(+), 74 deletions(-) diff --git a/regression-test/data/insert_p0/insert_group_commit_into.out b/regression-test/data/insert_p0/insert_group_commit_into.out index 44e6a32fd56158..640eca5d361c32 100644 --- a/regression-test/data/insert_p0/insert_group_commit_into.out +++ b/regression-test/data/insert_p0/insert_group_commit_into.out @@ -94,9 +94,6 @@ q 50 -- !sql -- 0 service_46da0dab-e27d-4820-aea2-9bfc15741615 1697032066304 0 3229b7cd-f3a2-4359-aa24-946388c9cc54 0 CgEwEiQzMjI5YjdjZC1mM2EyLTQzNTktYWEyNC05NDYzODhjOWNjNTQaggQY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAEY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAIY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAMY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAQY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAUY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAYY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAcY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAgY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAkY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAoY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAsY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECAwY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECA0Y/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECA4Y/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECA8Y/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECBAY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECBEY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECBIY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5GoQECBMY/6n597ExIP+p+fexMWIWCgh0YWdLZXlfMBIKdGFnVmFsdWVfMGIWCgh0YWdLZXlfMRIKdGFnVmFsdWVfMWIWCgh0YWdLZXlfMhIKdGFnVmFsdWVfMmIWCgh0YWdLZXlfMxIKdGFnVmFsdWVfM2IWCgh0YWdLZXlfNBIKdGFnVmFsdWVfNGIWCgh0YWdLZXlfNRIKdGFnVmFsdWVfNWIWCgh0YWdLZXlfNhIKdGFnVmFsdWVfNmIWCgh0YWdLZXlfNxIKdGFnVmFsdWVfN2IWCgh0YWdLZXlfOBIKdGFnVmFsdWVfOGIWCgh0YWdLZXlfORIKdGFnVmFsdWVfOWIYCgl0YWdLZXlfMTASC3RhZ1ZhbHVlXzEwYhgKCXRhZ0tleV8xMRILdGFnVmFsdWVfMTFiGAoJdGFnS2V5XzEyEgt0YWdWYWx1ZV8xMmIYCgl0YWdLZXlfMTMSC3RhZ1ZhbHVlXzEzYhgKCXRhZ0tleV8xNBILdGFnVmFsdWVfMTRiGAoJdGFnS2V5XzE1Egt0YWdWYWx1ZV8xNWIYCgl0YWdLZXlfMTYSC3RhZ1ZhbHVlXzE2YhgKCXRhZ0tleV8xNxILdGFnVmFsdWVfMTdiGAoJdGFnS2V5XzE4Egt0YWdWYWx1ZV8xOGIYCgl0YWdLZXlfMTkSC3RhZ1ZhbHVlXzE5IixzZXJ2aWNlXzQ2ZGEwZGFiLWUyN2QtNDgyMC1hZWEyLTliZmMxNTc0MTYxNSo0c2VydmljZV9pbnN0YW5jZWFjODlhNGI3LTgxZjctNDNlOC04NWVkLWQyYjU3OGQ5ODA1MA== 1697032066304 36b2d9ff-4c25-49f3-a726-eea812564411 355f96cd-b1b1-4688-a5f6-a8e3f3a55c9a false 3 service_instanceac89a4b7-81f7-43e8-85ed-d2b578d98050 statement: b9903670-3821-4f4c-a587-bbcf02c04b77 ["[tagKey_5=tagValue_5, tagKey_3=tagValue_3, tagKey_1=tagValue_1, tagKey_16=tagValue_16, tagKey_8=tagValue_8, tagKey_15=tagValue_15, tagKey_6=tagValue_6, tagKey_11=tagValue_11, tagKey_10=tagValue_10, tagKey_4=tagValue_4, tagKey_13=tagValue_13, tagKey_14=tagValue_14, tagKey_2=tagValue_2, tagKey_17=tagValue_17, tagKey_19=tagValue_19, tagKey_0=tagValue_0, tagKey_18=tagValue_18, tagKey_9=tagValue_9, tagKey_7=tagValue_7, tagKey_12=tagValue_12]"] --- !order -- -2023-06-10 cib2205045_1_1s 0.0000 168939.0 0.0000 0.0 0.0000 0.0 0.0000 day - -- !order2 -- 1 1 1 2 2 2 diff --git a/regression-test/data/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.out b/regression-test/data/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.out index 1d906c6bd613c9..1234a227bfeb9f 100644 --- a/regression-test/data/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.out +++ b/regression-test/data/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.out @@ -19,8 +19,8 @@ 1 11 -- !select_3m -- -1 12 -1 12 +2 12 +2 12 -- !select_4 -- 1 11 1 diff --git a/regression-test/suites/insert_p0/group_commit/replay_wal_restart_fe.groovy b/regression-test/suites/insert_p0/group_commit/replay_wal_restart_fe.groovy index 33664f52f2b0e9..ee31b9eda1a234 100644 --- a/regression-test/suites/insert_p0/group_commit/replay_wal_restart_fe.groovy +++ b/regression-test/suites/insert_p0/group_commit/replay_wal_restart_fe.groovy @@ -61,9 +61,8 @@ suite("replay_wal_restart_fe", 'docker') { cluster.stopBackends() cluster.restartFrontends() sleep(30000) - context.reconnectFe() - check_schema_change('RUNNING') cluster.startBackends() + context.reconnectFe() // check schema change status and row count check_schema_change('FINISHED') diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy b/regression-test/suites/insert_p0/insert_group_commit_into.groovy index bade0c8f279da0..90318a5226b2fc 100644 --- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy +++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy @@ -370,29 +370,9 @@ suite("insert_group_commit_into") { "replication_allocation" = "tag.location.default: 1", "group_commit_interval_ms" = "200" );""" - sql """drop table if exists ${table_tmp};""" - sql """CREATE TABLE ${table_tmp} ( - `dnt` varchar(200) NULL, - `ordernum` varchar(200) NULL, - `type` varchar(20) NULL, - `powers` double SUM NULL, - `p0` double REPLACE NULL, - `heatj` double SUM NULL, - `j0` double REPLACE NULL, - `heatg` double SUM NULL, - `g0` double REPLACE NULL, - `solar` double SUM NULL - ) ENGINE=OLAP - AGGREGATE KEY(`dnt`, `ordernum`, `type`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`ordernum`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "group_commit_interval_ms" = "200" - ); """ sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON ${table};""" createMV("""create materialized view ods_zn_dnt_max1 as - select ordernum,max(dnt) as dnt from ${table} + select ordernum as a1,max(dnt) as dntm from ${table} group by ordernum ORDER BY ordernum;""") connect( context.config.jdbcUser, context.config.jdbcPassword, context.config.jdbcUrl) { @@ -419,42 +399,6 @@ suite("insert_group_commit_into") { getRowCount(4) - qt_order """select - '2023-06-10', - tmp.ordernum, - cast(nvl(if(tmp.p0-tmp1.p0>0,tmp.p0-tmp1.p0,tmp.p0-tmp.p1),0) as decimal(10,4)), - nvl(tmp.p0,0), - cast(nvl(if(tmp.j0-tmp1.j0>0,tmp.j0-tmp1.j0,tmp.j0-tmp.j1)*277.78,0) as decimal(10,4)), - nvl(tmp.j0,0), - cast(nvl(if(tmp.g0-tmp1.g0>0,tmp.g0-tmp1.g0,tmp.g0-tmp.g1)*277.78,0) as decimal(10,4)), - nvl(tmp.g0,0), - cast(nvl(tmp.solar,0) as decimal(20,4)), - 'day' - from - ( - select - ordernum, - max(ljrl1) g0,min(ljrl1) g1, - max(ljrl2) j0,min(ljrl2) j1, - max(db1) p0,min(db1) p1, - max(fzl)*1600*0.278 solar - from( - select ordernum,dnt, - cast(if(json_extract(data,'\$.LJRL1')=0 or json_extract(data,'\$.LJRL1') like '%E%',null,json_extract(data,'\$.LJRL1')) as double) ljrl1, - cast(if(json_extract(data,'\$.LJRL2')=0 or json_extract(data,'\$.LJRL2') like '%E%',null,json_extract(data,'\$.LJRL2')) as double) ljrl2, - first_value(cast(if(json_extract(data,'\$.FZL')=0 or json_extract(data,'\$.FZL') like '%E%',null, - json_extract(data,'\$.FZL')) as double)) over (partition by ordernum order by dnt desc) fzl, - cast(if(json_extract(data,'\$.DB1')=0 or json_extract(data,'\$.DB1') like '%E%',null,json_extract(data,'\$.DB1')) as double) db1 - from ${table} - )a1 - group by ordernum - )tmp left join ( - select - ordernum,MAX(p0) p0,MAX(j0) j0,MAX(g0) g0 - from ${table_tmp} - group by ordernum - )tmp1 - on tmp.ordernum=tmp1.ordernum;""" qt_order2 """ SELECT row_number() over(partition by add_date order by pc_num desc) diff --git a/regression-test/suites/insert_p0/test_group_commit_timeout.groovy b/regression-test/suites/insert_p0/test_group_commit_timeout.groovy index add4d3c1eece44..e33f6b7c9226fa 100644 --- a/regression-test/suites/insert_p0/test_group_commit_timeout.groovy +++ b/regression-test/suites/insert_p0/test_group_commit_timeout.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_group_commit_timeout", "nonConcurrent") { +suite("test_group_commit_timeout") { def tableName = "test_group_commit_timeout" sql """ CREATE TABLE if not exists ${tableName} ( @@ -31,14 +31,14 @@ suite("test_group_commit_timeout", "nonConcurrent") { ); """ - def query_timeout = sql """show variables where variable_name = 'query_timeout';""" + /*def query_timeout = sql """show variables where variable_name = 'query_timeout';""" def insert_timeout = sql """show variables where variable_name = 'insert_timeout';""" - logger.info("query_timeout: ${query_timeout}, insert_timeout: ${insert_timeout}") + logger.info("query_timeout: ${query_timeout}, insert_timeout: ${insert_timeout}")*/ long start = System.currentTimeMillis() try { - sql "SET global query_timeout = 5" - sql "SET global insert_timeout = 5" + sql "SET query_timeout = 5" + sql "SET insert_timeout = 5" sql "set group_commit = sync_mode" sql "insert into ${tableName} values(1, 'a', 10)" @@ -49,7 +49,7 @@ suite("test_group_commit_timeout", "nonConcurrent") { assertTrue(e.getMessage().contains("FragmentMgr cancel worker going to cancel timeout instance") || e.getMessage().contains("Execute timeout") || e.getMessage().contains("timeout")) assertTrue(end - start <= 60000) } finally { - sql "SET global query_timeout = ${query_timeout[0][1]}" - sql "SET global insert_timeout = ${insert_timeout[0][1]}" + /*sql "SET query_timeout = ${query_timeout[0][1]}" + sql "SET insert_timeout = ${insert_timeout[0][1]}"*/ } } diff --git a/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.groovy b/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.groovy index d2537bfe8c6a55..89f3905280aac2 100644 --- a/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.groovy +++ b/regression-test/suites/insert_p0/transaction/txn_insert_restart_fe_with_schema_change.groovy @@ -80,11 +80,11 @@ suite("txn_insert_restart_fe_with_schema_change", 'docker') { } result = sql_return_maparray 'SHOW PROC "/transactions"' - runningTxn = result.find { it.DbName.indexOf(dbName) >= 0 }.RunningTransactionNum as int + def runningTxn = result.find { it.DbName.indexOf(dbName) >= 0 }.RunningTransactionNum as int assertEquals(4, runningTxn) sql "ALTER TABLE tbl_2 ADD COLUMN k3 INT DEFAULT '-1'" - sql 'CREATE MATERIALIZED VIEW tbl_3_mv AS SELECT k1, k1 + k2 FROM tbl_3' + sql 'CREATE MATERIALIZED VIEW tbl_3_mv AS SELECT k1 + 1, k1 + k2 FROM tbl_3' sql 'ALTER TABLE tbl_4 ADD ROLLUP tbl_3_r1(k1, v)' sql 'ALTER TABLE tbl_5 ORDER BY (k2, k1)' @@ -112,7 +112,7 @@ suite("txn_insert_restart_fe_with_schema_change", 'docker') { // should publish visible order_qt_select_2 'SELECT k1, k2 FROM tbl_2' order_qt_select_3 'SELECT * FROM tbl_3' - order_qt_select_3m 'SELECT k1, k1 + k2 FROM tbl_3' + order_qt_select_3m 'SELECT k1 + 1, k1 + k2 FROM tbl_3' order_qt_select_4 'SELECT * FROM tbl_4' order_qt_select_4r 'SELECT k1, v FROM tbl_4' order_qt_select_5 'SELECT k1, k2 FROM tbl_5' From ec174c1e54397f7777f4ee3396c0750363826294 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:05:56 +0800 Subject: [PATCH 358/572] branch-3.0: [chore](cloud) Optimize fuzzy test for txn lazy commit #54382 (#54574) Cherry-picked from #54382 Co-authored-by: Gavin Chou --- cloud/src/common/config.h | 5 ++++- cloud/src/meta-service/meta_service_txn.cpp | 14 +++++--------- cloud/test/txn_lazy_commit_test.cpp | 17 +++-------------- .../pipeline/cloud_p0/conf/ms_custom.conf | 2 +- .../pipeline/cloud_p1/conf/ms_custom.conf | 2 +- 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index b767d696034322..c52f4100f3c2be 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -281,7 +281,10 @@ CONF_Int32(txn_lazy_max_rowsets_per_batch, "1000"); // max TabletIndexPB num for batch get CONF_Int32(max_tablet_index_num_per_batch, "1000"); -CONF_Bool(enable_cloud_txn_lazy_commit_fuzzy_test, "false"); +// the possibility to use a lazy commit for a doris txn, ranges from 0 to 100, +// usually for testing +// 0 for never, 100 for always +CONF_mInt32(cloud_txn_lazy_commit_fuzzy_possibility, "0"); CONF_Bool(enable_check_instance_id, "true"); diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 28e262226ed59f..b2326e395c8a2a 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -2585,15 +2585,10 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* response->mutable_txn_info()->CopyFrom(txn_info); } // end commit_txn_with_sub_txn -static bool fuzzy_random() { - return std::chrono::steady_clock::now().time_since_epoch().count() & 0x01; -} - static bool force_txn_lazy_commit() { - if (config::enable_cloud_txn_lazy_commit_fuzzy_test) [[unlikely]] { - return fuzzy_random(); - } - return false; + static std::mt19937 rng(20250806 /* seed */); + static std::uniform_int_distribution dist(1, 100); + return dist(rng) <= config::cloud_txn_lazy_commit_fuzzy_possibility; } void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, @@ -2638,7 +2633,8 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, while ((!enable_txn_lazy_commit_feature || (tmp_rowsets_meta.size() <= config::txn_lazy_commit_rowsets_thresold))) { if (force_txn_lazy_commit()) { - LOG(INFO) << "fuzzy test force_txn_lazy_commit, txn_id=" << txn_id; + LOG(INFO) << "fuzzy test force_txn_lazy_commit, txn_id=" << txn_id + << " force_posibility=" << config::cloud_txn_lazy_commit_fuzzy_possibility; break; } diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index d2bf82435d8adc..e07485d8c0db8f 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1909,21 +1909,10 @@ TEST(TxnLazyCommitTest, RowsetMetaSizeExceedTest) { ASSERT_EQ(res.status().code(), MetaServiceCode::PROTOBUF_PARSE_ERR); } } -TEST(TxnLazyCommitTest, FuzzyRandom) { - int counter = 0; - for (size_t i = 0; i < 100000; i++) { - if (fuzzy_random()) { - counter++; - } - } - LOG(INFO) << "fuzzy_random counter: " << counter; - ASSERT_GT(counter, 30000); - ASSERT_LT(counter, 70000); -} TEST(TxnLazyCommitTest, ForceTxnLazyCommit) { int counter = 0; - config::enable_cloud_txn_lazy_commit_fuzzy_test = false; + config::cloud_txn_lazy_commit_fuzzy_possibility = 0; for (size_t i = 0; i < 100000; i++) { if (force_txn_lazy_commit()) { counter++; @@ -1932,7 +1921,7 @@ TEST(TxnLazyCommitTest, ForceTxnLazyCommit) { LOG(INFO) << "force_txn_lazy_commit counter: " << counter; ASSERT_EQ(counter, 0); - config::enable_cloud_txn_lazy_commit_fuzzy_test = true; + config::cloud_txn_lazy_commit_fuzzy_possibility = 50; counter = 0; for (size_t i = 0; i < 100000; i++) { if (force_txn_lazy_commit()) { @@ -1942,7 +1931,7 @@ TEST(TxnLazyCommitTest, ForceTxnLazyCommit) { LOG(INFO) << "force_txn_lazy_commit counter: " << counter; ASSERT_GT(counter, 30000); ASSERT_LT(counter, 70000); - config::enable_cloud_txn_lazy_commit_fuzzy_test = false; + config::cloud_txn_lazy_commit_fuzzy_possibility = 0; } TEST(TxnLazyCommitTest, RecycleTmpRowsetsCase1) { diff --git a/regression-test/pipeline/cloud_p0/conf/ms_custom.conf b/regression-test/pipeline/cloud_p0/conf/ms_custom.conf index 39b9d6c348d100..53873da71edfc7 100644 --- a/regression-test/pipeline/cloud_p0/conf/ms_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/ms_custom.conf @@ -1,3 +1,3 @@ # below lines will be appended to the default doris_cloud.conf when deploying meta service meta_schema_value_version = 1 -enable_cloud_txn_lazy_commit_fuzzy_test = true +cloud_txn_lazy_commit_fuzzy_possibility = 50 diff --git a/regression-test/pipeline/cloud_p1/conf/ms_custom.conf b/regression-test/pipeline/cloud_p1/conf/ms_custom.conf index 8a2f8734fb45cb..65080d400d2078 100644 --- a/regression-test/pipeline/cloud_p1/conf/ms_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/ms_custom.conf @@ -1,2 +1,2 @@ # below lines will be appended to the default doris_cloud.conf when deploying meta service -enable_cloud_txn_lazy_commit_fuzzy_test = true +cloud_txn_lazy_commit_fuzzy_possibility = 50 From 28120ed99d767591a9727dddeb792adde02279e1 Mon Sep 17 00:00:00 2001 From: Xin Liao Date: Tue, 12 Aug 2025 10:07:57 +0800 Subject: [PATCH 359/572] branch-3.0 [opt](load) S3 Load and TVF support access without AKSK (#53592) (#54040) --- be/src/util/s3_util.cpp | 19 ++- .../org/apache/doris/common/util/S3Util.java | 6 + .../property/PropertyConverter.java | 8 ++ .../property/S3ClientBEProperties.java | 7 ++ .../property/constants/S3Properties.java | 2 +- .../tablefunction/S3TableValuedFunction.java | 4 +- .../broker_load/test_s3_load_without_aksk.out | 45 +++++++ .../load_p0/tvf/test_tvf_without_aksk.out | 45 +++++++ ...ain_connection_and_ak_sk_correction.groovy | 55 --------- .../test_s3_load_without_aksk.groovy | 111 ++++++++++++++++++ .../load_p0/tvf/test_tvf_without_aksk.groovy | 60 ++++++++++ 11 files changed, 299 insertions(+), 63 deletions(-) create mode 100644 regression-test/data/load_p0/broker_load/test_s3_load_without_aksk.out create mode 100644 regression-test/data/load_p0/tvf/test_tvf_without_aksk.out create mode 100644 regression-test/suites/load_p0/broker_load/test_s3_load_without_aksk.groovy create mode 100644 regression-test/suites/load_p0/tvf/test_tvf_without_aksk.groovy diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index f650558333794d..d80e668c05c012 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -82,12 +82,17 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) { } if (conf.role_arn.empty()) { - if (conf.ak.empty()) { - return Status::InvalidArgument("Invalid s3 conf, empty ak"); - } - if (conf.sk.empty()) { + // Allow anonymous access when both ak and sk are empty + bool hasAk = !conf.ak.empty(); + bool hasSk = !conf.sk.empty(); + + // Either both credentials are provided or both are empty (anonymous access) + if (hasAk && conf.sk.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty sk"); } + if (hasSk && conf.ak.empty()) { + return Status::InvalidArgument("Invalid s3 conf, empty ak"); + } } return Status::OK(); } @@ -282,6 +287,12 @@ std::shared_ptr S3ClientFactory::get_aws_cred s3_conf.role_arn, Aws::String(), s3_conf.external_id, Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); } + + // Support anonymous access for public datasets when no credentials are provided + if (s3_conf.ak.empty() && s3_conf.sk.empty()) { + return std::make_shared(); + } + return std::make_shared(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index e204fab2e2247b..6ca4a99120b7da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -20,6 +20,7 @@ import org.apache.doris.common.credentials.CloudCredential; import com.google.common.base.Strings; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -144,6 +145,11 @@ private static AwsCredentialsProvider getAwsCredencialsProvider(URI endpoint, St } }).build(); } + + // For anonymous access (no credentials required) + if (Strings.isNullOrEmpty(accessKey) && Strings.isNullOrEmpty(secretKey)) { + return AnonymousCredentialsProvider.create(); + } return AwsCredentialsProviderChain.of(SystemPropertyCredentialsProvider.create(), EnvironmentVariableCredentialsProvider.create(), WebIdentityTokenFileCredentialsProvider.create(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index aa22a83e1b0067..e0b8520d5ce2c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem; import org.apache.hadoop.fs.obs.OBSConstants; import org.apache.hadoop.fs.obs.OBSFileSystem; +import org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider; @@ -318,6 +319,13 @@ private static void setS3FsAccess(Map s3Properties, Map getBeFSProperties(Map properti || properties.containsKey(GCSProperties.ENDPOINT) || properties.containsKey(CosProperties.ENDPOINT)) { return getBeAWSPropertiesFromS3(S3Properties.prefixToS3(properties)); + } else if (properties.containsKey(S3Properties.Env.ENDPOINT)) { + if (!properties.containsKey(S3Properties.Env.REGION)) { + String endpoint = properties.get(S3Properties.Env.ENDPOINT); + String region = PropertyConverter.checkRegion(endpoint, properties.get(S3Properties.Env.REGION), + S3Properties.Env.REGION); + properties.put(S3Properties.Env.REGION, region); + } } return properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 82a71548b69293..a727346e3256f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -74,7 +74,7 @@ public class S3Properties extends BaseProperties { public static final String VALIDITY_CHECK = "s3_validity_check"; public static final String PROVIDER = "provider"; public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT); - public static final List TVF_REQUIRED_FIELDS = Arrays.asList(ACCESS_KEY, SECRET_KEY); + public static final List TVF_REQUIRED_FIELDS = Arrays.asList(); public static final List FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN, ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 93dc28d84a35b2..3619821920eb14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -84,12 +84,10 @@ public S3TableValuedFunction(Map properties) throws AnalysisExce // Azure could run without region region = s3uri.getRegion().orElse("DUMMY-REGION"); } else { - region = s3uri.getRegion().orElseThrow(() -> new AnalysisException( - String.format("Properties '%s' is required.", S3Properties.REGION))); + region = PropertyConverter.checkRegion(endpoint, s3uri.getRegion().orElse(""), S3Properties.REGION); } otherProps.put(S3Properties.REGION, region); } - checkNecessaryS3Properties(otherProps); CloudCredentialWithEndpoint credential = new CloudCredentialWithEndpoint(endpoint, getOrDefaultAndRemove(otherProps, S3Properties.REGION, ""), getOrDefaultAndRemove(otherProps, S3Properties.ACCESS_KEY, ""), diff --git a/regression-test/data/load_p0/broker_load/test_s3_load_without_aksk.out b/regression-test/data/load_p0/broker_load/test_s3_load_without_aksk.out new file mode 100644 index 00000000000000..8b0290edbdd483 --- /dev/null +++ b/regression-test/data/load_p0/broker_load/test_s3_load_without_aksk.out @@ -0,0 +1,45 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 Emily 25 +2 Benjamin 35 +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql -- +1 Emily 25 +1 Emily 25 +2 Benjamin 35 +2 Benjamin 35 +3 Olivia 28 +3 Olivia 28 +4 Alexander 60 +4 Alexander 60 +5 Ava 17 +5 Ava 17 +6 William 69 +6 William 69 +7 Sophia 32 +7 Sophia 32 +8 James 64 +8 James 64 +9 Emma 37 +9 Emma 37 +10 Liam 64 +10 Liam 64 +11 Alexander 34 +12 Isabella 43 +13 Benjamin 56 +14 Sophia 12 +15 Christopher 33 +16 Emma 23 +17 Michael 11 +18 Olivia 38 +19 Daniel 19 +20 Ava 28 + diff --git a/regression-test/data/load_p0/tvf/test_tvf_without_aksk.out b/regression-test/data/load_p0/tvf/test_tvf_without_aksk.out new file mode 100644 index 00000000000000..8b0290edbdd483 --- /dev/null +++ b/regression-test/data/load_p0/tvf/test_tvf_without_aksk.out @@ -0,0 +1,45 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 Emily 25 +2 Benjamin 35 +3 Olivia 28 +4 Alexander 60 +5 Ava 17 +6 William 69 +7 Sophia 32 +8 James 64 +9 Emma 37 +10 Liam 64 + +-- !sql -- +1 Emily 25 +1 Emily 25 +2 Benjamin 35 +2 Benjamin 35 +3 Olivia 28 +3 Olivia 28 +4 Alexander 60 +4 Alexander 60 +5 Ava 17 +5 Ava 17 +6 William 69 +6 William 69 +7 Sophia 32 +7 Sophia 32 +8 James 64 +8 James 64 +9 Emma 37 +9 Emma 37 +10 Liam 64 +10 Liam 64 +11 Alexander 34 +12 Isabella 43 +13 Benjamin 56 +14 Sophia 12 +15 Christopher 33 +16 Emma 23 +17 Michael 11 +18 Olivia 38 +19 Daniel 19 +20 Ava 28 + diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 36073df4b07a2c..dbefce7a8c3adb 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -105,61 +105,6 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) } - label = UUID.randomUUID().toString().replace("-", "") - try { - result = sql """ - LOAD LABEL ${label} - ( - DATA INFILE("s3://${getS3BucketName()}/regression/tpch/sf1/part.tbl") - INTO TABLE ${tableName} - COLUMNS TERMINATED BY "|" - (p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, temp) - ) - WITH S3 - ( - "AWS_ENDPOINT" = "${getS3Endpoint()}", - "AWS_ACCESS_KEY" = "${getS3AK()}1", - "AWS_SECRET_KEY" = "${getS3SK()}", - "AWS_REGION" = "${getS3Region()}", - "PROVIDER" = "${getS3Provider()}" - ); - """ - logger.info("the third sql result is {}", result) - assertTrue(false. "AK is wrong, so the correction of AKSK test should fale") - } catch (Exception e) { - logger.info("the third sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) - } - - label = UUID.randomUUID().toString().replace("-", "") - try { - result = sql """ - LOAD LABEL ${label} - ( - DATA INFILE("s3://${getS3BucketName()}/regression/tpch/sf1/part.tbl") - INTO TABLE ${tableName} - COLUMNS TERMINATED BY "|" - (p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, temp), - DATA INFILE("s3://${getS3BucketName()}1/regression/tpch/sf1/orders.tbl.1", "s3://${getS3BucketName()}/regression/tpch/sf1/orders.tbl.2") - INTO TABLE ${tableNameOrders} - COLUMNS TERMINATED BY "|" - (o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, temp) - ) - WITH S3 - ( - "AWS_ENDPOINT" = "${getS3Endpoint()}", - "AWS_ACCESS_KEY" = "${getS3AK()}", - "AWS_SECRET_KEY" = "${getS3SK()}", - "AWS_REGION" = "${getS3Region()}", - "PROVIDER" = "${getS3Provider()}" - ); - """ - logger.info("the fourth sql result is {}", result) - assertTrue(false. "in the second DATA INFILE, the first bucket is wrong, so the sql should fail") - } catch (Exception e) { - logger.info("the fourth sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) - } sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" } diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load_without_aksk.groovy b/regression-test/suites/load_p0/broker_load/test_s3_load_without_aksk.groovy new file mode 100644 index 00000000000000..60b39c03657940 --- /dev/null +++ b/regression-test/suites/load_p0/broker_load/test_s3_load_without_aksk.groovy @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_s3_load_without_aksk", "load_p0") { + def tableName = "tbl_without_aksk" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + user_id BIGINT NOT NULL COMMENT "用户 ID", + name VARCHAR(20) COMMENT "用户姓名", + age INT COMMENT "用户年龄" + ) DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + def label = UUID.randomUUID().toString().replace("-", "0") + + def sql_str = """ + LOAD LABEL $label ( + DATA INFILE("s3://${s3BucketName}/regression/load/data/example_0.csv") + INTO TABLE $tableName + COLUMNS TERMINATED BY "," + ) + WITH S3 ( + "AWS_ENDPOINT" = "${getS3Endpoint()}", + "PROVIDER" = "${getS3Provider()}" + ) + """ + logger.info("submit sql: ${sql_str}"); + sql """${sql_str}""" + + def max_try_milli_secs = 600000 + while (max_try_milli_secs > 0) { + String[][] result = sql """ show load where label="$label" order by createtime desc limit 1; """ + if (result[0][2].equals("FINISHED")) { + logger.info("Load FINISHED " + label) + break + } + if (result[0][2].equals("CANCELLED")) { + def reason = result[0][7] + logger.info("load failed, reason:$reason") + assertTrue(1 == 2) + break + } + Thread.sleep(1000) + max_try_milli_secs -= 1000 + if(max_try_milli_secs <= 0) { + assertTrue(1 == 2, "load Timeout: $label") + } + } + + qt_sql """ SELECT * FROM ${tableName} order by user_id """ + + label = UUID.randomUUID().toString().replace("-", "0") + + sql_str = """ + LOAD LABEL $label ( + DATA INFILE("s3://${s3BucketName}/regression/load/data/example_*.csv") + INTO TABLE $tableName + COLUMNS TERMINATED BY "," + ) + WITH S3 ( + "s3.endpoint" = "${getS3Endpoint()}", + "PROVIDER" = "${getS3Provider()}" + ) + """ + logger.info("submit sql: ${sql_str}"); + sql """${sql_str}""" + + max_try_milli_secs = 600000 + while (max_try_milli_secs > 0) { + String[][] result = sql """ show load where label="$label" order by createtime desc limit 1; """ + if (result[0][2].equals("FINISHED")) { + logger.info("Load FINISHED " + label) + break + } + if (result[0][2].equals("CANCELLED")) { + def reason = result[0][7] + logger.info("load failed, reason:$reason") + assertTrue(1 == 2) + break + } + Thread.sleep(1000) + max_try_milli_secs -= 1000 + if(max_try_milli_secs <= 0) { + assertTrue(1 == 2, "load Timeout: $label") + } + } + + qt_sql """ SELECT * FROM ${tableName} order by user_id """ + +} diff --git a/regression-test/suites/load_p0/tvf/test_tvf_without_aksk.groovy b/regression-test/suites/load_p0/tvf/test_tvf_without_aksk.groovy new file mode 100644 index 00000000000000..622fd5b8276cc1 --- /dev/null +++ b/regression-test/suites/load_p0/tvf/test_tvf_without_aksk.groovy @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_tvf_without_aksk", "load_p0") { + def tableName = "tbl_without_aksk" + + sql """ DROP TABLE IF EXISTS ${tableName} """ + + sql """ + CREATE TABLE ${tableName} ( + user_id BIGINT NOT NULL COMMENT "用户 ID", + name VARCHAR(20) COMMENT "用户姓名", + age INT COMMENT "用户年龄" + ) DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + def label = UUID.randomUUID().toString().replace("-", "0") + + sql """ + INSERT INTO ${tableName} + SELECT * FROM S3 + ( + "uri" = "s3://${s3BucketName}/regression/load/data/example_0.csv", + "s3.endpoint" = "${getS3Endpoint()}", + "column_separator" = ",", + "format" = "csv" + ); + """ + qt_sql """ SELECT * FROM ${tableName} order by user_id """ + + sql """ + INSERT INTO ${tableName} + SELECT * FROM S3 + ( + "uri" = "s3://${s3BucketName}/regression/load/data/example_*.csv", + "s3.endpoint" = "${getS3Endpoint()}", + "column_separator" = ",", + "format" = "csv" + ); + """ + qt_sql """ SELECT * FROM ${tableName} order by user_id """ +} From 5a0e74cc0c60c0af4137a2c1038d381d7b6c0ee7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:14:09 +0800 Subject: [PATCH 360/572] branch-3.0: [fix](cloud) Fix warm up `ConcurrentModificationException` exception #53192 (#54051) Cherry-picked from #53192 Co-authored-by: deardeng --- .../cloud/catalog/CloudTabletRebalancer.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index 17d72dd7446a44..e7f080ea329804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -157,7 +157,6 @@ private class InfightTask { public Tablet pickedTablet; public long srcBe; public long destBe; - public boolean isGlobal; public Map> beToTablets; public long startTimestamp; BalanceType balanceType; @@ -168,21 +167,22 @@ private class TransferPairInfo { public long destBe; public long minTabletsNum; public long maxTabletsNum; - public boolean srcDecommissioned; } public Set getSnapshotTabletsInPrimaryByBeId(Long beId) { Set tabletIds = Sets.newHashSet(); Set tablets = beToTabletsGlobal.get(beId); if (tablets != null) { - for (Tablet tablet : tablets) { + // Create a copy + for (Tablet tablet : new HashSet<>(tablets)) { tabletIds.add(tablet.getId()); } } - tablets = beToColocateTabletsGlobal.get(beId); - if (tablets != null) { - for (Tablet tablet : tablets) { + Set colocateTablets = beToColocateTabletsGlobal.get(beId); + if (colocateTablets != null) { + // Create a copy + for (Tablet tablet : new HashSet<>(colocateTablets)) { tabletIds.add(tablet.getId()); } } @@ -194,7 +194,8 @@ public Set getSnapshotTabletsInSecondaryByBeId(Long beId) { Set tabletIds = Sets.newHashSet(); Set tablets = beToTabletsGlobalInSecondary.get(beId); if (tablets != null) { - for (Tablet tablet : tablets) { + // Create a copy + for (Tablet tablet : new HashSet<>(tablets)) { tabletIds.add(tablet.getId()); } } @@ -212,8 +213,10 @@ public int getTabletNumByBackendId(long beId) { Set tablets = beToTabletsGlobal.get(beId); Set colocateTablets = beToColocateTabletsGlobal.get(beId); - return (tablets == null ? 0 : tablets.size()) - + (colocateTablets == null ? 0 : colocateTablets.size()); + int tabletsSize = (tablets == null) ? 0 : tablets.size(); + int colocateTabletsSize = (colocateTablets == null) ? 0 : colocateTablets.size(); + + return tabletsSize + colocateTabletsSize; } // 1 build cluster to backends info From 7c6722b17dd08487f849e3ff3e6d47cc1c428a68 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:17:31 +0800 Subject: [PATCH 361/572] branch-3.0: [fix](array) Fix array distance functions #54348 (#54371) Cherry-picked from #54348 Co-authored-by: zhiqiang --- .../functions/array/function_array_distance.h | 20 ++++-- .../functions/scalar/CosineDistance.java | 5 +- .../functions/scalar/InnerProduct.java | 5 +- .../functions/scalar/L1Distance.java | 5 +- .../functions/scalar/L2Distance.java | 5 +- .../test_array_distance_functions.out | 58 +++++++++++++++++ .../test_array_distance_functions.groovy | 64 +++++++++++++++++++ 7 files changed, 145 insertions(+), 17 deletions(-) diff --git a/be/src/vec/functions/array/function_array_distance.h b/be/src/vec/functions/array/function_array_distance.h index 4349e93c97c966..5a855c04988c82 100644 --- a/be/src/vec/functions/array/function_array_distance.h +++ b/be/src/vec/functions/array/function_array_distance.h @@ -141,25 +141,35 @@ class FunctionArrayDistance : public IFunction { } dst_null_data[row] = false; - if (offsets1[row] != offsets2[row]) [[unlikely]] { + + // Calculate actual array sizes for current row. + // For nullable arrays, we cannot compare absolute offset values directly because: + // 1. When a row is null, its offset might equal the previous offset (no elements added) + // 2. Or it might include the array size even if the row is null (implementation dependent) + // Therefore, we must calculate the actual array size as: offsets[row] - offsets[row-1] + ssize_t size1 = offsets1[row] - offsets1[row - 1]; + ssize_t size2 = offsets2[row] - offsets2[row - 1]; + + if (size1 != size2) [[unlikely]] { return Status::InvalidArgument( "function {} have different input element sizes of array: {} and {}", - get_name(), offsets1[row] - offsets1[row - 1], - offsets2[row] - offsets2[row - 1]); + get_name(), size1, size2); } typename DistanceImpl::State st; for (ssize_t pos = offsets1[row - 1]; pos < offsets1[row]; ++pos) { + // Calculate corresponding position in the second array + ssize_t pos2 = offsets2[row - 1] + (pos - offsets1[row - 1]); if (arr1.nested_nullmap_data && arr1.nested_nullmap_data[pos]) { dst_null_data[row] = true; break; } - if (arr2.nested_nullmap_data && arr2.nested_nullmap_data[pos]) { + if (arr2.nested_nullmap_data && arr2.nested_nullmap_data[pos2]) { dst_null_data[row] = true; break; } DistanceImpl::accumulate(st, nested_col1->get_element(pos), - nested_col2->get_element(pos)); + nested_col2->get_element(pos2)); } if (!dst_null_data[row]) { dst_data[row] = DistanceImpl::finalize(st); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CosineDistance.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CosineDistance.java index 9761f949297233..14c388bb933377 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CosineDistance.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/CosineDistance.java @@ -20,9 +20,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForArrayItemAgg; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +35,7 @@ * cosine_distance function */ public class CosineDistance extends ScalarFunction implements ExplicitlyCastableSignature, - ComputePrecisionForArrayItemAgg, UnaryExpression, AlwaysNullable { + BinaryExpression, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/InnerProduct.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/InnerProduct.java index 5d226951cb13e3..a56d5d5a5229f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/InnerProduct.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/InnerProduct.java @@ -20,9 +20,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForArrayItemAgg; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +35,7 @@ * inner_product function */ public class InnerProduct extends ScalarFunction implements ExplicitlyCastableSignature, - ComputePrecisionForArrayItemAgg, UnaryExpression, AlwaysNullable { + BinaryExpression, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L1Distance.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L1Distance.java index 7423702fc7beaa..66a6ebd2bf41ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L1Distance.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L1Distance.java @@ -20,9 +20,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForArrayItemAgg; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +35,7 @@ * l1_distance function */ public class L1Distance extends ScalarFunction implements ExplicitlyCastableSignature, - ComputePrecisionForArrayItemAgg, UnaryExpression, AlwaysNullable { + BinaryExpression, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java index 14ffee389ae77d..a9775f59ad77c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java @@ -20,9 +20,8 @@ import org.apache.doris.catalog.FunctionSignature; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; -import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForArrayItemAgg; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +35,7 @@ * l2_distance function */ public class L2Distance extends ScalarFunction implements ExplicitlyCastableSignature, - ComputePrecisionForArrayItemAgg, UnaryExpression, AlwaysNullable { + BinaryExpression, AlwaysNullable { public static final List SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/regression-test/data/query_p0/sql_functions/array_functions/test_array_distance_functions.out b/regression-test/data/query_p0/sql_functions/array_functions/test_array_distance_functions.out index 10c9f2ae306b98..4a14b8c9a407c3 100644 --- a/regression-test/data/query_p0/sql_functions/array_functions/test_array_distance_functions.out +++ b/regression-test/data/query_p0/sql_functions/array_functions/test_array_distance_functions.out @@ -20,3 +20,61 @@ -- !sql -- \N +-- !sql -- +\N + +-- !sql -- +2.8284271247461903 + +-- !sql -- +0.025368153802923787 + +-- !sql -- +23.0 + +-- !sql -- +\N + +-- !sql -- +\N + +-- !sql -- +4.0 \N + +-- !sql -- +0.0 \N + +-- !sql -- +0.0 + +-- !sql -- +0.0 + +-- !sql -- +1 4.0 +2 \N +3 \N +4 4.0 +5 \N + +-- !sql -- +1 2.8284271247461903 +2 \N +3 \N +4 2.8284271247461903 +5 \N + +-- !sql -- +1 0.01613008990009257 +2 \N +3 \N +4 4.1697940139862055E-5 +5 \N + +-- !sql -- +1 11.0 +2 \N +3 \N +4 219.0 +5 \N + diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_distance_functions.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_distance_functions.groovy index cf196e0d61a316..9010750a2ec500 100644 --- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_distance_functions.groovy +++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_distance_functions.groovy @@ -25,6 +25,21 @@ suite("test_array_distance_functions") { qt_sql "SELECT l2_distance([1, 2, 3], NULL)" qt_sql "SELECT cosine_distance([1, 2, 3], [0, NULL, 0])" + // Test cases for nullable arrays with different null distributions + // These test the fix for correct array size comparison when nulls are present + qt_sql "SELECT l1_distance(NULL, NULL)" + qt_sql "SELECT l2_distance([1.0, 2.0], [3.0, 4.0])" + qt_sql "SELECT cosine_distance([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])" + qt_sql "SELECT inner_product([2.0, 3.0], [4.0, 5.0])" + + // Test arrays with NULL elements inside + qt_sql "SELECT l1_distance([1.0, NULL, 3.0], [4.0, NULL, 6.0])" + qt_sql "SELECT l2_distance([NULL, 2.0], [NULL, 5.0])" + + // Test mixed nullable scenarios - these should work correctly after the fix + qt_sql "SELECT l1_distance([1.0, 2.0], [3.0, 4.0]) as result1, l1_distance(NULL, [5.0, 6.0]) as result2" + qt_sql "SELECT cosine_distance([1.0], [2.0]) as result1, cosine_distance([3.0], NULL) as result2" + // abnormal test cases try { sql "SELECT l2_distance([0, 0], [1])" @@ -37,4 +52,53 @@ suite("test_array_distance_functions") { } catch (Exception ex) { assert("${ex}".contains("function cosine_distance have different input element sizes")) } + + // Test cases for the nullable array offset fix + // These cases specifically test scenarios where absolute offsets might differ + // but actual array sizes are the same (should pass) or different (should fail) + try { + sql "SELECT l1_distance([1.0, 2.0, 3.0], [4.0, 5.0])" + } catch (Exception ex) { + assert("${ex}".contains("function l1_distance have different input element sizes")) + } + + try { + sql "SELECT inner_product([1.0], [2.0, 3.0, 4.0])" + } catch (Exception ex) { + assert("${ex}".contains("function inner_product have different input element sizes")) + } + + // Edge case: empty arrays should work + qt_sql "SELECT l1_distance(CAST([] as ARRAY), CAST([] as ARRAY))" + qt_sql "SELECT l2_distance(CAST([] as ARRAY), CAST([] as ARRAY))" + + // Comprehensive test for the offset fix: test with table data containing mixed nulls + // This specifically tests the scenario where offsets might differ due to null distribution + // but actual array sizes are the same + sql """ + DROP TABLE IF EXISTS test_array_distance_nullable + """ + sql """ + CREATE TABLE test_array_distance_nullable ( + id INT, + arr1 ARRAY, + arr2 ARRAY + ) PROPERTIES ( + "replication_num" = "1" + ) + """ + sql """ + INSERT INTO test_array_distance_nullable VALUES + (1, [1.0, 2.0], [3.0, 4.0]), + (2, NULL, [5.0, 6.0]), + (3, [7.0, 8.0], NULL), + (4, [9.0, 10.0], [11.0, 12.0]), + (5, NULL, NULL) + """ + + // These queries should work correctly after the fix + qt_sql "SELECT id, l1_distance(arr1, arr2) FROM test_array_distance_nullable ORDER BY id" + qt_sql "SELECT id, l2_distance(arr1, arr2) FROM test_array_distance_nullable ORDER BY id" + qt_sql "SELECT id, cosine_distance(arr1, arr2) FROM test_array_distance_nullable ORDER BY id" + qt_sql "SELECT id, inner_product(arr1, arr2) FROM test_array_distance_nullable ORDER BY id" } From e1d64b3cfb2a254672c394b6f252f1e7e07926d6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:18:44 +0800 Subject: [PATCH 362/572] branch-3.0: [fix](job) allow job can auto resume when meet can not find be error #53848 (#53929) Cherry-picked from #53848 Co-authored-by: hui lai --- .../org/apache/doris/load/routineload/RoutineLoadManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index 169af97670f6e3..124c3a20ed17aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -506,7 +506,7 @@ public long getAvailableBeForTask(long jobId, long previousBeId) throws UserExce String msg = "no available BE found for job " + jobId + "please check the BE status and user's cluster or tags"; job.updateState(RoutineLoadJob.JobState.PAUSED, - new ErrorReason(InternalErrorCode.MANUAL_PAUSE_ERR, msg), false /* not replay */); + new ErrorReason(InternalErrorCode.INTERNAL_ERR, msg), false /* not replay */); } return -1L; } From 3f25f27cbfe37f3fe74dbaf11b7d08f19f2e6efc Mon Sep 17 00:00:00 2001 From: zhangdong Date: Tue, 12 Aug 2025 10:19:44 +0800 Subject: [PATCH 363/572] branch-3.0: [fix](auth)Fix the issue of incorrectly checking base table permissions when querying external views #53786 (#54003) pick: #53786 --- .../java/org/apache/doris/catalog/View.java | 7 +- .../java/org/apache/doris/catalog/ViewIf.java | 23 ++ .../apache/doris/datasource/ExternalView.java | 204 ++++++++++++++++++ .../nereids/rules/analysis/BindRelation.java | 4 +- .../trees/plans/logical/LogicalView.java | 12 +- .../test_select_external_view_auth.groovy | 75 +++++++ 6 files changed, 315 insertions(+), 10 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/ViewIf.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalView.java create mode 100644 regression-test/suites/auth_p0/test_select_external_view_auth.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java index 96b0fdf290dbb0..a27f23cd3c9ecd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/View.java @@ -52,7 +52,7 @@ * Refreshing or invalidating a view will reload the view's definition but will not * affect the metadata of the underlying tables (if any). */ -public class View extends Table implements GsonPostProcessable { +public class View extends Table implements GsonPostProcessable, ViewIf { private static final Logger LOG = LogManager.getLogger(View.class); // The original SQL-string given as view definition. Set during analysis. @@ -216,6 +216,11 @@ public boolean hasColLabels() { return colLabels != null; } + @Override + public String getViewText() { + return inlineViewDef; + } + // Get the md5 of signature string of this view. // This method is used to determine whether the views have the same schema. // Contains: diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ViewIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ViewIf.java new file mode 100644 index 00000000000000..e4c66fa467dc7f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ViewIf.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +public interface ViewIf extends TableIf { + String getViewText(); +} + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalView.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalView.java new file mode 100644 index 00000000000000..7c89b968aedcca --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalView.java @@ -0,0 +1,204 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.TableIndexes; +import org.apache.doris.catalog.ViewIf; +import org.apache.doris.common.Pair; +import org.apache.doris.statistics.AnalysisInfo; +import org.apache.doris.statistics.BaseAnalysisTask; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.thrift.TTableDescriptor; + +import java.io.DataOutput; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +public class ExternalView implements ViewIf { + private String viewText; + private ExternalTable externalTable; + + public ExternalView(ExternalTable externalTable, String viewText) { + this.viewText = viewText; + this.externalTable = externalTable; + } + + @Override + public String getViewText() { + return viewText; + } + + public ExternalTable getExternalTable() { + return externalTable; + } + + + @Override + public long getId() { + return externalTable.getId(); + } + + public String getName() { + return externalTable.getName(); + } + + @Override + public TableType getType() { + return externalTable.getType(); + } + + @Override + public List getFullSchema() { + return externalTable.getFullSchema(); + } + + @Override + public List getBaseSchema() { + return externalTable.getBaseSchema(); + } + + @Override + public List getBaseSchema(boolean full) { + return externalTable.getBaseSchema(); + } + + @Override + public void setNewFullSchema(List newSchema) { + externalTable.setNewFullSchema(newSchema); + } + + @Override + public Column getColumn(String name) { + return externalTable.getColumn(name); + } + + @Override + public String getMysqlType() { + return externalTable.getMysqlType(); + } + + @Override + public String getEngine() { + return externalTable.getEngine(); + } + + @Override + public String getComment() { + return externalTable.getComment(); + } + + @Override + public long getCreateTime() { + return externalTable.getCreateTime(); + } + + @Override + public long getUpdateTime() { + return externalTable.getUpdateTime(); + } + + @Override + public long getRowCount() { + return externalTable.getRowCount(); + } + + @Override + public long getCachedRowCount() { + return externalTable.getCachedRowCount(); + } + + @Override + public long fetchRowCount() { + return externalTable.fetchRowCount(); + } + + @Override + public long getDataLength() { + return externalTable.getDataLength(); + } + + @Override + public long getAvgRowLength() { + return externalTable.getAvgRowLength(); + } + + @Override + public long getIndexLength() { + return externalTable.getIndexLength(); + } + + @Override + public long getLastCheckTime() { + return externalTable.getLastCheckTime(); + } + + @Override + public String getComment(boolean escapeQuota) { + return externalTable.getComment(); + } + + @Override + public TTableDescriptor toThrift() { + return externalTable.toThrift(); + } + + @Override + public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) { + return externalTable.createAnalysisTask(info); + } + + @Override + public DatabaseIf getDatabase() { + return externalTable.getDatabase(); + } + + @Override + public Optional getColumnStatistic(String colName) { + return externalTable.getColumnStatistic(colName); + } + + @Override + public Set> getColumnIndexPairs(Set columns) { + return externalTable.getColumnIndexPairs(columns); + } + + @Override + public List getChunkSizes() { + return externalTable.getChunkSizes(); + } + + @Override + public void write(DataOutput out) throws IOException { + externalTable.write(out); + } + + @Override + public boolean autoAnalyzeEnabled() { + return externalTable.autoAnalyzeEnabled(); + } + + @Override + public TableIndexes getTableIndexes() { + return externalTable.getTableIndexes(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index c1d7a81e28a77d..8db2e6f0c7a8ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -33,6 +33,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.ExternalView; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.nereids.CTEContext; @@ -464,7 +465,8 @@ private Plan parseAndAnalyzeHiveView( ctx.changeDefaultCatalog(hiveCatalog); ctx.setDatabase(hiveDb); try { - return parseAndAnalyzeView(table, ddlSql, cascadesContext); + return new LogicalView<>(new ExternalView(table, ddlSql), + parseAndAnalyzeView(table, ddlSql, cascadesContext)); } finally { // restore catalog and db in connect context ctx.changeDefaultCatalog(previousCatalog); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java index 0eaec7c3c18b2c..5f80bfed336976 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalView.java @@ -17,7 +17,7 @@ package org.apache.doris.nereids.trees.plans.logical; -import org.apache.doris.catalog.View; +import org.apache.doris.catalog.ViewIf; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DataTrait; @@ -38,10 +38,10 @@ /** LogicalView */ public class LogicalView extends LogicalUnary { - private final View view; + private final ViewIf view; /** LogicalView */ - public LogicalView(View view, BODY body) { + public LogicalView(ViewIf view, BODY body) { super(PlanType.LOGICAL_VIEW, Optional.empty(), Optional.empty(), body); this.view = Objects.requireNonNull(view, "catalog can not be null"); if (!(body instanceof LogicalPlan)) { @@ -71,11 +71,7 @@ public String getName() { return view.getName(); } - public String getViewString() { - return view.getInlineViewDef(); - } - - public View getView() { + public ViewIf getView() { return view; } diff --git a/regression-test/suites/auth_p0/test_select_external_view_auth.groovy b/regression-test/suites/auth_p0/test_select_external_view_auth.groovy new file mode 100644 index 00000000000000..ebd7a3bea9733a --- /dev/null +++ b/regression-test/suites/auth_p0/test_select_external_view_auth.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_select_external_view_auth","p0,auth") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + for (String hivePrefix : ["hive2", "hive3"]) { + try { + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalogName = "${hivePrefix}_test_mtmv" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalogName}""" + sql """create catalog if not exists ${catalogName} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + String suiteName = "test_select_external_view_auth" + String user = "${suiteName}_user" + String pwd = 'C123_567p' + String dbName = "`default`" + String tableName = "sale_table" + String viewName = "test_view1" + + try_sql("drop user ${user}") + sql """create user '${user}' IDENTIFIED by '${pwd}'""" + sql """grant select_priv on regression_test to ${user}""" + + //cloud-mode + if (isCloudMode()) { + def clusters = sql " SHOW CLUSTERS; " + assertTrue(!clusters.isEmpty()) + def validCluster = clusters[0][0] + sql """GRANT USAGE_PRIV ON CLUSTER `${validCluster}` TO ${user}"""; + } + + sql """grant select_priv on ${catalogName}.${dbName}.${tableName} to ${user}""" + // table column + connect(user, "${pwd}", context.config.jdbcUrl) { + try { + sql "select * from ${catalogName}.${dbName}.${viewName}" + } catch (Exception e) { + log.info(e.getMessage()) + assertTrue(e.getMessage().contains("denied")) + } + } + sql """revoke select_priv on ${catalogName}.${dbName}.${tableName} from ${user}""" + sql """grant select_priv on ${catalogName}.${dbName}.${viewName} to ${user}""" + connect(user, "${pwd}", context.config.jdbcUrl) { + sql "select * from ${catalogName}.${dbName}.${viewName}" + } + try_sql("drop user ${user}") + sql """drop catalog if exists ${catalogName}""" + } finally { + } + } +} From ae68a3d08deac54b67ef76e691bcef5e7b138ea7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:20:40 +0800 Subject: [PATCH 364/572] branch-3.0: [fix](filecache) warm up download data less than actual #54363 (#54420) Cherry-picked from #54363 Co-authored-by: zhengyu --- be/src/io/cache/block_file_cache_downloader.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index 96f507816a6b5b..c7cc54764368c3 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -265,10 +265,12 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met std::unique_ptr buffer(new char[one_single_task_size]); + size_t task_offset = 0; for (size_t i = 0; i < task_num; i++) { - size_t offset = meta.offset + i * one_single_task_size; - size_t size = - std::min(one_single_task_size, static_cast(meta.download_size - offset)); + size_t offset = meta.offset + task_offset; + + size_t size = std::min(one_single_task_size, + static_cast(meta.download_size - task_offset)); size_t bytes_read; VLOG_DEBUG << "download_segment_file, path=" << meta.path << ", read_at offset=" << offset << ", size=" << size; @@ -285,6 +287,7 @@ void FileCacheBlockDownloader::download_segment_file(const DownloadFileMeta& met g_file_cache_download_failed_num << 1; return; } + task_offset += size; g_file_cache_download_finished_size << size; } From 4a202c7ffb1b092ecd884aebba06228e46f21ba4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:21:24 +0800 Subject: [PATCH 365/572] branch-3.0: [Fix](nereids) Fix incorrect results in GROUP BY with Modulo (%) operations #54153 (#54193) Cherry-picked from #54153 Co-authored-by: Jensen --- .../rules/rewrite/SimplifyAggGroupBy.java | 17 +++++++++++-- .../rules/rewrite/SimplifyAggGroupByTest.java | 24 +++++++++++++++++++ .../aggregate/aggregate_groupby_simplify.out | 9 +++++++ .../aggregate_groupby_simplify.groovy | 23 ++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/nereids_p0/aggregate/aggregate_groupby_simplify.out create mode 100644 regression-test/suites/nereids_p0/aggregate/aggregate_groupby_simplify.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupBy.java index 6dc446d88ca882..37d4d4806f087a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupBy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupBy.java @@ -20,13 +20,18 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.TreeNode; +import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.BinaryArithmetic; +import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; import java.util.List; @@ -40,11 +45,15 @@ * GROUP BY ClientIP */ public class SimplifyAggGroupBy extends OneRewriteRuleFactory { + private static final ImmutableSet> supportedFunctions + = ImmutableSet.of(Add.class, Subtract.class, Multiply.class, Divide.class); + @Override public Rule build() { return logicalAggregate() .when(agg -> agg.getGroupByExpressions().size() > 1 - && ExpressionUtils.allMatch(agg.getGroupByExpressions(), this::isBinaryArithmeticSlot)) + && ExpressionUtils.allMatch(agg.getGroupByExpressions(), + SimplifyAggGroupBy::isBinaryArithmeticSlot)) .then(agg -> { List groupByExpressions = agg.getGroupByExpressions(); ImmutableSet.Builder inputSlots @@ -61,13 +70,17 @@ public Rule build() { .toRule(RuleType.SIMPLIFY_AGG_GROUP_BY); } - private boolean isBinaryArithmeticSlot(TreeNode expr) { + @VisibleForTesting + protected static boolean isBinaryArithmeticSlot(TreeNode expr) { if (expr instanceof Slot) { return true; } if (!(expr instanceof BinaryArithmetic)) { return false; } + if (!supportedFunctions.contains(expr.getClass())) { + return false; + } return ExpressionUtils.isSlotOrCastOnSlot(expr.child(0)).isPresent() && expr.child(1) instanceof Literal || ExpressionUtils.isSlotOrCastOnSlot(expr.child(1)).isPresent() && expr.child(0) instanceof Literal; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupByTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupByTest.java index 34c3b012e76a3b..32c2cc4356d048 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupByTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SimplifyAggGroupByTest.java @@ -18,10 +18,13 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Mod; import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.functions.agg.Count; import org.apache.doris.nereids.trees.expressions.functions.scalar.Abs; import org.apache.doris.nereids.trees.expressions.literal.Literal; @@ -35,6 +38,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.List; @@ -132,4 +136,24 @@ void testAbs() { logicalProject(logicalAggregate().when(a -> a.getGroupByExpressions().size() == 2)) ); } + + @Test + void testisBinaryArithmeticSlot() { + Slot id = scan1.getOutput().get(0); + + Mod mod = new Mod(id, Literal.of(2)); + Assertions.assertFalse(SimplifyAggGroupBy.isBinaryArithmeticSlot(mod)); + + Add add = new Add(id, Literal.of(2)); + Assertions.assertTrue(SimplifyAggGroupBy.isBinaryArithmeticSlot(add)); + + Subtract subtract = new Subtract(id, Literal.of(2)); + Assertions.assertTrue(SimplifyAggGroupBy.isBinaryArithmeticSlot(subtract)); + + Multiply multiply = new Multiply(id, Literal.of(2)); + Assertions.assertTrue(SimplifyAggGroupBy.isBinaryArithmeticSlot(multiply)); + + Divide divide = new Divide(id, Literal.of(2)); + Assertions.assertTrue(SimplifyAggGroupBy.isBinaryArithmeticSlot(divide)); + } } diff --git a/regression-test/data/nereids_p0/aggregate/aggregate_groupby_simplify.out b/regression-test/data/nereids_p0/aggregate/aggregate_groupby_simplify.out new file mode 100644 index 00000000000000..0a2425d53f9217 --- /dev/null +++ b/regression-test/data/nereids_p0/aggregate/aggregate_groupby_simplify.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !aggregate -- +0 0 +0 1 +1 0 +1 1 +2 0 +2 1 + diff --git a/regression-test/suites/nereids_p0/aggregate/aggregate_groupby_simplify.groovy b/regression-test/suites/nereids_p0/aggregate/aggregate_groupby_simplify.groovy new file mode 100644 index 00000000000000..0951d9dd2ddf81 --- /dev/null +++ b/regression-test/suites/nereids_p0/aggregate/aggregate_groupby_simplify.groovy @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("aggregate_groupby_simplify") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + qt_aggregate "select number % 3 as a, number % 2 as b from numbers('number' = '10') group by a, b order by a, b;" +} From 7e4f29227be4c0f25ea7291b48d120b4599dd320 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 12 Aug 2025 10:22:27 +0800 Subject: [PATCH 366/572] branch-3.0: [fix](Nereids) self join not always could do colocate join #54323 (#54352) picked from #54323 --- fe/fe-core/pom.xml | 10 + .../apache/doris/nereids/util/JoinUtils.java | 7 +- .../doris/nereids/util/JoinUtilsTest.java | 272 ++++++++++++++++++ fe/pom.xml | 11 + 4 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 2938d032765ed3..54143c8a8f883e 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -954,6 +954,16 @@ under the License. sdk-core ${awssdk.version} + + org.mockito + mockito-core + test + + + org.mockito + mockito-inline + test + diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java index 770905bf199719..4de6696bca10e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/JoinUtils.java @@ -303,11 +303,8 @@ public static boolean couldColocateJoin(DistributionSpecHash leftHashSpec, Distr boolean noNeedCheckColocateGroup = hitSameIndex && (leftTablePartitions.equals(rightTablePartitions)) && (leftTablePartitions.size() <= 1); ColocateTableIndex colocateIndex = Env.getCurrentColocateIndex(); - if (noNeedCheckColocateGroup) { - return true; - } - if (!colocateIndex.isSameGroup(leftTableId, rightTableId) - || colocateIndex.isGroupUnstable(colocateIndex.getGroup(leftTableId))) { + if (!noNeedCheckColocateGroup && (!colocateIndex.isSameGroup(leftTableId, rightTableId) + || colocateIndex.isGroupUnstable(colocateIndex.getGroup(leftTableId)))) { return false; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java new file mode 100644 index 00000000000000..c1e9c77252a976 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/JoinUtilsTest.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.util; + +import org.apache.doris.catalog.ColocateTableIndex; +import org.apache.doris.catalog.ColocateTableIndex.GroupId; +import org.apache.doris.catalog.Env; +import org.apache.doris.nereids.properties.DistributionSpecHash; +import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.TinyIntType; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Lists; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; + +public class JoinUtilsTest { + + @Test + public void testCouldColocateJoinForSameTable() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), ShuffleType.NATURAL, + 1L, 1L, Collections.emptySet()); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), ShuffleType.NATURAL, + 1L, 1L, Collections.emptySet()); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression leftKey2 = new SlotReference(new ExprId(3), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey2 = new SlotReference(new ExprId(4), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts; + + // key same with distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertTrue(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1), new EqualTo(leftKey2, rightKey2)); + Assertions.assertTrue(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and NOT have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey2), new EqualTo(leftKey2, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key not contains distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } + + @Test + public void testCouldColocateJoinForDiffTableInSameGroupAndGroupIsStable() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + // same group and group is statble + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + GroupId groupId = new GroupId(1L, 1L); + ColocateTableIndex colocateIndex = Mockito.mock(ColocateTableIndex.class); + Mockito.when(colocateIndex.isSameGroup(1L, 2L)).thenReturn(true); + Mockito.when(colocateIndex.getGroup(1L)).thenReturn(groupId); + Mockito.when(colocateIndex.isGroupUnstable(groupId)).thenReturn(false); + mockedEnv.when(() -> Env.getCurrentColocateIndex()).thenReturn(colocateIndex); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet()); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), + ShuffleType.NATURAL, 2L, 2L, Collections.emptySet()); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression leftKey2 = new SlotReference(new ExprId(3), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey2 = new SlotReference(new ExprId(4), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts; + + // key same with distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertTrue(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1), new EqualTo(leftKey2, rightKey2)); + Assertions.assertTrue(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and NOT have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey2), new EqualTo(leftKey2, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key not contains distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } + } + + @Test + public void testCouldColocateJoinForNotNaturalHashDstribution() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + // same group and group is statble + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + GroupId groupId = new GroupId(1L, 1L); + ColocateTableIndex colocateIndex = Mockito.mock(ColocateTableIndex.class); + Mockito.when(colocateIndex.isSameGroup(1L, 2L)).thenReturn(true); + Mockito.when(colocateIndex.getGroup(1L)).thenReturn(groupId); + Mockito.when(colocateIndex.isGroupUnstable(groupId)).thenReturn(false); + mockedEnv.when(() -> Env.getCurrentColocateIndex()).thenReturn(colocateIndex); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet()); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), + ShuffleType.EXECUTION_BUCKETED, 2L, 2L, Collections.emptySet()); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression leftKey2 = new SlotReference(new ExprId(3), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey2 = new SlotReference(new ExprId(4), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts; + + // key same with distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1), new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and NOT have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey2), new EqualTo(leftKey2, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key not contains distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } + } + + @Test + public void testCouldColocateJoinForDiffTableInSameGroupAndGroupIsUnstable() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + // same group and group is statble + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + GroupId groupId = new GroupId(1L, 1L); + ColocateTableIndex colocateIndex = Mockito.mock(ColocateTableIndex.class); + Mockito.when(colocateIndex.isSameGroup(1L, 2L)).thenReturn(true); + Mockito.when(colocateIndex.getGroup(1L)).thenReturn(groupId); + Mockito.when(colocateIndex.isGroupUnstable(groupId)).thenReturn(true); + mockedEnv.when(() -> Env.getCurrentColocateIndex()).thenReturn(colocateIndex); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), + ShuffleType.NATURAL, 1L, 1L, Collections.emptySet()); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), + ShuffleType.NATURAL, 2L, 2L, Collections.emptySet()); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression leftKey2 = new SlotReference(new ExprId(3), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey2 = new SlotReference(new ExprId(4), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts; + + // key same with distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1), new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and NOT have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey2), new EqualTo(leftKey2, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key not contains distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } + } + + @Test + public void testCouldColocateJoinForDiffTableNotInSameGroup() { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + // same group and group is statble + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + GroupId groupId = new GroupId(1L, 1L); + ColocateTableIndex colocateIndex = Mockito.mock(ColocateTableIndex.class); + Mockito.when(colocateIndex.isSameGroup(1L, 2L)).thenReturn(true); + Mockito.when(colocateIndex.getGroup(1L)).thenReturn(groupId); + Mockito.when(colocateIndex.isGroupUnstable(groupId)).thenReturn(true); + mockedEnv.when(() -> Env.getCurrentColocateIndex()).thenReturn(colocateIndex); + + DistributionSpecHash left = new DistributionSpecHash(Lists.newArrayList(new ExprId(1)), ShuffleType.NATURAL, + 1L, 1L, Collections.emptySet()); + DistributionSpecHash right = new DistributionSpecHash(Lists.newArrayList(new ExprId(2)), ShuffleType.NATURAL, + 2L, 2L, Collections.emptySet()); + + Expression leftKey1 = new SlotReference(new ExprId(1), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey1 = new SlotReference(new ExprId(2), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression leftKey2 = new SlotReference(new ExprId(3), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + Expression rightKey2 = new SlotReference(new ExprId(4), "c1", + TinyIntType.INSTANCE, false, Lists.newArrayList()); + + List conjuncts; + + // key same with distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey1), new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key contains distribute key, and NOT have distribute key = distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey1, rightKey2), new EqualTo(leftKey2, rightKey1)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + + // key not contains distribute key + conjuncts = Lists.newArrayList(new EqualTo(leftKey2, rightKey2)); + Assertions.assertFalse(JoinUtils.couldColocateJoin(left, right, conjuncts)); + } + } +} diff --git a/fe/pom.xml b/fe/pom.xml index e88c6b383ff5a2..b4ce8194482d70 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -382,6 +382,7 @@ under the License. 3.15.0 2.29.26 0.1.4 + 4.11.0 @@ -1702,6 +1703,16 @@ under the License. semver4j ${semver4j.version} + + org.mockito + mockito-core + ${mockito.version} + + + org.mockito + mockito-inline + ${mockito.version} + From dc49270319c8c1d4260a64057204820eb64632b0 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 12 Aug 2025 10:23:14 +0800 Subject: [PATCH 367/572] branch-3.0: [Fix](mow) Should use rowset's schema when calculate delete bitmaps between segments (#54351) (#54368) pick https://github.com/apache/doris/pull/54351 --- .../cloud_engine_calc_delete_bitmap_task.cpp | 4 +- be/src/olap/base_tablet.cpp | 16 ++-- be/src/olap/base_tablet.h | 3 +- be/src/olap/calc_delete_bitmap_executor.cpp | 6 +- be/src/olap/calc_delete_bitmap_executor.h | 2 +- be/src/olap/rowset_builder.cpp | 8 +- be/src/olap/txn_manager.cpp | 3 +- be/src/service/backend_service.cpp | 4 +- .../test_mow_alter_seq_multi_segments.out | 10 ++ .../test_mow_alter_seq_multi_segments.groovy | 94 +++++++++++++++++++ 10 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 regression-test/data/fault_injection_p0/test_mow_alter_seq_multi_segments.out create mode 100644 regression-test/suites/fault_injection_p0/test_mow_alter_seq_multi_segments.groovy diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp index dc8ecaf26b451a..67dac3f7052a8b 100644 --- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp +++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp @@ -272,8 +272,8 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const { LOG_INFO("inject error when CloudTabletCalcDeleteBitmapTask::_handle_rowset"); return Status::MemoryLimitExceeded("injected MemoryLimitExceeded error"); }); - RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(), - segments, delete_bitmap)); + RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments( + rowset->tablet_schema(), rowset->rowset_id(), segments, delete_bitmap)); } status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id, txn_expiration); diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 6738a9c41c2c33..49d1f2851f0d09 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -368,8 +368,8 @@ void BaseTablet::generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta) } Status BaseTablet::calc_delete_bitmap_between_segments( - const RowsetId& rowset_id, const std::vector& segments, - DeleteBitmapPtr delete_bitmap) { + TabletSchemaSPtr schema, const RowsetId& rowset_id, + const std::vector& segments, DeleteBitmapPtr delete_bitmap) { size_t const num_segments = segments.size(); if (num_segments < 2) { return Status::OK(); @@ -377,12 +377,12 @@ Status BaseTablet::calc_delete_bitmap_between_segments( OlapStopWatch watch; size_t seq_col_length = 0; - if (_tablet_meta->tablet_schema()->has_sequence_col()) { - auto seq_col_idx = _tablet_meta->tablet_schema()->sequence_col_idx(); - seq_col_length = _tablet_meta->tablet_schema()->column(seq_col_idx).length() + 1; + if (schema->has_sequence_col()) { + auto seq_col_idx = schema->sequence_col_idx(); + seq_col_length = schema->column(seq_col_idx).length() + 1; } size_t rowid_length = 0; - if (!_tablet_meta->tablet_schema()->cluster_key_idxes().empty()) { + if (!schema->cluster_key_idxes().empty()) { rowid_length = PrimaryKeyIndexReader::ROW_ID_LENGTH; } @@ -1499,8 +1499,8 @@ Status BaseTablet::update_delete_bitmap_without_lock( // calculate delete bitmap between segments if necessary. DeleteBitmapPtr delete_bitmap = std::make_shared(self->tablet_id()); - RETURN_IF_ERROR(self->calc_delete_bitmap_between_segments(rowset->rowset_id(), segments, - delete_bitmap)); + RETURN_IF_ERROR(self->calc_delete_bitmap_between_segments( + rowset->tablet_schema(), rowset->rowset_id(), segments, delete_bitmap)); // get all base rowsets to calculate on std::vector specified_rowsets; diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h index 6c797e0478bcf7..4779600364da3b 100644 --- a/be/src/olap/base_tablet.h +++ b/be/src/olap/base_tablet.h @@ -186,7 +186,8 @@ class BaseTablet { RowsetWriter* rowset_writer); Status calc_delete_bitmap_between_segments( - const RowsetId& rowset_id, const std::vector& segments, + TabletSchemaSPtr schema, const RowsetId& rowset_id, + const std::vector& segments, DeleteBitmapPtr delete_bitmap); static Status commit_phase_update_delete_bitmap( diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp b/be/src/olap/calc_delete_bitmap_executor.cpp index 89e668c4c9480a..0cb6b5c0078b3b 100644 --- a/be/src/olap/calc_delete_bitmap_executor.cpp +++ b/be/src/olap/calc_delete_bitmap_executor.cpp @@ -58,7 +58,8 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetSharedPtr cur_ }); } -Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetId rowset_id, +Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, TabletSchemaSPtr schema, + RowsetId rowset_id, const std::vector& segments, DeleteBitmapPtr delete_bitmap) { { @@ -68,7 +69,8 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetId rowset_id, } return _thread_token->submit_func([=, this]() { SCOPED_ATTACH_TASK(_query_thread_context); - auto st = tablet->calc_delete_bitmap_between_segments(rowset_id, segments, delete_bitmap); + auto st = tablet->calc_delete_bitmap_between_segments(schema, rowset_id, segments, + delete_bitmap); if (!st.ok()) { LOG(WARNING) << "failed to calc delete bitmap between segments, tablet_id: " << tablet->tablet_id() << " rowset: " << rowset_id diff --git a/be/src/olap/calc_delete_bitmap_executor.h b/be/src/olap/calc_delete_bitmap_executor.h index 5d471e2587e7f6..4dccdf001439f9 100644 --- a/be/src/olap/calc_delete_bitmap_executor.h +++ b/be/src/olap/calc_delete_bitmap_executor.h @@ -56,7 +56,7 @@ class CalcDeleteBitmapToken { DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer); // calculate delete bitmap between `segments` - Status submit(BaseTabletSPtr tablet, RowsetId rowset_id, + Status submit(BaseTabletSPtr tablet, TabletSchemaSPtr schema, RowsetId rowset_id, const std::vector& segments, DeleteBitmapPtr delete_bitmap); diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index e9b518eaae02e3..dfe4da0d55fdd2 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -271,11 +271,11 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() { if (segments.size() > 1) { // calculate delete bitmap between segments if (config::enable_calc_delete_bitmap_between_segments_concurrently) { - RETURN_IF_ERROR(_calc_delete_bitmap_token->submit(_tablet, _rowset->rowset_id(), - segments, _delete_bitmap)); + RETURN_IF_ERROR(_calc_delete_bitmap_token->submit( + _tablet, _tablet_schema, _rowset->rowset_id(), segments, _delete_bitmap)); } else { - RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments(_rowset->rowset_id(), - segments, _delete_bitmap)); + RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments( + _tablet_schema, _rowset->rowset_id(), segments, _delete_bitmap)); } } diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp index f47bd776fac0b5..718df6c0c095b0 100644 --- a/be/src/olap/txn_manager.cpp +++ b/be/src/olap/txn_manager.cpp @@ -549,7 +549,8 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId partition_id, std::vector segments; RETURN_IF_ERROR(std::static_pointer_cast(rowset)->load_segments(&segments)); RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments( - rowset->rowset_id(), segments, tablet_txn_info->delete_bitmap)); + rowset->tablet_schema(), rowset->rowset_id(), segments, + tablet_txn_info->delete_bitmap)); } RETURN_IF_ERROR( diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index 116d4951723b0a..7be7642e753503 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -591,8 +591,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg* arg) { } if (segments.size() > 1) { // calculate delete bitmap between segments - status = local_tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(), - segments, delete_bitmap); + status = local_tablet->calc_delete_bitmap_between_segments( + rowset->tablet_schema(), rowset->rowset_id(), segments, delete_bitmap); if (!status) { LOG(WARNING) << "failed to calculate delete bitmap" << ". tablet_id: " << local_tablet->tablet_id() diff --git a/regression-test/data/fault_injection_p0/test_mow_alter_seq_multi_segments.out b/regression-test/data/fault_injection_p0/test_mow_alter_seq_multi_segments.out new file mode 100644 index 00000000000000..aa44268bf31897 --- /dev/null +++ b/regression-test/data/fault_injection_p0/test_mow_alter_seq_multi_segments.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 1 1 + +-- !sql -- +4064 + +-- !dup_key_count -- +0 + diff --git a/regression-test/suites/fault_injection_p0/test_mow_alter_seq_multi_segments.groovy b/regression-test/suites/fault_injection_p0/test_mow_alter_seq_multi_segments.groovy new file mode 100644 index 00000000000000..d6b17bd5c73a9b --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_mow_alter_seq_multi_segments.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_mow_alter_seq_multi_segments", "nonConcurrent") { + def table1 = "test_mow_alter_seq_multi_segments" + sql "DROP TABLE IF EXISTS ${table1} FORCE;" + sql """ CREATE TABLE IF NOT EXISTS ${table1} ( + `k1` int NOT NULL, + `c1` int, + `c2` int + )UNIQUE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", + "replication_num" = "1"); """ + + sql """insert into ${table1} values(1,1,1);""" + qt_sql "select * from ${table1} order by k1;" + sql """alter table ${table1} ENABLE FEATURE "SEQUENCE_LOAD" WITH PROPERTIES ("function_column.sequence_type"="int");""" + + // to cause multi segments and segment compaction + def customBeConfig = [ + doris_scanner_row_bytes : 1 + ] + + setBeConfigTemporary(customBeConfig) { + try { + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + // batch_size is 4164 in csv_reader.cpp + // _batch_size is 8192 in vtablet_writer.cpp + // to cause multi segments + GetDebugPoint().enableDebugPointForAllBEs("MemTable.need_flush") + + Thread.sleep(1000) + + int rows = 4064 + // load data that will have multi segments and there are duplicate keys between segments + String content = "" + (1..rows).each { + int x = it + content += "${x},${x},${x},1\n" + } + (1..rows).each { + int x = it + content += "${x},${x},${x},2\n" + } + def t1 = Thread.start { + streamLoad { + table "${table1}" + set 'column_separator', ',' + set 'columns', 'k1,c1,c2,seq' + set 'function_column.sequence_col', 'seq' + inputStream new ByteArrayInputStream(content.getBytes()) + time 30000 + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + def json = parseJson(result) + assert "success" == json.Status.toLowerCase() + } + } + } + t1.join() + qt_sql "select count() from ${table1};" + // qt_sql "select *,__DORIS_VERSION_COL__ as ver, __DORIS_DELETE_SIGN__ as del,__DORIS_SEQUENCE_COL__ as seq from ${table1} where k1<=10 order by k1,__DORIS_VERSION_COL__;" + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY';" + qt_dup_key_count "select count() from (select k1,count() as cnt from ${table1} group by k1 having cnt > 1) A;" + } catch(Exception e) { + logger.info(e.getMessage()) + throw e + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + } + } +} From b389dcc73fa9e1347b778755d497d103fb6cb149 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:24:18 +0800 Subject: [PATCH 368/572] branch-3.0: [fix](mtmv)Fix the mapping issue where base tables could still reference a dropped MTMV #53383 (#54080) Cherry-picked from #53383 Co-authored-by: zhangdong --- .../org/apache/doris/catalog/Database.java | 7 +- .../doris/mtmv/MTMVRelationManager.java | 3 + .../org/apache/doris/mtmv/MTMVService.java | 18 +++- .../apache/doris/mtmv/MTMVConcurrentTest.java | 85 +++++++++++++++++++ 4 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVConcurrentTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index a5d8d5e9665514..10e20da5efbe59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -457,13 +457,14 @@ public void unregisterTable(String tableName) { } Table table = getTableNullable(tableName); if (table != null) { - if (table instanceof MTMV) { - Env.getCurrentEnv().getMtmvService().unregisterMTMV((MTMV) table); - } this.nameToTable.remove(tableName); this.idToTable.remove(table.getId()); this.lowerCaseToTableName.remove(tableName.toLowerCase()); table.markDropped(); + // will check mtmv if exist by markDrop, so unregisterMTMV() need after markDropped() + if (table instanceof MTMV) { + Env.getCurrentEnv().getMtmvService().unregisterMTMV((MTMV) table); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index ba498636d73619..05ab6eb01f1946 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -251,6 +251,9 @@ public void refreshMTMV(RefreshMTMVInfo info) throws DdlException, MetaNotFoundE public void refreshComplete(MTMV mtmv, MTMVRelation relation, MTMVTask task) { if (task.getStatus() == TaskStatus.SUCCESS) { Objects.requireNonNull(relation); + if (mtmv.isDropped) { + return; + } refreshMTMVCache(relation, new BaseTableInfo(mtmv)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java index 2ee42c0aa2c833..9496bff0a51917 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java @@ -87,8 +87,13 @@ public void registerMTMV(MTMV mtmv, Long dbId) { public void unregisterMTMV(MTMV mtmv) { Objects.requireNonNull(mtmv, "mtmv can not be null"); LOG.info("deregisterMTMV: " + mtmv.getName()); - for (MTMVHookService mtmvHookService : hooks.values()) { - mtmvHookService.unregisterMTMV(mtmv); + mtmv.writeMvLock(); + try { + for (MTMVHookService mtmvHookService : hooks.values()) { + mtmvHookService.unregisterMTMV(mtmv); + } + } finally { + mtmv.writeMvUnlock(); } } @@ -147,8 +152,13 @@ public void refreshComplete(MTMV mtmv, MTMVRelation cache, MTMVTask task) { Objects.requireNonNull(mtmv, "mtmv can not be null"); Objects.requireNonNull(task, "task can not be null"); LOG.info("refreshComplete: " + mtmv.getName()); - for (MTMVHookService mtmvHookService : hooks.values()) { - mtmvHookService.refreshComplete(mtmv, cache, task); + mtmv.writeMvLock(); + try { + for (MTMVHookService mtmvHookService : hooks.values()) { + mtmvHookService.refreshComplete(mtmv, cache, task); + } + } finally { + mtmv.writeMvUnlock(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVConcurrentTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVConcurrentTest.java new file mode 100644 index 00000000000000..97f341e8e593ec --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVConcurrentTest.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Table; +import org.apache.doris.job.common.TaskStatus; +import org.apache.doris.job.extensions.mtmv.MTMVTask; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +public class MTMVConcurrentTest extends TestWithFeService { + @Test + public void testAlterMTMV() throws Exception { + createDatabaseAndUse("mtmv_concurrent_test"); + + createTable("CREATE TABLE `stu` (`sid` int(32) NULL, `sname` varchar(32) NULL)\n" + + "ENGINE=OLAP\n" + + "DUPLICATE KEY(`sid`)\n" + + "DISTRIBUTED BY HASH(`sid`) BUCKETS 1\n" + + "PROPERTIES ('replication_allocation' = 'tag.location.default: 1')"); + + createMvByNereids("CREATE MATERIALIZED VIEW mv_a BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + "DISTRIBUTED BY HASH(`sid`) BUCKETS 1\n" + + "PROPERTIES ('replication_allocation' = 'tag.location.default: 1') " + + "AS select * from stu limit 1"); + + + MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager(); + Table table = Env.getCurrentInternalCatalog().getDb("mtmv_concurrent_test").get() + .getTableOrMetaException("stu"); + BaseTableInfo baseTableInfo = new BaseTableInfo(table); + MTMV mtmv = (MTMV) Env.getCurrentInternalCatalog().getDb("mtmv_concurrent_test").get() + .getTableOrMetaException("mv_a"); + MTMVRelation relation = mtmv.getRelation(); + Set mtmvsByBaseTable = relationManager.getMtmvsByBaseTable(baseTableInfo); + Assertions.assertEquals(1, mtmvsByBaseTable.size()); + MTMVTask mtmvTask = new MTMVTask(mtmv, relation, null); + mtmvTask.setStatus(TaskStatus.SUCCESS); + + // Create threads for concurrent operations + Thread dropThread = new Thread(() -> { + try { + dropMvByNereids("drop materialized view mv_a"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Thread refreshThread = new Thread(() -> { + relationManager.refreshComplete(mtmv, relation, mtmvTask); + }); + + // Start both threads + dropThread.start(); + refreshThread.start(); + + // Wait for both threads to complete + dropThread.join(); + refreshThread.join(); + + mtmvsByBaseTable = relationManager.getMtmvsByBaseTable(baseTableInfo); + Assertions.assertEquals(0, mtmvsByBaseTable.size()); + } +} From d74d2c1bc27103250c525bff5714e203ba21470a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:28:05 +0800 Subject: [PATCH 369/572] branch-3.0: [fix](report) disk should be reported before tablet #54190 (#54237) Cherry-picked from #54190 Co-authored-by: Yongqiang YANG --- be/src/common/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 4bddb99e0a413b..491929a7a83362 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -245,7 +245,7 @@ DEFINE_mInt32(report_task_interval_seconds, "10"); // the interval time(seconds) for refresh storage policy from FE DEFINE_mInt32(storage_refresh_storage_policy_task_interval_seconds, "5"); // the interval time(seconds) for agent report disk state to FE -DEFINE_mInt32(report_disk_state_interval_seconds, "60"); +DEFINE_mInt32(report_disk_state_interval_seconds, "30"); // the interval time(seconds) for agent report olap table to FE DEFINE_mInt32(report_tablet_interval_seconds, "60"); // the max download speed(KB/s) From 8f0d86b2a3c693d6fd8b913bc4d13d452e097613 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 12 Aug 2025 10:30:06 +0800 Subject: [PATCH 370/572] =?UTF-8?q?[refactor](pipeline)=20Re-construct=20o?= =?UTF-8?q?wnership=20of=20pipeline=20components=20(#=E2=80=A6=20(#52873)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …49753) In this PR, ownership of core components in pipeline engine is re-constructed following the rules below: ![image](https://github.com/user-attachments/assets/04f842b7-f052-4e3c-9005-7df3103b4d7b) ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/pipeline/dependency.cpp | 18 +-- be/src/pipeline/dependency.h | 8 +- be/src/pipeline/pipeline.h | 1 + be/src/pipeline/pipeline_fragment_context.cpp | 15 +-- be/src/pipeline/pipeline_fragment_context.h | 2 +- be/src/pipeline/pipeline_task.cpp | 103 ++++++++++++------ be/src/pipeline/pipeline_task.h | 32 +++--- be/src/pipeline/task_queue.cpp | 20 ++-- be/src/pipeline/task_queue.h | 28 ++--- be/src/pipeline/task_scheduler.cpp | 38 ++++--- be/src/pipeline/task_scheduler.h | 2 +- 11 files changed, 159 insertions(+), 108 deletions(-) diff --git a/be/src/pipeline/dependency.cpp b/be/src/pipeline/dependency.cpp index b15005e03a8863..945ee3d4521569 100644 --- a/be/src/pipeline/dependency.cpp +++ b/be/src/pipeline/dependency.cpp @@ -46,8 +46,9 @@ Dependency* BasicSharedState::create_sink_dependency(int dest_id, int node_id, s return sink_deps.back().get(); } -void Dependency::_add_block_task(PipelineTask* task) { - DCHECK(_blocked_task.empty() || _blocked_task[_blocked_task.size() - 1] != task) +void Dependency::_add_block_task(std::shared_ptr task) { + DCHECK(_blocked_task.empty() || _blocked_task[_blocked_task.size() - 1].lock() == nullptr || + _blocked_task[_blocked_task.size() - 1].lock().get() != task.get()) << "Duplicate task: " << task->debug_string(); _blocked_task.push_back(task); } @@ -57,7 +58,7 @@ void Dependency::set_ready() { return; } _watcher.stop(); - std::vector local_block_task {}; + std::vector> local_block_task {}; { std::unique_lock lc(_task_lock); if (_ready) { @@ -66,12 +67,15 @@ void Dependency::set_ready() { _ready = true; local_block_task.swap(_blocked_task); } - for (auto* task : local_block_task) { - task->wake_up(); + for (auto task : local_block_task) { + if (auto t = task.lock()) { + std::unique_lock lc(_task_lock); + t->wake_up(); + } } } -Dependency* Dependency::is_blocked_by(PipelineTask* task) { +Dependency* Dependency::is_blocked_by(std::shared_ptr task) { std::unique_lock lc(_task_lock); auto ready = _ready.load(); if (!ready && task) { @@ -105,7 +109,7 @@ std::string RuntimeFilterDependency::debug_string(int indentation_level) { return fmt::to_string(debug_string_buffer); } -Dependency* RuntimeFilterDependency::is_blocked_by(PipelineTask* task) { +Dependency* RuntimeFilterDependency::is_blocked_by(std::shared_ptr task) { std::unique_lock lc(_task_lock); auto ready = _ready.load(); if (!ready && task) { diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h index ea6cacf51b1067..9f07076497f11f 100644 --- a/be/src/pipeline/dependency.h +++ b/be/src/pipeline/dependency.h @@ -106,7 +106,7 @@ class Dependency : public std::enable_shared_from_this { [[nodiscard]] int64_t watcher_elapse_time() { return _watcher.elapsed_time(); } // Which dependency current pipeline task is blocked by. `nullptr` if this dependency is ready. - [[nodiscard]] virtual Dependency* is_blocked_by(PipelineTask* task = nullptr); + [[nodiscard]] virtual Dependency* is_blocked_by(std::shared_ptr task = nullptr); // Notify downstream pipeline tasks this dependency is ready. void set_ready(); void set_ready_to_read() { @@ -151,7 +151,7 @@ class Dependency : public std::enable_shared_from_this { } protected: - void _add_block_task(PipelineTask* task); + void _add_block_task(std::shared_ptr task); const int _id; const int _node_id; @@ -162,7 +162,7 @@ class Dependency : public std::enable_shared_from_this { MonotonicStopWatch _watcher; std::mutex _task_lock; - std::vector _blocked_task; + std::vector> _blocked_task; // If `_always_ready` is true, `block()` will never block tasks. std::atomic _always_ready = false; @@ -282,7 +282,7 @@ class RuntimeFilterDependency final : public Dependency { : Dependency(id, node_id, name), _runtime_filter(runtime_filter) {} std::string debug_string(int indentation_level = 0) override; - Dependency* is_blocked_by(PipelineTask* task) override; + Dependency* is_blocked_by(std::shared_ptr task) override; private: const IRuntimeFilter* _runtime_filter = nullptr; diff --git a/be/src/pipeline/pipeline.h b/be/src/pipeline/pipeline.h index 061a62ea99b330..abfe883a804613 100644 --- a/be/src/pipeline/pipeline.h +++ b/be/src/pipeline/pipeline.h @@ -135,6 +135,7 @@ class Pipeline : public std::enable_shared_from_this { } int num_tasks_of_parent() const { return _num_tasks_of_parent; } + std::string& name() { return _name; } private: void _init_profile(); diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index a8e62f4f6d3524..950dbd87a4390c 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -133,18 +133,17 @@ PipelineFragmentContext::~PipelineFragmentContext() { // The memory released by the query end is recorded in the query mem tracker. SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(_query_ctx->query_mem_tracker); auto st = _query_ctx->exec_status(); - _query_ctx.reset(); for (size_t i = 0; i < _tasks.size(); i++) { if (!_tasks[i].empty()) { _call_back(_tasks[i].front()->runtime_state(), &st); } } - _tasks.clear(); for (auto& runtime_states : _task_runtime_states) { for (auto& runtime_state : runtime_states) { runtime_state.reset(); } } + _tasks.clear(); _dag.clear(); _pip_id_to_pipeline.clear(); _pipelines.clear(); @@ -154,6 +153,7 @@ PipelineFragmentContext::~PipelineFragmentContext() { _runtime_filter_states.clear(); _runtime_filter_mgr_map.clear(); _op_id_to_le_state.clear(); + _query_ctx.reset(); } bool PipelineFragmentContext::is_timeout(timespec now) const { @@ -445,10 +445,11 @@ Status PipelineFragmentContext::_build_pipeline_tasks(const doris::TPipelineFrag auto cur_task_id = _total_tasks++; task_runtime_state->set_task_id(cur_task_id); task_runtime_state->set_task_num(pipeline->num_tasks()); - auto task = std::make_unique(pipeline, cur_task_id, - task_runtime_state.get(), this, - pipeline_id_to_profile[pip_idx].get(), - get_local_exchange_state(pipeline), i); + auto task = std::make_shared( + pipeline, cur_task_id, task_runtime_state.get(), + std::dynamic_pointer_cast(shared_from_this()), + pipeline_id_to_profile[pip_idx].get(), get_local_exchange_state(pipeline), + i); pipeline->incr_created_tasks(i, task.get()); task_runtime_state->set_task(task.get()); pipeline_id_to_task.insert({pipeline->id(), task.get()}); @@ -1671,7 +1672,7 @@ Status PipelineFragmentContext::submit() { auto* scheduler = _query_ctx->get_pipe_exec_scheduler(); for (auto& task : _tasks) { for (auto& t : task) { - st = scheduler->schedule_task(t.get()); + st = scheduler->schedule_task(t); if (!st) { cancel(Status::InternalError("submit context to executor fail")); std::lock_guard l(_task_mutex); diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h index f119384b2fbb95..b08a3f90f95c4e 100644 --- a/be/src/pipeline/pipeline_fragment_context.h +++ b/be/src/pipeline/pipeline_fragment_context.h @@ -228,7 +228,7 @@ class PipelineFragmentContext : public TaskExecutionContext { OperatorPtr _root_op = nullptr; // this is a [n * m] matrix. n is parallelism of pipeline engine and m is the number of pipelines. - std::vector>> _tasks; + std::vector>> _tasks; // TODO: remove the _sink and _multi_cast_stream_sink_senders to set both // of it in pipeline task not the fragment_context diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp index 835387bc84ca26..c94ddcc6f41cce 100644 --- a/be/src/pipeline/pipeline_task.cpp +++ b/be/src/pipeline/pipeline_task.cpp @@ -48,12 +48,19 @@ namespace doris::pipeline { PipelineTask::PipelineTask( PipelinePtr& pipeline, uint32_t task_id, RuntimeState* state, - PipelineFragmentContext* fragment_context, RuntimeProfile* parent_profile, + std::shared_ptr fragment_context, RuntimeProfile* parent_profile, std::map, std::shared_ptr>> le_state_map, int task_idx) - : _index(task_id), + : +#ifdef BE_TEST + _query_id(fragment_context ? fragment_context->get_query_id() : TUniqueId()), +#else + _query_id(fragment_context->get_query_id()), +#endif + _pip_id(pipeline->id()), + _index(task_id), _pipeline(pipeline), _opened(false), _state(state), @@ -65,7 +72,8 @@ PipelineTask::PipelineTask( _sink(pipeline->sink_shared_pointer()), _le_state_map(std::move(le_state_map)), _task_idx(task_idx), - _execution_dep(state->get_query_ctx()->get_execution_dependency()) { + _execution_dep(state->get_query_ctx()->get_execution_dependency()), + _pipeline_name(_pipeline->name()) { _pipeline_task_watcher.start(); auto shared_state = _sink->create_shared_state(); @@ -120,8 +128,12 @@ Status PipelineTask::prepare(const TPipelineInstanceParams& local_params, const std::unique_lock lc(_dependency_lock); filter_dependencies.swap(_filter_dependencies); } - if (query_context()->is_cancelled()) { - clear_blocking_state(); + if (auto fragment = _fragment_context.lock()) { + if (fragment->get_query_ctx()->is_cancelled()) { + clear_blocking_state(); + } + } else { + return Status::InternalError("Fragment already finished! Query: {}", print_id(_query_id)); } return Status::OK(); } @@ -225,14 +237,14 @@ bool PipelineTask::_wait_to_start() { // Before task starting, we should make sure // 1. Execution dependency is ready (which is controlled by FE 2-phase commit) // 2. Runtime filter dependencies are ready - _blocked_dep = _execution_dep->is_blocked_by(this); + _blocked_dep = _execution_dep->is_blocked_by(shared_from_this()); if (_blocked_dep != nullptr) { static_cast(_blocked_dep)->start_watcher(); return true; } for (auto* op_dep : _filter_dependencies) { - _blocked_dep = op_dep->is_blocked_by(this); + _blocked_dep = op_dep->is_blocked_by(shared_from_this()); if (_blocked_dep != nullptr) { _blocked_dep->start_watcher(); return true; @@ -253,7 +265,7 @@ bool PipelineTask::_is_blocked() { for (int i = _read_dependencies.size() - 1; i >= 0; i--) { // `_read_dependencies` is organized according to operators. For each operator, running condition is met iff all dependencies are ready. for (auto* dep : _read_dependencies[i]) { - _blocked_dep = dep->is_blocked_by(this); + _blocked_dep = dep->is_blocked_by(shared_from_this()); if (_blocked_dep != nullptr) { _blocked_dep->start_watcher(); return true; @@ -272,7 +284,7 @@ bool PipelineTask::_is_blocked() { } for (auto* op_dep : _write_dependencies) { - _blocked_dep = op_dep->is_blocked_by(this); + _blocked_dep = op_dep->is_blocked_by(shared_from_this()); if (_blocked_dep != nullptr) { _blocked_dep->start_watcher(); return true; @@ -282,6 +294,10 @@ bool PipelineTask::_is_blocked() { } Status PipelineTask::execute(bool* eos) { + auto fragment_context = _fragment_context.lock(); + if (!fragment_context) { + return Status::InternalError("Fragment already finished! Query: {}", print_id(_query_id)); + } if (_eos) { *eos = true; return Status::OK(); @@ -304,11 +320,11 @@ Status PipelineTask::execute(bool* eos) { } int64_t delta_cpu_time = cpu_time_stop_watch.elapsed_time(); _task_cpu_timer->update(delta_cpu_time); - auto cpu_qs = query_context()->get_cpu_statistics(); + auto cpu_qs = fragment_context->get_query_ctx()->get_cpu_statistics(); if (cpu_qs) { cpu_qs->add_cpu_nanos(delta_cpu_time); } - query_context()->update_cpu_time(delta_cpu_time); + fragment_context->get_query_ctx()->update_cpu_time(delta_cpu_time); }}; if (_wait_to_start()) { if (config::enable_prefetch_tablet) { @@ -318,7 +334,7 @@ Status PipelineTask::execute(bool* eos) { } // The status must be runnable - if (!_opened && !_fragment_context->is_canceled()) { + if (!_opened && !fragment_context->is_canceled()) { DBUG_EXECUTE_IF("PipelineTask::execute.open_sleep", { auto required_pipeline_id = DebugPoints::instance()->get_debug_param_or_default( @@ -350,7 +366,7 @@ Status PipelineTask::execute(bool* eos) { _task_profile->add_info_string("TaskState", "Runnable"); _task_profile->add_info_string("BlockedByDependency", ""); - while (!_fragment_context->is_canceled()) { + while (!fragment_context->is_canceled()) { SCOPED_RAW_TIMER(&time_spent); if (_is_blocked()) { return Status::OK(); @@ -359,7 +375,7 @@ Status PipelineTask::execute(bool* eos) { /// When a task is cancelled, /// its blocking state will be cleared and it will transition to a ready state (though it is not truly ready). /// Here, checking whether it is cancelled to prevent tasks in a blocking state from being re-executed. - if (_fragment_context->is_canceled()) { + if (fragment_context->is_canceled()) { break; } @@ -428,7 +444,7 @@ Status PipelineTask::execute(bool* eos) { } } - RETURN_IF_ERROR(get_task_queue()->push_back(this)); + RETURN_IF_ERROR(get_task_queue()->push_back(shared_from_this())); return Status::OK(); } @@ -489,12 +505,34 @@ bool PipelineTask::should_revoke_memory(RuntimeState* state, int64_t revocable_m } } +void PipelineTask::stop_if_finished() { + auto fragment = _fragment_context.lock(); + if (!fragment) { + return; + } + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(fragment->get_query_ctx()->query_mem_tracker); + if (auto sink = _sink) { + if (sink->is_finished(_state)) { + clear_blocking_state(); + } + } +} + void PipelineTask::finalize() { + auto fragment = _fragment_context.lock(); + if (!fragment) { + return; + } + SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(fragment->get_query_ctx()->query_mem_tracker); std::unique_lock lc(_dependency_lock); _finalized = true; _sink_shared_state.reset(); _op_shared_states.clear(); _le_state_map.clear(); + _block.reset(); + _operators.clear(); + _sink.reset(); + _pipeline.reset(); } Status PipelineTask::close(Status exec_status, bool close_sink) { @@ -529,31 +567,37 @@ Status PipelineTask::close(Status exec_status, bool close_sink) { } std::string PipelineTask::debug_string() { - std::unique_lock lc(_dependency_lock); fmt::memory_buffer debug_string_buffer; - fmt::format_to(debug_string_buffer, "QueryId: {}\n", print_id(query_context()->query_id())); + fmt::format_to(debug_string_buffer, "QueryId: {}\n", print_id(_query_id)); fmt::format_to(debug_string_buffer, "InstanceId: {}\n", print_id(_state->fragment_instance_id())); + fmt::format_to( + debug_string_buffer, + "PipelineTask[this = {}, id = {}, open = {}, eos = {}, finalized = {}, dry run = " + "{}, _wake_up_early = {}, is running = {}]", + (void*)this, _index, _opened, _eos, _finalized, _dry_run, _wake_up_early.load(), + is_running()); + std::unique_lock lc(_dependency_lock); auto* cur_blocked_dep = _blocked_dep; - auto elapsed = _fragment_context->elapsed_time() / 1000000000.0; + auto fragment = _fragment_context.lock(); + if (is_finalized() || !fragment) { + fmt::format_to(debug_string_buffer, " pipeline name = {}", _pipeline_name); + return fmt::to_string(debug_string_buffer); + } + auto elapsed = fragment->elapsed_time() / NANOS_PER_SEC; fmt::format_to(debug_string_buffer, - "PipelineTask[this = {}, id = {}, open = {}, eos = {}, finish = {}, dry run = " - "{}, elapse time = {}s, _wake_up_early = {}], block dependency = {}, is " - "running = {}\noperators: ", - (void*)this, _index, _opened, _eos, _finalized, _dry_run, elapsed, - _wake_up_early.load(), - cur_blocked_dep && !_finalized ? cur_blocked_dep->debug_string() : "NULL", - is_running()); + " elapse time = {}s, block dependency = [{}]\noperators: ", elapsed, + cur_blocked_dep && !is_finalized() ? cur_blocked_dep->debug_string() : "NULL"); for (size_t i = 0; i < _operators.size(); i++) { fmt::format_to(debug_string_buffer, "\n{}", _opened && !_finalized ? _operators[i]->debug_string(_state, i) : _operators[i]->debug_string(i)); } fmt::format_to(debug_string_buffer, "\n{}\n", - _opened && !_finalized ? _sink->debug_string(_state, _operators.size()) - : _sink->debug_string(_operators.size())); + _opened && !is_finalized() ? _sink->debug_string(_state, _operators.size()) + : _sink->debug_string(_operators.size())); if (_finalized) { return fmt::to_string(debug_string_buffer); } @@ -588,10 +632,7 @@ std::string PipelineTask::debug_string() { void PipelineTask::wake_up() { // call by dependency - static_cast(get_task_queue()->push_back(this)); + static_cast(get_task_queue()->push_back(shared_from_this())); } -QueryContext* PipelineTask::query_context() { - return _fragment_context->get_query_ctx(); -} } // namespace doris::pipeline diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h index 94a553e2fa14ab..15b2a254bbea4d 100644 --- a/be/src/pipeline/pipeline_task.h +++ b/be/src/pipeline/pipeline_task.h @@ -45,10 +45,11 @@ class TaskQueue; class PriorityTaskQueue; class Dependency; -class PipelineTask { +class PipelineTask : public std::enable_shared_from_this { public: PipelineTask(PipelinePtr& pipeline, uint32_t task_id, RuntimeState* state, - PipelineFragmentContext* fragment_context, RuntimeProfile* parent_profile, + std::shared_ptr fragment_context, + RuntimeProfile* parent_profile, std::map, std::shared_ptr>> le_state_map, @@ -63,9 +64,7 @@ class PipelineTask { // must be call after all pipeline task is finish to release resource Status close(Status exec_status, bool close_sink = true); - PipelineFragmentContext* fragment_context() { return _fragment_context; } - - QueryContext* query_context(); + std::weak_ptr& fragment_context() { return _fragment_context; } int get_previous_core_id() const { return _previous_schedule_id != -1 ? _previous_schedule_id @@ -87,7 +86,7 @@ class PipelineTask { bool is_pending_finish() { for (auto* fin_dep : _finish_dependencies) { - _blocked_dep = fin_dep->is_blocked_by(this); + _blocked_dep = fin_dep->is_blocked_by(shared_from_this()); if (_blocked_dep != nullptr) { _blocked_dep->start_watcher(); return true; @@ -138,10 +137,11 @@ class PipelineTask { void set_wake_up_early() { _wake_up_early = true; } void clear_blocking_state() { + auto fragment = _fragment_context.lock(); _state->get_query_ctx()->get_execution_dependency()->set_always_ready(); // We use a lock to assure all dependencies are not deconstructed here. std::unique_lock lc(_dependency_lock); - if (!_finalized) { + if (!_finalized && fragment) { _execution_dep->set_always_ready(); for (auto* dep : _filter_dependencies) { dep->set_always_ready(); @@ -229,13 +229,9 @@ class PipelineTask { std::string task_name() const { return fmt::format("task{}({})", _index, _pipeline->_name); } - void stop_if_finished() { - if (_sink->is_finished(_state)) { - clear_blocking_state(); - } - } + void stop_if_finished(); - PipelineId pipeline_id() const { return _pipeline->id(); } + PipelineId pipeline_id() const { return _pip_id; } bool wake_up_early() const { return _wake_up_early; } @@ -249,7 +245,9 @@ class PipelineTask { void _fresh_profile_counter(); Status _open(); - uint32_t _index; + const TUniqueId _query_id; + const PipelineId _pip_id; + const uint32_t _index; PipelinePtr _pipeline; bool _has_exceed_timeout = false; bool _opened; @@ -257,7 +255,7 @@ class PipelineTask { int _previous_schedule_id = -1; uint32_t _schedule_time = 0; std::unique_ptr _block; - PipelineFragmentContext* _fragment_context = nullptr; + std::weak_ptr _fragment_context; TaskQueue* _task_queue = nullptr; // used for priority queue @@ -320,6 +318,10 @@ class PipelineTask { std::atomic _running = false; std::atomic _eos = false; std::atomic _wake_up_early = false; + const std::string _pipeline_name; }; +using PipelineTaskSPtr = std::shared_ptr; +using PipelineTaskWPtr = std::weak_ptr; + } // namespace doris::pipeline diff --git a/be/src/pipeline/task_queue.cpp b/be/src/pipeline/task_queue.cpp index b91b74dfdef9ba..43984d03a9d4d4 100644 --- a/be/src/pipeline/task_queue.cpp +++ b/be/src/pipeline/task_queue.cpp @@ -30,7 +30,7 @@ namespace doris::pipeline { TaskQueue::~TaskQueue() = default; -PipelineTask* SubTaskQueue::try_take(bool is_steal) { +PipelineTaskSPtr SubTaskQueue::try_take(bool is_steal) { if (_queue.empty()) { return nullptr; } @@ -56,7 +56,7 @@ void PriorityTaskQueue::close() { DorisMetrics::instance()->pipeline_task_queue_size->increment(-_total_task_size); } -PipelineTask* PriorityTaskQueue::_try_take_unprotected(bool is_steal) { +PipelineTaskSPtr PriorityTaskQueue::_try_take_unprotected(bool is_steal) { if (_total_task_size == 0 || _closed) { return nullptr; } @@ -93,13 +93,13 @@ int PriorityTaskQueue::_compute_level(uint64_t runtime) { return SUB_QUEUE_LEVEL - 1; } -PipelineTask* PriorityTaskQueue::try_take(bool is_steal) { +PipelineTaskSPtr PriorityTaskQueue::try_take(bool is_steal) { // TODO other efficient lock? e.g. if get lock fail, return null_ptr std::unique_lock lock(_work_size_mutex); return _try_take_unprotected(is_steal); } -PipelineTask* PriorityTaskQueue::take(uint32_t timeout_ms) { +PipelineTaskSPtr PriorityTaskQueue::take(uint32_t timeout_ms) { std::unique_lock lock(_work_size_mutex); auto task = _try_take_unprotected(false); if (task) { @@ -114,7 +114,7 @@ PipelineTask* PriorityTaskQueue::take(uint32_t timeout_ms) { } } -Status PriorityTaskQueue::push(PipelineTask* task) { +Status PriorityTaskQueue::push(PipelineTaskSPtr task) { if (_closed) { return Status::InternalError("WorkTaskQueue closed"); } @@ -154,8 +154,8 @@ void MultiCoreTaskQueue::close() { } } -PipelineTask* MultiCoreTaskQueue::take(int core_id) { - PipelineTask* task = nullptr; +PipelineTaskSPtr MultiCoreTaskQueue::take(int core_id) { + PipelineTaskSPtr task = nullptr; while (!_closed) { DCHECK(_prio_task_queue_list->size() > core_id) << " list size: " << _prio_task_queue_list->size() << " core_id: " << core_id @@ -181,7 +181,7 @@ PipelineTask* MultiCoreTaskQueue::take(int core_id) { return task; } -PipelineTask* MultiCoreTaskQueue::_steal_take( +PipelineTaskSPtr MultiCoreTaskQueue::_steal_take( int core_id, std::vector>& prio_task_queue_list) { DCHECK(core_id < _core_size); int next_id = core_id; @@ -200,7 +200,7 @@ PipelineTask* MultiCoreTaskQueue::_steal_take( return nullptr; } -Status MultiCoreTaskQueue::push_back(PipelineTask* task) { +Status MultiCoreTaskQueue::push_back(PipelineTaskSPtr task) { int core_id = task->get_previous_core_id(); if (core_id < 0) { core_id = _next_core.fetch_add(1) % _core_size; @@ -208,7 +208,7 @@ Status MultiCoreTaskQueue::push_back(PipelineTask* task) { return push_back(task, core_id); } -Status MultiCoreTaskQueue::push_back(PipelineTask* task, int core_id) { +Status MultiCoreTaskQueue::push_back(PipelineTaskSPtr task, int core_id) { DCHECK(core_id < _core_size); task->put_in_runnable_queue(); return (*_prio_task_queue_list)[core_id]->push(task); diff --git a/be/src/pipeline/task_queue.h b/be/src/pipeline/task_queue.h index b389ebc2c51517..d8b4ee27a7f94f 100644 --- a/be/src/pipeline/task_queue.h +++ b/be/src/pipeline/task_queue.h @@ -40,13 +40,13 @@ class TaskQueue { virtual void close() = 0; // Get the task by core id. // TODO: To think the logic is useful? - virtual PipelineTask* take(int core_id) = 0; + virtual PipelineTaskSPtr take(int core_id) = 0; // push from scheduler - virtual Status push_back(PipelineTask* task) = 0; + virtual Status push_back(PipelineTaskSPtr task) = 0; // push from worker - virtual Status push_back(PipelineTask* task, int core_id) = 0; + virtual Status push_back(PipelineTaskSPtr task, int core_id) = 0; virtual void update_statistics(PipelineTask* task, int64_t time_spent) {} @@ -61,9 +61,9 @@ class SubTaskQueue { friend class PriorityTaskQueue; public: - void push_back(PipelineTask* task) { _queue.emplace(task); } + void push_back(PipelineTaskSPtr task) { _queue.emplace(task); } - PipelineTask* try_take(bool is_steal); + PipelineTaskSPtr try_take(bool is_steal); void set_level_factor(double level_factor) { _level_factor = level_factor; } @@ -79,7 +79,7 @@ class SubTaskQueue { bool empty() { return _queue.empty(); } private: - std::queue _queue; + std::queue _queue; // depends on LEVEL_QUEUE_TIME_FACTOR double _level_factor = 1; @@ -93,18 +93,18 @@ class PriorityTaskQueue { void close(); - PipelineTask* try_take(bool is_steal); + PipelineTaskSPtr try_take(bool is_steal); - PipelineTask* take(uint32_t timeout_ms = 0); + PipelineTaskSPtr take(uint32_t timeout_ms = 0); - Status push(PipelineTask* task); + Status push(PipelineTaskSPtr task); void inc_sub_queue_runtime(int level, uint64_t runtime) { _sub_queues[level].inc_runtime(runtime); } private: - PipelineTask* _try_take_unprotected(bool is_steal); + PipelineTaskSPtr _try_take_unprotected(bool is_steal); static constexpr auto LEVEL_QUEUE_TIME_FACTOR = 2; static constexpr size_t SUB_QUEUE_LEVEL = 6; SubTaskQueue _sub_queues[SUB_QUEUE_LEVEL]; @@ -133,17 +133,17 @@ class MultiCoreTaskQueue : public TaskQueue { void close() override; // Get the task by core id. - PipelineTask* take(int core_id) override; + PipelineTaskSPtr take(int core_id) override; // TODO combine these methods to `push_back(task, core_id = -1)` - Status push_back(PipelineTask* task) override; + Status push_back(PipelineTaskSPtr task) override; - Status push_back(PipelineTask* task, int core_id) override; + Status push_back(PipelineTaskSPtr task, int core_id) override; void update_statistics(PipelineTask* task, int64_t time_spent) override; private: - PipelineTask* _steal_take( + PipelineTaskSPtr _steal_take( int core_id, std::vector>& prio_task_queue_list); std::shared_ptr>> _prio_task_queue_list; diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp index 1fb3fbb3c36865..399bf416aec227 100644 --- a/be/src/pipeline/task_scheduler.cpp +++ b/be/src/pipeline/task_scheduler.cpp @@ -66,19 +66,16 @@ Status TaskScheduler::start() { return Status::OK(); } -Status TaskScheduler::schedule_task(PipelineTask* task) { +Status TaskScheduler::schedule_task(PipelineTaskSPtr task) { return _task_queue->push_back(task); } // after _close_task, task maybe destructed. -void _close_task(PipelineTask* task, Status exec_status) { +void _close_task(PipelineTask* task, Status exec_status, PipelineFragmentContext* ctx) { // Has to attach memory tracker here, because the close task will also release some memory. // Should count the memory to the query or the query's memory will not decrease when part of // task finished. SCOPED_ATTACH_TASK(task->runtime_state()); - // close_a_pipeline may delete fragment context and will core in some defer - // code, because the defer code will access fragment context it self. - auto lock_for_context = task->fragment_context()->shared_from_this(); // is_pending_finish does not check status, so has to check status in close API. // For example, in async writer, the writer may failed during dealing with eos_block // but it does not return error status. Has to check the error status in close API. @@ -86,16 +83,16 @@ void _close_task(PipelineTask* task, Status exec_status) { // for pending finish now. So that could call close directly. Status status = task->close(exec_status); if (!status.ok()) { - task->fragment_context()->cancel(status); + ctx->cancel(status); } task->finalize(); task->set_running(false); - task->fragment_context()->close_a_pipeline(task->pipeline_id()); + ctx->close_a_pipeline(task->pipeline_id()); } void TaskScheduler::_do_work(size_t index) { while (_markers[index]) { - auto* task = _task_queue->take(index); + auto task = _task_queue->take(index); if (!task) { continue; } @@ -106,11 +103,15 @@ void TaskScheduler::_do_work(size_t index) { if (task->is_finalized()) { continue; } + auto fragment_context = task->fragment_context().lock(); + if (!fragment_context) { + // Fragment already finishedquery + continue; + } task->log_detail_if_need(); task->set_running(true); task->set_task_queue(_task_queue.get()); - auto* fragment_ctx = task->fragment_context(); - bool canceled = fragment_ctx->is_canceled(); + bool canceled = fragment_context->is_canceled(); // If the state is PENDING_FINISH, then the task is come from blocked queue, its is_pending_finish // has to return false. The task is finished and need to close now. @@ -121,7 +122,8 @@ void TaskScheduler::_do_work(size_t index) { // If pipeline is canceled, it will report after pipeline closed, and will propagate // errors to downstream through exchange. So, here we needn't send_report. // fragment_ctx->send_report(true); - _close_task(task, fragment_ctx->get_query_ctx()->exec_status()); + _close_task(task.get(), fragment_context->get_query_ctx()->exec_status(), + fragment_context.get()); continue; } @@ -137,7 +139,7 @@ void TaskScheduler::_do_work(size_t index) { ASSIGN_STATUS_IF_CATCH_EXCEPTION( //TODO: use a better enclose to abstracting these if (ExecEnv::GetInstance()->pipeline_tracer_context()->enabled()) { - TUniqueId query_id = task->query_context()->query_id(); + TUniqueId query_id = fragment_context->get_query_id(); std::string task_name = task->task_name(); std::thread::id tid = std::this_thread::get_id(); @@ -160,14 +162,14 @@ void TaskScheduler::_do_work(size_t index) { // LOG(WARNING)<< "task:\n"<debug_string(); // exec failed，cancel all fragment instance - fragment_ctx->cancel(status); + fragment_context->cancel(status); LOG(WARNING) << fmt::format("Pipeline task failed. query_id: {} reason: {}", - print_id(task->query_context()->query_id()), + print_id(fragment_context->get_query_ctx()->query_id()), status.to_string()); - _close_task(task, status); + _close_task(task.get(), status, fragment_context.get()); continue; } - fragment_ctx->trigger_report_if_necessary(); + fragment_context->trigger_report_if_necessary(); if (eos) { // is pending finish will add the task to dependency's blocking queue, and then the task will be @@ -176,8 +178,8 @@ void TaskScheduler::_do_work(size_t index) { // Only meet eos, should set task to PENDING_FINISH state task->set_running(false); } else { - Status exec_status = fragment_ctx->get_query_ctx()->exec_status(); - _close_task(task, exec_status); + Status exec_status = fragment_context->get_query_ctx()->exec_status(); + _close_task(task.get(), exec_status, fragment_context.get()); } continue; } diff --git a/be/src/pipeline/task_scheduler.h b/be/src/pipeline/task_scheduler.h index 9a20807ea268e8..0307d0603ec286 100644 --- a/be/src/pipeline/task_scheduler.h +++ b/be/src/pipeline/task_scheduler.h @@ -55,7 +55,7 @@ class TaskScheduler { ~TaskScheduler(); - Status schedule_task(PipelineTask* task); + Status schedule_task(PipelineTaskSPtr task); Status start(); From 5c6e2a6052d9ff22f4d690ad6a66be51c880d226 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 12 Aug 2025 10:31:25 +0800 Subject: [PATCH 371/572] branch-3.0: [fix](nereids) Fix the expr id are same but different expr when agg table with random distribute #52993 (#53619) picked from #52993 --- .../nereids/rules/analysis/BindRelation.java | 5 ++- .../rules/analysis/BindRelationTest.java | 43 +++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 8db2e6f0c7a8ee..274801f7fd6b90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -59,6 +59,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.expressions.functions.AggCombinerFunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; @@ -282,8 +283,8 @@ private LogicalPlan preAggForRandomDistribution(LogicalOlapScan olapScan) { if (function == null) { return olapScan; } - Alias alias = new Alias(exprId, ImmutableList.of(function), col.getName(), - olapScan.qualified(), true); + Alias alias = new Alias(StatementScopeIdGenerator.newExprId(), ImmutableList.of(function), + col.getName(), olapScan.qualified(), true); outputExpressions.add(alias); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index eaeaa3b2edda8b..428254a91c472a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -20,10 +20,16 @@ import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.RulePromise; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; +import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanRewriter; import org.apache.doris.utframe.TestWithFeService; @@ -31,6 +37,11 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + class BindRelationTest extends TestWithFeService implements GeneratedPlanPatterns { private static final String DB1 = "db1"; private static final String DB2 = "db2"; @@ -93,6 +104,38 @@ void bindRandomAggTable() { plan.getOutput().get(1).getQualifier()); } + @Test + void testBindRandomAggTableExprIdSame() { + connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION"); + connectContext.getState().setIsQuery(true); + PlanChecker.from(connectContext) + .checkPlannerResult("select * from db1.tagg", + planner -> { + List collectedAlias = new ArrayList<>(); + planner.getCascadesContext().getRewritePlan().accept( + new DefaultPlanVisitor>() { + @Override + public Void visitLogicalAggregate(LogicalAggregate aggregate, + List context) { + for (Expression expression : aggregate.getExpressions()) { + collectedAlias.addAll( + expression.collectToList(Alias.class::isInstance)); + } + return super.visitLogicalAggregate(aggregate, context); + } + }, collectedAlias); + for (Alias alias : collectedAlias) { + for (Expression child : alias.children()) { + Set childExpressionSet = + child.collectToSet(NamedExpression.class::isInstance).stream() + .map(expr -> ((NamedExpression) expr).getExprId()) + .collect(Collectors.toSet()); + Assertions.assertFalse(childExpressionSet.contains(alias.getExprId())); + } + } + }); + } + @Override public RulePromise defaultPromise() { return RulePromise.REWRITE; From 7d6787f4933b484ad260b7f399bc213c3afb04f4 Mon Sep 17 00:00:00 2001 From: morrySnow Date: Tue, 12 Aug 2025 10:32:38 +0800 Subject: [PATCH 372/572] branch-3.0: [fix](Nereids) not generate duplicate exprid after convert outer to anti rule #52798 (#53898) cherry picked from #52798 --- .../doris/nereids/jobs/executor/Rewriter.java | 7 +- .../apache/doris/nereids/rules/RuleSet.java | 2 - .../rules/expression/ExpressionRewrite.java | 190 +++++++++++++++++- .../rewrite/ConvertOuterJoinToAntiJoin.java | 86 +++++--- .../nereids/rules/rewrite/ExprIdRewriter.java | 184 ----------------- .../StatementScopeIdGenerator.java | 2 +- .../plans/logical/LogicalCTEConsumer.java | 17 ++ .../ConvertOuterJoinToAntiJoinTest.java | 22 +- .../rules/rewrite/EliminateOuterJoinTest.java | 5 +- .../doris/nereids/util/PlanChecker.java | 9 + .../transform_outer_join_to_anti.groovy | 8 + 11 files changed, 303 insertions(+), 229 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 60522e3da39d9f..4e58cd170121c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.rules.rewrite.CollectFilterAboveConsumer; import org.apache.doris.nereids.rules.rewrite.ColumnPruning; import org.apache.doris.nereids.rules.rewrite.ConvertInnerOrCrossJoin; +import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin; import org.apache.doris.nereids.rules.rewrite.CountDistinctRewrite; import org.apache.doris.nereids.rules.rewrite.CountLiteralRewrite; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; @@ -446,8 +447,7 @@ public class Rewriter extends AbstractBatchJobExecutor { new CollectCteConsumerOutput() ) ), - topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new) - ) + topic("Collect used column", custom(RuleType.COLLECT_COLUMNS, QueryColumnCollector::new)) ) ); @@ -455,6 +455,7 @@ public class Rewriter extends AbstractBatchJobExecutor { ImmutableSet.of(LogicalCTEAnchor.class), () -> jobs( // after variant sub path pruning, we need do column pruning again + bottomUp(RuleSet.PUSH_DOWN_FILTERS), custom(RuleType.COLUMN_PRUNING, ColumnPruning::new), bottomUp(ImmutableList.of( new PushDownFilterThroughProject(), @@ -548,6 +549,8 @@ private static List getWholeTreeRewriteJobs( topic("rewrite cte sub-tree before sub path push down", custom(RuleType.REWRITE_CTE_CHILDREN, () -> new RewriteCteChildren(beforePushDownJobs)) ))); + rewriteJobs.addAll(jobs(topic("convert outer join to anti", + custom(RuleType.CONVERT_OUTER_JOIN_TO_ANTI, ConvertOuterJoinToAntiJoin::new)))); if (needOrExpansion) { rewriteJobs.addAll(jobs(topic("or expansion", custom(RuleType.OR_EXPANSION, () -> OrExpansion.INSTANCE)))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index bcd12ac17d2579..15943a25a90f86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -86,7 +86,6 @@ import org.apache.doris.nereids.rules.implementation.LogicalTopNToPhysicalTopN; import org.apache.doris.nereids.rules.implementation.LogicalUnionToPhysicalUnion; import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow; -import org.apache.doris.nereids.rules.rewrite.ConvertOuterJoinToAntiJoin; import org.apache.doris.nereids.rules.rewrite.CreatePartitionTopNFromWindow; import org.apache.doris.nereids.rules.rewrite.EliminateFilter; import org.apache.doris.nereids.rules.rewrite.EliminateOuterJoin; @@ -148,7 +147,6 @@ public class RuleSet { new PushDownFilterThroughGenerate(), new PushDownProjectThroughLimit(), new EliminateOuterJoin(), - new ConvertOuterJoinToAntiJoin(), new MergeProjects(), new MergeFilters(), new MergeGenerates(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java index e69f93bf84551e..a5c77e2e47ef8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionRewrite.java @@ -20,6 +20,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.nereids.pattern.ExpressionPatternRules; import org.apache.doris.nereids.pattern.ExpressionPatternTraverseListeners; +import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; @@ -28,25 +29,39 @@ import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.OrderExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.functions.Function; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; +import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -79,7 +94,19 @@ public List buildRules() { new JoinExpressionRewrite().build(), new SortExpressionRewrite().build(), new LogicalRepeatRewrite().build(), - new HavingExpressionRewrite().build()); + new HavingExpressionRewrite().build(), + new LogicalPartitionTopNExpressionRewrite().build(), + new LogicalTopNExpressionRewrite().build(), + new LogicalSetOperationRewrite().build(), + new LogicalWindowRewrite().build(), + new LogicalCteConsumerRewrite().build(), + new LogicalResultSinkRewrite().build(), + new LogicalFileSinkRewrite().build(), + new LogicalHiveTableSinkRewrite().build(), + new LogicalIcebergTableSinkRewrite().build(), + new LogicalJdbcTableSinkRewrite().build(), + new LogicalOlapTableSinkRewrite().build(), + new LogicalDeferMaterializeResultSinkRewrite().build()); } private class GenerateExpressionRewrite extends OneRewriteRuleFactory { @@ -264,7 +291,166 @@ public Rule build() { } } - private class LogicalRepeatRewrite extends OneRewriteRuleFactory { + private class LogicalWindowRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalWindow().thenApply(ctx -> { + LogicalWindow window = ctx.root; + List windowExpressions = window.getWindowExpressions(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List result = rewriteAll(windowExpressions, rewriter, context); + return window.withExpressionsAndChild(result, window.child()); + }) + .toRule(RuleType.REWRITE_WINDOW_EXPRESSION); + } + } + + private class LogicalSetOperationRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalSetOperation().thenApply(ctx -> { + LogicalSetOperation setOperation = ctx.root; + List> slotsList = setOperation.getRegularChildrenOutputs(); + List> newSlotsList = new ArrayList<>(); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + for (List slots : slotsList) { + List result = rewriteAll(slots, rewriter, context); + newSlotsList.add(result); + } + return setOperation.withChildrenAndTheirOutputs(setOperation.children(), newSlotsList); + }) + .toRule(RuleType.REWRITE_SET_OPERATION_EXPRESSION); + } + } + + private class LogicalTopNExpressionRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalTopN().thenApply(ctx -> { + LogicalTopN topN = ctx.root; + List orderKeys = topN.getOrderKeys(); + ImmutableList.Builder rewrittenOrderKeys + = ImmutableList.builderWithExpectedSize(orderKeys.size()); + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + boolean changed = false; + for (OrderKey k : orderKeys) { + Expression expression = rewriter.rewrite(k.getExpr(), context); + changed |= expression != k.getExpr(); + rewrittenOrderKeys.add(new OrderKey(expression, k.isAsc(), k.isNullFirst())); + } + return changed ? topN.withOrderKeys(rewrittenOrderKeys.build()) : topN; + }).toRule(RuleType.REWRITE_TOPN_EXPRESSION); + } + } + + private class LogicalPartitionTopNExpressionRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalPartitionTopN().thenApply(ctx -> { + LogicalPartitionTopN partitionTopN = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List newOrderExpressions = new ArrayList<>(); + for (OrderExpression orderExpression : partitionTopN.getOrderKeys()) { + OrderKey orderKey = orderExpression.getOrderKey(); + Expression expr = rewriter.rewrite(orderKey.getExpr(), context); + OrderKey newOrderKey = new OrderKey(expr, orderKey.isAsc(), orderKey.isNullFirst()); + newOrderExpressions.add(new OrderExpression(newOrderKey)); + } + List result = rewriteAll(partitionTopN.getPartitionKeys(), rewriter, context); + return partitionTopN.withPartitionKeysAndOrderKeys(result, newOrderExpressions); + }).toRule(RuleType.REWRITE_PARTITION_TOPN_EXPRESSION); + } + } + + private class LogicalCteConsumerRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalCTEConsumer().thenApply(ctx -> { + LogicalCTEConsumer consumer = ctx.root; + boolean changed = false; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + ImmutableMap.Builder cToPBuilder = ImmutableMap.builder(); + ImmutableMultimap.Builder pToCBuilder = ImmutableMultimap.builder(); + for (Map.Entry entry : consumer.getConsumerToProducerOutputMap().entrySet()) { + Slot key = (Slot) rewriter.rewrite(entry.getKey(), context); + Slot value = (Slot) rewriter.rewrite(entry.getValue(), context); + cToPBuilder.put(key, value); + pToCBuilder.put(value, key); + if (!key.equals(entry.getKey()) || !value.equals(entry.getValue())) { + changed = true; + } + } + return changed ? consumer.withTwoMaps(cToPBuilder.build(), pToCBuilder.build()) : consumer; + }).toRule(RuleType.REWRITE_TOPN_EXPRESSION); + } + } + + private class LogicalResultSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalResultSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private class LogicalFileSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalFileSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private class LogicalHiveTableSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalHiveTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private class LogicalIcebergTableSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalIcebergTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private class LogicalJdbcTableSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalJdbcTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private class LogicalOlapTableSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalOlapTableSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private class LogicalDeferMaterializeResultSinkRewrite extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalDeferMaterializeResultSink().thenApply(ExpressionRewrite.this::applyRewriteToSink) + .toRule(RuleType.REWRITE_SINK_EXPRESSION); + } + } + + private LogicalSink applyRewriteToSink(MatchingContext> ctx) { + LogicalSink sink = ctx.root; + ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); + List outputExprs = sink.getOutputExprs(); + List result = rewriteAll(outputExprs, rewriter, context); + return sink.withOutputExprs(result); + } + + /** LogicalRepeatRewrite */ + public class LogicalRepeatRewrite extends OneRewriteRuleFactory { @Override public Rule build() { return logicalRepeat().thenApply(ctx -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java index c9185fd1a3cfea..46445573055df2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoin.java @@ -17,9 +17,9 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.jobs.JobContext; import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; @@ -28,9 +28,14 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.util.TypeUtils; -import java.util.List; +import com.google.common.collect.ImmutableList; + +import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -42,18 +47,41 @@ * project(A.*) * - LeftAntiJoin(A, B) */ -public class ConvertOuterJoinToAntiJoin extends OneRewriteRuleFactory { +public class ConvertOuterJoinToAntiJoin extends DefaultPlanRewriter> implements CustomRewriter { + private ExprIdRewriter exprIdReplacer; + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + if (!plan.containsType(LogicalJoin.class)) { + return plan; + } + Map replaceMap = new HashMap<>(); + ExprIdRewriter.ReplaceRule replaceRule = new ExprIdRewriter.ReplaceRule(replaceMap); + exprIdReplacer = new ExprIdRewriter(replaceRule, jobContext); + return plan.accept(this, replaceMap); + } @Override - public Rule build() { - return logicalFilter(logicalJoin() - .when(join -> join.getJoinType().isOuterJoin())) - .then(this::toAntiJoin) - .toRule(RuleType.CONVERT_OUTER_JOIN_TO_ANTI); + public Plan visit(Plan plan, Map replaceMap) { + plan = visitChildren(this, plan, replaceMap); + plan = exprIdReplacer.rewriteExpr(plan, replaceMap); + return plan; } - private Plan toAntiJoin(LogicalFilter> filter) { + @Override + public Plan visitLogicalFilter(LogicalFilter filter, Map replaceMap) { + filter = (LogicalFilter) visit(filter, replaceMap); + if (!(filter.child() instanceof LogicalJoin)) { + return filter; + } + return toAntiJoin((LogicalFilter>) filter, replaceMap); + } + + private Plan toAntiJoin(LogicalFilter> filter, Map replaceMap) { LogicalJoin join = filter.child(); + if (!join.getJoinType().isLeftOuterJoin() && !join.getJoinType().isRightOuterJoin()) { + return filter; + } Set alwaysNullSlots = filter.getConjuncts().stream() .filter(p -> TypeUtils.isNull(p).isPresent()) @@ -66,33 +94,37 @@ private Plan toAntiJoin(LogicalFilter> filter) { .filter(s -> alwaysNullSlots.contains(s) && !s.nullable()) .collect(Collectors.toSet()); - Plan newJoin = null; + Plan newChild = null; if (join.getJoinType().isLeftOuterJoin() && !rightAlwaysNullSlots.isEmpty()) { - newJoin = join.withJoinTypeAndContext(JoinType.LEFT_ANTI_JOIN, join.getJoinReorderContext()); + newChild = join.withJoinTypeAndContext(JoinType.LEFT_ANTI_JOIN, join.getJoinReorderContext()); } if (join.getJoinType().isRightOuterJoin() && !leftAlwaysNullSlots.isEmpty()) { - newJoin = join.withJoinTypeAndContext(JoinType.RIGHT_ANTI_JOIN, join.getJoinReorderContext()); + newChild = join.withJoinTypeAndContext(JoinType.RIGHT_ANTI_JOIN, join.getJoinReorderContext()); } - if (newJoin == null) { - return null; + if (newChild == null) { + return filter; } - if (!newJoin.getOutputSet().containsAll(filter.getInputSlots())) { + if (!newChild.getOutputSet().containsAll(filter.getInputSlots())) { // if there are slots that don't belong to join output, we use null alias to replace them // such as: // project(A.id, null as B.id) // - (A left anti join B) - Set joinOutput = newJoin.getOutputSet(); - List projects = filter.getOutput().stream() - .map(s -> { - if (joinOutput.contains(s)) { - return s; - } else { - return new Alias(s.getExprId(), new NullLiteral(s.getDataType()), s.getName()); - } - }).collect(Collectors.toList()); - newJoin = new LogicalProject<>(projects, newJoin); + Set joinOutputs = newChild.getOutputSet(); + ImmutableList.Builder projectsBuilder = ImmutableList.builder(); + for (NamedExpression e : filter.getOutput()) { + if (joinOutputs.contains(e)) { + projectsBuilder.add(e); + } else { + Alias newAlias = new Alias(new NullLiteral(e.getDataType()), e.getName(), e.getQualifier()); + replaceMap.put(e.getExprId(), newAlias.getExprId()); + projectsBuilder.add(newAlias); + } + } + newChild = new LogicalProject<>(projectsBuilder.build(), newChild); + return exprIdReplacer.rewriteExpr(filter.withChildren(newChild), replaceMap); + } else { + return filter.withChildren(newChild); } - return filter.withChildren(newJoin); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java index 60c9da4bc6eec5..5e065fa3724b08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExprIdRewriter.java @@ -18,32 +18,20 @@ package org.apache.doris.nereids.rules.rewrite; import org.apache.doris.nereids.jobs.JobContext; -import org.apache.doris.nereids.pattern.MatchingContext; import org.apache.doris.nereids.pattern.Pattern; -import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.rules.expression.ExpressionPatternMatcher; import org.apache.doris.nereids.rules.expression.ExpressionPatternRuleFactory; import org.apache.doris.nereids.rules.expression.ExpressionRewrite; -import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext; import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.OrderExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; -import org.apache.doris.nereids.trees.plans.logical.LogicalSetOperation; -import org.apache.doris.nereids.trees.plans.logical.LogicalSink; -import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; -import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; import com.google.common.collect.ImmutableList; -import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -58,26 +46,6 @@ public ExprIdRewriter(ReplaceRule replaceRule, JobContext jobContext) { this.jobContext = jobContext; } - @Override - public List buildRules() { - ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(super.buildRules()); - builder.addAll(ImmutableList.of( - new LogicalPartitionTopNExpressionRewrite().build(), - new LogicalTopNExpressionRewrite().build(), - new LogicalSetOperationRewrite().build(), - new LogicalWindowRewrite().build(), - new LogicalResultSinkRewrite().build(), - new LogicalFileSinkRewrite().build(), - new LogicalHiveTableSinkRewrite().build(), - new LogicalIcebergTableSinkRewrite().build(), - new LogicalJdbcTableSinkRewrite().build(), - new LogicalOlapTableSinkRewrite().build(), - new LogicalDeferMaterializeResultSinkRewrite().build() - )); - return builder.build(); - } - /**rewriteExpr*/ public Plan rewriteExpr(Plan plan, Map replaceMap) { if (replaceMap.isEmpty()) { @@ -129,156 +97,4 @@ public List> buildRules() { ); } } - - private class LogicalResultSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalResultSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalFileSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalFileSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalHiveTableSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalHiveTableSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalIcebergTableSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalIcebergTableSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalJdbcTableSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalJdbcTableSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalOlapTableSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalOlapTableSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalDeferMaterializeResultSinkRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalDeferMaterializeResultSink().thenApply(ExprIdRewriter.this::applyRewrite) - .toRule(RuleType.REWRITE_SINK_EXPRESSION); - } - } - - private class LogicalSetOperationRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalSetOperation().thenApply(ctx -> { - LogicalSetOperation setOperation = ctx.root; - List> slotsList = setOperation.getRegularChildrenOutputs(); - List> newSlotsList = new ArrayList<>(); - ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - for (List slots : slotsList) { - List newSlots = rewriteAll(slots, rewriter, context); - newSlotsList.add(newSlots); - } - if (newSlotsList.equals(slotsList)) { - return setOperation; - } - return setOperation.withChildrenAndTheirOutputs(setOperation.children(), newSlotsList); - }) - .toRule(RuleType.REWRITE_SET_OPERATION_EXPRESSION); - } - } - - private class LogicalWindowRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalWindow().thenApply(ctx -> { - LogicalWindow window = ctx.root; - List windowExpressions = window.getWindowExpressions(); - ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - List newWindowExpressions = rewriteAll(windowExpressions, rewriter, context); - if (newWindowExpressions.equals(windowExpressions)) { - return window; - } - return window.withExpressionsAndChild(newWindowExpressions, window.child()); - }) - .toRule(RuleType.REWRITE_WINDOW_EXPRESSION); - } - } - - private class LogicalTopNExpressionRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalTopN().thenApply(ctx -> { - LogicalTopN topN = ctx.root; - List orderKeys = topN.getOrderKeys(); - ImmutableList.Builder rewrittenOrderKeys - = ImmutableList.builderWithExpectedSize(orderKeys.size()); - ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - boolean changed = false; - for (OrderKey k : orderKeys) { - Expression expression = rewriter.rewrite(k.getExpr(), context); - changed |= expression != k.getExpr(); - rewrittenOrderKeys.add(new OrderKey(expression, k.isAsc(), k.isNullFirst())); - } - return changed ? topN.withOrderKeys(rewrittenOrderKeys.build()) : topN; - }).toRule(RuleType.REWRITE_TOPN_EXPRESSION); - } - } - - private class LogicalPartitionTopNExpressionRewrite extends OneRewriteRuleFactory { - @Override - public Rule build() { - return logicalPartitionTopN().thenApply(ctx -> { - LogicalPartitionTopN partitionTopN = ctx.root; - ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - List newOrderExpressions = new ArrayList<>(); - boolean changed = false; - for (OrderExpression orderExpression : partitionTopN.getOrderKeys()) { - OrderKey orderKey = orderExpression.getOrderKey(); - Expression expr = rewriter.rewrite(orderKey.getExpr(), context); - changed |= expr != orderKey.getExpr(); - OrderKey newOrderKey = new OrderKey(expr, orderKey.isAsc(), orderKey.isNullFirst()); - newOrderExpressions.add(new OrderExpression(newOrderKey)); - } - List newPartitionKeys = rewriteAll(partitionTopN.getPartitionKeys(), rewriter, context); - if (!newPartitionKeys.equals(partitionTopN.getPartitionKeys())) { - changed = true; - } - if (!changed) { - return partitionTopN; - } - return partitionTopN.withPartitionKeysAndOrderKeys(newPartitionKeys, newOrderExpressions); - }).toRule(RuleType.REWRITE_PARTITION_TOPN_EXPRESSION); - } - } - - private LogicalSink applyRewrite(MatchingContext> ctx) { - LogicalSink sink = ctx.root; - ExpressionRewriteContext context = new ExpressionRewriteContext(ctx.cascadesContext); - List outputExprs = sink.getOutputExprs(); - List newOutputExprs = rewriteAll(outputExprs, rewriter, context); - if (outputExprs.equals(newOutputExprs)) { - return sink; - } - return sink.withOutputExprs(newOutputExprs); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java index df7ef2ab69a100..cf0ecc3cb9b956 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/StatementScopeIdGenerator.java @@ -81,6 +81,6 @@ public static void clear() throws Exception { if (ConnectContext.get() != null) { ConnectContext.get().setStatementContext(new StatementContext()); } - statementContext = new StatementContext(); + statementContext = new StatementContext(10000); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java index 415fdddf80b449..6148f62378e65a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCTEConsumer.java @@ -198,4 +198,21 @@ public String toString() { "relationId", relationId, "name", name); } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + LogicalCTEConsumer that = (LogicalCTEConsumer) o; + return Objects.equals(consumerToProducerOutputMap, that.consumerToProducerOutputMap); + } + + @Override + public int hashCode() { + return super.hashCode(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java index 1159fc2a7cec6d..b3166c2224052b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/ConvertOuterJoinToAntiJoinTest.java @@ -32,17 +32,21 @@ import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; class ConvertOuterJoinToAntiJoinTest implements MemoPatternMatchSupported { - private final LogicalOlapScan scan1; - private final LogicalOlapScan scan2; + private LogicalOlapScan scan1; + private LogicalOlapScan scan2; - public ConvertOuterJoinToAntiJoinTest() throws Exception { + @BeforeEach + void setUp() throws Exception { // clear id so that slot id keep consistent every running + ConnectContext.remove(); StatementScopeIdGenerator.clear(); scan1 = PlanConstructor.newLogicalOlapScan(0, "t1", 0); scan2 = PlanConstructor.newLogicalOlapScan(1, "t2", 0); @@ -58,7 +62,7 @@ void testEliminateLeftWithProject() { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new InferFilterNotNull()) - .applyTopDown(new ConvertOuterJoinToAntiJoin()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() .matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin())); } @@ -73,7 +77,7 @@ void testEliminateRightWithProject() { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new InferFilterNotNull()) - .applyTopDown(new ConvertOuterJoinToAntiJoin()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() .matches(logicalJoin().when(join -> join.getJoinType().isRightAntiJoin())); } @@ -91,7 +95,7 @@ void testEliminateLeftWithLeftPredicate() { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new InferFilterNotNull()) - .applyTopDown(new ConvertOuterJoinToAntiJoin()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() .matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin())); } @@ -109,7 +113,7 @@ void testEliminateLeftWithRightPredicate() { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new InferFilterNotNull()) - .applyTopDown(new ConvertOuterJoinToAntiJoin()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() .matches(logicalJoin().when(join -> join.getJoinType().isLeftAntiJoin())); } @@ -127,7 +131,7 @@ void testEliminateLeftWithOrPredicate() { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new InferFilterNotNull()) - .applyTopDown(new ConvertOuterJoinToAntiJoin()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() .matches(logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin())); } @@ -146,7 +150,7 @@ void testEliminateLeftWithAndPredicate() { PlanChecker.from(MemoTestUtils.createConnectContext(), plan) .applyTopDown(new InferFilterNotNull()) - .applyTopDown(new ConvertOuterJoinToAntiJoin()) + .applyCustom(new ConvertOuterJoinToAntiJoin()) .printlnTree() .matches(logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin())); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java index 255f1e82e0061c..f0034410163649 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoinTest.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.And; import org.apache.doris.nereids.trees.expressions.GreaterThan; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; @@ -69,7 +70,7 @@ void testEliminateLeft() { void testEliminateRight() { LogicalPlan plan = new LogicalPlanBuilder(scan1) .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) // t1.id = t2.id - .filter(new GreaterThan(scan1.getOutput().get(0), Literal.of(1))) + .filter(new GreaterThan(scan1.getOutput().get(0), new IntegerLiteral(1))) .build(); PlanChecker.from(MemoTestUtils.createConnectContext(), plan) @@ -81,7 +82,7 @@ void testEliminateRight() { logicalFilter( logicalJoin().when(join -> join.getJoinType().isInnerJoin()) ).when(filter -> filter.getConjuncts().size() == 1) - .when(filter -> Objects.equals(filter.getConjuncts().toString(), "[(id#0 > 1)]")) + .when(filter -> Objects.equals(filter.getConjuncts().iterator().next(), new GreaterThan(scan1.getOutput().get(0), new IntegerLiteral(1)))) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 71d0f0101b0413..6962572d07a483 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -32,6 +32,7 @@ import org.apache.doris.nereids.jobs.executor.Optimizer; import org.apache.doris.nereids.jobs.executor.Rewriter; import org.apache.doris.nereids.jobs.joinorder.JoinOrderJob; +import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob; import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob; import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteTopDownJob; import org.apache.doris.nereids.jobs.rewrite.RootPlanTreeRewriteJob; @@ -202,6 +203,14 @@ public PlanChecker applyTopDown(List rule) { return this; } + public PlanChecker applyCustom(CustomRewriter customRewriter) { + CustomRewriteJob customRewriteJob = new CustomRewriteJob(() -> customRewriter, RuleType.TEST_REWRITE); + customRewriteJob.execute(cascadesContext.getCurrentJobContext()); + cascadesContext.toMemo(); + MemoValidator.validate(cascadesContext.getMemo()); + return this; + } + /** * apply a top down rewrite rule if you not care the ruleId * diff --git a/regression-test/suites/nereids_syntax_p0/transform_outer_join_to_anti.groovy b/regression-test/suites/nereids_syntax_p0/transform_outer_join_to_anti.groovy index ccbb8fd64a8f6f..f806f4ce5c7a5e 100644 --- a/regression-test/suites/nereids_syntax_p0/transform_outer_join_to_anti.groovy +++ b/regression-test/suites/nereids_syntax_p0/transform_outer_join_to_anti.groovy @@ -84,4 +84,12 @@ suite("transform_outer_join_to_anti") { sql("select * from eliminate_outer_join_A right outer join eliminate_outer_join_B on eliminate_outer_join_B.b = eliminate_outer_join_A.a where eliminate_outer_join_A.a is null and eliminate_outer_join_B.null_b is null and eliminate_outer_join_A.null_a is null") contains "ANTI JOIN" } + + explain { + sql """with temp as ( + select * from eliminate_outer_join_A left outer join eliminate_outer_join_B on eliminate_outer_join_B.b = eliminate_outer_join_A.a where eliminate_outer_join_B.b is null + ) + select * from temp t1 join temp t2""" + contains "ANTI JOIN" + } } From 3b90dacf10ba3516069352fc546359d956794e8d Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 12 Aug 2025 10:33:23 +0800 Subject: [PATCH 373/572] branch-3.0: [opt](nereids) opt range inference for or expression when out of order #46303 (#53989) picked from part of #46303 --- .../rules/expression/rules/SimplifyRange.java | 44 ++++++++++++++++--- .../rules/expression/SimplifyRangeTest.java | 40 +++++++++++------ .../mv/BaseMaterializedIndexSelectTest.java | 9 ++++ .../rules/rewrite/mv/SelectMvIndexTest.java | 8 +++- 4 files changed, 81 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java index d5fd8e24783c28..434f7a6f5bda41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java @@ -46,7 +46,9 @@ import com.google.common.collect.Multimap; import com.google.common.collect.Multimaps; import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; +import com.google.common.collect.TreeRangeSet; import java.util.ArrayList; import java.util.Collection; @@ -157,18 +159,18 @@ public ValueDesc visitInPredicate(InPredicate inPredicate, ExpressionRewriteCont @Override public ValueDesc visitAnd(And and, ExpressionRewriteContext context) { return simplify(context, and, ExpressionUtils.extractConjunction(and), - ValueDesc::intersect, ExpressionUtils::and); + ValueDesc::intersect, ExpressionUtils::and, true); } @Override public ValueDesc visitOr(Or or, ExpressionRewriteContext context) { return simplify(context, or, ExpressionUtils.extractDisjunction(or), - ValueDesc::union, ExpressionUtils::or); + ValueDesc::union, ExpressionUtils::or, false); } private ValueDesc simplify(ExpressionRewriteContext context, Expression originExpr, List predicates, - BinaryOperator op, BinaryOperator exprOp) { + BinaryOperator op, BinaryOperator exprOp, boolean isAnd) { Multimap groupByReference = Multimaps.newListMultimap(new LinkedHashMap<>(), ArrayList::new); @@ -181,7 +183,9 @@ private ValueDesc simplify(ExpressionRewriteContext context, List valuePerRefs = Lists.newArrayList(); for (Entry> referenceValues : groupByReference.asMap().entrySet()) { List valuePerReference = (List) referenceValues.getValue(); - + if (!isAnd) { + valuePerReference = unionDiscreteAndRange(context, referenceValues.getKey(), valuePerReference); + } // merge per reference ValueDesc simplifiedValue = valuePerReference.get(0); for (int i = 1; i < valuePerReference.size(); i++) { @@ -200,6 +204,30 @@ private ValueDesc simplify(ExpressionRewriteContext context, } } + /** merge discrete and ranges only, no merge other value desc */ + public static List unionDiscreteAndRange(ExpressionRewriteContext context, + Expression reference, List valueDescs) { + List result = Lists.newArrayListWithExpectedSize(valueDescs.size()); + + // for (a >= 8 and a < 9) or (a >=12 and a < 13) or (a >=13 and a < 14) can convert to + // (a >= 8 and a < 9) or (a >=12 and a < 14) + RangeSet rangeSet = TreeRangeSet.create(); + for (ValueDesc valueDesc : valueDescs) { + if (valueDesc instanceof RangeValue) { + Range range = ((RangeValue) valueDesc).range; + rangeSet.add(range); + } else { + result.add(valueDesc); + } + } + for (Range range : rangeSet.asRanges()) { + RangeValue rangeValue = new RangeValue(context, reference, RangeValue.toExpression(range, reference)); + rangeValue.range = range; + result.add(rangeValue); + } + return result; + } + private abstract static class ValueDesc { ExpressionRewriteContext context; Expression toExpr; @@ -357,7 +385,9 @@ public ValueDesc intersect(ValueDesc other) { if (range.isConnected(o.range)) { RangeValue rangeValue = new RangeValue(context, reference, originExpr); rangeValue.range = range.intersection(o.range); - return rangeValue; + if (!rangeValue.range.isEmpty()) { + return rangeValue; + } } return new EmptyValue(context, reference, originExpr); } @@ -372,6 +402,10 @@ public ValueDesc intersect(ValueDesc other) { @Override public Expression toExpression() { + return toExpression(this.range, this.reference); + } + + public static Expression toExpression(Range range, Expression reference) { List result = Lists.newArrayList(); if (range.hasLowerBound()) { if (range.lowerBoundType() == BoundType.CLOSED) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java index 79906880f53d5a..ca2cab9905b1e9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java @@ -65,6 +65,8 @@ public void testSimplify() { executor = new ExpressionRuleExecutor(ImmutableList.of( bottomUp(SimplifyRange.INSTANCE) )); + assertRewrite("(TA >= 8 and TA < 8) or (TA >= 8 and TA < 8)", "TA is null and null"); + assertRewrite("(TA >=12 and TA < 13) or (TA >= 15 and TA < 16) or (TA >= 16 and TA < 17)", "(TA >=12 and TA < 13) or (TA >=15 and TA < 17)"); assertRewrite("TA", "TA"); assertRewrite("TA > 3 or TA > null", "TA > 3 OR NULL"); assertRewrite("TA > 3 or TA < null", "TA > 3 OR NULL"); @@ -85,16 +87,16 @@ public void testSimplify() { assertRewrite("(TA > 3 and TA < 1) or (TA > 7 and TA < 5)", "TA is null and null"); assertRewriteNotNull("TA > 3 and TA < 1", "FALSE"); assertRewrite("TA > 3 and TA < 1", "TA is null and null"); - assertRewrite("TA >= 3 and TA < 3", "TA >= 3 and TA < 3"); + assertRewrite("TA >= 3 and TA < 3", "TA is null and null"); assertRewriteNotNull("TA = 1 and TA > 10", "FALSE"); assertRewrite("TA = 1 and TA > 10", "TA is null and null"); - assertRewrite("TA > 5 or TA < 1", "TA > 5 or TA < 1"); + assertRewrite("TA > 5 or TA < 1", "TA < 1 or TA > 5"); assertRewrite("TA > 5 or TA > 1 or TA > 10", "TA > 1"); assertRewrite("TA > 5 or TA > 1 or TA < 10", "TA is not null or null"); assertRewriteNotNull("TA > 5 or TA > 1 or TA < 10", "TRUE"); assertRewrite("TA > 5 and TA > 1 and TA > 10", "TA > 10"); assertRewrite("TA > 5 and TA > 1 and TA < 10", "TA > 5 and TA < 10"); - assertRewrite("TA > 1 or TA < 1", "TA > 1 or TA < 1"); + assertRewrite("TA > 1 or TA < 1", "TA < 1 or TA > 1"); assertRewrite("TA > 1 or TA < 10", "TA is not null or null"); assertRewriteNotNull("TA > 1 or TA < 10", "TRUE"); assertRewrite("TA > 5 and TA < 10", "TA > 5 and TA < 10"); @@ -109,7 +111,7 @@ public void testSimplify() { assertRewrite("(TA > 10 or TA > 20) and (TB > 10 and TB > 20)", "TA > 10 and TB > 20"); assertRewrite("((TB > 30 and TA > 40) and TA > 20) and (TB > 10 and TB > 20)", "TB > 30 and TA > 40"); assertRewrite("(TA > 10 and TB > 10) or (TB > 10 and TB > 20)", "TA > 10 and TB > 10 or TB > 20"); - assertRewrite("((TA > 10 or TA > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA > 5 and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))"); + assertRewrite("((TA > 10 or TA > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA > 5 and TB > 10) or (TB > 10 and (TB < 10 or TB > 20))"); assertRewriteNotNull("TA in (1,2,3) and TA > 10", "FALSE"); assertRewrite("TA in (1,2,3) and TA > 10", "TA is null and null"); assertRewrite("TA in (1,2,3) and TA >= 1", "TA in (1,2,3)"); @@ -147,15 +149,15 @@ public void testSimplify() { assertRewrite("(TA + TC > 3 and TA + TC < 1) or (TA + TC > 7 and TA + TC < 5)", "(TA + TC) is null and null"); assertRewriteNotNull("TA + TC > 3 and TA + TC < 1", "FALSE"); assertRewrite("TA + TC > 3 and TA + TC < 1", "(TA + TC) is null and null"); - assertRewrite("TA + TC >= 3 and TA + TC < 3", "TA + TC >= 3 and TA + TC < 3"); + assertRewrite("TA + TC >= 3 and TA + TC < 3", "TA + TC is null and null"); assertRewriteNotNull("TA + TC = 1 and TA + TC > 10", "FALSE"); assertRewrite("TA + TC = 1 and TA + TC > 10", "(TA + TC) is null and null"); - assertRewrite("TA + TC > 5 or TA + TC < 1", "TA + TC > 5 or TA + TC < 1"); + assertRewrite("TA + TC > 5 or TA + TC < 1", "TA + TC < 1 or TA + TC > 5"); assertRewrite("TA + TC > 5 or TA + TC > 1 or TA + TC > 10", "TA + TC > 1"); assertRewrite("TA + TC > 5 or TA + TC > 1 or TA + TC < 10", "(TA + TC) is not null or null"); assertRewrite("TA + TC > 5 and TA + TC > 1 and TA + TC > 10", "TA + TC > 10"); assertRewrite("TA + TC > 5 and TA + TC > 1 and TA + TC < 10", "TA + TC > 5 and TA + TC < 10"); - assertRewrite("TA + TC > 1 or TA + TC < 1", "TA + TC > 1 or TA + TC < 1"); + assertRewrite("TA + TC > 1 or TA + TC < 1", "TA + TC < 1 or TA + TC > 1"); assertRewrite("TA + TC > 1 or TA + TC < 10", "(TA + TC) is not null or null"); assertRewrite("TA + TC > 5 and TA + TC < 10", "TA + TC > 5 and TA + TC < 10"); assertRewrite("TA + TC > 5 and TA + TC > 10", "TA + TC > 10"); @@ -168,7 +170,7 @@ public void testSimplify() { assertRewrite("(TA + TC > 10 or TA + TC > 20) and (TB > 10 and TB > 20)", "TA + TC > 10 and TB > 20"); assertRewrite("((TB > 30 and TA + TC > 40) and TA + TC > 20) and (TB > 10 and TB > 20)", "TB > 30 and TA + TC > 40"); assertRewrite("(TA + TC > 10 and TB > 10) or (TB > 10 and TB > 20)", "TA + TC > 10 and TB > 10 or TB > 20"); - assertRewrite("((TA + TC > 10 or TA + TC > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA + TC > 5 and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))"); + assertRewrite("((TA + TC > 10 or TA + TC > 5) and TB > 10) or (TB > 10 and (TB > 20 or TB < 10))", "(TA + TC > 5 and TB > 10) or (TB > 10 and (TB < 10 or TB > 20))"); assertRewriteNotNull("TA + TC in (1,2,3) and TA + TC > 10", "FALSE"); assertRewrite("TA + TC in (1,2,3) and TA + TC > 10", "(TA + TC) is null and null"); assertRewrite("TA + TC in (1,2,3) and TA + TC >= 1", "TA + TC in (1,2,3)"); @@ -204,6 +206,9 @@ public void testSimplifyDate() { executor = new ExpressionRuleExecutor(ImmutableList.of( bottomUp(SimplifyRange.INSTANCE) )); + assertRewrite( + "(AA >= date '2024-01-01' and AA < date '2024-01-02') or (AA >= date '2024-01-05' and AA < date '2024-01-06') or (AA >= date '2024-01-06' and AA < date '2024-01-07')", + "(AA >= date '2024-01-01' and AA < date '2024-01-02') or (AA >= date '2024-01-05' and AA < date '2024-01-07')"); assertRewrite("AA", "AA"); assertRewrite( "(AA >= date '2024-01-01' and AA <= date '2024-01-03') or (AA > date '2024-01-05' and AA < date '2024-01-07')", @@ -217,11 +222,13 @@ public void testSimplifyDate() { assertRewriteNotNull("AA > date '2024-01-03' and AA < date '2024-01-01'", "FALSE"); assertRewrite("AA > date '2024-01-03' and AA < date '2024-01-01'", "AA is null and null"); assertRewrite("AA >= date '2024-01-01' and AA < date '2024-01-01'", - "AA >= date '2024-01-01' and AA < date '2024-01-01'"); + "AA is null and null"); + assertRewrite("(AA >= date '2024-01-01' and AA < date '2024-01-01') or (AA >= date '2024-01-01' and AA < date '2024-01-01')", + "AA is null and null"); assertRewriteNotNull("AA = date '2024-01-01' and AA > date '2024-01-10'", "FALSE"); assertRewrite("AA = date '2024-01-01' and AA > date '2024-01-10'", "AA is null and null"); assertRewrite("AA > date '2024-01-05' or AA < date '2024-01-01'", - "AA > date '2024-01-05' or AA < date '2024-01-01'"); + "AA < date '2024-01-01' or AA > date '2024-01-05'"); assertRewrite("AA > date '2024-01-05' or AA > date '2024-01-01' or AA > date '2024-01-10'", "AA > date '2024-01-01'"); assertRewrite("AA > date '2024-01-05' or AA > date '2024-01-01' or AA < date '2024-01-10'", "AA is not null or null"); @@ -231,7 +238,7 @@ public void testSimplifyDate() { assertRewrite("AA > date '2024-01-05' and AA > date '2024-01-01' and AA < date '2024-01-10'", "AA > date '2024-01-05' and AA < date '2024-01-10'"); assertRewrite("AA > date '2024-01-05' or AA < date '2024-01-05'", - "AA > date '2024-01-05' or AA < date '2024-01-05'"); + "AA < date '2024-01-05' or AA > date '2024-01-05'"); assertRewrite("AA > date '2024-01-01' or AA < date '2024-01-10'", "AA is not null or null"); assertRewriteNotNull("AA > date '2024-01-01' or AA < date '2024-01-10'", "TRUE"); assertRewrite("AA > date '2024-01-05' and AA < date '2024-01-10'", @@ -285,6 +292,9 @@ public void testSimplifyDateTime() { bottomUp(SimplifyRange.INSTANCE) )); assertRewrite("CA", "CA"); + assertRewrite( + "(CA >= timestamp '2024-01-01 00:00:00' and CA < timestamp '2024-01-02 00:00:00') or (CA >= timestamp '2024-01-05 00:00:00' and CA < timestamp '2024-01-07 00:00:00') or (CA >= timestamp '2024-01-07 00:00:00' and CA < timestamp '2024-01-08 00:00:00')", + "(CA >= timestamp '2024-01-01 00:00:00' and CA < timestamp '2024-01-02 00:00:00') or (CA >= timestamp '2024-01-05 00:00:00' and CA < timestamp '2024-01-08 00:00:00')"); assertRewrite( "(CA >= timestamp '2024-01-01 00:00:00' and CA <= timestamp '2024-01-03 00:00:00') or (CA > timestamp '2024-01-05 00:00:00' and CA < timestamp '2024-01-07 00:00:00')", "(CA >= timestamp '2024-01-01 00:00:00' and CA <= timestamp '2024-01-03 00:00:00') or (CA > timestamp '2024-01-05 00:00:00' and CA < timestamp '2024-01-07 00:00:00')"); @@ -297,11 +307,13 @@ public void testSimplifyDateTime() { assertRewriteNotNull("CA > timestamp '2024-01-03 00:00:10' and CA < timestamp '2024-01-01 01:00:00'", "FALSE"); assertRewrite("CA > timestamp '2024-01-03 00:00:10' and CA < timestamp '2024-01-01 01:00:00'", "CA is null and null"); assertRewrite("CA >= timestamp '2024-01-01 00:00:10' and CA < timestamp '2024-01-01 00:00:10'", - "CA >= timestamp '2024-01-01 00:00:10' and CA < timestamp '2024-01-01 00:00:10'"); + "CA is null and null"); + assertRewrite("(CA >= timestamp '2024-01-01 00:00:10' and CA < timestamp '2024-01-01 00:00:10') or (CA >= timestamp '2024-01-01 00:00:10' and CA < timestamp '2024-01-01 00:00:10')", + "CA is null and null"); assertRewriteNotNull("CA = timestamp '2024-01-01 10:00:10' and CA > timestamp '2024-01-10 00:00:10'", "FALSE"); assertRewrite("CA = timestamp '2024-01-01 10:00:10' and CA > timestamp '2024-01-10 00:00:10'", "CA is null and null"); assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-01 00:00:10'", - "CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-01 00:00:10'"); + "CA < timestamp '2024-01-01 00:00:10' or CA > timestamp '2024-01-05 00:00:10'"); assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA > timestamp '2024-01-01 00:00:10' or CA > timestamp '2024-01-10 00:00:10'", "CA > timestamp '2024-01-01 00:00:10'"); assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA > timestamp '2024-01-01 00:00:10' or CA < timestamp '2024-01-10 00:00:10'", "CA is not null or null"); @@ -311,7 +323,7 @@ public void testSimplifyDateTime() { assertRewrite("CA > timestamp '2024-01-05 00:00:10' and CA > timestamp '2024-01-01 00:00:10' and CA < timestamp '2024-01-10 00:00:10'", "CA > timestamp '2024-01-05 00:00:10' and CA < timestamp '2024-01-10 00:00:10'"); assertRewrite("CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-05 00:00:10'", - "CA > timestamp '2024-01-05 00:00:10' or CA < timestamp '2024-01-05 00:00:10'"); + "CA < timestamp '2024-01-05 00:00:10' or CA > timestamp '2024-01-05 00:00:10'"); assertRewrite("CA > timestamp '2024-01-01 00:02:10' or CA < timestamp '2024-01-10 00:02:10'", "CA is not null or null"); assertRewriteNotNull("CA > timestamp '2024-01-01 00:00:00' or CA < timestamp '2024-01-10 00:00:00'", "TRUE"); assertRewrite("CA > timestamp '2024-01-05 01:00:00' and CA < timestamp '2024-01-10 01:00:00'", diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/BaseMaterializedIndexSelectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/BaseMaterializedIndexSelectTest.java index 4550431e398610..3ab49e2b08037a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/BaseMaterializedIndexSelectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/BaseMaterializedIndexSelectTest.java @@ -25,6 +25,7 @@ import org.junit.jupiter.api.Assertions; import java.util.List; +import java.util.Set; import java.util.function.Consumer; /** @@ -38,6 +39,14 @@ protected void singleTableTest(String sql, String indexName, boolean preAgg) { }); } + // any index in indexNameSet is ok + protected void singleTableTest(String sql, Set indexNameSet, boolean preAgg) { + singleTableTest(sql, scan -> { + Assertions.assertEquals(preAgg, scan.isPreAggregation()); + Assertions.assertTrue(indexNameSet.contains(scan.getSelectedIndexName())); + }); + } + protected void singleTableTest(String sql, Consumer scanConsumer) { PlanChecker.from(connectContext).checkPlannerResult(sql, planner -> { List scans = planner.getScanNodes(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java index c769e1c210c804..6dfb2a96fff34d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMvIndexTest.java @@ -40,6 +40,7 @@ import org.apache.doris.utframe.DorisAssert; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -1144,7 +1145,8 @@ void selectBitmapMvWithProjectMultiMv() throws Exception { createMv("create materialized view mv2 as" + " select a, c, bitmap_union(to_bitmap(b)) from selectBitmapMvWithProjectMultiMv group by a, c;"); - testMv("select a, bitmap_union_count(to_bitmap(b)) as cnt from selectBitmapMvWithProjectMultiMv group by a", "mv"); + testMv("select a, bitmap_union_count(to_bitmap(b)) as cnt from selectBitmapMvWithProjectMultiMv group by a", + ImmutableSet.of("mv", "mv2")); dropTable("selectBitmapMvWithProjectMultiMv", true); } @@ -1212,6 +1214,10 @@ private void testMv(String sql, String indexName) { singleTableTest(sql, indexName, true); } + private void testMv(String sql, Set indexNameSet) { + singleTableTest(sql, indexNameSet, true); + } + private void assertOneAggFuncType(LogicalAggregate agg, Class aggFuncType) { Set aggFuncs = agg.getOutputExpressions() .stream() From 86015384cf1dd8880c4b67f640be2b2d445d1bbd Mon Sep 17 00:00:00 2001 From: amory Date: Tue, 12 Aug 2025 10:34:53 +0800 Subject: [PATCH 374/572] branch-3.0 cherry-pick [fix](variant) fix the reading core caused by inserting nested column and scalar column in variant sub-column (#53897) ### What problem does this PR solve? backport: https://github.com/apache/doris/pull/53083 Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/cloud/cloud_meta_mgr.cpp | 13 +- .../cloud/schema_cloud_dictionary_cache.cpp | 36 +- be/src/olap/base_tablet.cpp | 4 +- be/src/olap/rowset/beta_rowset_writer.cpp | 7 +- be/src/olap/rowset/beta_rowset_writer.h | 2 +- .../segment_v2/hierarchical_data_reader.h | 7 +- be/src/service/internal_service.cpp | 18 +- be/src/vec/common/schema_util.cpp | 96 ++- be/src/vec/common/schema_util.h | 17 +- be/src/vec/json/parse2column.cpp | 10 + .../test_schema_cloud_dictionary_cache.cpp | 54 ++ be/test/common/schema_util_test.cpp | 311 +++++++++ .../doris/datasource/InternalCatalog.java | 10 + .../org/apache/doris/qe/SessionVariable.java | 15 + gensrc/thrift/AgentService.thrift | 2 +- .../data/variant_p0/nested/load.out | 647 ++++++++++++++++++ .../data/variant_p0/nested/sql/q01.out | 31 + .../suites/variant_p0/delete_update.groovy | 1 + .../suites/variant_p0/nested.groovy | 2 +- .../suites/variant_p0/nested/load.groovy | 198 ++++++ .../suites/variant_p0/nested/sql/q01.sql | 13 + .../suites/variant_p0/nested2.groovy | 8 + ...est_double_write_when_schema_change.groovy | 1 + 23 files changed, 1460 insertions(+), 43 deletions(-) create mode 100644 regression-test/data/variant_p0/nested/load.out create mode 100644 regression-test/data/variant_p0/nested/sql/q01.out create mode 100644 regression-test/suites/variant_p0/nested/load.groovy create mode 100644 regression-test/suites/variant_p0/nested/sql/q01.sql diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 2fa1ecf4a8616a..fe47bb7c8e0628 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -1007,11 +1007,18 @@ Status CloudMetaMgr::commit_rowset(RowsetMeta& rs_meta, const std::string& job_i // Replace schema dictionary keys based on the rowset's index ID to maintain schema consistency. CloudStorageEngine& engine = ExecEnv::GetInstance()->storage_engine().to_cloud(); // if not enable dict cache, then directly return true to avoid refresh - bool replaced = + Status replaced_st = config::variant_use_cloud_schema_dict_cache ? engine.get_schema_cloud_dictionary_cache().replace_schema_to_dict_keys( rs_meta_pb.index_id(), req.mutable_rowset_meta()) - : true; + : Status::OK(); + // if the replaced_st is not ok and alse not NotFound, then we need to just return the replaced_st + VLOG_DEBUG << "replace schema to dict keys, replaced_st: " << replaced_st.to_string() + << ", replaced_st.is(): " + << replaced_st.is(); + if (!replaced_st.ok() && !replaced_st.is()) { + return replaced_st; + } Status st = retry_rpc("commit rowset", req, &resp, &MetaService_Stub::commit_rowset); if (!st.ok() && resp.status().code() == MetaServiceCode::ALREADY_EXISTED) { if (existed_rs_meta != nullptr && resp.has_existed_rowset_meta()) { @@ -1025,7 +1032,7 @@ Status CloudMetaMgr::commit_rowset(RowsetMeta& rs_meta, const std::string& job_i // If dictionary replacement fails, it may indicate that the local schema dictionary is outdated. // Refreshing the dictionary here ensures that the rowset metadata is updated with the latest schema definitions, // which is critical for maintaining consistency between the rowset and its corresponding schema. - if (!replaced) { + if (replaced_st.is()) { RETURN_IF_ERROR( engine.get_schema_cloud_dictionary_cache().refresh_dict(rs_meta_pb.index_id())); } diff --git a/be/src/cloud/schema_cloud_dictionary_cache.cpp b/be/src/cloud/schema_cloud_dictionary_cache.cpp index 25f0b2327024fd..9fdde420ecbf66 100644 --- a/be/src/cloud/schema_cloud_dictionary_cache.cpp +++ b/be/src/cloud/schema_cloud_dictionary_cache.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -62,6 +63,27 @@ SchemaCloudDictionarySPtr SchemaCloudDictionaryCache::_lookup(int64_t index_id) return dict; } +Status check_path_amibigus(const SchemaCloudDictionary& schema, RowsetMetaCloudPB* rowset_meta) { + // if enable_variant_flatten_nested is false, then we don't need to check path amibigus + if (!rowset_meta->tablet_schema().enable_variant_flatten_nested()) { + return Status::OK(); + } + // try to get all the paths in the rowset meta + vectorized::PathsInData all_paths; + for (const auto& column : rowset_meta->tablet_schema().column()) { + vectorized::PathInData path_in_data; + path_in_data.from_protobuf(column.column_path_info()); + all_paths.push_back(path_in_data); + } + // try to get all the paths in the schema dict + for (const auto& [_, column] : schema.column_dict()) { + vectorized::PathInData path_in_data; + path_in_data.from_protobuf(column.column_path_info()); + all_paths.push_back(path_in_data); + } + RETURN_IF_ERROR(vectorized::schema_util::check_variant_has_no_ambiguous_paths(all_paths)); + return Status::OK(); +} /** * Processes dictionary entries by matching items from the given item map. * It maps items to their dictionary keys, then adds these keys to the rowset metadata. @@ -101,7 +123,7 @@ Status process_dictionary(SchemaCloudDictionary& dict, return output; }; - google::protobuf::RepeatedPtrField none_ext_items; + google::protobuf::RepeatedPtrField none_extracted_items; std::unordered_map reversed_dict; for (const auto& [key, val] : item_dict) { reversed_dict[serialize_fn(val)] = key; @@ -110,7 +132,7 @@ Status process_dictionary(SchemaCloudDictionary& dict, for (const auto& item : items) { if (filter(item)) { // Filter none extended items, mainly extended columns and extended indexes - *none_ext_items.Add() = item; + *none_extracted_items.Add() = item; continue; } const std::string serialized_key = serialize_fn(item); @@ -127,7 +149,7 @@ Status process_dictionary(SchemaCloudDictionary& dict, } // clear extended items to prevent writing them to fdb if (result != nullptr) { - result->Swap(&none_ext_items); + result->Swap(&none_extracted_items); } return Status::OK(); } @@ -137,11 +159,15 @@ Status SchemaCloudDictionaryCache::replace_schema_to_dict_keys(int64_t index_id, if (!rowset_meta->has_variant_type_in_schema()) { return Status::OK(); } + // first attempt to get dict from cache auto dict = _lookup(index_id); if (!dict) { - g_schema_dict_cache_miss_count << 1; - return Status::NotFound("Not found dict {}", index_id); + // if not found the dict in cache, then refresh the dict from remote meta service + RETURN_IF_ERROR(refresh_dict(index_id, &dict)); } + // here we should have the dict + DCHECK(dict); + RETURN_IF_ERROR(check_path_amibigus(*dict, rowset_meta)); auto* dict_list = rowset_meta->mutable_schema_dict_key_list(); // Process column dictionary: add keys for non-extended columns. auto column_filter = [&](const doris::ColumnPB& col) -> bool { return col.unique_id() >= 0; }; diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp index 49d1f2851f0d09..59fe2dc87c17fd 100644 --- a/be/src/olap/base_tablet.cpp +++ b/be/src/olap/base_tablet.cpp @@ -189,11 +189,13 @@ Status BaseTablet::update_by_least_common_schema(const TabletSchemaSPtr& update_ CHECK(_max_version_schema->schema_version() >= update_schema->schema_version()); TabletSchemaSPtr final_schema; bool check_column_size = true; + VLOG_DEBUG << "dump _max_version_schema: " << _max_version_schema->dump_full_schema(); + VLOG_DEBUG << "dump update_schema: " << update_schema->dump_full_schema(); RETURN_IF_ERROR(vectorized::schema_util::get_least_common_schema( {_max_version_schema, update_schema}, _max_version_schema, final_schema, check_column_size)); _max_version_schema = final_schema; - VLOG_DEBUG << "dump updated tablet schema: " << final_schema->dump_structure(); + VLOG_DEBUG << "dump updated tablet schema: " << final_schema->dump_full_schema(); return Status::OK(); } diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 6f14f45138d32d..e230a25c874bad 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -856,13 +856,13 @@ int64_t BetaRowsetWriter::_num_seg() const { // Eg. rowset schema: A(int), B(float), C(int), D(int) // _tabelt->tablet_schema: A(bigint), B(double) // => update_schema: A(bigint), B(double), C(int), D(int) -void BaseBetaRowsetWriter::update_rowset_schema(TabletSchemaSPtr flush_schema) { +Status BaseBetaRowsetWriter::update_rowset_schema(TabletSchemaSPtr flush_schema) { std::lock_guard lock(*(_context.schema_lock)); TabletSchemaSPtr update_schema; if (_context.merged_tablet_schema == nullptr) { _context.merged_tablet_schema = _context.tablet_schema; } - static_cast(vectorized::schema_util::get_least_common_schema( + RETURN_IF_ERROR(vectorized::schema_util::get_least_common_schema( {_context.merged_tablet_schema, flush_schema}, nullptr, update_schema)); CHECK_GE(update_schema->num_columns(), flush_schema->num_columns()) << "Rowset merge schema columns count is " << update_schema->num_columns() @@ -871,6 +871,7 @@ void BaseBetaRowsetWriter::update_rowset_schema(TabletSchemaSPtr flush_schema) { << " flush_schema: " << flush_schema->dump_structure(); _context.merged_tablet_schema.swap(update_schema); VLOG_DEBUG << "dump rs schema: " << _context.tablet_schema->dump_structure(); + return Status::OK(); } Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num) { @@ -1085,7 +1086,7 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati } // tablet schema updated if (flush_schema != nullptr) { - update_rowset_schema(flush_schema); + RETURN_IF_ERROR(update_rowset_schema(flush_schema)); } if (_context.mow_context != nullptr) { // ensure that the segment file writing is complete diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h index 19e3c4da31db41..11a3ecc8f01000 100644 --- a/be/src/olap/rowset/beta_rowset_writer.h +++ b/be/src/olap/rowset/beta_rowset_writer.h @@ -198,7 +198,7 @@ class BaseBetaRowsetWriter : public RowsetWriter { } private: - void update_rowset_schema(TabletSchemaSPtr flush_schema); + Status update_rowset_schema(TabletSchemaSPtr flush_schema); // build a tmp rowset for load segment to calc delete_bitmap // for this segment protected: diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h index 6b0d4d1234541d..3254bef25e6539 100644 --- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h +++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h @@ -145,8 +145,11 @@ class HierarchicalDataReader : public ColumnIterator { PathInData relative_path = node.path.copy_pop_nfront(_path.get_parts().size()); if (node.path.has_nested_part()) { - CHECK_EQ(getTypeName(remove_nullable(node.data.type)->get_type_id()), - getTypeName(TypeIndex::Array)); + if (remove_nullable(node.data.type)->get_type_id() != TypeIndex::Array) { + return Status::InternalError( + "Meet none array column when flatten nested array, path {}, type {}", + node.path.get_path(), node.data.type->get_name()); + } PathInData parent_path = node.path.get_nested_prefix_path().copy_pop_nfront( _path.get_parts().size()); nested_subcolumns[parent_path].emplace_back(relative_path, column->get_ptr(), diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp index 6d5d9055fc3a3e..14e07ebc699774 100644 --- a/be/src/service/internal_service.cpp +++ b/be/src/service/internal_service.cpp @@ -1173,8 +1173,13 @@ void PInternalService::fetch_remote_tablet_schema(google::protobuf::RpcControlle if (!schemas.empty() && st.ok()) { // merge all TabletSchemaSPtr merged_schema; - static_cast(vectorized::schema_util::get_least_common_schema(schemas, nullptr, - merged_schema)); + st = vectorized::schema_util::get_least_common_schema(schemas, nullptr, + merged_schema); + if (!st.ok()) { + LOG(WARNING) << "Failed to get least common schema: " << st.to_string(); + st = Status::InternalError("Failed to get least common schema: {}", + st.to_string()); + } VLOG_DEBUG << "dump schema:" << merged_schema->dump_structure(); merged_schema->reserve_extracted_columns(); merged_schema->to_schema_pb(response->mutable_merged_schema()); @@ -1210,8 +1215,13 @@ void PInternalService::fetch_remote_tablet_schema(google::protobuf::RpcControlle if (!tablet_schemas.empty()) { // merge all TabletSchemaSPtr merged_schema; - static_cast(vectorized::schema_util::get_least_common_schema( - tablet_schemas, nullptr, merged_schema)); + st = vectorized::schema_util::get_least_common_schema(tablet_schemas, nullptr, + merged_schema); + if (!st.ok()) { + LOG(WARNING) << "Failed to get least common schema: " << st.to_string(); + st = Status::InternalError("Failed to get least common schema: {}", + st.to_string()); + } merged_schema->to_schema_pb(response->mutable_merged_schema()); VLOG_DEBUG << "dump schema:" << merged_schema->dump_structure(); } diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index c1255318e7a2be..c6d0e5e73a3d8a 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -248,10 +248,64 @@ TabletColumn get_column_by_type(const vectorized::DataTypePtr& data_type, const return result; } -void update_least_schema_internal(const std::map& subcolumns_types, - TabletSchemaSPtr& common_schema, bool update_sparse_column, - int32_t variant_col_unique_id, - std::set* path_set = nullptr) { +// check if two paths which same prefix have different structure +static bool has_different_structure_in_same_path(const PathInData::Parts& lhs, + const PathInData::Parts& rhs) { + if (lhs.size() != rhs.size()) { + return false; // different size means different structure + } + // Since we group by path string, lhs and rhs must have the same size and keys + // We only need to check if they have different nested structure + for (size_t i = 0; i < lhs.size(); ++i) { + if (lhs[i] != rhs[i]) { + VLOG_DEBUG << fmt::format( + "Check different structure: {} vs {}, lhs[i].is_nested: {}, rhs[i].is_nested: " + "{}", + lhs[i].key, rhs[i].key, lhs[i].is_nested, rhs[i].is_nested); + return true; + } + } + return false; +} + +Status check_variant_has_no_ambiguous_paths(const PathsInData& tuple_paths) { + // Group paths by their string representation to reduce comparisons + std::unordered_map> path_groups; + + for (size_t i = 0; i < tuple_paths.size(); ++i) { + // same path should have same structure, so we group them by path + path_groups[tuple_paths[i].get_path()].push_back(i); + // print part of tuple_paths[i] + VLOG_DEBUG << "tuple_paths[i]: " << tuple_paths[i].get_path(); + } + + // Only compare paths within the same group + for (const auto& [path_str, indices] : path_groups) { + if (indices.size() <= 1) { + continue; // No conflicts possible + } + + // Compare all pairs within this group + for (size_t i = 0; i < indices.size(); ++i) { + for (size_t j = 0; j < i; ++j) { + if (has_different_structure_in_same_path(tuple_paths[indices[i]].get_parts(), + tuple_paths[indices[j]].get_parts())) { + return Status::DataQualityError( + "Ambiguous paths: {} vs {} with different nested part {} vs {}", + tuple_paths[indices[i]].get_path(), tuple_paths[indices[j]].get_path(), + tuple_paths[indices[i]].has_nested_part(), + tuple_paths[indices[j]].has_nested_part()); + } + } + } + } + return Status::OK(); +} + +Status update_least_schema_internal(const std::map& subcolumns_types, + TabletSchemaSPtr& common_schema, bool update_sparse_column, + int32_t variant_col_unique_id, + std::set* path_set = nullptr) { PathsInData tuple_paths; DataTypes tuple_types; CHECK(common_schema.use_count() == 1); @@ -302,13 +356,18 @@ void update_least_schema_internal(const std::map& subcolu path_set->insert(tuple_paths[i]); } } + return Status::OK(); } -void update_least_common_schema(const std::vector& schemas, - TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, - std::set* path_set) { +Status update_least_common_schema(const std::vector& schemas, + TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, + std::set* path_set) { // Types of subcolumns by path from all tuples. std::map subcolumns_types; + + // Collect all paths first to enable batch checking + std::vector all_paths; + for (const TabletSchemaSPtr& schema : schemas) { for (const TabletColumnPtr& col : schema->columns()) { // Get subcolumns of this variant @@ -316,9 +375,14 @@ void update_least_common_schema(const std::vector& schemas, col->parent_unique_id() == variant_col_unique_id) { subcolumns_types[*col->path_info_ptr()].push_back( DataTypeFactory::instance().create_data_type(*col, col->is_nullable())); + all_paths.push_back(*col->path_info_ptr()); } } } + + // Batch check for conflicts + RETURN_IF_ERROR(check_variant_has_no_ambiguous_paths(all_paths)); + for (const TabletSchemaSPtr& schema : schemas) { if (schema->field_index(variant_col_unique_id) == -1) { // maybe dropped @@ -336,13 +400,13 @@ void update_least_common_schema(const std::vector& schemas, } } } - update_least_schema_internal(subcolumns_types, common_schema, false, variant_col_unique_id, - path_set); + return update_least_schema_internal(subcolumns_types, common_schema, false, + variant_col_unique_id, path_set); } -void update_least_sparse_column(const std::vector& schemas, - TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, - const std::set& path_set) { +Status update_least_sparse_column(const std::vector& schemas, + TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, + const std::set& path_set) { // Types of subcolumns by path from all tuples. std::map subcolumns_types; for (const TabletSchemaSPtr& schema : schemas) { @@ -361,7 +425,8 @@ void update_least_sparse_column(const std::vector& schemas, } } } - update_least_schema_internal(subcolumns_types, common_schema, true, variant_col_unique_id); + return update_least_schema_internal(subcolumns_types, common_schema, true, + variant_col_unique_id); } void inherit_column_attributes(const TabletColumn& source, TabletColumn& target, @@ -416,7 +481,6 @@ Status get_least_common_schema(const std::vector& schemas, const TabletSchemaSPtr& base_schema, TabletSchemaSPtr& output_schema, bool check_schema_size) { std::vector variant_column_unique_id; - // Construct a schema excluding the extracted columns and gather unique identifiers for variants. // Ensure that the output schema also excludes these extracted columns. This approach prevents // duplicated paths following the update_least_common_schema process. @@ -463,9 +527,9 @@ Status get_least_common_schema(const std::vector& schemas, std::set path_set; // 1. cast extracted column to common type // path set is used to record the paths of those sparse columns that have been merged into the extracted columns, eg: v:b - update_least_common_schema(schemas, output_schema, unique_id, &path_set); + RETURN_IF_ERROR(update_least_common_schema(schemas, output_schema, unique_id, &path_set)); // 2. cast sparse column to common type, exclude the columns from the path set - update_least_sparse_column(schemas, output_schema, unique_id, path_set); + RETURN_IF_ERROR(update_least_sparse_column(schemas, output_schema, unique_id, path_set)); } inherit_column_attributes(output_schema); diff --git a/be/src/vec/common/schema_util.h b/be/src/vec/common/schema_util.h index 7c228ed2cc0a95..481b72ca52ee87 100644 --- a/be/src/vec/common/schema_util.h +++ b/be/src/vec/common/schema_util.h @@ -88,6 +88,11 @@ Status parse_variant_columns(Block& block, const std::vector& variant_pos, const ParseConfig& config); Status encode_variant_sparse_subcolumns(ColumnObject& column); +// check if the tuple_paths has ambiguous paths +// situation: +// throw exception if there exists a prefix with matched names, but not matched structure (is Nested, number of dimensions). +Status check_variant_has_no_ambiguous_paths(const std::vector& paths); + // Pick the tablet schema with the highest schema version as the reference. // Then update all variant columns to there least common types. // Return the final merged schema as common schema. @@ -98,13 +103,13 @@ Status get_least_common_schema(const std::vector& schemas, // Get least common types for extracted columns which has Path info, // with a speicified variant column's unique id -void update_least_common_schema(const std::vector& schemas, - TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, - std::unordered_set* path_set); +Status update_least_common_schema(const std::vector& schemas, + TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, + std::unordered_set* path_set); -void update_least_sparse_column(const std::vector& schemas, - TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, - const std::unordered_set& path_set); +Status update_least_sparse_column(const std::vector& schemas, + TabletSchemaSPtr& common_schema, int32_t variant_col_unique_id, + const std::unordered_set& path_set); // inherit attributes like index/agg info from it's parent column void inherit_column_attributes(TabletSchemaSPtr& schema); diff --git a/be/src/vec/json/parse2column.cpp b/be/src/vec/json/parse2column.cpp index ba18083a95c5f6..d9cc9256ed7be2 100644 --- a/be/src/vec/json/parse2column.cpp +++ b/be/src/vec/json/parse2column.cpp @@ -155,6 +155,16 @@ void parse_json_to_variant(IColumn& column, const char* src, size_t length, auto& [paths, values] = *result; assert(paths.size() == values.size()); size_t old_num_rows = column_object.size(); + if (config.enable_flatten_nested) { + // here we should check the paths in variant and paths in result, + // if two paths which same prefix have different structure, we should throw an exception + std::vector check_paths; + for (const auto& entry : column_object.get_subcolumns()) { + check_paths.push_back(entry->path); + } + check_paths.insert(check_paths.end(), paths.begin(), paths.end()); + THROW_IF_ERROR(vectorized::schema_util::check_variant_has_no_ambiguous_paths(check_paths)); + } for (size_t i = 0; i < paths.size(); ++i) { FieldInfo field_info; get_field_info(values[i], &field_info); diff --git a/be/test/cloud/test_schema_cloud_dictionary_cache.cpp b/be/test/cloud/test_schema_cloud_dictionary_cache.cpp index 3d05eb67e457fa..0fc4fd0c3f55fd 100644 --- a/be/test/cloud/test_schema_cloud_dictionary_cache.cpp +++ b/be/test/cloud/test_schema_cloud_dictionary_cache.cpp @@ -15,9 +15,11 @@ // specific language governing permissions and limitations // under the License. +#include "cloud/schema_cloud_dictionary_cache.cpp" #include "cloud/schema_cloud_dictionary_cache.h" #include "gen_cpp/olap_file.pb.h" #include "gtest/gtest.h" +#include "vec/json/path_in_data.h" namespace doris { @@ -175,4 +177,56 @@ TEST(SchemaCloudDictionaryCacheTest, ReplaceDictKeysToSchema_RefreshFailure) { EXPECT_FALSE(st.ok()); } +// Test case 5: replace_schema_to_dict_keys with tablet_schema.enable_variant_flatten_nested = true +TEST(SchemaCloudDictionaryCacheTest, ProcessDictionary_VariantPathConflict_Throws) { + SchemaCloudDictionarySPtr dict = std::make_shared(); + // construct two variant columns with same unique_id but different path_info + auto& col_dict = *dict->mutable_column_dict(); + ColumnPB* col1 = &(col_dict)[101]; + col1->set_unique_id(101); + vectorized::PathInDataBuilder builder1; + builder1.append("v", false).append("nested", true).append("a", false); + vectorized::PathInData path_in_data1 = builder1.build(); + segment_v2::ColumnPathInfo path_info1; + path_in_data1.to_protobuf(&path_info1, 0); + col1->mutable_column_path_info()->CopyFrom(path_info1); + { + RowsetMetaCloudPB rs_meta; + rs_meta.set_has_variant_type_in_schema(true); + auto* schema = rs_meta.mutable_tablet_schema(); + schema->set_enable_variant_flatten_nested(true); + // add two columns with same key but different is_nested value + auto* col_schema1 = schema->add_column(); + col_schema1->set_unique_id(101); + // create pathIndata with same key but different is_nested value + vectorized::PathInDataBuilder builder3; + builder3.append("v", false).append("nested", false).append("a", false); + vectorized::PathInData path_in_data3 = builder3.build(); + segment_v2::ColumnPathInfo path_info3; + path_in_data3.to_protobuf(&path_info3, 0); + col_schema1->mutable_column_path_info()->CopyFrom(path_info3); + auto st = check_path_amibigus(*dict, &rs_meta); + EXPECT_FALSE(st.ok()); + EXPECT_EQ(st.code(), TStatusCode::DATA_QUALITY_ERROR); + } + + { + RowsetMetaCloudPB rs_meta; + rs_meta.set_has_variant_type_in_schema(true); + auto* schema = rs_meta.mutable_tablet_schema(); + // add two columns with same key but same is_nested value + auto* col_schema3 = schema->add_column(); + col_schema3->set_unique_id(101); + vectorized::PathInDataBuilder builder5; + builder5.append("v", false).append("nested", true).append("a", false); + vectorized::PathInData path_in_data5 = builder5.build(); + segment_v2::ColumnPathInfo path_info5; + path_in_data5.to_protobuf(&path_info5, 0); + col_schema3->mutable_column_path_info()->CopyFrom(path_info5); + // assert no exception + auto st = check_path_amibigus(*dict, &rs_meta); + EXPECT_TRUE(st.ok()) << st.to_string(); + } +} + } // namespace doris \ No newline at end of file diff --git a/be/test/common/schema_util_test.cpp b/be/test/common/schema_util_test.cpp index fb8b23c10cb19c..5ec1e8ffbaba45 100644 --- a/be/test/common/schema_util_test.cpp +++ b/be/test/common/schema_util_test.cpp @@ -19,6 +19,14 @@ #include +#include "vec/columns/column_object.h" +#include "vec/columns/column_string.h" +#include "vec/core/block.h" +#include "vec/core/column_with_type_and_name.h" +#include "vec/data_types/data_type_object.h" +#include "vec/data_types/data_type_string.h" +#include "vec/json/json_parser.h" + namespace doris { class SchemaUtilTest : public testing::Test {}; @@ -118,4 +126,307 @@ TEST_F(SchemaUtilTest, inherit_column_attributes) { } } +// Test has_different_structure_in_same_path function indirectly through check_variant_has_no_ambiguous_paths +TEST_F(SchemaUtilTest, has_different_structure_in_same_path_indirect) { + // Test case 1: Same structure and same length - should not detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", false).append("b", false).append("c", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("b", false).append("c", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 2: Different keys at same position - should not detect ambiguity (different keys) + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", false).append("b", false).append("c", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("d", false).append("c", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 3: Same keys but different nested structure - should detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", false).append("b", true); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("b", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("Ambiguous paths") != std::string::npos); + } + + // Test case 4: Same keys but different anonymous array levels - should detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", true).append("b", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("b", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("Ambiguous paths") != std::string::npos); + } + + // Test case 5: Same keys but different nested and anonymous levels - should detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", true).append("b", true); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("b", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("Ambiguous paths") != std::string::npos); + } + + // Test case 6: Different lengths - should not detect ambiguity (new behavior: only check same length paths) + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", false).append("b", false).append("c", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("b", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 7: Different lengths with structure difference - should not detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", false).append("b", true).append("c", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a", false).append("b", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 8: Complex nested structure difference with same length - should detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("user", false).append("address", true).append("street", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("user", false).append("address", false).append("street", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_FALSE(status.ok()); + EXPECT_TRUE(status.to_string().find("Ambiguous paths") != std::string::npos); + } + + // Test case 9: Multiple paths with different lengths - should not detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("config", false).append("database", false).append("host", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("config", false).append("database", false); + paths.emplace_back(builder2.build()); + + vectorized::PathInDataBuilder builder3; + builder3.append("config", false); + paths.emplace_back(builder3.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 10: Empty paths - should not detect ambiguity + { + vectorized::PathsInData paths; + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 11: Single path - should not detect ambiguity + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("single", false).append("path", false); + paths.emplace_back(builder1.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 12: we have path like '{"a.b": "UPPER CASE", "a.c": "lower case", "a" : {"b" : 123}, "a" : {"c" : 456}}' + { + vectorized::PathsInData paths; + vectorized::PathInDataBuilder builder1; + builder1.append("a", false).append("b", false); + paths.emplace_back(builder1.build()); + + vectorized::PathInDataBuilder builder2; + builder2.append("a.b", false); + paths.emplace_back(builder2.build()); + + auto status = vectorized::schema_util::check_variant_has_no_ambiguous_paths(paths); + EXPECT_TRUE(status.ok()) << status.to_string(); + } +} + +// Test check_path_conflicts_with_existing function indirectly through update_least_common_schema +TEST_F(SchemaUtilTest, check_path_conflicts_with_existing) { + // Test case 1: No conflicts - should succeed + { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + + // Create a variant column + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1, + "VARIANT", "v1", IndexType::INVERTED); + + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + std::vector subcolumns; + + // Add subcolumns with different paths + construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_STRING, 1, "v1.name", + &subcolumns); + construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_INT, 1, "v1.age", + &subcolumns); + + std::vector schemas = {tablet_schema}; + TabletSchemaSPtr output_schema; + + auto status = vectorized::schema_util::get_least_common_schema(schemas, nullptr, + output_schema, false); + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 2: Conflicts with same path but different structure - should fail + { + TabletSchemaPB schema_pb; + schema_pb.set_keys_type(KeysType::DUP_KEYS); + + // Create a variant column + construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001, "v1_index", 1, + "VARIANT", "v1", IndexType::INVERTED); + + TabletSchemaSPtr tablet_schema = std::make_shared(); + tablet_schema->init_from_pb(schema_pb); + + // Add subcolumns with same path but different structure + // This would require creating paths with different nested structure + // For now, we'll test the basic functionality + + std::vector schemas = {tablet_schema}; + TabletSchemaSPtr output_schema; + + auto status = vectorized::schema_util::get_least_common_schema(schemas, nullptr, + output_schema, false); + // This should succeed since we don't have conflicting paths in this simple case + EXPECT_TRUE(status.ok()) << status.to_string(); + } + + // Test case 3: Multiple schemas with conflicting paths - should fail + { + // Create first schema + TabletSchemaPB schema_pb1; + schema_pb1.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb1.add_column(), schema_pb1.add_index(), 10001, "v1_index", 1, + "VARIANT", "v1", IndexType::INVERTED); + + TabletSchemaSPtr tablet_schema1 = std::make_shared(); + tablet_schema1->init_from_pb(schema_pb1); + std::vector subcolumns; + construct_subcolumn(tablet_schema1, FieldType::OLAP_FIELD_TYPE_STRING, 1, "v1.address", + &subcolumns); + + // Create second schema with same path but different structure + TabletSchemaPB schema_pb2; + schema_pb2.set_keys_type(KeysType::DUP_KEYS); + construct_column(schema_pb2.add_column(), schema_pb2.add_index(), 10001, "v1_index", 1, + "VARIANT", "v1", IndexType::INVERTED); + + TabletSchemaSPtr tablet_schema2 = std::make_shared(); + tablet_schema2->init_from_pb(schema_pb2); + std::vector subcolumns2; + construct_subcolumn(tablet_schema2, FieldType::OLAP_FIELD_TYPE_INT, 1, "v1.address", + &subcolumns2); + + std::vector schemas = {tablet_schema1, tablet_schema2}; + TabletSchemaSPtr output_schema; + + auto status = vectorized::schema_util::get_least_common_schema(schemas, nullptr, + output_schema, false); + // This should succeed since the paths are the same and we're just checking for structure conflicts + EXPECT_TRUE(status.ok()) << status.to_string(); + } +} + +TEST_F(SchemaUtilTest, parse_variant_columns_ambiguous_paths) { + using namespace doris::vectorized; + // Prepare the string column with two rows + auto string_col = ColumnString::create(); + auto field1 = vectorized::Field(String("{\"nested\": [{\"a\": 2.5, \"b\": \"123.1\"}]}")); + auto field2 = vectorized::Field(String("{\"nested\": {\"a\": 2.5, \"b\": \"123.1\"}}")); + string_col->insert(field1); + string_col->insert(field2); + auto string_type = std::make_shared(); + + // Prepare the variant column with the string column as root + vectorized::ColumnObject::Subcolumns dynamic_subcolumns; + dynamic_subcolumns.create_root( + vectorized::ColumnObject::Subcolumn(string_col->assume_mutable(), string_type, true)); + + auto variant_col = ColumnObject::create(std::move(dynamic_subcolumns), true); + auto variant_type = std::make_shared(); + + // Construct the block + Block block; + block.insert( + vectorized::ColumnWithTypeAndName(variant_col->assume_mutable(), variant_type, "v")); + + // The variant column is at index 0 + std::vector variant_pos = {0}; + ParseConfig config; + config.enable_flatten_nested = true; + + // Should throw due to ambiguous paths + Status st = schema_util::parse_variant_columns(block, variant_pos, config); + EXPECT_FALSE(st.ok()); + EXPECT_TRUE(st.to_string().find("Ambiguous paths") != std::string::npos); +} + } // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index ce6a9b76e8c389..3be70a8c90be77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2645,6 +2645,16 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx boolean variantEnableFlattenNested = false; try { variantEnableFlattenNested = PropertyAnalyzer.analyzeVariantFlattenNested(properties); + // only if session variable: disable_variant_flatten_nested = false and + // table property: variant_enable_flatten_nested = true + // we can enable variant flatten nested otherwise throw error + if (ctx != null && !ctx.getSessionVariable().getDisableVariantFlattenNested() + && variantEnableFlattenNested) { + olapTable.setVariantEnableFlattenNested(variantEnableFlattenNested); + } else if (variantEnableFlattenNested) { + throw new DdlException("If you want to enable variant flatten nested, " + + "please set session variable: disable_variant_flatten_nested = false"); + } } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 4dcb828dc84483..50a70b67844a0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -681,6 +681,10 @@ public class SessionVariable implements Serializable, Writable { public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant"; + // disable variant flatten nested as session variable, default is true, + // which means disable variant flatten nested when create table + public static final String DISABLE_VARIANT_FLATTEN_NESTED = "disable_variant_flatten_nested"; + // CLOUD_VARIABLES_BEGIN public static final String CLOUD_CLUSTER = "cloud_cluster"; public static final String DISABLE_EMPTY_PARTITION_PRUNE = "disable_empty_partition_prune"; @@ -1280,6 +1284,9 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT, needForward = true) private boolean disableInvertedIndexV1ForVaraint = true; + @VariableMgr.VarAttr(name = DISABLE_VARIANT_FLATTEN_NESTED, needForward = true) + private boolean disableVariantFlattenNested = true; + public int getBeNumberForTest() { return beNumberForTest; } @@ -4787,6 +4794,14 @@ public boolean getDisableInvertedIndexV1ForVaraint() { return disableInvertedIndexV1ForVaraint; } + public void setDisableVariantFlattenNested(boolean disableVariantFlattenNested) { + this.disableVariantFlattenNested = disableVariantFlattenNested; + } + + public boolean getDisableVariantFlattenNested() { + return disableVariantFlattenNested; + } + public void checkSqlConvertorFeatures(String features) { if (Strings.isNullOrEmpty(features)) { return; diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 03172cf2917e0e..37845d814a6a51 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -48,7 +48,7 @@ struct TTabletSchema { // col unique id for row store column 20: optional list row_store_col_cids 21: optional i64 row_store_page_size = 16384 - 22: optional bool variant_enable_flatten_nested = false + 22: optional bool variant_enable_flatten_nested = false 23: optional i64 storage_page_size = 65536 } diff --git a/regression-test/data/variant_p0/nested/load.out b/regression-test/data/variant_p0/nested/load.out new file mode 100644 index 00000000000000..d0cb9d65faeaeb --- /dev/null +++ b/regression-test/data/variant_p0/nested/load.out @@ -0,0 +1,647 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_0 -- +1 {"nested":[{"a":1,"c":1.1},{"b":"1"}]} + +-- !sql_1 -- +[1, null] + +-- !sql_2 -- +[null, "1"] + +-- !sql_3 -- +[1.1, null] + +-- !sql_4 -- +[{"a":1,"c":1.1}, {"b":"1"}] + +-- !sql_8 -- +[1, null] 2 + +-- !sql_9 -- +[null, 1] 2 + +-- !sql_10 -- +[1, null] 2 + +-- !sql_11 -- +["1", null] 2 + +-- !sql_12 -- +[null, "1"] 2 + +-- !sql_13 -- +["1.1", null] 2 + +-- !sql_14 -- +[1, null] 2 + +-- !sql_15 -- +[null, 1] 2 + +-- !sql_16 -- +[1.1, null] 2 + +-- !sql_17 -- +\N \N \N + +-- !sql_18 -- +[1, NULL] ['NULL', '1'] [1.1, NULL] + +-- !sql_19 -- +\N \N \N + +-- !sql_0 -- +1 {"nested":[{"a":1.0,"c":1.1},{"b":"1"}]} +3 {"nested":[{"a":2.5,"b":"123.1"}]} +4 {"nested":[{"a":2.5,"b":123.1}]} +5 {"nested":[{"a":2.5,"c":"123.1"},{"b":"123.1"}]} +6 {"nested":[{"a":2.5},{"b":123.1}]} +7 {"nested":[{"a":2.5},{"c":123.1},{"b":"123.1"}]} + +-- !sql_1 -- +[1, null] +[2.5] +[2.5] +[2.5, null] +[2.5, null] +[2.5, null, null] + +-- !sql_2 -- +[null, ""1""] +[""123.1""] +["123.1"] +[null, ""123.1""] +[null, "123.1"] +[null, null, ""123.1""] + +-- !sql_3 -- +["1.1", null] +\N +\N +[""123.1"", null] +\N +[null, "123.1", null] + +-- !sql_4 -- +[{"a":1.0,"c":1.1}, {"b":"1"}] +[{"a":2.5,"b":"123.1"}] +[{"a":2.5,"b":123.1}] +[{"a":2.5,"c":"123.1"}, {"b":"123.1"}] +[{"a":2.5}, {"b":123.1}] +[{"a":2.5}, {"c":123.1}, {"b":"123.1"}] + +-- !sql_8 -- +[1, null] 2 +[2] 1 +[2] 1 +[2, null] 2 +[2, null] 2 +[2, null, null] 3 + +-- !sql_9 -- +[null, 1] 2 +[123] 1 +[123] 1 +[null, 123] 2 +[null, 123] 2 +[null, null, 123] 3 + +-- !sql_10 -- +[1, null] 2 +\N \N +\N \N +[123, null] 2 +\N \N +[null, 123, null] 3 + +-- !sql_11 -- +["1", null] 2 +["2.5"] 1 +["2.5"] 1 +["2.5", null] 2 +["2.5", null] 2 +["2.5", null, null] 3 + +-- !sql_12 -- +[null, "1"] 2 +["123.1"] 1 +["123.1"] 1 +[null, "123.1"] 2 +[null, "123.1"] 2 +[null, null, "123.1"] 3 + +-- !sql_13 -- +["1.1", null] 2 +\N \N +\N \N +["123.1", null] 2 +\N \N +[null, "123.1", null] 3 + +-- !sql_14 -- +[1, null] 2 +[2.5] 1 +[2.5] 1 +[2.5, null] 2 +[2.5, null] 2 +[2.5, null, null] 3 + +-- !sql_15 -- +[null, 1] 2 +[123.1] 1 +[123.1] 1 +[null, 123.1] 2 +[null, 123.1] 2 +[null, null, 123.1] 3 + +-- !sql_16 -- +[1.1, null] 2 +\N \N +\N \N +[123.1, null] 2 +\N \N +[null, 123.1, null] 3 + +-- !sql_17 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_18 -- +[1, NULL] ['NULL', '1'] [1.1, NULL] +[2.5] ['123.1'] \N +[2.5] [123.1] \N +[2.5, NULL] ['NULL', '123.1'] ['123.1', 'NULL'] +[2.5, NULL] [NULL, 123.1] \N +[2.5, NULL, NULL] ['NULL', 'NULL', '123.1'] [NULL, 123.1, NULL] + +-- !sql_19 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_0 -- +1 {"nested":[{"a":1.0,"c":1.1},{"b":"1"}]} +3 {"nested":[{"a":2.5,"b":"123.1"}]} +4 {"nested":[{"a":2.5,"b":123.1}]} +5 {"nested":[{"a":2.5,"c":"123.1"},{"b":"123.1"}]} +6 {"nested":[{"a":2.5},{"b":123.1}]} +7 {"nested":[{"a":2.5},{"c":123.1},{"b":"123.1"}]} + +-- !sql_1 -- +[1, null] +[2.5] +[2.5] +[2.5, null] +[2.5, null] +[2.5, null, null] + +-- !sql_2 -- +[null, ""1""] +[""123.1""] +["123.1"] +[null, ""123.1""] +[null, "123.1"] +[null, null, ""123.1""] + +-- !sql_3 -- +["1.1", null] +[null] +[null] +[""123.1"", null] +[null, null] +[null, "123.1", null] + +-- !sql_4 -- +[{"a":1.0,"c":1.1}, {"b":"1"}] +[{"a":2.5,"b":"123.1"}] +[{"a":2.5,"b":123.1}] +[{"a":2.5,"c":"123.1"}, {"b":"123.1"}] +[{"a":2.5}, {"b":123.1}] +[{"a":2.5}, {"c":123.1}, {"b":"123.1"}] + +-- !sql_8 -- +[1, null] 2 +[2] 1 +[2] 1 +[2, null] 2 +[2, null] 2 +[2, null, null] 3 + +-- !sql_9 -- +[null, 1] 2 +[123] 1 +[123] 1 +[null, 123] 2 +[null, 123] 2 +[null, null, 123] 3 + +-- !sql_10 -- +[1, null] 2 +[null] 1 +[null] 1 +[123, null] 2 +[null, null] 2 +[null, 123, null] 3 + +-- !sql_11 -- +["1", null] 2 +["2.5"] 1 +["2.5"] 1 +["2.5", null] 2 +["2.5", null] 2 +["2.5", null, null] 3 + +-- !sql_12 -- +[null, "1"] 2 +["123.1"] 1 +["123.1"] 1 +[null, "123.1"] 2 +[null, "123.1"] 2 +[null, null, "123.1"] 3 + +-- !sql_13 -- +["1.1", null] 2 +[null] 1 +[null] 1 +["123.1", null] 2 +[null, null] 2 +[null, "123.1", null] 3 + +-- !sql_14 -- +[1, null] 2 +[2.5] 1 +[2.5] 1 +[2.5, null] 2 +[2.5, null] 2 +[2.5, null, null] 3 + +-- !sql_15 -- +[null, 1] 2 +[123.1] 1 +[123.1] 1 +[null, 123.1] 2 +[null, 123.1] 2 +[null, null, 123.1] 3 + +-- !sql_16 -- +[1.1, null] 2 +[null] 1 +[null] 1 +[123.1, null] 2 +[null, null] 2 +[null, 123.1, null] 3 + +-- !sql_17 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_18 -- +[1, NULL] [NULL, "1"] [1.1, NULL] +[2.5] ["123.1"] [NULL] +[2.5] [123.1] [NULL] +[2.5, NULL] [NULL, "123.1"] ["123.1", NULL] +[2.5, NULL] [NULL, 123.1] [NULL, NULL] +[2.5, NULL, NULL] [NULL, NULL, "123.1"] [NULL, 123.1, NULL] + +-- !sql_19 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_0 -- +1 {"nested":{"a":2.5,"b":"123.1"}} + +-- !sql_1 -- +2.5 + +-- !sql_2 -- +123.1 + +-- !sql_3 -- +\N + +-- !sql_4 -- +{"a":2.5,"b":"123.1"} + +-- !sql_8 -- +\N \N + +-- !sql_9 -- +\N \N + +-- !sql_10 -- +\N \N + +-- !sql_11 -- +\N \N + +-- !sql_12 -- +\N \N + +-- !sql_13 -- +\N \N + +-- !sql_14 -- +\N \N + +-- !sql_15 -- +\N \N + +-- !sql_16 -- +\N \N + +-- !sql_17 -- +2 123 \N + +-- !sql_18 -- +2.5 123.1 \N + +-- !sql_19 -- +2.5 123.1 \N + +-- !sql_0 -- +1 {"nested":{"a":2.5,"b":"123.1"}} +3 {"nested":{"a":2.5,"b":123.1}} +4 {"nested":{"a":2.5,"c":"123.1"}} +5 {"nested":{"a":2.5,"c":123.1}} +6 {"nested":{"a":2.5,"c":"123.1"}} +7 {"nested":{"a":2.5,"b":"123.1","c":123.1}} + +-- !sql_1 -- +2.5 +2.5 +2.5 +2.5 +2.5 +2.5 + +-- !sql_2 -- +"123.1" +123.1 +\N +\N +\N +"123.1" + +-- !sql_3 -- +\N +\N +"123.1" +123.1 +"123.1" +123.1 + +-- !sql_4 -- +{"a":2.5,"b":"123.1"} +{"a":2.5,"b":123.1} +{"a":2.5,"c":"123.1"} +{"a":2.5,"c":123.1} +{"a":2.5,"c":"123.1"} +{"a":2.5,"b":"123.1","c":123.1} + +-- !sql_8 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_9 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_10 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_11 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_12 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_13 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_14 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_15 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_16 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_17 -- +2 123 \N +2 123 \N +2 \N 123 +2 \N 123 +2 \N 123 +2 123 123 + +-- !sql_18 -- +2.5 123.1 \N +2.5 123.1 \N +2.5 \N 123.1 +2.5 \N 123.1 +2.5 \N 123.1 +2.5 123.1 123.1 + +-- !sql_19 -- +2.5 123.1 \N +2.5 123.1 \N +2.5 \N 123.1 +2.5 \N 123.1 +2.5 \N 123.1 +2.5 123.1 123.1 + +-- !sql_0 -- +1 {"nested":{"a":2.5,"b":"123.1"}} +3 {"nested":{"a":2.5,"b":123.1}} +4 {"nested":{"a":2.5,"c":"123.1"}} +5 {"nested":{"a":2.5,"c":123.1}} +6 {"nested":{"a":2.5,"c":"123.1"}} +7 {"nested":{"a":2.5,"b":"123.1","c":123.1}} + +-- !sql_1 -- +2.5 +2.5 +2.5 +2.5 +2.5 +2.5 + +-- !sql_2 -- +"123.1" +123.1 +\N +\N +\N +"123.1" + +-- !sql_3 -- +\N +\N +"123.1" +123.1 +"123.1" +123.1 + +-- !sql_4 -- +{"a":2.5,"b":"123.1"} +{"a":2.5,"b":123.1} +{"a":2.5,"c":"123.1"} +{"a":2.5,"c":123.1} +{"a":2.5,"c":"123.1"} +{"a":2.5,"b":"123.1","c":123.1} + +-- !sql_8 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_9 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_10 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_11 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_12 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_13 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_14 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_15 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_16 -- +\N \N +\N \N +\N \N +\N \N +\N \N +\N \N + +-- !sql_17 -- +2 123 \N +2 123 \N +2 \N 123 +2 \N 123 +2 \N 123 +2 123 123 + +-- !sql_18 -- +2.5 123.1 \N +2.5 123.1 \N +2.5 \N 123.1 +2.5 \N 123.1 +2.5 \N 123.1 +2.5 123.1 123.1 + +-- !sql_19 -- +2.5 123.1 \N +2.5 123.1 \N +2.5 \N 123.1 +2.5 \N 123.1 +2.5 \N 123.1 +2.5 123.1 123.1 + diff --git a/regression-test/data/variant_p0/nested/sql/q01.out b/regression-test/data/variant_p0/nested/sql/q01.out new file mode 100644 index 00000000000000..ea77db963fe272 --- /dev/null +++ b/regression-test/data/variant_p0/nested/sql/q01.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +[1, null] + +-- !q01_2 -- +[null, "1"] + +-- !q01_3 -- +[1.1, null] + +-- !q01_4 -- +[{"a":1,"c":1.1}, {"b":"1"}] + +-- !q01_5 -- +[1, null] 2 + +-- !q01_6 -- +[null, 1] 2 + +-- !q01_7 -- +[1, null] 2 + +-- !q01_8 -- +["1", null] 2 + +-- !q01_9 -- +[null, "1"] 2 + +-- !q01_10 -- +["1.1", null] 2 + diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy index 92da76ad3ce7cf..dcae6c628bf6e4 100644 --- a/regression-test/suites/variant_p0/delete_update.groovy +++ b/regression-test/suites/variant_p0/delete_update.groovy @@ -21,6 +21,7 @@ suite("regression_test_variant_delete_and_update", "variant_type"){ // MOR def table_name = "var_delete_update" sql "DROP TABLE IF EXISTS ${table_name}" + sql """ set disable_variant_flatten_nested = false """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy index 40f30fca016506..30559859f6c726 100644 --- a/regression-test/suites/variant_p0/nested.groovy +++ b/regression-test/suites/variant_p0/nested.groovy @@ -24,7 +24,7 @@ suite("regression_test_variant_nested", "p0"){ def table_name = "var_nested" sql "DROP TABLE IF EXISTS ${table_name}" - + sql "set disable_variant_flatten_nested = false" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, diff --git a/regression-test/suites/variant_p0/nested/load.groovy b/regression-test/suites/variant_p0/nested/load.groovy new file mode 100644 index 00000000000000..0fee39d4b3d8aa --- /dev/null +++ b/regression-test/suites/variant_p0/nested/load.groovy @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// this test is used to test the load of nested array +suite("variant_nested_type_load", "p0"){ + + try { + + // create a table with conflict variant which insert same nested subcolumn and scalar subcolumn data + def table_name = "var_nested_load_conflict" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """set describe_extend_variant_column = true""" + + // set disable_variant_flatten_nested = true to disable variant flatten nested which is default behavior + sql """ set disable_variant_flatten_nested = true """ + test { + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case + properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + """ + exception "If you want to enable variant flatten nested, please set session variable" + } + + + // set disable_variant_flatten_nested = false to enable variant flatten nested + sql """ set disable_variant_flatten_nested = false """ + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case + properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + """ + sql """ insert into ${table_name} values (1, '{"nested": [{"a": 1, "c": 1.1}, {"b": "1"}]}'); """ + + def desc_table = { tn -> + sql """ set describe_extend_variant_column = true """ + sql """ select * from ${tn} order by k """ + qt_sql_desc """ desc ${tn} """ + } + + def sql_select_batch = { tn -> + qt_sql_0 """select * from ${tn} order by k""" + + qt_sql_1 """select v['nested']['a'] from ${tn} order by k""" + qt_sql_2 """select v['nested']['b'] from ${tn} order by k""" + qt_sql_3 """select v['nested']['c'] from ${tn} order by k""" + + qt_sql_4 """select v['nested'] from ${tn} order by k""" + } + + def sql_test_cast_to_array = { tn -> + // test cast to array + qt_sql_8 """select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from ${tn} order by k""" + qt_sql_9 """select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from ${tn} order by k""" + qt_sql_10 """select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from ${tn} order by k""" + + // test cast to array + qt_sql_11 """select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from ${tn} order by k""" + qt_sql_12 """select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from ${tn} order by k""" + qt_sql_13 """select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from ${tn} order by k""" + + // test cast to array + qt_sql_14 """select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from ${tn} order by k""" + qt_sql_15 """select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from ${tn} order by k""" + qt_sql_16 """select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from ${tn} order by k""" + + } + + def sql_test_cast_to_scalar = { tn -> + qt_sql_17 """select cast(v['nested']['a'] as int), cast(v['nested']['b'] as int), cast(v['nested']['c'] as int) from ${tn} order by k""" + qt_sql_18 """select cast(v['nested']['a'] as string), cast(v['nested']['b'] as string), cast(v['nested']['c'] as string) from ${tn} order by k""" + qt_sql_19 """select cast(v['nested']['a'] as double), cast(v['nested']['b'] as double), cast(v['nested']['c'] as double) from ${tn} order by k""" + } + + /// insert a array of object for a, b, c + // insert structure conflict in one row + // a , b, c is Nested array, + def table_name_1 = "var_nested_load_no_conflict" + sql "DROP TABLE IF EXISTS ${table_name_1}" + sql """set describe_extend_variant_column = true""" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name_1} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case + properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + """ + // insert a array of object for a, b, c first then insert structure conflict in one row + // insert structure conflict in one row + // a , b, c is Nested array, + sql """ + insert into ${table_name_1} values (1, '{"nested": [{"a": 1, "c": 1.1}, {"b": "1"}]}'); + """ + sql_select_batch(table_name_1) + sql_test_cast_to_array(table_name_1) + sql_test_cast_to_scalar(table_name_1) + // insert structure conflict in one row + test { + sql """ + insert into ${table_name_1} values (2, '{"nested": {"a": 2.5, "b": "123.1"}}'); + """ + exception "Ambiguous paths" + } + // insert more different combination data for a, b, c + sql """ + insert into ${table_name_1} values (3, '{"nested": [{"a": 2.5, "b": "123.1"}]}'); + """ + sql """ + insert into ${table_name_1} values (4, '{"nested": [{"a": 2.5, "b": 123.1}]}'); + """ + sql """ + insert into ${table_name_1} values (5, '{"nested": [{"a": 2.5, "c": "123.1"}, {"b": "123.1"}]}'); + """ + sql """ + insert into ${table_name_1} values (6, '{"nested": [{"a": 2.5}, {"b": 123.1}]}'); + """ + sql """ + insert into ${table_name_1} values (7, '{"nested": [{"a": 2.5}, {"c": 123.1}, {"b": "123.1"}]}'); + """ + sql_select_batch(table_name_1) + sql_test_cast_to_array(table_name_1) + sql_test_cast_to_scalar(table_name_1) + // trigger and wait compaction + trigger_and_wait_compaction("${table_name_1}", "full") + sql_select_batch(table_name_1) + sql_test_cast_to_array(table_name_1) + sql_test_cast_to_scalar(table_name_1) + + // drop table + sql """ drop table ${table_name_1} """ + sql """ create table ${table_name_1} (k bigint, v variant) duplicate key(k) distributed by hash(k) buckets 1 properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true") """ + // insert scalar data first then insert structure conflict in one row + sql """ + insert into ${table_name_1} values (1, '{"nested": {"a": 2.5, "b": "123.1"}}'); + """ + sql_select_batch(table_name_1) + sql_test_cast_to_array(table_name_1) + sql_test_cast_to_scalar(table_name_1) + // insert structure conflict in one row: a array of object for a, b, c + test { + sql """ + insert into ${table_name_1} values (2, '{"nested": [{"a": 2.5, "b": "123.1"}]}'); + """ + exception "Ambiguous paths" + } + // insert more different combination data for a, b, c in scalar + sql """ + insert into ${table_name_1} values (3, '{"nested": {"a": 2.5, "b": 123.1}}'); + """ + sql """ + insert into ${table_name_1} values (4, '{"nested": {"a": 2.5, "c": "123.1"}}'); + """ + sql """ + insert into ${table_name_1} values (5, '{"nested": {"a": 2.5, "c": 123.1}}'); + """ + sql """ + insert into ${table_name_1} values (6, '{"nested": {"a": 2.5, "c": "123.1"}}'); + """ + sql """ + insert into ${table_name_1} values (7, '{"nested": {"a": 2.5, "b": "123.1", "c": 123.1}}'); + """ + sql_select_batch(table_name_1) + sql_test_cast_to_array(table_name_1) + sql_test_cast_to_scalar(table_name_1) + // trigger and wait compaction + trigger_and_wait_compaction("${table_name_1}", "full") + sql_select_batch(table_name_1) + sql_test_cast_to_array(table_name_1) + sql_test_cast_to_scalar(table_name_1) + + } finally { + } + +} diff --git a/regression-test/suites/variant_p0/nested/sql/q01.sql b/regression-test/suites/variant_p0/nested/sql/q01.sql new file mode 100644 index 00000000000000..71ee81428edf97 --- /dev/null +++ b/regression-test/suites/variant_p0/nested/sql/q01.sql @@ -0,0 +1,13 @@ +-- TABLES: var_nested_load_conflict +select v['nested']['a'] from var_nested_load_conflict order by k; +select v['nested']['b'] from var_nested_load_conflict order by k; +select v['nested']['c'] from var_nested_load_conflict order by k; +select v['nested'] from var_nested_load_conflict order by k; + +select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from var_nested_load_conflict order by k; +select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from var_nested_load_conflict order by k; +select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from var_nested_load_conflict order by k; + +select cast(v['nested']['a'] as array), size(cast(v['nested']['a'] as array)) from var_nested_load_conflict order by k; +select cast(v['nested']['b'] as array), size(cast(v['nested']['b'] as array)) from var_nested_load_conflict order by k; +select cast(v['nested']['c'] as array), size(cast(v['nested']['c'] as array)) from var_nested_load_conflict order by k; \ No newline at end of file diff --git a/regression-test/suites/variant_p0/nested2.groovy b/regression-test/suites/variant_p0/nested2.groovy index 8d48fcfce9b1c0..099b1c903f15d8 100644 --- a/regression-test/suites/variant_p0/nested2.groovy +++ b/regression-test/suites/variant_p0/nested2.groovy @@ -24,6 +24,7 @@ suite("variant_nested_type_conflict", "p0"){ sql "DROP TABLE IF EXISTS ${table_name}" sql """set describe_extend_variant_column = true""" + sql """ set disable_variant_flatten_nested = false """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, @@ -67,6 +68,13 @@ suite("variant_nested_type_conflict", "p0"){ """ exception "Nesting of array in Nested array within variant subcolumns is currently not supported." } + // insert batch different structure in same path + test { + sql """ + insert into ${table_name} values (3, '{"nested": [{"a": 2.5, "b": "123.1"}]}'), (4, '{"nested": {"a": 2.5, "b": "123.1"}}'); + """ + exception "Ambiguous paths" + } /// insert a array of object for a, b, c // insert type conflict in multiple rows sql """ diff --git a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy index 5c9d85fb8edefa..a8b78bdd258582 100644 --- a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy +++ b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy @@ -57,6 +57,7 @@ suite("double_write_schema_change_with_variant", "nonConcurrent") { def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" + sql "set disable_variant_flatten_nested = false" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, From d061d683b3e560f36ee26dcf6df53c7684ce1ab8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:35:53 +0800 Subject: [PATCH 375/572] branch-3.0: [Bug](fold) skip INET6_ATON function in fold by BE #53423 (#53779) Cherry-picked from #53423 Co-authored-by: zhangstar333 --- .../rules/expression/rules/FoldConstantRuleOnBE.java | 7 +++++-- .../fold_constant/fold_constant_by_be.groovy | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java index 0ed2116d1c5583..6d95fb8667a4e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnBE.java @@ -40,6 +40,8 @@ import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.FromBase64; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv6StringToNumOrDefault; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Ipv6StringToNumOrNull; import org.apache.doris.nereids.trees.expressions.functions.scalar.NonNullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Nullable; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sleep; @@ -231,8 +233,9 @@ private static boolean shouldSkipFold(Expression expr) { return true; } - // Skip from_base64 function to avoid incorrect binary data processing during constant folding - if (expr instanceof FromBase64) { + // Skip those function to avoid incorrect binary data processing during constant folding + if (expr instanceof FromBase64 || expr instanceof Ipv6StringToNumOrNull + || expr instanceof Ipv6StringToNumOrDefault) { return true; } diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy index f3b1b0cdcd5b16..9b1a902b5ec127 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_by_be.groovy @@ -86,4 +86,14 @@ suite("fold_constant_by_be") { "from table_200_undef_partitions2_keys3_properties4_distributed_by53;") notContains("mask") } + + sql 'set enable_fold_constant_by_be=true;' + explain { + sql "select IS_IPV4_MAPPED(NULLABLE(INET6_ATON('192.168.1.1')));" + contains "192.168.1.1" + } + explain { + sql "select IS_IPV4_MAPPED(NULLABLE(ipv6_string_to_num_or_default('192.168.1.1')));" + contains "192.168.1.1" + } } From 6b40ab0d28ae53c826e18268b59c5ce30e749037 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:36:31 +0800 Subject: [PATCH 376/572] branch-3.0: [fix](nereids)alter sql block rule lost sql pattern info #53559 (#54091) Cherry-picked from #53559 Co-authored-by: starocean999 --- .../org/apache/doris/blockrule/SqlBlockRuleMgr.java | 2 ++ .../sql_block_rule_p0/test_sql_block_rule.groovy | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java index 13df2eb9377b17..ff64c1999b80d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/blockrule/SqlBlockRuleMgr.java @@ -47,6 +47,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -169,6 +170,7 @@ public void alterSqlBlockRule(AlterSqlBlockRuleStmt stmt) throws AnalysisExcepti if (sqlBlockRule.getEnable() == null) { sqlBlockRule.setEnable(originRule.getEnable()); } + sqlBlockRule.setSqlPattern(Pattern.compile(sqlBlockRule.getSql())); verifyLimitations(sqlBlockRule); SqlBlockUtil.checkAlterValidate(sqlBlockRule); diff --git a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy index 651ee946bd92bb..2864305d25aea1 100644 --- a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy +++ b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy @@ -248,4 +248,17 @@ suite("test_sql_block_rule", "nonConcurrent") { """ } + multi_sql """ + drop SQL_BLOCK_RULE if exists rule_drop_r; + + CREATE SQL_BLOCK_RULE rule_drop + PROPERTIES( + "sql"="select \\* from order_analysis", + "global"="true", + "enable"="true"); + + ALTER SQL_BLOCK_RULE rule_drop PROPERTIES("global"="true"); + + select NULL; + """ } From 21bcd31b3149ea5f622f76ef3620e725369eeeee Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:37:21 +0800 Subject: [PATCH 377/572] branch-3.0: [fix](fe) Fix S3Resource cannot alter iam role arn and ak/sk #54156 (#54468) Cherry-picked from #54156 Co-authored-by: Lei Zhang --- .../org/apache/doris/catalog/S3Resource.java | 18 ++ .../test_alter_resource_with_role.groovy | 233 ++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 regression-test/suites/aws_iam_role_p0/test_alter_resource_with_role.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 22df2c9082f5cd..a767eac5775248 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -28,6 +28,7 @@ import org.apache.doris.fs.obj.S3ObjStorage; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -201,12 +202,29 @@ public void modifyProperties(Map properties) throws DdlException // modify properties writeLock(); + for (Map.Entry kv : properties.entrySet()) { replaceIfEffectiveValue(this.properties, kv.getKey(), kv.getValue()); if (kv.getKey().equals(S3Properties.Env.TOKEN) || kv.getKey().equals(S3Properties.SESSION_TOKEN)) { this.properties.put(kv.getKey(), kv.getValue()); } + + if (kv.getKey().equalsIgnoreCase(S3Properties.ROLE_ARN) + && !Strings.isNullOrEmpty(kv.getValue())) { + this.properties.remove(S3Properties.ACCESS_KEY); + this.properties.remove(S3Properties.Env.ACCESS_KEY); + this.properties.remove(S3Properties.SECRET_KEY); + this.properties.remove(S3Properties.Env.SECRET_KEY); + } + + if (kv.getKey().equalsIgnoreCase(S3Properties.ACCESS_KEY) + && !Strings.isNullOrEmpty(kv.getValue())) { + this.properties.remove(S3Properties.ROLE_ARN); + this.properties.remove(S3Properties.Env.ROLE_ARN); + this.properties.remove(S3Properties.EXTERNAL_ID); + this.properties.remove(S3Properties.Env.EXTERNAL_ID); + } } ++version; writeUnlock(); diff --git a/regression-test/suites/aws_iam_role_p0/test_alter_resource_with_role.groovy b/regression-test/suites/aws_iam_role_p0/test_alter_resource_with_role.groovy new file mode 100644 index 00000000000000..092ec9e28adb9a --- /dev/null +++ b/regression-test/suites/aws_iam_role_p0/test_alter_resource_with_role.groovy @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.base.Strings; +import groovy.json.JsonSlurper + +suite("test_alter_resource_with_role") { + if (Strings.isNullOrEmpty(context.config.awsRoleArn)) { + logger.info("skip ${name} case, because awsRoleArn is null or empty") + return + } + + + if (isCloudMode()) { + logger.info("skip ${name} case, because it is cloud mode") + return + } + + def tableName = "test_alter_resource_with_role" + def randomStr = UUID.randomUUID().toString().replace("-", "") + def resourceName = "alter_resource_${randomStr}" + def policyName = "alter_policy_${randomStr}" + + def awsEndpoint = context.config.awsEndpoint + def region = context.config.awsRegion + def bucket = context.config.awsBucket + def roleArn = context.config.awsRoleArn + def externalId = context.config.awsExternalId + def prefix = context.config.awsPrefix + + def awsAccessKey = context.config.awsAccessKey + def awsSecretKey = context.config.awsSecretKey + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resourceName}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${awsEndpoint}", + "AWS_REGION" = "${region}", + "AWS_BUCKET" = "${bucket}", + "AWS_ROOT_PATH" = "${prefix}/aws_iam_role_p0/test_alter_resource_with_role/${randomStr}", + "AWS_ACCESS_KEY" = "error_ak", + "AWS_SECRET_KEY" = "error_sk", + "s3_validity_check" = "false" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policyName} + PROPERTIES( + "storage_resource" = "${resourceName}", + "cooldown_ttl" = "1" + ) + """ + + sql """ + DROP TABLE IF EXISTS ${tableName} FORCE; + """ + + sql """ + CREATE TABLE ${tableName} + ( + siteid INT DEFAULT '10', + citycode SMALLINT NOT NULL, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_policy" = "${policyName}" + ) + """ + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (2, 2, "yyy", 2), + (3, 3, "zzz", 3) + """ + + def result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """ + log.info("result:${result}") + assertTrue(!result.toString().contains(roleArn)) + assertTrue(!result.toString().contains(externalId)); + assertTrue(result.toString().contains("error_ak")); + + // data_sizes is one arrayList, t is tablet + def fetchDataSize = {List data_sizes, Map t -> + def tabletId = t.TabletId + def meta_url = t.MetaUrl + def clos = { respCode, body -> + logger.info("test ttl expired resp Code {}", "${respCode}".toString()) + assertEquals("${respCode}".toString(), "200") + String out = "${body}".toString() + logger.info("test fetchBeHttp, resp body: ${out}") + def obj = new JsonSlurper().parseText(out) + data_sizes[0] = obj.local_data_size + data_sizes[1] = obj.remote_data_size + } + meta_url = meta_url.replace("header", "data_size") + + def i = meta_url.indexOf("/api") + def endPoint = meta_url.substring(0, i) + def metaUri = meta_url.substring(i) + logger.info("test fetchBeHttp, endpoint:${endPoint}, metaUri:${metaUri}") + i = endPoint.lastIndexOf('/') + endPoint = endPoint.substring(i + 1) + + httpTest { + endpoint {endPoint} + uri metaUri + op "get" + check clos + } + } + + List sizes = [-1, -1] + def tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + log.info( "test tablets not empty:${tablets}") + fetchDataSize(sizes, tablets[0]) + def retry = 12 + while (sizes[1] == 0 && retry-- > 0) { + log.info( "test remote size is zero, sleep 10s") + sleep(10000) + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] == 0, "remote size expected is zero, but got ${sizes[1]}") + assertTrue(tablets.size() > 0) + + // alter resource to arn role + sql """ truncate table ${tableName} """ + sql """ALTER RESOURCE "${resourceName}" + PROPERTIES( + "AWS_ROLE_ARN" = "${roleArn}", + "AWS_EXTERNAL_ID" = "${externalId}" + );""" + + result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """ + log.info("result:${result}") + assertTrue(result.toString().contains(roleArn)) + assertTrue(result.toString().contains(externalId)); + assertTrue(!result.toString().contains("error_ak")); + + // report_tablet_interval_seconds = 60 sleep wait for report policy + sleep(120 * 1000) + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (4, 4, "yyy", 5), + (5, 5, "zzz", 5), + (6, 6, "xxx", 6) + """ + + sizes = [-1, -1] + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + log.info( "test tablets not empty:${tablets}") + fetchDataSize(sizes, tablets[0]) + retry = 100 + while (sizes[1] == 0 && retry-- > 0) { + log.info( "test remote size is zero, sleep 10s") + sleep(10000) + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] != 0, "remote size is still zero, maybe some error occurred") + assertTrue(tablets.size() > 0) + log.info( "test remote size not zero") + + // alter resource to correct ak/sk + sql """ truncate table ${tableName} """ + sql """ALTER RESOURCE "${resourceName}" + PROPERTIES( + "AWS_ACCESS_KEY" = "${awsAccessKey}", + "AWS_SECRET_KEY" = "${awsSecretKey}" + );""" + + result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """ + log.info("result:${result}") + assertTrue(!result.toString().contains(roleArn)) + assertTrue(!result.toString().contains(externalId)); + assertTrue(result.toString().contains(awsAccessKey)); + + // report_tablet_interval_seconds = 60 sleep wait for report policy + sleep(120 * 1000) + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (4, 4, "yyy", 5), + (5, 5, "zzz", 5), + (6, 6, "xxx", 6), + (7, 7, "aaa", 8) + """ + + sizes = [-1, -1] + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + log.info( "test tablets not empty:${tablets}") + fetchDataSize(sizes, tablets[0]) + retry = 100 + while (sizes[1] == 0 && retry-- > 0) { + log.info( "test remote size is zero, sleep 10s") + sleep(10000) + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] != 0, "remote size is still zero, maybe some error occurred") + assertTrue(tablets.size() > 0) + log.info( "test remote size not zero") +} \ No newline at end of file From 4b7944d3335389bd8d3132b25348c93434cd2e12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:38:10 +0800 Subject: [PATCH 378/572] branch-3.0: [fix](nereids) Fix query err when topn push through union_all #54158 (#54257) Cherry-picked from #54158 Co-authored-by: seawinde --- .../rewrite/PushDownTopNThroughUnion.java | 9 +- .../push_down_top_n_through_union.out | 22 ++ .../push_down_top_n_through_union.groovy | 238 ++++++++++++++++++ 3 files changed, 266 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java index c13c1143efd877..b02d7a1e45df91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java @@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier; import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; @@ -63,11 +64,13 @@ public List buildRules() { .then(topN -> { LogicalUnion union = topN.child(); List newChildren = new ArrayList<>(); - for (Plan child : union.children()) { + for (int j = 0; j < union.children().size(); j++) { + Plan child = union.child(j); + List regularChildOutput = union.getRegularChildOutput(j); Map replaceMap = new HashMap<>(); - for (int i = 0; i < union.getOutputs().size(); ++i) { + for (int i = 0; i < regularChildOutput.size(); ++i) { NamedExpression output = union.getOutputs().get(i); - replaceMap.put(output, child.getOutput().get(i)); + replaceMap.put(output, regularChildOutput.get(i)); } List orderKeys = topN.getOrderKeys().stream() diff --git a/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out b/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out index be8053d8349085..6a32bbce857730 100644 --- a/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out +++ b/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out @@ -129,3 +129,25 @@ PhysicalResultSink ------------filter((t2.id < 5) and (t2.score < 20)) --------------PhysicalOlapScan[table1] +-- !union_all_push_down_top_n -- +error_code=500&message=Database+connection+failed {"component":"database","severity":"critical"} 500 Database connection error 192.168.1.103 /api/v1/products 192.168.1.103 api.example.com {"app":"api-gateway","dependencies":["mysql","redis"],"env":"production"} {"city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503},"country":"JP"} {"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"} {"region":"ap-northeast-1","server":"api-node-05"} https://app.example.com/products Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36 {"source":"mobile","tags":["error","database"]} {"context":{"attempts":3,"database":"products_db","duration_ms":5000,"error":"Connection timeout"},"level":"ERROR","message":"Failed to connect to database","timestamp":"2025-07-31T16:45:33Z"} 2025-07-31T16:45:33 +error_code=500&message=Internal+Server+Error {"severity":"high","type":"error"} 500 Internal server error occurred 192.168.1.104 /api/v1/users 192.168.1.104 api.example.com {"app":"web","env":"production"} {"city":"Berlin","country":"Germany"} {"Content-Type":"application/json","X-Request-ID":"req123"} {"hostname":"web-server-02"} https://client.example.com Mozilla/5.0 (Windows NT 10.0; Win64; x64) {"input_type":"error","tags":["error","prod"]} {"level":"error","message":"server error","timestamp":"2025-07-31T18:15:55Z"} 2025-07-31T18:15:55 +order_id=789&amount=99.99 {"method":"credit_card","type":"payment"} 201 Payment processed 192.168.1.102 /api/v1/payments 192.168.1.102 pay.example.com {"app":"payment","env":"production"} {"city":"London","country":"UK"} {"Content-Type":"application/json","X-CSRF-Token":"xyz789"} {"hostname":"payment-gateway-01"} https://checkout.example.com Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) {"input_type":"api","tags":["payment","prod"]} {"level":"info","message":"payment success","timestamp":"2025-07-31T14:45:20Z"} 2025-07-31T14:45:20 +product_id=456&category=electronics {"device":"mobile","type":"product_view"} 304 Product page viewed 192.168.1.101 /products/456 192.168.1.101 shop.example.com {"app":"ecommerce","env":"staging"} {"city":"New York","country":"USA"} {"Cache-Control":"max-age=3600","Content-Type":"text/html"} {"hostname":"shop-server-02"} https://search.example.com Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) {"input_type":"http","tags":["shop","mobile"]} {"level":"info","message":"product view","timestamp":"2025-07-31T11:30:45Z"} 2025-07-31T11:30:45 +product_id=789&quantity=2&price=59.99 {"event":"purchase","payment_method":"credit_card"} 201 Order processed successfully 192.168.1.101 /api/v1/orders 192.168.1.101 shop.example.com {"app":"ecommerce","env":"production","version":"3.2.1"} {"city":"New York","coordinates":{"lat":40.7128,"lon":-74.006},"country":"US"} {"Accept":"application/json","Content-Type":"application/json","X-CSRF-Token":"csrf_abc123"} {"region":"us-east-1","server":"shop-node-03"} https://shop.example.com/product/789 Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1 {"source":"mobile","tags":["transaction","order"]} {"context":{"amount":119.98,"currency":"USD","customer_id":"cust_123abc","order_id":"ord_789xyz"},"level":"INFO","message":"New order created","timestamp":"2025-07-31T11:30:45Z"} 2025-07-31T11:30:45 +query=smartphone&page=1&sort=price_asc {"filters":["in_stock","free_shipping"],"search_type":"product"} 200 Search results returned 192.168.1.104 /api/v1/search 192.168.1.104 search.example.com {"app":"search","env":"production","features":["autocomplete","faceted_search"]} {"city":"Berlin","coordinates":{"lat":52.52,"lon":13.405},"country":"DE"} {"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104, 203.0.113.5"} {"region":"eu-central-1","server":"search-node-01"} https://www.example.com/search Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/125.0 {"source":"web","tags":["search","query"]} {"context":{"filters_applied":2,"query":"smartphone","response_time_ms":125,"results_count":42},"level":"INFO","message":"Search query processed","timestamp":"2025-07-31T19:05:07Z"} 2025-07-31T19:05:07 +search_term=laptop&page=2 {"results_count":25,"type":"search"} 200 Search results returned 192.168.1.103 /search 192.168.1.103 search.example.com {"app":"search","env":"production"} {"city":"Tokyo","country":"Japan"} {"Accept-Language":"en-US","Content-Type":"application/json"} {"hostname":"search-server-03"} https://www.example.com Mozilla/5.0 (Linux; Android 11; SM-G991B) {"input_type":"search","tags":["search","prod"]} {"level":"info","message":"search query","timestamp":"2025-07-31T16:30:10Z"} 2025-07-31T16:30:10 +transaction_id=txn_987654&amount=199.99 {"currency":"USD","gateway":"stripe","type":"payment"} 202 Payment processing started 192.168.1.102 /api/v1/payments 192.168.1.102 pay.example.com {"app":"payment","env":"production","features":["3d_secure","refunds"]} {"city":"London","coordinates":{"lat":51.5074,"lon":-0.1278},"country":"GB"} {"Content-Type":"application/json","Idempotency-Key":"idemp_123","X-Request-ID":"req_987xyz"} {"region":"eu-west-2","server":"payment-gw-02"} https://checkout.example.com/payment Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 {"source":"web","tags":["finance","payment"]} {"context":{"amount":199.99,"currency":"USD","customer":"cust_456def","transaction_id":"txn_987654"},"level":"INFO","message":"Payment initiated","timestamp":"2025-07-31T14:20:18Z"} 2025-07-31T14:20:18 + +-- !union_all_push_down_top_n_shape -- +------------PhysicalOlapScan[t1] +------------PhysicalOlapScan[t2] +----------PhysicalTopN[LOCAL_SORT] +----------PhysicalTopN[LOCAL_SORT] +--------PhysicalTopN[MERGE_SORT] +--------PhysicalTopN[MERGE_SORT] +------PhysicalUnion +----PhysicalTopN[LOCAL_SORT] +--PhysicalTopN[MERGE_SORT] +PhysicalResultSink + diff --git a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy index fa0eacd5b301ba..b8555432865346 100644 --- a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy +++ b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy @@ -74,4 +74,242 @@ suite("push_down_top_n_through_union") { qt_push_down_topn_union_complex_conditions """ explain shape plan select * from (select * from table1 t1 where t1.score > 10 and t1.name = 'Test' union all select * from table1 t2 where t2.id < 5 and t2.score < 20) sub order by id limit 10; """ + + + sql "DROP TABLE IF EXISTS t1"; + + sql """ + CREATE TABLE `t1` ( + + `logTimestamp` datetime NULL, + + `args1` varchar(65533) NULL, + + `args2` variant NULL, + + `args3` int NULL, + + `args4` text NULL, + + `args5` varchar(65533) NULL, + + `args6` varchar(65533) NULL, + + `args7` varchar(200) NULL, + + `args8` varchar(65533) NULL, + + `args9` variant NULL, + + `args10` variant NULL, + + `args11` variant NULL, + + `args12` variant NULL, + + `args13` varchar(65533) NULL, + + `args14` text NULL, + + `args15` variant NULL, + + `log` variant NULL + ) ENGINE=OLAP + + DISTRIBUTED BY RANDOM BUCKETS 10 + + PROPERTIES ( + + "replication_num" = "1" + + );""" + + sql """ + INSERT INTO t1 ( + `logTimestamp`, `args1`, `args2`, `args3`, `args4`, + `args5`, `args6`, `args7`, `args8`, `args9`, + `args10`, `args11`, `args12`, `args13`, `args14`, + `args15`, `log` + ) VALUES + + ('2025-07-31 09:15:22', 'username=admin&password=*****', '{"action":"login","status":"success"}', 200, 'User authentication successful', + '192.168.1.100', '/api/v1/auth/login', '192.168.1.100', 'auth.example.com', '{"app":"authentication","env":"production"}', + '{"country":"CN","city":"Beijing","coordinates":{"lat":39.9042,"lon":116.4074}}', '{"Content-Type":"application/json","Authorization":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"}', '{"server":"auth-node-01","region":"east-1"}', 'https://portal.example.com/login', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', + '{"source":"web","tags":["security","auth"]}', '{"timestamp":"2025-07-31T09:15:22Z","level":"INFO","message":"Successful login for user admin","context":{"ip":"192.168.1.100","user_agent":"Chrome/125.0.0.0","session_id":"sess_xyz123"}}'), + + + ('2025-07-31 11:30:45', 'product_id=789&quantity=2&price=59.99', '{"event":"purchase","payment_method":"credit_card"}', 201, 'Order processed successfully', + '192.168.1.101', '/api/v1/orders', '192.168.1.101', 'shop.example.com', '{"app":"ecommerce","env":"production","version":"3.2.1"}', + '{"country":"US","city":"New York","coordinates":{"lat":40.7128,"lon":-74.0060}}', '{"Content-Type":"application/json","X-CSRF-Token":"csrf_abc123","Accept":"application/json"}', '{"server":"shop-node-03","region":"us-east-1"}', 'https://shop.example.com/product/789', 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1', + '{"source":"mobile","tags":["transaction","order"]}', '{"timestamp":"2025-07-31T11:30:45Z","level":"INFO","message":"New order created","context":{"order_id":"ord_789xyz","amount":119.98,"currency":"USD","customer_id":"cust_123abc"}}'), + + + ('2025-07-31 14:20:18', 'transaction_id=txn_987654&amount=199.99', '{"type":"payment","gateway":"stripe","currency":"USD"}', 202, 'Payment processing started', + '192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com', '{"app":"payment","env":"production","features":["3d_secure","refunds"]}', + '{"country":"GB","city":"London","coordinates":{"lat":51.5074,"lon":-0.1278}}', '{"Content-Type":"application/json","X-Request-ID":"req_987xyz","Idempotency-Key":"idemp_123"}', '{"server":"payment-gw-02","region":"eu-west-2"}', 'https://checkout.example.com/payment', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36', + '{"source":"web","tags":["finance","payment"]}', '{"timestamp":"2025-07-31T14:20:18Z","level":"INFO","message":"Payment initiated","context":{"transaction_id":"txn_987654","amount":199.99,"currency":"USD","customer":"cust_456def"}}'), + + + ('2025-07-31 16:45:33', 'error_code=500&message=Database+connection+failed', '{"severity":"critical","component":"database"}', 500, 'Database connection error', + '192.168.1.103', '/api/v1/products', '192.168.1.103', 'api.example.com', '{"app":"api-gateway","env":"production","dependencies":["mysql","redis"]}', + '{"country":"JP","city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503}}', '{"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"}', '{"server":"api-node-05","region":"ap-northeast-1"}', 'https://app.example.com/products', 'Mozilla/5.0 (Linux; Android 13; SM-S901B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36', + '{"source":"mobile","tags":["error","database"]}', '{"timestamp":"2025-07-31T16:45:33Z","level":"ERROR","message":"Failed to connect to database","context":{"error":"Connection timeout","attempts":3,"database":"products_db","duration_ms":5000}}'), + + + ('2025-07-31 19:05:07', 'query=smartphone&page=1&sort=price_asc', '{"search_type":"product","filters":["in_stock","free_shipping"]}', 200, 'Search results returned', + '192.168.1.104', '/api/v1/search', '192.168.1.104', 'search.example.com', '{"app":"search","env":"production","features":["autocomplete","faceted_search"]}', + '{"country":"DE","city":"Berlin","coordinates":{"lat":52.5200,"lon":13.4050}}', '{"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104, 203.0.113.5"}', '{"server":"search-node-01","region":"eu-central-1"}', 'https://www.example.com/search', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/125.0', + '{"source":"web","tags":["search","query"]}', '{"timestamp":"2025-07-31T19:05:07Z","level":"INFO","message":"Search query processed","context":{"query":"smartphone","results_count":42,"response_time_ms":125,"filters_applied":2}}'); + """ + + sql """ + DROP TABLE IF EXISTS t2; + """ + + sql """ + CREATE TABLE `t2` ( + + `logTimestamp` datetime NULL, + + `args1` varchar(65533) NULL, + + `args2` variant NULL, + + `args3` int NULL, + + `args4` text NULL, + + `args5` varchar(65533) NULL, + + `args6` varchar(65533) NULL, + + `args7` varchar(200) NULL, + + `args8` varchar(65533) NULL, + + `args9` variant NULL, + + `args10` variant NULL, + + `args11` variant NULL, + + `args12` variant NULL, + + `args13` varchar(65533) NULL, + + `args14` text NULL, + + `args15` variant NULL, + + `log` variant NULL + + ) ENGINE=OLAP + + DUPLICATE KEY(`logTimestamp`) + + DISTRIBUTED BY RANDOM BUCKETS 10 + + PROPERTIES ( + + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO t2 ( + `logTimestamp`, `args1`, `args2`, `args3`, `args4`, + `args5`, `args6`, `args7`, `args8`, `args9`, + `args10`, `args11`, `args12`, `args13`, `args14`, + `args15`, `log` + ) VALUES + + ('2025-07-31 10:15:30', 'user_id=123&action=login', '{"type":"authentication","status":"success"}', 200, 'User login successful', + '192.168.1.100', '/api/v1/login', '192.168.1.100', 'api.example.com', '{"app":"web","env":"production"}', + '{"country":"China","city":"Beijing"}', '{"Content-Type":"application/json","Authorization":"Bearer abc123"}', '{"hostname":"web-server-01"}', 'https://referer.example.com', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + '{"input_type":"http","tags":["auth","prod"]}', '{"timestamp":"2025-07-31T10:15:30Z","level":"info","message":"login success"}'), + + + ('2025-07-31 11:30:45', 'product_id=456&category=electronics', '{"type":"product_view","device":"mobile"}', 304, 'Product page viewed', + '192.168.1.101', '/products/456', '192.168.1.101', 'shop.example.com', '{"app":"ecommerce","env":"staging"}', + '{"country":"USA","city":"New York"}', '{"Content-Type":"text/html","Cache-Control":"max-age=3600"}', '{"hostname":"shop-server-02"}', 'https://search.example.com', 'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X)', + '{"input_type":"http","tags":["shop","mobile"]}', '{"timestamp":"2025-07-31T11:30:45Z","level":"info","message":"product view"}'), + + + ('2025-07-31 14:45:20', 'order_id=789&amount=99.99', '{"type":"payment","method":"credit_card"}', 201, 'Payment processed', + '192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com', '{"app":"payment","env":"production"}', + '{"country":"UK","city":"London"}', '{"Content-Type":"application/json","X-CSRF-Token":"xyz789"}', '{"hostname":"payment-gateway-01"}', 'https://checkout.example.com', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)', + '{"input_type":"api","tags":["payment","prod"]}', '{"timestamp":"2025-07-31T14:45:20Z","level":"info","message":"payment success"}'), + + + ('2025-07-31 16:30:10', 'search_term=laptop&page=2', '{"type":"search","results_count":25}', 200, 'Search results returned', + '192.168.1.103', '/search', '192.168.1.103', 'search.example.com', '{"app":"search","env":"production"}', + '{"country":"Japan","city":"Tokyo"}', '{"Content-Type":"application/json","Accept-Language":"en-US"}', '{"hostname":"search-server-03"}', 'https://www.example.com', 'Mozilla/5.0 (Linux; Android 11; SM-G991B)', + '{"input_type":"search","tags":["search","prod"]}', '{"timestamp":"2025-07-31T16:30:10Z","level":"info","message":"search query"}'), + + + ('2025-07-31 18:15:55', 'error_code=500&message=Internal+Server+Error', '{"type":"error","severity":"high"}', 500, 'Internal server error occurred', + '192.168.1.104', '/api/v1/users', '192.168.1.104', 'api.example.com', '{"app":"web","env":"production"}', + '{"country":"Germany","city":"Berlin"}', '{"Content-Type":"application/json","X-Request-ID":"req123"}', '{"hostname":"web-server-02"}', 'https://client.example.com', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)', + '{"input_type":"error","tags":["error","prod"]}', '{"timestamp":"2025-07-31T18:15:55Z","level":"error","message":"server error"}'); + """ + + + sql "DROP VIEW IF EXISTS view_test" + + + sql """ + CREATE VIEW `view_test`AS + SELECT t1.`logTimestamp`, + t1.`args1`, + t1.`args2`, + t1.`args3`, + t1.`args4`, + t1.`args5`, + t1.`args6`, + t1.`args7`, + t1.`args8`, + t1.`args9`, + t1.`args10`, + t1.`args11`, + t1.`args12`, + t1.`args13`, + t1.`args14`, + t1.`args15`, + t1.`log` + FROM t1 + UNION all + SELECT t2.`logTimestamp`, + t2.`args1`, + t2.`args2`, + t2.`args3`, + t2.`args4`, + t2.`args5`, + t2.`args6`, + t2.`args7`, + t2.`args8`, + t2.`args9`, + t2.`args10`, + t2.`args11`, + t2.`args12`, + t2.`args13`, + t2.`args14`, + t2.`args15`, + t2.`log` + FROM t2; + """ + + + order_qt_union_all_push_down_top_n """ + SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`, `log`, logTimestamp + FROM view_test + ORDER BY logTimestamp desc + LIMIT 8;""" + + + order_qt_union_all_push_down_top_n_shape """ + explain shape plan SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`, `log`, logTimestamp + FROM view_test + ORDER BY logTimestamp desc + LIMIT 8; + """ } \ No newline at end of file From 0804a8887537beea1f509f0050aeb518856f1708 Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Mon, 11 Aug 2025 19:38:59 -0700 Subject: [PATCH 379/572] branch-3.0: [fix](query tvf): resolve column mismatch error in JDBC query function (#54077) (#54248) bp #54077 Co-authored-by: XnY-wei <68527868+XnY-wei@users.noreply.github.com> Co-authored-by: weixingyu12 --- be/src/pipeline/exec/jdbc_scan_operator.cpp | 3 +- be/src/pipeline/exec/jdbc_scan_operator.h | 1 + be/src/vec/exec/scan/new_jdbc_scanner.cpp | 7 +- be/src/vec/exec/scan/new_jdbc_scanner.h | 3 +- be/src/vec/exec/vjdbc_connector.cpp | 7 +- be/src/vec/exec/vjdbc_connector.h | 1 + .../apache/doris/jdbc/BaseJdbcExecutor.java | 94 +++++++++++++--- .../apache/doris/jdbc/MySQLJdbcExecutor.java | 105 ++++++++++-------- .../datasource/jdbc/source/JdbcScanNode.java | 1 + gensrc/thrift/PlanNodes.thrift | 1 + gensrc/thrift/Types.thrift | 1 + .../jdbc/test_doris_jdbc_catalog.out | 24 +++- .../jdbc/test_doris_jdbc_catalog.groovy | 36 +++--- 13 files changed, 200 insertions(+), 84 deletions(-) diff --git a/be/src/pipeline/exec/jdbc_scan_operator.cpp b/be/src/pipeline/exec/jdbc_scan_operator.cpp index 54f76396343944..f8d1ff1c4a7e81 100644 --- a/be/src/pipeline/exec/jdbc_scan_operator.cpp +++ b/be/src/pipeline/exec/jdbc_scan_operator.cpp @@ -31,7 +31,7 @@ std::string JDBCScanLocalState::name_suffix() const { Status JDBCScanLocalState::_init_scanners(std::list* scanners) { auto& p = _parent->cast(); std::unique_ptr scanner = vectorized::NewJdbcScanner::create_unique( - state(), this, p._limit, p._tuple_id, p._query_string, p._table_type, + state(), this, p._limit, p._tuple_id, p._query_string, p._table_type, p._is_tvf, _scanner_profile.get()); RETURN_IF_ERROR(scanner->prepare(state(), _conjuncts)); scanners->push_back(std::move(scanner)); @@ -46,6 +46,7 @@ JDBCScanOperatorX::JDBCScanOperatorX(ObjectPool* pool, const TPlanNode& tnode, i _query_string(tnode.jdbc_scan_node.query_string), _table_type(tnode.jdbc_scan_node.table_type) { _output_tuple_id = tnode.jdbc_scan_node.tuple_id; + _is_tvf = tnode.jdbc_scan_node.__isset.is_tvf ? tnode.jdbc_scan_node.is_tvf : false; } } // namespace doris::pipeline diff --git a/be/src/pipeline/exec/jdbc_scan_operator.h b/be/src/pipeline/exec/jdbc_scan_operator.h index 9ceff69ef4cd4b..ac8f582a7eb404 100644 --- a/be/src/pipeline/exec/jdbc_scan_operator.h +++ b/be/src/pipeline/exec/jdbc_scan_operator.h @@ -60,6 +60,7 @@ class JDBCScanOperatorX final : public ScanOperatorX { TupleId _tuple_id; std::string _query_string; TOdbcTableType::type _table_type; + bool _is_tvf; }; } // namespace doris::pipeline diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.cpp b/be/src/vec/exec/scan/new_jdbc_scanner.cpp index a470af27179e92..480b34b3bf410e 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.cpp +++ b/be/src/vec/exec/scan/new_jdbc_scanner.cpp @@ -38,13 +38,15 @@ namespace doris::vectorized { NewJdbcScanner::NewJdbcScanner(RuntimeState* state, doris::pipeline::JDBCScanLocalState* local_state, int64_t limit, const TupleId& tuple_id, const std::string& query_string, - TOdbcTableType::type table_type, RuntimeProfile* profile) + TOdbcTableType::type table_type, bool is_tvf, + RuntimeProfile* profile) : VScanner(state, local_state, limit, profile), _jdbc_eos(false), _tuple_id(tuple_id), _query_string(query_string), _tuple_desc(nullptr), - _table_type(table_type) { + _table_type(table_type), + _is_tvf(is_tvf) { _init_profile(local_state->_scanner_profile); } @@ -84,6 +86,7 @@ Status NewJdbcScanner::prepare(RuntimeState* state, const VExprContextSPtrs& con _jdbc_param.query_string = std::move(_query_string); _jdbc_param.use_transaction = false; // not useful for scanner but only sink. _jdbc_param.table_type = _table_type; + _jdbc_param.is_tvf = _is_tvf; _jdbc_param.connection_pool_min_size = jdbc_table->connection_pool_min_size(); _jdbc_param.connection_pool_max_size = jdbc_table->connection_pool_max_size(); _jdbc_param.connection_pool_max_life_time = jdbc_table->connection_pool_max_life_time(); diff --git a/be/src/vec/exec/scan/new_jdbc_scanner.h b/be/src/vec/exec/scan/new_jdbc_scanner.h index e3e172596efe49..eed5505512b427 100644 --- a/be/src/vec/exec/scan/new_jdbc_scanner.h +++ b/be/src/vec/exec/scan/new_jdbc_scanner.h @@ -47,7 +47,7 @@ class NewJdbcScanner : public VScanner { NewJdbcScanner(RuntimeState* state, doris::pipeline::JDBCScanLocalState* parent, int64_t limit, const TupleId& tuple_id, const std::string& query_string, - TOdbcTableType::type table_type, RuntimeProfile* profile); + TOdbcTableType::type table_type, bool is_tvf, RuntimeProfile* profile); Status open(RuntimeState* state) override; Status close(RuntimeState* state) override; @@ -83,6 +83,7 @@ class NewJdbcScanner : public VScanner { const TupleDescriptor* _tuple_desc = nullptr; // the sql query database type: like mysql, PG... TOdbcTableType::type _table_type; + bool _is_tvf; // Scanner of JDBC. std::unique_ptr _jdbc_connector; JdbcConnectorParam _jdbc_param; diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index a8d8d6fd210405..17e4781a5a6fe8 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -151,6 +151,7 @@ Status JdbcConnector::open(RuntimeState* state, bool read) { ctor_params.__set_connection_pool_cache_clear_time( config::jdbc_connection_pool_cache_clear_time_sec); ctor_params.__set_connection_pool_keep_alive(_conn_param.connection_pool_keep_alive); + ctor_params.__set_is_tvf(_conn_param.is_tvf); jbyteArray ctor_params_bytes; // Pushed frame will be popped when jni_frame goes out-of-scope. @@ -216,8 +217,10 @@ Status JdbcConnector::query() { return Status::InternalError("GetJniExceptionMsg meet error, query={}, msg={}", _conn_param.query_string, status.to_string()); } - if (colunm_count != materialize_num) { - return Status::InternalError("input and output column num not equal of jdbc query."); + if (colunm_count < materialize_num) { + return Status::InternalError( + "JDBC query returned fewer columns ({}) than required ({}).", colunm_count, + materialize_num); } } diff --git a/be/src/vec/exec/vjdbc_connector.h b/be/src/vec/exec/vjdbc_connector.h index 3505d830a363bc..8838ce5984d322 100644 --- a/be/src/vec/exec/vjdbc_connector.h +++ b/be/src/vec/exec/vjdbc_connector.h @@ -56,6 +56,7 @@ struct JdbcConnectorParam { std::string table_name; bool use_transaction = false; TOdbcTableType::type table_type; + bool is_tvf = false; int32_t connection_pool_min_size = -1; int32_t connection_pool_max_size = -1; int32_t connection_pool_max_wait_time = -1; diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java index 4688602d07a5f5..3e71cc5abddb0b 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java @@ -57,6 +57,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -80,6 +81,19 @@ public abstract class BaseJdbcExecutor implements JdbcExecutor { protected String jdbcDriverVersion; private static final Map classLoaderMap = Maps.newConcurrentMap(); + // col name(lowercase) -> index in resultSetMetaData + // this map is only used for "query()" tvf, so only valid if isTvf is true. + // Because for "query()" tvf, the sql string is written by user, so the column name in resultSetMetaData + // maybe larger than the column name in outputTable. + // For example, if the sql is "select a from query('select a,b from tbl')", + // the column num in resultSetMetaData is 2, but the outputTable only has 1 column "a". + // But if the sql is "select a from (select a,b from tbl)x", + // the column num in resultSetMetaData is 1, and the outputTable also has 1 column "a". + // Because the planner will do the column pruning before generating the sql string. + // So, for query() tvf, we need to map the column name in outputTable to the column index in resultSetMetaData. + private Map resultSetColumnMap = null; + private boolean isTvf = false; + public BaseJdbcExecutor(byte[] thriftParams) throws Exception { setJdbcDriverSystemProperties(); TJdbcExecutorCtorParams request = new TJdbcExecutorCtorParams(); @@ -108,6 +122,7 @@ public BaseJdbcExecutor(byte[] thriftParams) throws Exception { JdbcDataSource.getDataSource().setCleanupInterval(request.connection_pool_cache_clear_time); init(config, request.statement); this.jdbcDriverVersion = getJdbcDriverVersion(); + this.isTvf = request.isSetIsTvf() ? request.is_tvf : false; } public void close() throws Exception { @@ -207,32 +222,59 @@ public long getBlockAddress(int batchSize, Map outputParams) thr if (isNullableString == null || replaceString == null) { throw new IllegalArgumentException( - "Output parameters 'is_nullable' and 'replace_string' are required."); + "Output parameters 'is_nullable' and 'replace_string' are required."); } String[] nullableList = isNullableString.split(","); String[] replaceStringList = replaceString.split(","); curBlockRows = 0; - int columnCount = resultSetMetaData.getColumnCount(); - initializeBlock(columnCount, replaceStringList, batchSize, outputTable); + int outputColumnCount = outputTable.getColumns().length; + initializeBlock(outputColumnCount, replaceStringList, batchSize, outputTable); + + // the resultSetColumnMap is only for "query()" tvf + if (this.isTvf && this.resultSetColumnMap == null) { + this.resultSetColumnMap = new HashMap<>(); + int resultSetColumnCount = resultSetMetaData.getColumnCount(); + for (int i = 1; i <= resultSetColumnCount; i++) { + String columnName = resultSetMetaData.getColumnName(i).trim().toLowerCase(); + resultSetColumnMap.put(columnName, i); + } + } do { - for (int i = 0; i < columnCount; ++i) { - ColumnType type = outputTable.getColumnType(i); - block.get(i)[curBlockRows] = getColumnValue(i, type, replaceStringList); + for (int i = 0; i < outputColumnCount; ++i) { + String outputColumnName = outputTable.getFields()[i]; + int columnIndex = getRealColumnIndex(outputColumnName, i); + if (columnIndex > -1) { + ColumnType type = convertTypeIfNecessary(i, outputTable.getColumnType(i), replaceStringList); + block.get(i)[curBlockRows] = getColumnValue(columnIndex, type, replaceStringList); + } else { + throw new RuntimeException("Column not found in resultSetColumnMap: " + outputColumnName); + } } curBlockRows++; } while (curBlockRows < batchSize && resultSet.next()); - for (int i = 0; i < columnCount; ++i) { - ColumnType type = outputTable.getColumnType(i); - Object[] columnData = block.get(i); - Class componentType = columnData.getClass().getComponentType(); - Object[] newColumn = (Object[]) Array.newInstance(componentType, curBlockRows); - System.arraycopy(columnData, 0, newColumn, 0, curBlockRows); - boolean isNullable = Boolean.parseBoolean(nullableList[i]); - outputTable.appendData(i, newColumn, getOutputConverter(type, replaceStringList[i]), isNullable); + for (int i = 0; i < outputColumnCount; ++i) { + String outputColumnName = outputTable.getFields()[i]; + int columnIndex = getRealColumnIndex(outputColumnName, i); + if (columnIndex > -1) { + ColumnType type = outputTable.getColumnType(i); + Object[] columnData = block.get(i); + Class componentType = columnData.getClass().getComponentType(); + Object[] newColumn = (Object[]) Array.newInstance(componentType, curBlockRows); + System.arraycopy(columnData, 0, newColumn, 0, curBlockRows); + boolean isNullable = Boolean.parseBoolean(nullableList[i]); + outputTable.appendData( + i, + newColumn, + getOutputConverter(type, replaceStringList[i]), + isNullable + ); + } else { + throw new RuntimeException("Column not found in resultSetColumnMap: " + outputColumnName); + } } } catch (Exception e) { LOG.warn("jdbc get block address exception: ", e); @@ -243,6 +285,14 @@ public long getBlockAddress(int batchSize, Map outputParams) thr return outputTable.getMetaAddress(); } + private int getRealColumnIndex(String outputColumnName, int indexInOutputTable) { + // -1 because ResultSetMetaData column index starts from 1, but index in outputTable starts from 0. + int columnIndex = this.isTvf + ? resultSetColumnMap.getOrDefault(outputColumnName.toLowerCase(), 0) - 1 : indexInOutputTable; + return columnIndex; + } + + protected void initializeBlock(int columnCount, String[] replaceStringList, int batchSizeNum, VectorTable outputTable) { for (int i = 0; i < columnCount; ++i) { @@ -458,6 +508,19 @@ protected void initializeStatement(Connection conn, JdbcDataSourceConfig config, protected abstract Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException; + /** + * Some special column types (like bitmap/hll in Doris) may need to be converted to string. + * Subclass can override this method to handle such conversions. + * + * @param outputIdx + * @param origType + * @param replaceStringList + * @return + */ + protected ColumnType convertTypeIfNecessary(int outputIdx, ColumnType origType, String[] replaceStringList) { + return origType; + } + /* | Type | Java Array Type | |---------------------------------------------|----------------------------| @@ -666,3 +729,6 @@ protected String defaultByteArrayToHexString(byte[] bytes) { return hexString.toString(); } } + + + diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java index 4e5af95211b508..9ceb613425a972 100644 --- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/MySQLJdbcExecutor.java @@ -101,60 +101,67 @@ protected void initializeBlock(int columnCount, String[] replaceStringList, int } } + @Override + protected ColumnType convertTypeIfNecessary(int outputIdx, ColumnType origType, String[] replaceStringList) { + if (replaceStringList[outputIdx].equals("bitmap") || replaceStringList[outputIdx].equals("hll")) { + return new ColumnType(origType.getName(), Type.BYTE); + } + return origType; + } + @Override protected Object getColumnValue(int columnIndex, ColumnType type, String[] replaceStringList) throws SQLException { - if (replaceStringList[columnIndex].equals("bitmap") || replaceStringList[columnIndex].equals("hll")) { - byte[] data = resultSet.getBytes(columnIndex + 1); - if (resultSet.wasNull()) { - return null; - } - return data; - } else { - switch (type.getType()) { - case BOOLEAN: - return resultSet.getObject(columnIndex + 1, Boolean.class); - case TINYINT: - case SMALLINT: - case LARGEINT: + switch (type.getType()) { + case BOOLEAN: + return resultSet.getObject(columnIndex + 1, Boolean.class); + case TINYINT: + case SMALLINT: + case LARGEINT: + return resultSet.getObject(columnIndex + 1); + case INT: + return resultSet.getObject(columnIndex + 1, Integer.class); + case BIGINT: + return resultSet.getObject(columnIndex + 1, Long.class); + case FLOAT: + return resultSet.getObject(columnIndex + 1, Float.class); + case DOUBLE: + return resultSet.getObject(columnIndex + 1, Double.class); + case DECIMALV2: + case DECIMAL32: + case DECIMAL64: + case DECIMAL128: + return resultSet.getObject(columnIndex + 1, BigDecimal.class); + case DATE: + case DATEV2: + return resultSet.getObject(columnIndex + 1, LocalDate.class); + case DATETIME: + case DATETIMEV2: + return resultSet.getObject(columnIndex + 1, LocalDateTime.class); + case CHAR: + case VARCHAR: + case ARRAY: + return resultSet.getObject(columnIndex + 1, String.class); + case STRING: { + int jdbcType = resultSetMetaData.getColumnType(columnIndex + 1); + // If it is a time type in mysql, or use mysql driver connect mariadb + // We need to obtain the string directly to ensure that we can obtain a time other than 24 hours. + // If it is another database, such as oceanbase, this processing will lose precision information, + // so the original processing method will be maintained for the time being. + if (jdbcType == Types.TIME && config.getTableType() == TOdbcTableType.MYSQL) { + return resultSet.getString(columnIndex + 1); + } else { return resultSet.getObject(columnIndex + 1); - case INT: - return resultSet.getObject(columnIndex + 1, Integer.class); - case BIGINT: - return resultSet.getObject(columnIndex + 1, Long.class); - case FLOAT: - return resultSet.getObject(columnIndex + 1, Float.class); - case DOUBLE: - return resultSet.getObject(columnIndex + 1, Double.class); - case DECIMALV2: - case DECIMAL32: - case DECIMAL64: - case DECIMAL128: - return resultSet.getObject(columnIndex + 1, BigDecimal.class); - case DATE: - case DATEV2: - return resultSet.getObject(columnIndex + 1, LocalDate.class); - case DATETIME: - case DATETIMEV2: - return resultSet.getObject(columnIndex + 1, LocalDateTime.class); - case CHAR: - case VARCHAR: - case ARRAY: - return resultSet.getObject(columnIndex + 1, String.class); - case STRING: { - int jdbcType = resultSetMetaData.getColumnType(columnIndex + 1); - // If it is a time type in mysql, or use mysql driver connect mariadb - // We need to obtain the string directly to ensure that we can obtain a time other than 24 hours. - // If it is another database, such as oceanbase, this processing will lose precision information, - // so the original processing method will be maintained for the time being. - if (jdbcType == Types.TIME && config.getTableType() == TOdbcTableType.MYSQL) { - return resultSet.getString(columnIndex + 1); - } else { - return resultSet.getObject(columnIndex + 1); - } } - default: - throw new IllegalArgumentException("Unsupported column type: " + type.getType()); } + case BYTE: { + byte[] data = resultSet.getBytes(columnIndex + 1); + if (resultSet.wasNull()) { + return null; + } + return data; + } + default: + throw new IllegalArgumentException("Unsupported column type: " + type.getType()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java index 019cceca6a7229..338f564b8045f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java @@ -293,6 +293,7 @@ protected void toThrift(TPlanNode msg) { msg.jdbc_scan_node.setQueryString(getJdbcQueryStr()); } msg.jdbc_scan_node.setTableType(jdbcType); + msg.jdbc_scan_node.setIsTvf(isTableValuedFunction); super.toThrift(msg); } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 10ca220d943bae..304f07a5328672 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -642,6 +642,7 @@ struct TJdbcScanNode { 2: optional string table_name 3: optional string query_string 4: optional Types.TOdbcTableType table_type + 5: optional bool is_tvf } struct TBrokerScanNode { diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index ca4aa9ef68bf87..028aafed19b7d4 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -455,6 +455,7 @@ struct TJdbcExecutorCtorParams { 15: optional bool connection_pool_keep_alive 16: optional i64 catalog_id 17: optional string jdbc_driver_checksum + 18: optional bool is_tvf } struct TJavaUdfExecutorCtorParams { diff --git a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out index 9695f628fee739..220c756a0e28b4 100644 --- a/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out +++ b/regression-test/data/external_table_p0/jdbc/test_doris_jdbc_catalog.out @@ -141,14 +141,34 @@ char_col char(85) Yes true \N NONE varchar_col char(85) Yes true \N NONE json_col text Yes true \N NONE --- !sql -- +-- !sql1 -- \N \N a 1 --- !sql -- +-- !sql2 -- \N \N 1 a +-- !sql3 -- +\N +a + +-- !sql4 -- +\N +1 + +-- !sql5 -- +\N +1 + +-- !sql6 -- +\N \N +1 a + +-- !sql7 -- +\N +1 + -- !sql -- doris_jdbc_catalog diff --git a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy index e57ee5ecb37358..44c2cb62174f5a 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_doris_jdbc_catalog.groovy @@ -231,20 +231,30 @@ suite("test_doris_jdbc_catalog", "p0,external,doris,external_docker,external_doc // test query tvf qt_sql """desc function query("catalog" = "doris_jdbc_catalog", "query" = "select * from regression_test_jdbc_catalog_p0.base");""" - order_qt_sql """ select varchar_col,tinyint_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base");""" + order_qt_sql1 """ select varchar_col,tinyint_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base");""" - order_qt_sql """ select tinyint_col,varchar_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base");""" + order_qt_sql2 """ select tinyint_col,varchar_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base");""" - //clean - qt_sql """select current_catalog()""" - sql "switch internal" - qt_sql """select current_catalog()""" - sql "use regression_test_jdbc_catalog_p0" - sql """ drop table if exists test_doris_jdbc_doris_in_tb """ - sql """ drop table if exists bowen_hll_test """ - sql """ drop table if exists base """ - sql """ drop table if exists all_null_tbl """ - sql """ drop table if exists arr """ - sql """ drop table if exists test_insert_order """ + order_qt_sql3 """ select varchar_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base");""" + + order_qt_sql4 """ select tinyint_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base");""" + + order_qt_sql5 """ with tmp as (select varchar_col,tinyint_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base")) select tinyint_col from tmp;""" + + order_qt_sql6 """ with tmp as (select varchar_col,tinyint_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base")) select tinyint_col,varchar_col from tmp;""" + + order_qt_sql7 """ with tmp as (select tinyint_col,varchar_col from query("catalog" = "doris_jdbc_catalog", "query" = "select varchar_col,tinyint_col from regression_test_jdbc_catalog_p0.base")) select tinyint_col from tmp;""" + + // //clean + // qt_sql """select current_catalog()""" + // sql "switch internal" + // qt_sql """select current_catalog()""" + // sql "use regression_test_jdbc_catalog_p0" + // sql """ drop table if exists test_doris_jdbc_doris_in_tb """ + // sql """ drop table if exists bowen_hll_test """ + // sql """ drop table if exists base """ + // sql """ drop table if exists all_null_tbl """ + // sql """ drop table if exists arr """ + // sql """ drop table if exists test_insert_order """ } From 307d4e5267a03553c39b94ec38968e8f575dc156 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Tue, 12 Aug 2025 10:39:57 +0800 Subject: [PATCH 380/572] branch-30:[Bug](function) fix to_ipv6 cause stack-buffer-overflow error (#53713) (#54011) ### What problem does this PR solve? Problem Summary: cherry-pick from (#53713) ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/common/format_ip.h | 6 +++--- be/src/vec/functions/function_ip.h | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/be/src/vec/common/format_ip.h b/be/src/vec/common/format_ip.h index 45f90d3bdad761..4d9b104fd63ca7 100644 --- a/be/src/vec/common/format_ip.h +++ b/be/src/vec/common/format_ip.h @@ -124,12 +124,12 @@ inline void format_ipv4(const unsigned char* src, char*& dst, uint8_t mask_tail_ */ template requires(std::is_same::type, char>::value) -inline bool parse_ipv4(T*& src, EOFfunction eof, unsigned char* dst, int64_t first_octet = -1) { +inline bool parse_ipv4(T*& src, EOFfunction eof, unsigned char* dst, int32_t first_octet = -1) { if (src == nullptr || first_octet > IPV4_MAX_OCTET_VALUE) { return false; } - int64_t result = 0; + UInt32 result = 0; int offset = (IPV4_BINARY_LENGTH - 1) * IPV4_OCTET_BITS; if (first_octet >= 0) { result |= first_octet << offset; @@ -141,7 +141,7 @@ inline bool parse_ipv4(T*& src, EOFfunction eof, unsigned char* dst, int64_t fir return false; } - int64_t value = 0; + UInt32 value = 0; size_t len = 0; while (is_numeric_ascii(*src) && len <= 3) { value = value * DECIMAL_BASE + (*src - '0'); diff --git a/be/src/vec/functions/function_ip.h b/be/src/vec/functions/function_ip.h index 9c4737c84a78f2..be0f4d7ba47116 100644 --- a/be/src/vec/functions/function_ip.h +++ b/be/src/vec/functions/function_ip.h @@ -150,10 +150,8 @@ ColumnPtr convert_to_ipv4(ColumnPtr column, const PaddedPODArray* null_ma vec_null_map_to = &col_null_map_to->get_data(); } - auto col_res = ToColumn::create(); - + auto col_res = ToColumn::create(column_size, 0); auto& vec_res = col_res->get_data(); - vec_res.resize(column_size); const ColumnString::Chars& vec_src = column_string->get_chars(); const ColumnString::Offsets& offsets_src = column_string->get_offsets(); From e850f5f50c89c9caf3e071166aac9ca2216db88f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:40:42 +0800 Subject: [PATCH 381/572] branch-3.0: [fix](mtmv)Fix incorrect result of show create MTMV when partition by date_trunc #53858 (#53956) Cherry-picked from #53858 Co-authored-by: zhangdong --- .../java/org/apache/doris/catalog/Env.java | 5 +++-- .../mtmv/MTMVPartitionExprDateTrunc.java | 5 +++++ .../doris/mtmv/MTMVPartitionExprService.java | 7 ++++++ .../mtmv_p0/test_show_create_mtmv.groovy | 22 +++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index ee74323ddfa6f7..4f2ef46d49fff1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -186,6 +186,7 @@ import org.apache.doris.meta.MetaContext; import org.apache.doris.metric.MetricRepo; import org.apache.doris.mtmv.MTMVAlterOpType; +import org.apache.doris.mtmv.MTMVPartitionExprFactory; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVRefreshPartitionSnapshot; @@ -3468,7 +3469,7 @@ private static void addMTMVKeyInfo(MTMV mtmv, StringBuilder sb) { } } - private static void addMTMVPartitionInfo(MTMV mtmv, StringBuilder sb) { + private static void addMTMVPartitionInfo(MTMV mtmv, StringBuilder sb) throws AnalysisException { MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo(); if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { return; @@ -3477,7 +3478,7 @@ private static void addMTMVPartitionInfo(MTMV mtmv, StringBuilder sb) { if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) { sb.append("`" + mvPartitionInfo.getPartitionCol() + "`"); } else { - sb.append(mvPartitionInfo.getExpr().toSql()); + sb.append(MTMVPartitionExprFactory.getExprService(mvPartitionInfo.getExpr()).toSql(mvPartitionInfo)); } sb.append(")"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java index 95a8717e01c4c7..01b1c21043d76d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java @@ -83,6 +83,11 @@ public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException } } + @Override + public String toSql(MTMVPartitionInfo mvPartitionInfo) { + return String.format("date_trunc(`%s`, '%s')", mvPartitionInfo.getPartitionCol(), timeUnit); + } + @Override public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map mvProperties) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java index e6974343ef2444..670368f8954496 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java @@ -57,4 +57,11 @@ PartitionKeyDesc generateRollUpPartitionKeyDesc( * @throws AnalysisException */ void analyze(MTMVPartitionInfo mtmvPartitionInfo) throws AnalysisException; + + /** + * for show create MTMV + * @param mvPartitionInfo + * @return + */ + String toSql(MTMVPartitionInfo mvPartitionInfo); } diff --git a/regression-test/suites/mtmv_p0/test_show_create_mtmv.groovy b/regression-test/suites/mtmv_p0/test_show_create_mtmv.groovy index 41f249e5fe382d..d3ad928393fd23 100644 --- a/regression-test/suites/mtmv_p0/test_show_create_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_show_create_mtmv.groovy @@ -121,6 +121,28 @@ suite("test_show_create_mtmv","mtmv") { def showCreateMTMVResultAgain = sql """show CREATE MATERIALIZED VIEW ${mvName}""" logger.info("showCreateMTMVAgainResult: " + showCreateMTMVResultAgain.toString()) assertEquals(showCreateMTMVResult.toString(), showCreateMTMVResultAgain.toString()) + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(date_trunc(`col1`, 'day')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS select date_trunc(`k2`, 'day') as col1 from ${tableName}; + """ + showCreateMTMVResult = sql """show CREATE MATERIALIZED VIEW ${mvName}""" + logger.info("showCreateMTMVResult: " + showCreateMTMVResult.toString()) + assertTrue(showCreateMTMVResult.toString().contains("date_trunc(`col1`, 'day')")) + sql """drop materialized view if exists ${mvName};""" + sql """ + ${showCreateMTMVResult[0][1]} + """ + showCreateMTMVResultAgain = sql """show CREATE MATERIALIZED VIEW ${mvName}""" + logger.info("showCreateMTMVAgainResult: " + showCreateMTMVResultAgain.toString()) + assertEquals(showCreateMTMVResult.toString(), showCreateMTMVResultAgain.toString()) sql """drop table if exists `${tableName}`""" sql """drop materialized view if exists ${mvName};""" From 4632d4894e0dd9b903cce9dad6c9a280e4244c6c Mon Sep 17 00:00:00 2001 From: James Date: Tue, 12 Aug 2025 10:42:49 +0800 Subject: [PATCH 382/572] branch-3.0: [fix](statistics)Fix auto analyze job appender comparator. (#54029) (#54118) backport: https://github.com/apache/doris/pull/54029 --- .../statistics/StatisticsJobAppender.java | 12 +++++++-- .../statistics/StatisticsJobAppenderTest.java | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java index 4a3e93550fc9c1..7597c25633f53e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java @@ -30,10 +30,12 @@ import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -160,8 +162,7 @@ protected void appendToLowJobs(Map>> lowPrio if (!db.isPresent()) { continue; } - List
tables = db.get().getTables().stream() - .sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList()); + List
tables = sortTables(db.get().getTables()); for (Table t : tables) { if (!(t instanceof OlapTable) || t.getId() <= currentTableId) { continue; @@ -213,6 +214,13 @@ protected void appendToLowJobs(Map>> lowPrio lastRoundFinishTime = System.currentTimeMillis(); } + protected List
sortTables(List
tables) { + if (tables == null) { + return Lists.newArrayList(); + } + return tables.stream().sorted(Comparator.comparingLong(Table::getId)).collect(Collectors.toList()); + } + @VisibleForTesting public boolean doAppend(Map>> jobMap, Set> columnIndexPairs, diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java index 5b890795f01076..94c50c91ee3f19 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java @@ -379,4 +379,30 @@ public void testDoAppend() { Assertions.assertEquals(1, jobMap.get(tableName2).size()); Assertions.assertTrue(jobMap.get(tableName2).contains(pair3)); } + + @Test + public void testSortTables() { + Column column1 = new Column("placeholder", PrimitiveType.INT); + List schema = new ArrayList<>(); + schema.add(column1); + OlapTable table1 = new OlapTable(1340000000000L, "testTable", schema, null, null, null); + OlapTable table2 = new OlapTable(3000000000L, "testTable2", schema, null, null, null); + OlapTable table3 = new OlapTable(5000000000L, "testTable3", schema, null, null, null); + OlapTable table4 = new OlapTable(1, "testTable4", schema, null, null, null); + List
tables = Lists.newArrayList(); + tables.add(table1); + tables.add(table2); + tables.add(table3); + tables.add(table4); + StatisticsJobAppender appender = new StatisticsJobAppender(); + List
sortedTables = appender.sortTables(tables); + Assertions.assertEquals(4, sortedTables.size()); + Assertions.assertEquals(1, sortedTables.get(0).getId()); + Assertions.assertEquals(3000000000L, sortedTables.get(1).getId()); + Assertions.assertEquals(5000000000L, sortedTables.get(2).getId()); + Assertions.assertEquals(1340000000000L, sortedTables.get(3).getId()); + + sortedTables = appender.sortTables(null); + Assertions.assertEquals(0, sortedTables.size()); + } } From d646d2c46e0ef229b5e2eb95f9a66b0fcde144d4 Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 12 Aug 2025 10:45:05 +0800 Subject: [PATCH 383/572] branch-3.0: [fix](nereids) project distinct to agg checker exclude window function #54133 (#54541) cherry pick from #54133 --- .../nereids/rules/analysis/CheckAnalysis.java | 3 +- .../OneRowRelationExtractAggregate.java | 25 +++------ .../analysis/ProjectToGlobalAggregate.java | 16 ++---- .../ProjectWithDistinctToAggregate.java | 9 +-- .../doris/nereids/util/ExpressionUtils.java | 20 +++++++ .../project_distinct_to_agg.out | 56 +++++++++++++++++++ regression-test/plugins/plugin_planner.groovy | 28 ++++++++++ .../project_distinct_to_agg.groovy | 32 +++++++++++ 8 files changed, 152 insertions(+), 37 deletions(-) create mode 100644 regression-test/data/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.out create mode 100644 regression-test/plugins/plugin_planner.groovy create mode 100644 regression-test/suites/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java index ae7bb232e83552..e35f420ee7be1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAnalysis.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -173,7 +174,7 @@ private void checkAggregate(LogicalAggregate aggregate) { "The query contains multi count distinct or sum distinct, each can't have multi columns"); } for (Expression expr : aggregate.getGroupByExpressions()) { - if (expr.anyMatch(AggregateFunction.class::isInstance)) { + if (ExpressionUtils.hasNonWindowAggregateFunction(expr)) { throw new AnalysisException( "GROUP BY expression must not contain aggregate functions: " + expr.toSql()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java index 37aaf22ce76031..b5eb0649ee0f85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/OneRowRelationExtractAggregate.java @@ -19,16 +19,13 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalRelation; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; -import java.util.List; - /** * OneRowRelationExtractAggregate. *

@@ -50,19 +47,13 @@ public class OneRowRelationExtractAggregate extends OneAnalysisRuleFactory { @Override public Rule build() { return RuleType.ONE_ROW_RELATION_EXTRACT_AGGREGATE.build( - logicalOneRowRelation().then(relation -> { - List outputs = relation.getOutputs(); - boolean needGlobalAggregate = outputs - .stream() - .anyMatch(p -> p.accept(ExpressionVisitors.CONTAINS_AGGREGATE_CHECKER, null)); - if (needGlobalAggregate) { - LogicalRelation newRelation = new LogicalOneRowRelation(relation.getRelationId(), - ImmutableList.of()); - return new LogicalAggregate<>(ImmutableList.of(), relation.getOutputs(), newRelation); - } else { - return relation; - } - }) + logicalOneRowRelation() + .when(relation -> ExpressionUtils.hasNonWindowAggregateFunction(relation.getOutputs())) + .then(relation -> { + LogicalRelation newRelation = new LogicalOneRowRelation(relation.getRelationId(), + ImmutableList.of()); + return new LogicalAggregate<>(ImmutableList.of(), relation.getOutputs(), newRelation); + }) ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java index da642e76610dc1..a600bf383d1390 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectToGlobalAggregate.java @@ -19,8 +19,8 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.ImmutableList; @@ -43,17 +43,9 @@ public class ProjectToGlobalAggregate extends OneAnalysisRuleFactory { @Override public Rule build() { return RuleType.PROJECT_TO_GLOBAL_AGGREGATE.build( - logicalProject().then(project -> { - boolean needGlobalAggregate = project.getProjects() - .stream() - .anyMatch(p -> p.accept(ExpressionVisitors.CONTAINS_AGGREGATE_CHECKER, null)); - - if (needGlobalAggregate) { - return new LogicalAggregate<>(ImmutableList.of(), project.getProjects(), project.child()); - } else { - return project; - } - }) + logicalProject() + .when(project -> ExpressionUtils.hasNonWindowAggregateFunction(project.getProjects())) + .then(project -> new LogicalAggregate<>(ImmutableList.of(), project.getProjects(), project.child())) ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java index f858820d612ca4..804e29f48c190f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ProjectWithDistinctToAggregate.java @@ -19,10 +19,9 @@ import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.ExpressionUtils; /** * ProjectWithDistinctToAggregate. @@ -46,12 +45,8 @@ public Rule build() { return RuleType.PROJECT_WITH_DISTINCT_TO_AGGREGATE.build( logicalProject() .when(LogicalProject::isDistinct) - .whenNot(project -> project.getProjects().stream().anyMatch(this::hasAggregateFunction)) + .whenNot(project -> ExpressionUtils.hasNonWindowAggregateFunction(project.getProjects())) .then(project -> new LogicalAggregate<>(project.getProjects(), false, project.child())) ); } - - private boolean hasAggregateFunction(Expression expression) { - return expression.anyMatch(AggregateFunction.class::isInstance); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java index fd5f99dfb44993..7ec0f287bff47e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java @@ -53,6 +53,7 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer; @@ -984,4 +985,23 @@ public static boolean unionConstExprsSatisfyConjuncts(LogicalUnion union, Set expressions) { + for (Expression expression : expressions) { + if (hasNonWindowAggregateFunction(expression)) { + return true; + } + } + return false; + } + + /** + * has aggregate function, exclude the window function + */ + public static boolean hasNonWindowAggregateFunction(Expression expression) { + return expression.accept(ExpressionVisitors.CONTAINS_AGGREGATE_CHECKER, null); + } } diff --git a/regression-test/data/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.out b/regression-test/data/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.out new file mode 100644 index 00000000000000..bba41f2bd6138a --- /dev/null +++ b/regression-test/data/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.out @@ -0,0 +1,56 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !with_windows_1_shape -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------PhysicalProject +--------PhysicalWindow +----------PhysicalProject +------------PhysicalOlapScan[tbl_project_distinct_to_agg] + +-- !with_windows_1_result -- +1 12 +2 12 + +-- !with_windows_2_shape -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------PhysicalProject +--------PhysicalWindow +----------PhysicalQuickSort[LOCAL_SORT] +------------PhysicalUnion + +-- !with_windows_2_result -- +100 + +-- !order_by_shape -- +PhysicalResultSink +--PhysicalQuickSort[MERGE_SORT] +----PhysicalQuickSort[LOCAL_SORT] +------hashAgg[GLOBAL] +--------hashAgg[LOCAL] +----------PhysicalUnion + +-- !order_by_result -- +101 + +-- !constant_shape -- +PhysicalResultSink +--PhysicalLimit[GLOBAL] +----PhysicalLimit[LOCAL] +------PhysicalProject +--------PhysicalStorageLayerAggregate[tbl_project_distinct_to_agg] + +-- !constant_result -- +1 2 3 + +-- !agg_shape -- +PhysicalResultSink +--hashAgg[GLOBAL] +----hashAgg[LOCAL] +------PhysicalOlapScan[tbl_project_distinct_to_agg] + +-- !agg_result -- +7 + diff --git a/regression-test/plugins/plugin_planner.groovy b/regression-test/plugins/plugin_planner.groovy new file mode 100644 index 00000000000000..5b0605da6d8f47 --- /dev/null +++ b/regression-test/plugins/plugin_planner.groovy @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.Suite + +Suite.metaClass.explainAndResult = { String tag, String sql -> + "qt_${tag}_shape" "explain shape plan ${sql}" + "qt_${tag}_result" "${sql}" +} + +Suite.metaClass.explainAndOrderResult = { String tag, String sql -> + "qt_${tag}_shape" "explain shape plan ${sql}" + "order_qt_${tag}_result" "${sql}" +} diff --git a/regression-test/suites/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.groovy b/regression-test/suites/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.groovy new file mode 100644 index 00000000000000..f6afb9539a92da --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/project_distinct_to_agg/project_distinct_to_agg.groovy @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('project_distinct_to_agg') { + def tbl = 'tbl_project_distinct_to_agg' + sql "SET ignore_shape_nodes='PhysicalDistribute'" + sql "drop table if exists ${tbl} force" + sql "create table ${tbl} (a int) distributed by hash(a) buckets 10 properties ('replication_num' = '1')" + sql "insert into ${tbl} values (1), (1), (1), (2), (2)" + + explainAndOrderResult 'with_windows_1', "select distinct a, max(a + 10) over() from ${tbl}" + explainAndOrderResult 'with_windows_2', 'select distinct sum(value) over(partition by id) from (select 100 value, 1 id union all select 100, 2)a' + explainAndResult 'order_by', 'select distinct value+1 from (select 100 value, 1 id union all select 100, 2)a order by value+1' + explainAndOrderResult 'constant', "select distinct 1, 2, 3 from ${tbl}" + explainAndOrderResult 'agg', "select distinct sum(a) from ${tbl}" + + sql "drop table if exists ${tbl} force" +} From 2f52e93d60b65f374298430218868415332523e0 Mon Sep 17 00:00:00 2001 From: starocean999 Date: Tue, 12 Aug 2025 10:45:46 +0800 Subject: [PATCH 384/572] branch-3.0:[fix](nereids) insert task should set isNereids to true in ConnectContext (#54451) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … pick https://github.com/apache/doris/pull/53919 Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../java/org/apache/doris/job/extensions/insert/InsertTask.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java index ff18f611c1ed8d..a9cd0185965411 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java @@ -156,6 +156,7 @@ public void before() throws JobException { ctx.setQualifiedUser(userIdentity.getQualifiedUser()); ctx.setCurrentUserIdentity(userIdentity); ctx.getState().reset(); + ctx.getState().setNereids(true); ctx.setThreadLocalInfo(); if (StringUtils.isNotEmpty(currentDb)) { ctx.setDatabase(currentDb); From eaf9d265fa97f29528440b2b4c4299b26dfa1631 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:47:36 +0800 Subject: [PATCH 385/572] branch-3.0: [fix](load) fix missing error url return for stream load #54115 (#54266) Cherry-picked from #54115 Co-authored-by: Xin Liao --- be/src/pipeline/pipeline_fragment_context.cpp | 4 ++++ .../stream_load/test_stream_load_with_filtered_rows.groovy | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp index 950dbd87a4390c..4a069483dc3b96 100644 --- a/be/src/pipeline/pipeline_fragment_context.cpp +++ b/be/src/pipeline/pipeline_fragment_context.cpp @@ -207,6 +207,10 @@ void PipelineFragmentContext::cancel(const Status reason) { auto stream_load_ctx = _exec_env->new_load_stream_mgr()->get(_query_id); if (stream_load_ctx != nullptr) { stream_load_ctx->pipe->cancel(reason.to_string()); + // Set error URL here because after pipe is cancelled, stream load execution may return early. + // We need to set the error URL at this point to ensure error information is properly + // propagated to the client. + stream_load_ctx->error_url = get_load_error_url(); } for (auto& tasks : _tasks) { diff --git a/regression-test/suites/load_p0/stream_load/test_stream_load_with_filtered_rows.groovy b/regression-test/suites/load_p0/stream_load/test_stream_load_with_filtered_rows.groovy index 1801d2be52de36..1a338a0cbf2460 100644 --- a/regression-test/suites/load_p0/stream_load/test_stream_load_with_filtered_rows.groovy +++ b/regression-test/suites/load_p0/stream_load/test_stream_load_with_filtered_rows.groovy @@ -21,7 +21,7 @@ import org.apache.http.util.EntityUtils import java.text.SimpleDateFormat -suite("test_stream_load_with_filtered_rows", "p2") { +suite("test_stream_load_with_filtered_rows", "p0") { sql "show tables" // test length of input is too long than schema. From b2483c73bb03a7688ff31bbb4b3b6cd72b08172d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:49:06 +0800 Subject: [PATCH 386/572] branch-3.0: [opt](object client) Print requestId when s3 request failed #54066 (#54510) Cherry-picked from #54066 Co-authored-by: Lei Zhang --- be/src/io/fs/s3_obj_storage_client.cpp | 197 ++++++++++++++++--------- cloud/src/recycler/s3_obj_client.cpp | 40 ++--- 2 files changed, 144 insertions(+), 93 deletions(-) diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index c6cd48f838625e..e9f81da5e90df1 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -114,30 +114,43 @@ using Aws::S3::Model::UploadPartOutcome; namespace doris::io { using namespace Aws::S3::Model; +static constexpr int S3_REQUEST_THRESHOLD_MS = 5000; + ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { - CreateMultipartUploadRequest create_request; - create_request.WithBucket(opts.bucket).WithKey(opts.key); - create_request.SetContentType("application/octet-stream"); + CreateMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + request.SetContentType("application/octet-stream"); + + MonotonicStopWatch watch; + watch.start(); - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->CreateMultipartUpload(create_request); }), - "s3_file_writer::create_multi_part_upload", std::cref(create_request).get()); + s3_put_rate_limit([&]() { return _client->CreateMultipartUpload(request); }), + "s3_file_writer::create_multi_part_upload", std::cref(request).get()); SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); + watch.stop(); - if (outcome.IsSuccess()) { - return ObjectStorageUploadResponse {.upload_id {outcome.GetResult().GetUploadId()}}; + s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) + << "CreateMultipartUpload cost=" << watch.elapsed_time_milliseconds() << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; + + if (!outcome.IsSuccess()) { + auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", + opts.path.native())); + LOG(WARNING) << st << " request_id=" << request_id; + return ObjectStorageUploadResponse { + .resp = {convert_to_obj_response(std::move(st)), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetRequestId()}, + }; } - return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response( - s3fs_error(outcome.GetError(), - fmt::format("failed to create multipart upload {} ", - opts.path.native()))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}, - }; + return ObjectStorageUploadResponse {.upload_id {outcome.GetResult().GetUploadId()}}; } ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOptions& opts, @@ -150,68 +163,91 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti request.SetBody(string_view_stream); request.SetContentLength(stream.size()); request.SetContentType("application/octet-stream"); - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - auto response = SYNC_POINT_HOOK_RETURN_VALUE( + + MonotonicStopWatch watch; + watch.start(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( s3_put_rate_limit([&]() { return _client->PutObject(request); }), "s3_file_writer::put_object", std::cref(request).get(), &stream); - if (!response.IsSuccess()) { - auto st = s3fs_error(response.GetError(), - fmt::format("failed to put object {}", opts.path.native())); - LOG(WARNING) << st; + + watch.stop(); + + s3_bvar::s3_put_latency << watch.elapsed_time_microseconds(); + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + if (!outcome.IsSuccess()) { + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to put object: {}", opts.path.native())); + LOG(WARNING) << st << ", request_id=" << request_id; return ObjectStorageResponse {convert_to_obj_response(std::move(st)), - static_cast(response.GetError().GetResponseCode()), - response.GetError().GetRequestId()}; + static_cast(outcome.GetError().GetResponseCode()), + request_id}; } + + LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) + << "PutObject cost=" << watch.elapsed_time_milliseconds() << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; return ObjectStorageResponse::OK(); } ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { - UploadPartRequest upload_request; - upload_request.WithBucket(opts.bucket) + UploadPartRequest request; + request.WithBucket(opts.bucket) .WithKey(opts.key) .WithPartNumber(part_num) .WithUploadId(*opts.upload_id); auto string_view_stream = std::make_shared(stream.data(), stream.size()); - upload_request.SetBody(string_view_stream); + request.SetBody(string_view_stream); Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - upload_request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + + request.SetContentLength(stream.size()); + request.SetContentType("application/octet-stream"); + + MonotonicStopWatch watch; + watch.start(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + s3_put_rate_limit([&]() { return _client->UploadPart(request); }), + "s3_file_writer::upload_part", std::cref(request).get(), &stream); - upload_request.SetContentLength(stream.size()); - upload_request.SetContentType("application/octet-stream"); + watch.stop(); - UploadPartOutcome upload_part_outcome; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - upload_part_outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->UploadPart(upload_request); }), - "s3_file_writer::upload_part", std::cref(upload_request).get(), &stream); - } - TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &upload_part_outcome); - if (!upload_part_outcome.IsSuccess()) { - auto s = Status::IOError( - "failed to upload part (bucket={}, key={}, part_num={}, up_load_id={}): {}, " - "exception {}, error code {}", + s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); + if (!outcome.IsSuccess()) { + auto st = Status::IOError( + "failed to UploadPart bucket={}, key={}, part_num={}, upload_id={}, message={}, " + "exception_name={}, response_code={}, request_id={}", opts.bucket, opts.path.native(), part_num, *opts.upload_id, - upload_part_outcome.GetError().GetMessage(), - upload_part_outcome.GetError().GetExceptionName(), - upload_part_outcome.GetError().GetResponseCode()); - LOG_WARNING(s.to_string()); + outcome.GetError().GetMessage(), outcome.GetError().GetExceptionName(), + outcome.GetError().GetResponseCode(), request_id); + + LOG(WARNING) << st << ", request_id=" << request_id; return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response(std::move(s)), - static_cast(upload_part_outcome.GetError().GetResponseCode()), - upload_part_outcome.GetError().GetRequestId()}}; + .resp = {convert_to_obj_response(std::move(st)), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetRequestId()}}; } - return ObjectStorageUploadResponse {.etag = upload_part_outcome.GetResult().GetETag()}; + + LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) + << "UploadPart cost=" << watch.elapsed_time_milliseconds() << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key + << ", part_num=" << part_num << ", upload_id=" << *opts.upload_id; + return ObjectStorageUploadResponse {.etag = outcome.GetResult().GetETag()}; } ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) { - CompleteMultipartUploadRequest complete_request; - complete_request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); + CompleteMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); CompletedMultipartUpload completed_upload; std::vector complete_parts; @@ -223,23 +259,35 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( return part; }); completed_upload.SetParts(std::move(complete_parts)); - complete_request.WithMultipartUpload(completed_upload); + request.WithMultipartUpload(completed_upload); TEST_SYNC_POINT_RETURN_WITH_VALUE("S3FileWriter::_complete:3", ObjectStorageResponse(), this); - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - auto complete_outcome = SYNC_POINT_HOOK_RETURN_VALUE( - s3_put_rate_limit([&]() { return _client->CompleteMultipartUpload(complete_request); }), - "s3_file_writer::complete_multi_part", std::cref(complete_request).get()); - - if (!complete_outcome.IsSuccess()) { - auto st = s3fs_error(complete_outcome.GetError(), - fmt::format("failed to complete multi part upload {}, upload_id={}", + + MonotonicStopWatch watch; + watch.start(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + s3_put_rate_limit([&]() { return _client->CompleteMultipartUpload(request); }), + "s3_file_writer::complete_multi_part", std::cref(request).get()); + + watch.stop(); + s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + if (!outcome.IsSuccess()) { + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", opts.path.native(), *opts.upload_id)); - LOG(WARNING) << st; + LOG(WARNING) << st << ", request_id=" << request_id; return {convert_to_obj_response(std::move(st)), - static_cast(complete_outcome.GetError().GetResponseCode()), - complete_outcome.GetError().GetRequestId()}; + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetRequestId()}; } + + LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) + << "CompleteMultipartUpload cost=" << watch.elapsed_time_milliseconds() << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key + << ", upload_id=" << *opts.upload_id; return ObjectStorageResponse::OK(); } @@ -287,9 +335,9 @@ ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOpti } *size_return = outcome.GetResult().GetContentLength(); if (*size_return != bytes_read) { - return {convert_to_obj_response( - Status::InternalError("failed to read from {}(bytes read: {}, bytes req: {})", - opts.path.native(), *size_return, bytes_read))}; + return {convert_to_obj_response(Status::InternalError( + "failed to read from {}(bytes read: {}, bytes req: {}), request_id: {}", + opts.path.native(), *size_return, bytes_read, outcome.GetResult().GetRequestId()))}; } return ObjectStorageResponse::OK(); } @@ -323,9 +371,10 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp } is_trucated = outcome.GetResult().GetIsTruncated(); if (is_trucated && outcome.GetResult().GetNextContinuationToken().empty()) { - return {convert_to_obj_response(Status::InternalError( - "failed to list {}, is_trucated is true, but next continuation token is empty", - opts.prefix))}; + return {convert_to_obj_response( + Status::InternalError("failed to list {}, is_trucated is true, but next " + "continuation token is empty, request_id={}", + opts.prefix, outcome.GetResult().GetRequestId()))}; } request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); @@ -358,8 +407,9 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath } if (!delete_outcome.GetResult().GetErrors().empty()) { const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response(Status::InternalError("failed to delete object {}: {}", - e.GetKey(), e.GetMessage()))}; + return {convert_to_obj_response( + Status::InternalError("failed to delete object {}: {}, request_id={}", e.GetKey(), + e.GetMessage(), delete_outcome.GetResult().GetRequestId()))}; } return ObjectStorageResponse::OK(); } @@ -423,7 +473,8 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( if (!delete_outcome.GetResult().GetErrors().empty()) { const auto& e = delete_outcome.GetResult().GetErrors().front(); return {convert_to_obj_response(Status::InternalError( - "failed to delete object {}: {}", opts.key, e.GetMessage()))}; + "failed to delete object {}: {}, request_id={}", opts.key, e.GetMessage(), + delete_outcome.GetResult().GetRequestId()))}; } } is_trucated = result.GetIsTruncated(); diff --git a/cloud/src/recycler/s3_obj_client.cpp b/cloud/src/recycler/s3_obj_client.cpp index a5a8977e17b5ff..fe4c368ceda230 100644 --- a/cloud/src/recycler/s3_obj_client.cpp +++ b/cloud/src/recycler/s3_obj_client.cpp @@ -96,13 +96,16 @@ class S3ObjListIterator final : public ObjectListIterator { return client_->ListObjectsV2(req_); }); + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); if (!outcome.IsSuccess()) { LOG_WARNING("failed to list objects") .tag("endpoint", endpoint_) .tag("bucket", req_.GetBucket()) .tag("prefix", req_.GetPrefix()) .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()); + .tag("error", outcome.GetError().GetMessage()) + .tag("request_id", request_id); is_valid_ = false; return false; } @@ -112,7 +115,8 @@ class S3ObjListIterator final : public ObjectListIterator { LOG_WARNING("failed to list objects, isTruncated but no continuation token") .tag("endpoint", endpoint_) .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()); + .tag("prefix", req_.GetPrefix()) + .tag("request_id", request_id); is_valid_ = false; return false; @@ -123,7 +127,8 @@ class S3ObjListIterator final : public ObjectListIterator { const_cast(outcome.GetResult().GetNextContinuationToken()))); auto&& content = outcome.GetResult().GetContents(); - DCHECK(!(has_more_ && content.empty())) << has_more_ << ' ' << content.empty(); + DCHECK(!(has_more_ && content.empty())) + << has_more_ << ' ' << content.empty() << " request_id=" << request_id; results_.reserve(content.size()); for (auto&& obj : std::ranges::reverse_view(content)) { @@ -178,7 +183,8 @@ ObjectStorageResponse S3ObjClient::put_object(ObjectStoragePathRef path, std::st .tag("bucket", path.bucket) .tag("key", path.key) .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()); + .tag("error", outcome.GetError().GetMessage()) + .tag("request_id", outcome.GetError().GetRequestId()); return -1; } return 0; @@ -204,7 +210,8 @@ ObjectStorageResponse S3ObjClient::head_object(ObjectStoragePathRef path, Object .tag("bucket", path.bucket) .tag("key", path.key) .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()); + .tag("error", outcome.GetError().GetMessage()) + .tag("request_id", outcome.GetError().GetRequestId()); return -1; } } @@ -243,18 +250,8 @@ ObjectStorageResponse S3ObjClient::delete_objects(const std::string& bucket, .tag("key[0]", delete_request.GetDelete().GetObjects().front().GetKey()) .tag("responseCode", static_cast(delete_outcome.GetError().GetResponseCode())) - .tag("error", delete_outcome.GetError().GetMessage()); - return -1; - } - - if (!delete_outcome.IsSuccess()) { - LOG_WARNING("failed to delete objects") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("key[0]", delete_request.GetDelete().GetObjects().front().GetKey()) - .tag("responseCode", - static_cast(delete_outcome.GetError().GetResponseCode())) - .tag("error", delete_outcome.GetError().GetMessage()); + .tag("error", delete_outcome.GetError().GetMessage()) + .tag("request_id", delete_outcome.GetError().GetRequestId()); return -1; } @@ -303,7 +300,8 @@ ObjectStorageResponse S3ObjClient::delete_object(ObjectStoragePathRef path) { .tag("key", path.key) .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) .tag("error", outcome.GetError().GetMessage()) - .tag("exception", outcome.GetError().GetExceptionName()); + .tag("exception", outcome.GetError().GetExceptionName()) + .tag("request_id", outcome.GetError().GetRequestId()); if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return {ObjectStorageResponse::NOT_FOUND, outcome.GetError().GetMessage()}; } @@ -339,7 +337,8 @@ ObjectStorageResponse S3ObjClient::get_life_cycle(const std::string& bucket, .tag("endpoint", endpoint_) .tag("bucket", bucket) .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()); + .tag("error", outcome.GetError().GetMessage()) + .tag("request_id", outcome.GetError().GetRequestId()); return -1; } @@ -370,7 +369,8 @@ ObjectStorageResponse S3ObjClient::check_versioning(const std::string& bucket) { .tag("endpoint", endpoint_) .tag("bucket", bucket) .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()); + .tag("error", outcome.GetError().GetMessage()) + .tag("request_id", outcome.GetError().GetRequestId()); return -1; } return 0; From 6f34e43b20af2e92a031edd4988d57df9c11a3cd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:50:19 +0800 Subject: [PATCH 387/572] branch-3.0: [fix](sc) retry on network error #54419 (#54488) Cherry-picked from #54419 Co-authored-by: Yongqiang YANG Co-authored-by: Yongqiang YANG --- .../main/java/org/apache/doris/common/Config.java | 10 +++++----- .../java/org/apache/doris/alter/AlterJobV2.java | 11 +++++++++++ .../java/org/apache/doris/alter/RollupJobV2.java | 9 +-------- .../org/apache/doris/alter/SchemaChangeJobV2.java | 13 +++---------- .../java/org/apache/doris/task/AgentBatchTask.java | 4 ++++ .../test_schema_change_with_compaction11.groovy | 3 ++- 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 94728fe3f9df43..51240074645d45 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3295,12 +3295,12 @@ public static int metaServiceRpcRetryTimes() { "Maximal concurrent num of get tablet stat job."}) public static int max_get_tablet_stat_task_threads_num = 4; - @ConfField(mutable = true, description = {"存算分离模式下schema change失败是否重试", - "Whether to enable retry when schema change failed in cloud model, default is true."}) - public static boolean enable_schema_change_retry_in_cloud_mode = true; + @ConfField(mutable = true, description = {"schema change job 失败是否重试", + "Whether to enable retry when a schema change job fails, default is true."}) + public static boolean enable_schema_change_retry = true; - @ConfField(mutable = true, description = {"存算分离模式下schema change重试次数", - "Max retry times when schema change failed in cloud model, default is 3."}) + @ConfField(mutable = true, description = {"schema change job 重试次数", + "Max retry times when a schema change job fails, default is 3."}) public static int schema_change_max_retry_time = 3; @ConfField(mutable = true, description = {"是否允许使用ShowCacheHotSpotStmt语句", diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java index b1ccf230526117..d13cacae06d20f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java @@ -30,6 +30,7 @@ import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.task.AgentTask; +import org.apache.doris.thrift.TStatusCode; import com.google.common.base.Strings; import com.google.common.collect.Maps; @@ -263,6 +264,16 @@ public final synchronized boolean cancel(String errMsg) { return cancelImpl(errMsg); } + protected int getRetryTimes(AgentTask task) { + int maxFailedTimes = 0; + if (Config.enable_schema_change_retry && task.getErrorCode() != null + && (task.getErrorCode().equals(TStatusCode.DELETE_BITMAP_LOCK_ERROR) + || task.getErrorCode().equals(TStatusCode.NETWORK_ERROR))) { + maxFailedTimes = Config.schema_change_max_retry_time; + } + return maxFailedTimes; + } + /** * should be call before executing the job. * return false if table is not stable. diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java index e5d97d1f97b051..f9a3ba16190090 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java @@ -64,7 +64,6 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.task.CreateReplicaTask; -import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; @@ -543,13 +542,7 @@ protected void runRunningJob() throws AlterCancelException { List tasks = rollupBatchTask.getUnfinishedTasks(2000); ensureCloudClusterExist(tasks); for (AgentTask task : tasks) { - int maxFailedTimes = 0; - if (Config.isCloudMode() && Config.enable_schema_change_retry_in_cloud_mode) { - if (task.getErrorCode() != null && task.getErrorCode() - .equals(TStatusCode.DELETE_BITMAP_LOCK_ERROR)) { - maxFailedTimes = Config.schema_change_max_retry_time; - } - } + int maxFailedTimes = getRetryTimes(task); if (task.getFailedTimes() > maxFailedTimes) { task.setFinished(true); AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java index 28d58e4f6fa5a8..2346cdea92a5b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java @@ -57,7 +57,6 @@ import org.apache.doris.task.AgentTaskQueue; import org.apache.doris.task.AlterReplicaTask; import org.apache.doris.task.CreateReplicaTask; -import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageFormat; import org.apache.doris.thrift.TStorageMedium; import org.apache.doris.thrift.TStorageType; @@ -587,19 +586,13 @@ protected void runRunningJob() throws AlterCancelException { List tasks = schemaChangeBatchTask.getUnfinishedTasks(2000); ensureCloudClusterExist(tasks); for (AgentTask task : tasks) { - int maxFailedTimes = 0; - if (Config.isCloudMode() && Config.enable_schema_change_retry_in_cloud_mode) { - if (task.getErrorCode() != null && task.getErrorCode() - .equals(TStatusCode.DELETE_BITMAP_LOCK_ERROR)) { - maxFailedTimes = Config.schema_change_max_retry_time; - } - } + int maxFailedTimes = getRetryTimes(task); if (task.getFailedTimes() > maxFailedTimes) { task.setFinished(true); if (!FeConstants.runningUnitTest) { AgentTaskQueue.removeTask(task.getBackendId(), TTaskType.ALTER, task.getSignature()); - LOG.warn("schema change task failed, failedTimes: {}, maxFailedTimes: {}, err: {}", - task.getFailedTimes(), maxFailedTimes, task.getErrorMsg()); + LOG.warn("schema change task failed, job: {}, failedTimes: {}, maxFailedTimes: {}, err: {}", + jobId, task.getFailedTimes(), maxFailedTimes, task.getErrorMsg()); List failedBackends = failedTabletBackends.get(task.getTabletId()); if (failedBackends == null) { failedBackends = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java index ead255ace368cd..a821aa143efd91 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/AgentBatchTask.java @@ -49,6 +49,7 @@ import org.apache.doris.thrift.TPushStoragePolicyReq; import org.apache.doris.thrift.TReleaseSnapshotRequest; import org.apache.doris.thrift.TSnapshotRequest; +import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TStorageMediumMigrateReq; import org.apache.doris.thrift.TTaskType; import org.apache.doris.thrift.TUpdateTabletMetaInfoReq; @@ -222,6 +223,9 @@ public void run() { List tasks = this.backendIdToTasks.get(backendId); for (AgentTask task : tasks) { task.failedWithMsg(errMsg); + if (errMsg.contains("Socket is closed")) { + task.setErrorCode(TStatusCode.NETWORK_ERROR); + } } } } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy index 4c1c772da37c93..0e765e46b7b82c 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy @@ -144,8 +144,9 @@ suite('test_schema_change_with_compaction11', 'docker') { GetDebugPoint().disableDebugPointForAllBEs(injectName) } int max_try_time = 3000 + def result = null while (max_try_time--){ - def result = getJobState("date") + result = getJobState("date") if (result == "FINISHED" || result == "CANCELLED") { sleep(3000) break From 8e8aff039bae10d4e95f54e2cb13b4b7b74cab74 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:52:08 +0800 Subject: [PATCH 388/572] branch-3.0: [enhance](recycler)collect fdb metric when recycler start #53656 (#54506) Cherry-picked from #53656 Co-authored-by: koarz <3577087577@qq.com> --- cloud/src/main.cpp | 12 ++++++++++++ cloud/src/meta-service/meta_server.cpp | 9 --------- cloud/src/meta-service/meta_server.h | 1 - cloud/test/meta_server_test.cpp | 10 ++-------- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 6d0c22f3204e80..aa057d6184e139 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -274,6 +275,7 @@ int main(int argc, char** argv) { std::unique_ptr meta_server; // meta-service std::unique_ptr recycler; + std::unique_ptr fdb_metric_exporter; std::thread periodiccally_log_thread; std::mutex periodiccally_log_thread_lock; std::condition_variable periodiccally_log_thread_cv; @@ -315,6 +317,7 @@ int main(int argc, char** argv) { periodiccally_log_thread = std::thread {periodiccally_log}; pthread_setname_np(periodiccally_log_thread.native_handle(), "recycler_periodically_log"); } + // start service brpc::ServerOptions options; if (config::brpc_idle_timeout_sec != -1) { @@ -331,6 +334,14 @@ int main(int argc, char** argv) { return -1; } end = steady_clock::now(); + + fdb_metric_exporter = std::make_unique(txn_kv); + ret = fdb_metric_exporter->start(); + if (ret != 0) { + LOG(WARNING) << "failed to start fdb metric exporter"; + return -2; + } + msg = "successfully started service listening on port=" + std::to_string(port) + " time_elapsed_ms=" + std::to_string(duration_cast(end - start).count()); LOG(INFO) << msg; @@ -344,6 +355,7 @@ int main(int argc, char** argv) { if (recycler) { recycler->stop(); } + fdb_metric_exporter->stop(); if (periodiccally_log_thread.joinable()) { { diff --git a/cloud/src/meta-service/meta_server.cpp b/cloud/src/meta-service/meta_server.cpp index 9677d70b8c042e..6606c41c7085c7 100644 --- a/cloud/src/meta-service/meta_server.cpp +++ b/cloud/src/meta-service/meta_server.cpp @@ -63,14 +63,6 @@ int MetaServer::start(brpc::Server* server) { return -1; } - fdb_metric_exporter_.reset(new FdbMetricExporter(txn_kv_)); - ret = fdb_metric_exporter_->start(); - TEST_SYNC_POINT_CALLBACK("MetaServer::start:3", &ret); - if (ret != 0) { - LOG(WARNING) << "failed to start fdb metric exporter"; - return -2; - } - auto rate_limiter = std::make_shared(); // Add service @@ -91,7 +83,6 @@ int MetaServer::start(brpc::Server* server) { void MetaServer::stop() { server_register_->stop(); - fdb_metric_exporter_->stop(); } void MetaServerRegister::prepare_registry(ServiceRegistryPB* reg) { diff --git a/cloud/src/meta-service/meta_server.h b/cloud/src/meta-service/meta_server.h index 556da7739185a1..15d4985d0977d9 100644 --- a/cloud/src/meta-service/meta_server.h +++ b/cloud/src/meta-service/meta_server.h @@ -48,7 +48,6 @@ class MetaServer { private: std::shared_ptr txn_kv_; std::unique_ptr server_register_; - std::unique_ptr fdb_metric_exporter_; }; class ServiceRegistryPB; diff --git a/cloud/test/meta_server_test.cpp b/cloud/test/meta_server_test.cpp index a6a44851f791e4..b59d39f2dacb66 100644 --- a/cloud/test/meta_server_test.cpp +++ b/cloud/test/meta_server_test.cpp @@ -168,10 +168,9 @@ TEST(MetaServerTest, StartAndStop) { auto sp = SyncPoint::get_instance(); - std::array sps {"MetaServer::start:1", "MetaServer::start:2", - "MetaServer::start:3"}; + std::array sps {"MetaServer::start:1", "MetaServer::start:2"}; // use structured binding for point alias (avoid multi lines of declaration) - auto [meta_server_start_1, meta_server_start_2, meta_server_start_3] = sps; + auto [meta_server_start_1, meta_server_start_2] = sps; sp->enable_processing(); DORIS_CLOUD_DEFER { for (auto& i : sps) { @@ -195,11 +194,6 @@ TEST(MetaServerTest, StartAndStop) { ASSERT_EQ(server->start(&brpc_server), -1); sp->clear_call_back(meta_server_start_2); - // failed to start fdb metrics exporter - sp->set_call_back(meta_server_start_3, foo); - ASSERT_EQ(server->start(&brpc_server), -2); - sp->clear_call_back(meta_server_start_3); - ASSERT_EQ(server->start(&brpc_server), 0); ASSERT_EQ(brpc_server.Start(0, &options), 0); auto addr = brpc_server.listen_address(); From 1a3733310843106d1851c88261e7b0a0a972b361 Mon Sep 17 00:00:00 2001 From: xy720 <22125576+xy720@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:53:36 +0800 Subject: [PATCH 389/572] branch-3.0: [bug](group commit) Fix group commit blocked after schema change throw exception (#54312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix group commit blocked after schema change throw exception Problem Summary: pick #54113 Reproduce step: 1、create table ``` CREATE TABLE `test_table_uniq` ( `company_id` varchar(32) NOT NULL, `date` datetime NOT NULL, `discount` decimal(19,10) NULL, ) ENGINE=OLAP UNIQUE KEY(`company_id`, `date`) DISTRIBUTED BY HASH(`company_id`) BUCKETS 8 ``` 2、group commit insert SUCCESS ``` SET group_commit = async_mode; INSERT INTO test_table_uniq (company_id, date, discount) VALUES(1, '2025-07-25', 10); ``` 3、create rollup and wait job SUCCESS ``` CREATE MATERIALIZED VIEW mv_company_day AS SELECT company_id, date FROM test_table_uniq; ``` 4、group commit insert SUCCESS ``` SET group_commit = async_mode; INSERT INTO test_table_uniq (company_id, date, discount) VALUES(2, '2025-07-25', 11); ``` 5、create rollup with same name and throw exception ``` CREATE MATERIALIZED VIEW mv_company_day AS SELECT company_id, date FROM test_table_uniq; ERROR 1105 (HY000): errCode = 2, detailMessage = Materialized view[mv_company_day] already exists ``` 6、group commit insert FAIL ``` SET group_commit = async_mode; INSERT INTO test_table_uniq (company_id, date, discount) VALUES(3, '2025-07-25', 12); ``` JDBC ERROR LOG: ``` java.sql.BatchUpdateException: errCode = 2, detailMessage = insert table 1753872336812 is blocked on schema change at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at com.mysql.cj.util.Util.handleNewInstance(Util.java:192) at com.mysql.cj.util.Util.getInstance(Util.java:167) at com.mysql.cj.util.Util.getInstance(Util.java:174) at com.mysql.cj.jdbc.exceptions.SQLError.createBatchUpdateException(SQLError.java:224) at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchedInserts(ClientPreparedStatement.java:755) at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchInternal(ClientPreparedStatement.java:426) at com.mysql.cj.jdbc.StatementImpl.executeBatch(StatementImpl.java:795) at DorisStressTest.main(DorisStressTest.java:78) Caused by: java.sql.SQLException: errCode = 2, detailMessage = insert table 1753872336812 is blocked on schema change at com.mysql.cj.jdbc.exceptions.SQLError.createSQLException(SQLError.java:129) at com.mysql.cj.jdbc.exceptions.SQLExceptionsMapping.translateException(SQLExceptionsMapping.java:122) at com.mysql.cj.jdbc.ServerPreparedStatement.serverExecute(ServerPreparedStatement.java:633) at com.mysql.cj.jdbc.ServerPreparedStatement.executeInternal(ServerPreparedStatement.java:417) at com.mysql.cj.jdbc.ClientPreparedStatement.executeUpdateInternal(ClientPreparedStatement.java:1098) at com.mysql.cj.jdbc.ClientPreparedStatement.executeUpdateInternal(ClientPreparedStatement.java:1046) at com.mysql.cj.jdbc.ClientPreparedStatement.executeLargeUpdate(ClientPreparedStatement.java:1371) at com.mysql.cj.jdbc.ClientPreparedStatement.executeBatchedInserts(ClientPreparedStatement.java:716) ``` master/3.0/2.1 ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/alter/MaterializedViewHandler.java | 15 +++++++++++++++ .../insert_p0/insert_group_commit_into.groovy | 19 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 64a747e99e4d6a..84def3c626680a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -162,6 +162,8 @@ private boolean removeAlterJobV2FromTableNotFinalStateJobMap(AlterJobV2 alterJob if (tableNotFinalStateJobIdset == null) { // This could happen when this job is already removed before. // return false, so that we will not set table's to NORMAL again. + LOG.warn("alter job is already removed before. tableId: {}, jobId: {}", + tableId, jobId); return false; } tableNotFinalStateJobIdset.remove(jobId); @@ -228,6 +230,11 @@ public void processCreateMaterializedView(CreateMaterializedViewStmt addMVClause Env.getCurrentEnv().getEditLog().logAlterJob(rollupJobV2); LOG.info("finished to create materialized view job: {}", rollupJobV2.getJobId()); } finally { + if (olapTable.getState() != OlapTableState.ROLLUP) { + // state is not ROLLUP, means encountered some exception before jobs submitted, + // so we need to unblock table here. + Env.getCurrentEnv().getGroupCommitManager().unblockTable(olapTable.getId()); + } olapTable.writeUnlock(); } } @@ -333,6 +340,11 @@ public void processBatchAddRollup(String rawSql, List alterClauses, } throw e; } finally { + if (olapTable.getState() != OlapTableState.ROLLUP) { + // state is not ROLLUP, means encountered some exception before jobs submitted, + // so we need to unblock table here. + Env.getCurrentEnv().getGroupCommitManager().unblockTable(olapTable.getId()); + } olapTable.writeUnlock(); } } @@ -1220,6 +1232,9 @@ private void onJobDone(AlterJobV2 alterJob) { changeTableStatus(alterJob.getDbId(), alterJob.getTableId(), OlapTableState.NORMAL); LOG.info("set table's state to NORMAL, table id: {}, job id: {}", alterJob.getTableId(), alterJob.getJobId()); + } else { + LOG.warn("Failed to remove job from tableNotFinalStateJobMap, table id: {}, job id: {}", + alterJob.getTableId(), alterJob.getJobId()); } } diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy b/regression-test/suites/insert_p0/insert_group_commit_into.groovy index 90318a5226b2fc..b486c4438817ad 100644 --- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy +++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy @@ -229,6 +229,23 @@ suite("insert_group_commit_into") { logger.info("row count: " + rowCount) assertEquals(23, rowCount[0][0]) + // 8. Test create rollup throw exception and group commit behavior + try { + sql """ alter table ${table} ADD ROLLUP r1(name, score); """ + assertTrue(false, "create rollup with duplicate name should fail.") + } catch (Exception e) { + logger.info("Expected create rollup error: " + e.getMessage()) + assertTrue(e.getMessage().contains("already exists")) + } + + group_commit_insert_with_retry """ insert into ${table}(id, name) values(2, 'b'); """, 1 + group_commit_insert_with_retry """ insert into ${table}(id) values(6); """, 1 + getRowCount(25) + + // Verify group commit works after add rollup throw exception + group_commit_insert """ insert into ${table}(id, name) values(2, 'b'); """, 1 + getRowCount(26) + // txn insert sql """ set enable_nereids_dml = true; """ sql """ set enable_nereids_planner=true; """ @@ -242,7 +259,7 @@ suite("insert_group_commit_into") { rowCount = sql "select count(*) from ${table}" logger.info("row count: " + rowCount) - assertEquals(rowCount[0][0], 25) + assertEquals(rowCount[0][0], 28) } } finally { // try_sql("DROP TABLE ${table}") From 0589df60aeb562b8f008856d66693ea6fc0feffe Mon Sep 17 00:00:00 2001 From: zhannngchen Date: Tue, 12 Aug 2025 10:57:01 +0800 Subject: [PATCH 390/572] branch-3.0: [log](cloud-mow) add debug log for error: partition info is empty, table may be dropped (#53016) (#53995) cherry-pick #53016 --- .../cloud/transaction/CloudGlobalTransactionMgr.java | 12 +++++++++++- .../transaction/DeleteBitmapUpdateLockContext.java | 8 +++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 944099409dc397..0f50ec8704f8f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -364,7 +364,7 @@ public void commitTransactionWithoutLock(long dbId, List

tableList, long if (!checkTransactionStateBeforeCommit(dbId, transactionId)) { return; } - DeleteBitmapUpdateLockContext lockContext = new DeleteBitmapUpdateLockContext(); + DeleteBitmapUpdateLockContext lockContext = new DeleteBitmapUpdateLockContext(transactionId); getDeleteBitmapUpdateLock(transactionId, mowTableList, tabletCommitInfos, lockContext); if (lockContext.getBackendToPartitionTablets().isEmpty()) { throw new UserException( @@ -764,6 +764,16 @@ private void getPartitionInfo(List tableList, partitionToTablets.get(partitionId).add(tabletIds.get(i)); lockContext.getPartitions().putIfAbsent(partitionId, tableMap.get(tableId).getPartition(partitionId)); } + if (!tableList.isEmpty() && !tabletCommitInfos.isEmpty() && lockContext.getTableToTabletList().isEmpty()) { + String tableListDebugStr = tableList.stream().map(table -> String.valueOf(table.getId())) + .collect(Collectors.joining(", ")); + String tabletMetaDebugStr = tabletMetaList.stream().map(TabletMeta::toString) + .collect(Collectors.joining("\n")); + LOG.warn( + "getPartitionInfo for lock_id: {} failed, LockContext.TableToTabletList is empty," + + " this should never happen. tableList: {}, tabletIds: {}, tabletMetaList: {}", + lockContext.getLockId(), tableListDebugStr, tabletIds.toString(), tabletMetaDebugStr); + } } private Map getPartitionVersions(Map partitionMap) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/DeleteBitmapUpdateLockContext.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/DeleteBitmapUpdateLockContext.java index 120715d627610b..9064947d148ac9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/DeleteBitmapUpdateLockContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/DeleteBitmapUpdateLockContext.java @@ -27,6 +27,7 @@ import java.util.Set; public class DeleteBitmapUpdateLockContext { + private long lockId; private Map baseCompactionCnts; private Map cumulativeCompactionCnts; private Map cumulativePoints; @@ -37,7 +38,8 @@ public class DeleteBitmapUpdateLockContext { private Map> tableToTabletList; private Map tabletToTabletMeta; - public DeleteBitmapUpdateLockContext() { + public DeleteBitmapUpdateLockContext(long lockId) { + this.lockId = lockId; baseCompactionCnts = Maps.newHashMap(); cumulativeCompactionCnts = Maps.newHashMap(); cumulativePoints = Maps.newHashMap(); @@ -49,6 +51,10 @@ public DeleteBitmapUpdateLockContext() { tabletToTabletMeta = Maps.newHashMap(); } + public long getLockId() { + return lockId; + } + public Map> getTableToTabletList() { return tableToTabletList; } From 7f573c3be01c0426fbd838e01e870ecb1b2ad3e3 Mon Sep 17 00:00:00 2001 From: TengJianPing Date: Tue, 12 Aug 2025 10:59:09 +0800 Subject: [PATCH 391/572] [fix](decimal) fix wrong result of decimal divide (#54453) --- be/src/vec/functions/function_binary_arithmetic.h | 8 +++++--- be/src/vec/functions/minus.cpp | 2 +- be/src/vec/functions/multiply.cpp | 2 +- be/src/vec/functions/plus.cpp | 2 +- .../decimalv3/test_arithmetic_expressions.out | 2 ++ .../decimalv3/test_arithmetic_expressions.groovy | 10 ++++++++++ 6 files changed, 20 insertions(+), 6 deletions(-) diff --git a/be/src/vec/functions/function_binary_arithmetic.h b/be/src/vec/functions/function_binary_arithmetic.h index 9f2af326f71901..c20d27767cd01a 100644 --- a/be/src/vec/functions/function_binary_arithmetic.h +++ b/be/src/vec/functions/function_binary_arithmetic.h @@ -238,7 +238,9 @@ struct DecimalBinaryOperation { using OpTraits = OperationTraits; using NativeResultType = typename NativeType::Type; - using Op = Operation; + using NativeLeftType = typename NativeType::Type; + using NativeRightType = typename NativeType::Type; + using Op = Operation; using Traits = NumberTraits::BinaryOperatorTraits; using ArrayC = typename ColumnDecimal::Container; @@ -654,11 +656,11 @@ struct DecimalBinaryOperation { } /// null_map for divide and mod - static ALWAYS_INLINE NativeResultType apply(NativeResultType a, NativeResultType b, + static ALWAYS_INLINE NativeResultType apply(const NativeLeftType& a, const NativeRightType& b, UInt8& is_null, const ResultType& max_result_number) { static_assert(OpTraits::is_division || OpTraits::is_mod); - if constexpr (IsDecimalV2 || IsDecimalV2) { + if constexpr (IsDecimalV2 && IsDecimalV2) { DecimalV2Value l(a); DecimalV2Value r(b); auto ans = Op::template apply(l, r, is_null); diff --git a/be/src/vec/functions/minus.cpp b/be/src/vec/functions/minus.cpp index 122842b2d9d7f6..b666fffd673c04 100644 --- a/be/src/vec/functions/minus.cpp +++ b/be/src/vec/functions/minus.cpp @@ -46,7 +46,7 @@ struct MinusImpl { /// Apply operation and check overflow. It's used for Decimal operations. @returns true if overflowed, false otherwise. template static inline bool apply(A a, B b, Result& c) { - return common::sub_overflow(static_cast(a), b, c); + return common::sub_overflow(static_cast(a), static_cast(b), c); } }; diff --git a/be/src/vec/functions/multiply.cpp b/be/src/vec/functions/multiply.cpp index f871a3a742cb6c..458df6ae8d06b6 100644 --- a/be/src/vec/functions/multiply.cpp +++ b/be/src/vec/functions/multiply.cpp @@ -99,7 +99,7 @@ struct MultiplyImpl { /// Apply operation and check overflow. It's used for Decimal operations. @returns true if overflowed, false otherwise. template static inline bool apply(A a, B b, Result& c) { - return common::mul_overflow(static_cast(a), b, c); + return common::mul_overflow(static_cast(a), static_cast(b), c); } }; diff --git a/be/src/vec/functions/plus.cpp b/be/src/vec/functions/plus.cpp index 480db82a58100a..6aeffacb025474 100644 --- a/be/src/vec/functions/plus.cpp +++ b/be/src/vec/functions/plus.cpp @@ -47,7 +47,7 @@ struct PlusImpl { /// Apply operation and check overflow. It's used for Decimal operations. @returns true if overflowed, false otherwise. template static inline bool apply(A a, B b, Result& c) { - return common::add_overflow(static_cast(a), b, c); + return common::add_overflow(static_cast(a), static_cast(b), c); } }; diff --git a/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out b/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out index 7b74bec40d499b..3d72fcc9eaca79 100644 --- a/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out +++ b/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out @@ -327,3 +327,5 @@ 76891560.464178000 5 0.464178000 0.464178000 277170831.851350000 7 0.851350000 0.851350000 +-- !fix_decimal_divide_0 -- +100.02 9223372036854775807 0.000000 diff --git a/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy b/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy index 5936cf29b45fba..cbe52a6b6e94c8 100644 --- a/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy +++ b/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy @@ -455,4 +455,14 @@ mysql [test]>select k3, CAST(k3 AS DECIMALV3(38, 10)) from test_arithmetic_expre qt_decimal256_mod """ select v1, v2, v1 % v2, v1 % v3 from test_arithmetic_expressions_256_5 ORDER BY id; """ + // bugfix, divide + sql "DROP TABLE IF EXISTS `fix_decimal_divide`" + sql """ + create table fix_decimal_divide(f1 decimalv3(5,2), f2 decimalv3(20,0)) properties("replication_num"="1"); + """ + sql """ + insert into fix_decimal_divide values(100.02, 9223372036854775807); + """ + qt_fix_decimal_divide_0 "select f1, f2, f1 / f2 from fix_decimal_divide order by 1,2,3;" + } From 7aa0f97438fa7a29071f1b909294c181e29a73aa Mon Sep 17 00:00:00 2001 From: MoanasDaddyXu Date: Tue, 12 Aug 2025 10:59:53 +0800 Subject: [PATCH 392/572] branch-3.0: [fix](case) sync master routine load cases to branch-3.0 (#53920) ### What problem does this PR solve? Issue Number: close #xxx Related PR: #53214 Problem Summary: ### Release note None ### Check List (For Author) - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [x] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../routine_load/test_black_list.groovy | 26 +++ .../routine_load/test_disable_load.groovy | 26 +++ ...multi_table_load_data_quality_error.groovy | 25 +++ .../test_multi_table_load_error.groovy | 25 +++ .../test_out_of_range_error.groovy | 25 +++ ...st_routin_load_abnormal_job_monitor.groovy | 25 +++ .../test_routine_load_alter.groovy | 25 +++ .../routine_load/test_routine_load_eof.groovy | 25 +++ .../test_routine_load_error.groovy | 25 +++ .../test_routine_load_error_info.groovy | 26 +++ .../test_routine_load_follower_fe.groovy | 178 ++++++++++++++++++ .../test_routine_load_metrics.groovy | 25 +++ .../test_routine_load_offset.groovy | 25 +++ .../test_routine_load_progress.groovy | 26 +++ .../test_routine_load_property.groovy | 25 +++ .../test_routine_load_restart_fe.groovy | 25 +++ .../test_routine_load_schedule.groovy | 29 ++- .../test_routine_load_timeout_value.groovy | 26 +++ .../test_routine_load_topic_change.groovy | 25 +++ .../test_routine_load_with_sc.groovy | 25 +++ .../test_routine_load_with_udf.groovy | 25 +++ .../test_routine_load_with_user.groovy | 25 +++ .../test_show_routine_load.groovy | 25 +++ 23 files changed, 735 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy diff --git a/regression-test/suites/load_p0/routine_load/test_black_list.groovy b/regression-test/suites/load_p0/routine_load/test_black_list.groovy index 807389e5ec924b..9b1e2726f9dc0e 100644 --- a/regression-test/suites/load_p0/routine_load/test_black_list.groovy +++ b/regression-test/suites/load_p0/routine_load/test_black_list.groovy @@ -34,7 +34,33 @@ suite("test_black_list","nonConcurrent,p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() diff --git a/regression-test/suites/load_p0/routine_load/test_disable_load.groovy b/regression-test/suites/load_p0/routine_load/test_disable_load.groovy index 022a9ae92d0a3c..38651775ab08cf 100644 --- a/regression-test/suites/load_p0/routine_load/test_disable_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_disable_load.groovy @@ -34,7 +34,33 @@ suite("test_disable_load","nonConcurrent,p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() diff --git a/regression-test/suites/load_p0/routine_load/test_multi_table_load_data_quality_error.groovy b/regression-test/suites/load_p0/routine_load/test_multi_table_load_data_quality_error.groovy index 549dbdf3f594b0..fbd334a1455984 100644 --- a/regression-test/suites/load_p0/routine_load/test_multi_table_load_data_quality_error.groovy +++ b/regression-test/suites/load_p0/routine_load/test_multi_table_load_data_quality_error.groovy @@ -34,8 +34,33 @@ suite("test_multi_table_load_data_quality_error","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy b/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy index 039e734466e33e..7ffa6efc149929 100644 --- a/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy +++ b/regression-test/suites/load_p0/routine_load/test_multi_table_load_error.groovy @@ -37,8 +37,33 @@ suite("test_multi_table_load_eror","nonConcurrent") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_out_of_range_error.groovy b/regression-test/suites/load_p0/routine_load/test_out_of_range_error.groovy index 1ae74b73301298..c16a1b0dae0cb4 100644 --- a/regression-test/suites/load_p0/routine_load/test_out_of_range_error.groovy +++ b/regression-test/suites/load_p0/routine_load/test_out_of_range_error.groovy @@ -36,8 +36,33 @@ suite("test_out_of_range","nonConcurrent") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routin_load_abnormal_job_monitor.groovy b/regression-test/suites/load_p0/routine_load/test_routin_load_abnormal_job_monitor.groovy index 7cc08b5b813feb..4cc3a743becdbc 100644 --- a/regression-test/suites/load_p0/routine_load/test_routin_load_abnormal_job_monitor.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routin_load_abnormal_job_monitor.groovy @@ -38,8 +38,33 @@ suite("test_routine_load_abnormal_job_monitor","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy index 0089aff61e56b4..b1d418180ebf32 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_alter.groovy @@ -34,8 +34,33 @@ suite("test_routine_load_alter","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_eof.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_eof.groovy index ac0b08248ef51b..b200cb22e5441c 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_eof.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_eof.groovy @@ -37,8 +37,33 @@ suite("test_routine_load_eof","nonConcurrent") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") def count = 0 while(true) { diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy index 844d4e5a1830e0..d103dc5eb0f83f 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_error.groovy @@ -40,8 +40,33 @@ suite("test_routine_load_error","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy index 2f018a937291dd..7394d72c41d245 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_error_info.groovy @@ -36,7 +36,33 @@ suite("test_routine_load_error_info","nonConcurrent") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy new file mode 100644 index 00000000000000..8f7ed9a4cf6bcb --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions +import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig + +suite("test_routine_load_follower_fe","docker") { + def options = new ClusterOptions() + // Configure 3 FE nodes cluster + options.setFeNum(3) + options.setBeNum(1) + + docker(options) { + def kafkaCsvTpoics = [ + "test_routine_load_follower_fe", + ] + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + // 1. send data to kafka + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + // Create kafka producer + def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") + + // Send test data to kafka topic + for (String kafkaCsvTopic in kafkaCsvTpoics) { + // Create simple test data + def testData = [ + "1,test_data_1,2023-01-01,value1,2023-01-01 10:00:00,extra1", + "2,test_data_2,2023-01-02,value2,2023-01-02 11:00:00,extra2", + "3,test_data_3,2023-01-03,value3,2023-01-03 12:00:00,extra3", + "4,test_data_4,2023-01-04,value4,2023-01-04 13:00:00,extra4", + "5,test_data_5,2023-01-05,value5,2023-01-05 14:00:00,extra5" + ] + + testData.each { line -> + logger.info("Sending data to kafka: ${line}") + def record = new ProducerRecord<>(kafkaCsvTopic, null, line) + producer.send(record) + } + } + producer.close() + + // 3. Connect to a follower FE and create table + def masterFe = cluster.getMasterFe() + def allFes = cluster.getAllFrontends() + def followerFes = allFes.findAll { fe -> fe.index != masterFe.index } + def followerFe = followerFes[0] + logger.info("Master FE: ${masterFe.host}") + logger.info("Using follower FE: ${followerFe.host}") + // Connect to follower FE + def url = String.format( + "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", + followerFe.host, followerFe.queryPort) + logger.info("Connecting to follower FE: ${url}") + context.connectTo(url, context.config.jdbcUser, context.config.jdbcPassword) + + sql "drop database if exists test_routine_load_follower_fe" + sql "create database test_routine_load_follower_fe" + sql "use test_routine_load_follower_fe" + def tableName = "test_routine_load_follower_fe" + def job = "test_follower_routine_load" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(20) NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + + try { + // 4. Create routine load job on follower FE + sql """ + CREATE ROUTINE LOAD ${job} ON ${tableName} + COLUMNS TERMINATED BY "," + PROPERTIES + ( + "max_batch_interval" = "20", + "max_batch_rows" = "300000", + "max_batch_size" = "209715200" + ) + FROM KAFKA + ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${kafkaCsvTpoics[0]}", + "property.group.id" = "test-follower-consumer-group", + "property.client.id" = "test-follower-client-id", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + + // 5. Wait for routine load to process data + def count = 0 + def maxWaitCount = 60 // Wait up to 60 seconds + while (count < maxWaitCount) { + def state = sql "show routine load for ${job}" + def routineLoadState = state[0][8].toString() + def statistic = state[0][14].toString() + logger.info("Routine load state: ${routineLoadState}") + logger.info("Routine load statistic: ${statistic}") + + def rowCount = sql "select count(*) from ${tableName}" + // Check if routine load is running and has processed some data + if (routineLoadState == "RUNNING" && rowCount[0][0] > 0) { + break + } + + sleep(1000) + count++ + } + } catch (Exception e) { + logger.error("Test failed with exception: ${e.message}") + } finally { + try { + sql "stop routine load for ${job}" + } catch (Exception e) { + logger.warn("Failed to stop routine load job: ${e.message}") + } + } + } + } +} \ No newline at end of file diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_metrics.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_metrics.groovy index bb1afb6dd348e1..33b5166a9a8e48 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_metrics.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_metrics.groovy @@ -38,8 +38,33 @@ suite("test_routine_load_metrics","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_offset.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_offset.groovy index 84d0509cea3ffc..1280d3dbe4cc6d 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_offset.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_offset.groovy @@ -34,8 +34,33 @@ suite("test_routine_load_offset","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_progress.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_progress.groovy index c372c5826b2464..a353b4da7b1aa3 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_progress.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_progress.groovy @@ -40,7 +40,33 @@ suite("test_routine_load_progress","docker") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_property.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_property.groovy index 9cc1fa0d2d96f1..cd10af8dfe5fd1 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_property.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_property.groovy @@ -36,8 +36,33 @@ suite("test_routine_load_property","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy index 104026fb16e3f6..cedbf998e47af6 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_restart_fe.groovy @@ -38,8 +38,33 @@ suite("test_routine_load_restart_fe", "docker") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_schedule.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_schedule.groovy index d0f01fc3d04e6b..1d37e975eeb9aa 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_schedule.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_schedule.groovy @@ -37,8 +37,33 @@ suite("test_routine_load_schedule","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text @@ -70,8 +95,8 @@ suite("test_routine_load_schedule","p0") { k07 LARGEINT NULL, k08 FLOAT NULL, k09 DOUBLE NULL, - k10 DECIMAL(9,1) NULL, - k11 DECIMALV3(9,1) NULL, + k10 DECIMAL(10,1) NULL, + k11 DECIMALV3(10,1) NULL, k12 DATETIME NULL, k13 DATEV2 NULL, k14 DATETIMEV2 NULL, diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy index 873b945857212c..937dba4424acc4 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_timeout_value.groovy @@ -36,7 +36,33 @@ suite("test_routine_load_timeout_value","nonConcurrent") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } + // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text def lines = txt.readLines() diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_topic_change.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_topic_change.groovy index 25bf9933d112ff..09a1f970e2fb48 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_topic_change.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_topic_change.groovy @@ -36,8 +36,33 @@ suite("test_routine_load_topic_change","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_sc.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_sc.groovy index 33c047062dd8b9..02220a2d1b89fe 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_sc.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_sc.groovy @@ -36,8 +36,33 @@ suite("test_routine_load_with_sc","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy index 9f4ae866e992e5..9bbb7ee208d680 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_udf.groovy @@ -34,8 +34,33 @@ suite("test_routine_load_with_udf","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy index 3611e1cc0d6035..754c5d82aae059 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_with_user.groovy @@ -39,8 +39,33 @@ suite("test_routine_load_with_user","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text diff --git a/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy b/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy index d6b31db11f9d27..ac3d14ffe370df 100644 --- a/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_show_routine_load.groovy @@ -34,8 +34,33 @@ suite("test_show_routine_load","p0") { props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + // add timeout config + props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, "10000") + props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "10000") + + // check conenction + def verifyKafkaConnection = { prod -> + try { + logger.info("=====try to connect Kafka========") + def partitions = prod.partitionsFor("__connection_verification_topic") + return partitions != null + } catch (Exception e) { + throw new Exception("Kafka connect fail: ${e.message}".toString()) + } + } // Create kafka producer def producer = new KafkaProducer<>(props) + try { + logger.info("Kafka connecting: ${kafka_broker}") + if (!verifyKafkaConnection(producer)) { + throw new Exception("can't get any kafka info") + } + } catch (Exception e) { + logger.error("FATAL: " + e.getMessage()) + producer.close() + throw e + } + logger.info("Kafka connect success") for (String kafkaCsvTopic in kafkaCsvTpoics) { def txt = new File("""${context.file.parent}/data/${kafkaCsvTopic}.csv""").text From 2165c9d179e32a2b40418b2abd729bf93a72cb80 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:01:58 +0800 Subject: [PATCH 393/572] branch-3.0: [chore](cloud) Check local ip before start metaservice #53793 (#53887) Cherry-picked from #53793 Co-authored-by: walter --- cloud/src/common/network_util.cpp | 1 + cloud/src/main.cpp | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/cloud/src/common/network_util.cpp b/cloud/src/common/network_util.cpp index 5ac8483cc1452d..14b6d0fa13d5ef 100644 --- a/cloud/src/common/network_util.cpp +++ b/cloud/src/common/network_util.cpp @@ -170,6 +170,7 @@ std::string get_local_ip(const std::string& priority_networks) { << "there may be multiple NICs for use, " << "please set priority_networks with a CIDR expression in doris_cloud.conf " << "to choose a non-loopback address accordingly"; + LOG(WARNING) << "process will exit ..."; exit(-1); }; if (priority_networks == "") { diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index aa057d6184e139..92ea5f72c50e49 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -37,6 +37,7 @@ #include "common/configbase.h" #include "common/encryption_util.h" #include "common/logging.h" +#include "common/network_util.h" #include "meta-service/meta_server.h" #include "meta-store/mem_txn_kv.h" #include "meta-store/txn_kv.h" @@ -229,6 +230,10 @@ int main(int argc, char** argv) { LOG(INFO) << build_info(); std::cout << build_info() << std::endl; + // Check the local ip before starting the meta service or recycler. + std::string ip = get_local_ip(config::priority_networks); + std::cout << "local ip: " << ip << std::endl; + if (!args.get(ARG_META_SERVICE) && !args.get(ARG_RECYCLER)) { std::get<0>(args.args()[ARG_META_SERVICE]) = true; std::get<0>(args.args()[ARG_RECYCLER]) = true; From f33e807aa1c4b80528eecfc841122ae89f80b218 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:03:30 +0800 Subject: [PATCH 394/572] branch-3.0: [fix](rpc) robust retry and exception handling for MetaService RPC #53670 (#53735) Cherry-picked from #53670 Co-authored-by: hui lai --- .../doris/cloud/rpc/MetaServiceProxy.java | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java index 95753821c35b05..dc6dc11d518e10 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/rpc/MetaServiceProxy.java @@ -22,7 +22,6 @@ import org.apache.doris.rpc.RpcException; import com.google.common.collect.Maps; -import io.grpc.Status; import io.grpc.StatusRuntimeException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -176,46 +175,51 @@ public MetaServiceClientWrapper(MetaServiceProxy proxy) { } public Response executeRequest(Function function) throws RpcException { - int tried = 0; - while (tried++ < Config.meta_service_rpc_retry_cnt) { + long maxRetries = Config.meta_service_rpc_retry_cnt; + for (long tried = 1; tried <= maxRetries; tried++) { MetaServiceClient client = null; try { client = proxy.getProxy(); return function.apply(client); } catch (StatusRuntimeException sre) { - LOG.info("failed to request meta servive code {}, msg {}, trycnt {}", sre.getStatus().getCode(), + LOG.warn("failed to request meta service code {}, msg {}, trycnt {}", sre.getStatus().getCode(), sre.getMessage(), tried); - if ((tried > Config.meta_service_rpc_retry_cnt - || (sre.getStatus().getCode() != Status.Code.UNAVAILABLE - && sre.getStatus().getCode() != Status.Code.UNKNOWN)) - && (tried > Config.meta_service_rpc_timeout_retry_times - || sre.getStatus().getCode() != Status.Code.DEADLINE_EXCEEDED)) { + boolean shouldRetry = false; + switch (sre.getStatus().getCode()) { + case UNAVAILABLE: + case UNKNOWN: + shouldRetry = true; + break; + case DEADLINE_EXCEEDED: + shouldRetry = tried <= Config.meta_service_rpc_timeout_retry_times; + break; + default: + shouldRetry = false; + } + if (!shouldRetry || tried >= maxRetries) { throw new RpcException("", sre.getMessage(), sre); } } catch (Exception e) { - LOG.info("failed to request meta servive trycnt {}", tried, e); - if (tried > Config.meta_service_rpc_retry_cnt) { + LOG.warn("failed to request meta servive trycnt {}", tried, e); + if (tried >= maxRetries) { throw new RpcException("", e.getMessage(), e); } - } catch (Throwable t) { - LOG.info("failed to request meta servive trycnt {}", tried, t); - if (tried > Config.meta_service_rpc_retry_cnt) { - throw new RpcException("", t.getMessage()); + } finally { + if (proxy.needReconn() && client != null) { + client.shutdown(true); } } - if (proxy.needReconn() && client != null) { - client.shutdown(true); - } - int delay = 20 + random.nextInt(200 - 20 + 1); try { Thread.sleep(delay); } catch (InterruptedException interruptedException) { - // ignore + Thread.currentThread().interrupt(); + throw new RpcException("", interruptedException.getMessage(), interruptedException); } } - return null; // impossible and unreachable, just make the compiler happy + // impossible and unreachable, just make the compiler happy + throw new RpcException("", "All retries exhausted", null); } } From c02d8d72dbba966fca71239485f005ecbae83e92 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:04:08 +0800 Subject: [PATCH 395/572] branch-3.0: [fix](regression) Set `be-cluster-id` default to false #53643 (#53813) Cherry-picked from #53643 Co-authored-by: deardeng --- docker/runtime/doris-compose/command.py | 20 +++++++------------ .../regression/suite/SuiteCluster.groovy | 4 ++-- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py index 57f302dcc45a8c..84febcf415ff74 100644 --- a/docker/runtime/doris-compose/command.py +++ b/docker/runtime/doris-compose/command.py @@ -480,19 +480,12 @@ def add_parser(self, args_parsers): "Do not set BE meta service endpoint in conf. Default is False." ) - if self._support_boolean_action(): - parser.add_argument( - "--be-cluster-id", - default=False, - action=self._get_parser_bool_action(False), - help="Do not set BE cluster ID in conf. Default is False.") - else: - parser.add_argument( - "--no-be-cluster-id", - dest='be_cluster_id', - default=False, - action=self._get_parser_bool_action(False), - help="Do not set BE cluster ID in conf. Default is False.") + # if default==False, use this style to parser, like --be-cluster-id + parser.add_argument( + "--be-cluster-id", + default=False, + action=self._get_parser_bool_action(True), + help="Do not set BE cluster ID in conf. Default is False.") parser.add_argument( "--fdb-version", @@ -500,6 +493,7 @@ def add_parser(self, args_parsers): default="7.1.26", help="fdb image version. Only use in cloud cluster.") + # if default==True, use this style to parser, like --detach if self._support_boolean_action(): parser.add_argument( "--detach", diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index 61758f9c5ddeb9..aa7d3e7afe7865 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -355,8 +355,8 @@ class SuiteCluster { if (!options.beMetaServiceEndpoint) { cmd += ['--no-be-metaservice-endpoint'] } - if (!options.beClusterId) { - cmd += ['--no-be-cluster-id'] + if (options.beClusterId) { + cmd += ['--be-cluster-id'] } cmd += ['--wait-timeout', String.valueOf(options.waitTimeout)] From 261dd7f2e870d27a23af5142c35c00732a960660 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:05:13 +0800 Subject: [PATCH 396/572] branch-3.0: [fix](p2) throw Exception if hdfs is not enable #52761 (#53740) Cherry-picked from #52761 Co-authored-by: yagagagaga --- .../cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy | 2 +- .../create_table_use_dynamic_partition_by_hdfs.groovy | 2 +- .../create_table_use_partition_policy_by_hdfs.groovy | 2 +- .../create_table_use_policy_by_hdfs.groovy | 2 +- .../suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy | 2 +- .../modify_replica_use_partition_by_hdfs.groovy | 2 +- .../table_modify_resouce_and_policy_by_hdfs.groovy | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy index 721c12d7dd463c..7354fc74895f4c 100644 --- a/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy @@ -20,7 +20,7 @@ import java.time.LocalDate; suite("add_drop_partition_by_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def fetchBeHttp = { check_func, meta_url -> def i = meta_url.indexOf("/api") diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy index 0e64132ba5140f..bf811b217fb60a 100644 --- a/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy @@ -20,7 +20,7 @@ import java.time.LocalDate; suite("cold_heat_dynamic_partition_by_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def fetchBeHttp = { check_func, meta_url -> def i = meta_url.indexOf("/api") diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy index ce4264480a81cb..8004b2dc7720de 100644 --- a/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy @@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("create_table_use_partition_policy_by_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def fetchBeHttp = { check_func, meta_url -> def i = meta_url.indexOf("/api") diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy index 408f2e6275b967..43e9e26cc94f87 100644 --- a/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy @@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("create_table_use_policy_by_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def fetchBeHttp = { check_func, meta_url -> def i = meta_url.indexOf("/api") diff --git a/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy index 8aa2ded305a665..089c89b72d5f36 100644 --- a/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy @@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("load_colddata_to_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def fetchBeHttp = { check_func, meta_url -> diff --git a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy index f085a640907680..6f69d913befad6 100644 --- a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy @@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("modify_replica_use_partition_by_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def replicaNum = getFeConfig("force_olap_table_replication_num") setFeConfig("force_olap_table_replication_num", 0) diff --git a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy index 601d42ab0d20e9..574d9ef44fc9f7 100644 --- a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy +++ b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy @@ -19,7 +19,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("table_modify_resouce_by_hdfs") { if (!enableHdfs()) { - logger.info("skip this case because hdfs is not enabled"); + throw new RuntimeException("Hdfs is not enabled, if you want to skip this case, please mute it in regression-conf.groovy"); } def fetchBeHttp = { check_func, meta_url -> def i = meta_url.indexOf("/api") From 5394bf363e4c9ae7322b5e04abfbd0a977d11cb1 Mon Sep 17 00:00:00 2001 From: lihangyu Date: Tue, 12 Aug 2025 11:06:05 +0800 Subject: [PATCH 397/572] branch-3.0: [Fix](SIMDJson) manually selecting the implementation for different CPU architecture (#53893) cherry-pick #52732 --- be/src/runtime/exec_env_init.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 8faaffd2831430..3ec7fed673f6da 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include // IWYU pragma: keep @@ -121,6 +122,7 @@ #endif namespace doris { + class PBackendService_Stub; class PFunctionService_Stub; @@ -168,6 +170,26 @@ Status ExecEnv::init(ExecEnv* env, const std::vector& store_paths, return env->_init(store_paths, spill_store_paths, broken_paths); } +// pick simdjson implementation based on CPU capabilities +inline void init_simdjson_parser() { + // haswell: AVX2 (2013 Intel Haswell or later, all AMD Zen processors) + const auto* haswell_implementation = simdjson::get_available_implementations()["haswell"]; + if (!haswell_implementation || !haswell_implementation->supported_by_runtime_system()) { + // pick available implementation + for (const auto* implementation : simdjson::get_available_implementations()) { + if (implementation->supported_by_runtime_system()) { + LOG(INFO) << "Using SimdJSON implementation : " << implementation->name() << ": " + << implementation->description(); + simdjson::get_active_implementation() = implementation; + return; + } + } + LOG(WARNING) << "No available SimdJSON implementation found."; + } else { + LOG(INFO) << "Using SimdJSON Haswell implementation"; + } +} + Status ExecEnv::_init(const std::vector& store_paths, const std::vector& spill_store_paths, const std::set& broken_paths) { @@ -356,6 +378,8 @@ Status ExecEnv::_init(const std::vector& store_paths, _runtime_query_statistics_mgr->start_report_thread(); _s_ready = true; + init_simdjson_parser(); + // Make aws-sdk-cpp InitAPI and ShutdownAPI called in the same thread S3ClientFactory::instance(); return Status::OK(); From 1d4022a7d1ff9944aa51024c99b6557fd771bce4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:07:42 +0800 Subject: [PATCH 398/572] branch-3.0: [test](metric)add meta-serivce metric format check #53649 (#54045) Cherry-picked from #53649 Co-authored-by: koarz --- .../regression/util/PromethuesChecker.groovy | 75 +++++++++++++++++++ .../cloud_p0/test_metrics_format.groovy | 41 ++++++++++ 2 files changed, 116 insertions(+) create mode 100644 regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy create mode 100644 regression-test/suites/cloud_p0/test_metrics_format.groovy diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy new file mode 100644 index 00000000000000..8a45052fb7fda1 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import groovy.transform.CompileStatic +import java.util.regex.Matcher +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@CompileStatic +class PromethuesChecker { + private static final Logger log = LoggerFactory.getLogger(PromethuesChecker.class) + + static boolean regexp(String s) { + if (s == null) return false + s =~ /^[a-zA-Z_][a-zA-Z0-9_]*(\{[a-zA-Z_][a-zA-Z0-9_]*="[^"]+"(,[a-zA-Z_][a-zA-Z0-9_]*="[^"]+")*\})?\s+-?([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)$/ + } + + static boolean check(String str) { + // counter gauge summary histogram + String type = "" + if (str == null || str.trim().isEmpty()) return false + + def lines = str.split('\n') + boolean allValid = true + + for (String line : lines) { + line = line.trim() + if (line.isEmpty()) continue + if (line.startsWith("# HELP ")) continue + + if (line.startsWith("# TYPE ")) { + def matcher = (line =~ /^# TYPE\s+([a-zA-Z_][a-zA-Z0-9_]*)\s+(counter|gauge|histogram|summary)$/) as Matcher + if (matcher.matches()) { + type = matcher.group(2)?.toLowerCase() + } else { + allValid = false + } + } else { + switch (type) { + case "counter": + case "gauge": + case "summary": + case "histogram": + if (!regexp(line)) { + log.info("invalid metric format ${line} type ${type}, please check regexp or metric format".toString()) + allValid = false + } + break + default: + allValid = false + log.info("unknow metric type ${type}".toString()) + break + } + } + } + + return allValid + } +} diff --git a/regression-test/suites/cloud_p0/test_metrics_format.groovy b/regression-test/suites/cloud_p0/test_metrics_format.groovy new file mode 100644 index 00000000000000..2b366c692c6b9f --- /dev/null +++ b/regression-test/suites/cloud_p0/test_metrics_format.groovy @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.PromethuesChecker + +suite('test_metrics_format') { + if (!isCloudMode()) { + log.info("not cloud mode just return") + return + } + def get_meta_service_metric = { check_func -> + httpTest { + op "get" + endpoint context.config.metaServiceHttpAddress + uri "/brpc_metrics" + check check_func + } + } + + get_meta_service_metric.call { + respCode, body -> + assertEquals("${respCode}".toString(), "200") + String out = "${body}".toString() + Boolean res = PromethuesChecker.check(out) + assertTrue(res) + } +} \ No newline at end of file From 685c1d58717bdd25a29a7e69549b5cf040115850 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:08:15 +0800 Subject: [PATCH 399/572] branch-3.0: [opt](recycler) Add recycler metrics for delete by prefix #52709 (#53927) Cherry-picked from #52709 Co-authored-by: Uniqueyou --- cloud/src/recycler/recycler.cpp | 48 +++++++++++++++------------------ 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index cdc43e85cfd393..33d5ec62952e18 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -1307,6 +1307,7 @@ int InstanceRecycler::recycle_partitions() { partition_version_keys.push_back(partition_version_key( {instance_id_, part_pb.db_id(), part_pb.table_id(), partition_id})); } + metrics_context.total_recycled_num = num_recycled; metrics_context.report(); } return ret; @@ -1729,7 +1730,11 @@ int InstanceRecycler::delete_rowset_data( const auto& rowset_id = rs.rowset_id_v2(); int64_t tablet_id = rs.tablet_id(); int64_t num_segments = rs.num_segments(); - if (num_segments <= 0) continue; + if (num_segments <= 0) { + metrics_context.total_recycled_num++; + metrics_context.total_recycled_data_size += rs.total_disk_size(); + continue; + } // Process inverted indexes std::vector> index_ids; @@ -2362,7 +2367,7 @@ int InstanceRecycler::recycle_rowsets() { int64_t earlest_ts = std::numeric_limits::max(); - auto handle_rowset_kv = [&, this](std::string_view k, std::string_view v) -> int { + auto handle_rowset_kv = [&](std::string_view k, std::string_view v) -> int { ++num_scanned; total_rowset_key_size += k.size(); total_rowset_value_size += v.size(); @@ -2372,13 +2377,13 @@ int InstanceRecycler::recycle_rowsets() { return -1; } - int final_expiration = calculate_rowset_expired_time(instance_id_, rowset, &earlest_ts); + int64_t current_time = ::time(nullptr); + int64_t expiration = calculate_rowset_expired_time(instance_id_, rowset, &earlest_ts); VLOG_DEBUG << "recycle rowset scan, key=" << hex(k) << " num_scanned=" << num_scanned - << " num_expired=" << num_expired << " expiration=" << final_expiration + << " num_expired=" << num_expired << " expiration=" << expiration << " RecycleRowsetPB=" << rowset.ShortDebugString(); - int64_t current_time = ::time(nullptr); - if (current_time < final_expiration) { // not expired + if (current_time < expiration) { // not expired return 0; } ++num_expired; @@ -2441,9 +2446,8 @@ int InstanceRecycler::recycle_rowsets() { } else { num_compacted += rowset.type() == RecycleRowsetPB::COMPACT; rowset_keys.emplace_back(k); - if (rowset_meta->num_segments() > 0) { // Skip empty rowset - rowsets.emplace(rowset_meta->rowset_id_v2(), std::move(*rowset_meta)); - } else { + rowsets.emplace(rowset_meta->rowset_id_v2(), std::move(*rowset_meta)); + if (rowset_meta->num_segments() <= 0) { // Skip empty rowset ++num_empty_rowset; } } @@ -2592,9 +2596,7 @@ int InstanceRecycler::recycle_tmp_rowsets() { << " num_expired=" << num_expired; tmp_rowset_keys.push_back(k); - if (rowset.num_segments() > 0) { // Skip empty rowset - tmp_rowsets.emplace(rowset.rowset_id_v2(), std::move(rowset)); - } + tmp_rowsets.emplace(rowset.rowset_id_v2(), std::move(rowset)); return 0; }; @@ -3779,15 +3781,10 @@ int InstanceRecycler::scan_and_statistics_rowsets() { return 0; } } - if (rowset.type() != RecycleRowsetPB::PREPARE) { - if (rowset_meta->num_segments() > 0) { - metrics_context.total_need_recycle_num++; - segment_metrics_context_.total_need_recycle_num += rowset_meta->num_segments(); - segment_metrics_context_.total_need_recycle_data_size += - rowset_meta->total_disk_size(); - metrics_context.total_need_recycle_data_size += rowset_meta->total_disk_size(); - } - } + metrics_context.total_need_recycle_num++; + metrics_context.total_need_recycle_data_size += rowset_meta->total_disk_size(); + segment_metrics_context_.total_need_recycle_num += rowset_meta->num_segments(); + segment_metrics_context_.total_need_recycle_data_size += rowset_meta->total_disk_size(); return 0; }; return scan_and_recycle(recyc_rs_key0, recyc_rs_key1, std::move(handle_rowset_kv), @@ -3831,16 +3828,13 @@ int InstanceRecycler::scan_and_statistics_tmp_rowsets() { if (rowset.num_segments() > 0) [[unlikely]] { // impossible return 0; } - metrics_context.total_need_recycle_num++; return 0; } metrics_context.total_need_recycle_num++; - if (rowset.num_segments() > 0) { - metrics_context.total_need_recycle_data_size += rowset.total_disk_size(); - segment_metrics_context_.total_need_recycle_data_size += rowset.total_disk_size(); - segment_metrics_context_.total_need_recycle_num += rowset.num_segments(); - } + metrics_context.total_need_recycle_data_size += rowset.total_disk_size(); + segment_metrics_context_.total_need_recycle_data_size += rowset.total_disk_size(); + segment_metrics_context_.total_need_recycle_num += rowset.num_segments(); return 0; }; return scan_and_recycle(tmp_rs_key0, tmp_rs_key1, std::move(handle_tmp_rowsets_kv), From a9ebcb80f0555713fc69b5afdd6811edb95f3655 Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 12 Aug 2025 11:09:02 +0800 Subject: [PATCH 400/572] branch-3.0:[fix](orc)fix core that iceberg-orc reader read miss iceberg.id orc file.(#49051) (#54167) ### What problem does this PR solve? pick #49051 but only fix: ``` terminate called after throwing an instance of 'std::range_error' what(): Key not found: iceberg.id *** Query id: 6a93d7cdc9f44370-a40b07934a14c81b *** *** is nereids: 1 *** *** tablet id: 0 *** *** Aborted at 1753842428 (unix time) try "date -d @1753842428" if you are using GNU date *** *** Current BE git commitID: 910c4249c5 *** *** SIGABRT unknown detail explain (@0x5a46f) received by PID 369775 (TID 371694 OR 0x7fad067ef640) from PID 369775; stack trace: *** terminate called recursively terminate called recursively terminate called recursively terminate called recursively 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/zcp/repo_center/doris_release/doris/be/src/common/signal_handler.h:421 1# 0x00007FB12263EBF0 in /lib64/libc.so.6 2# __pthread_kill_implementation in /lib64/libc.so.6 3# gsignal in /lib64/libc.so.6 4# abort in /lib64/libc.so.6 5# __gnu_cxx::__verbose_terminate_handler() [clone .cold] at ../../../../libstdc++-v3/libsupc++/vterminate.cc:75 6# __cxxabiv1::__terminate(void (*)()) at ../../../../libstdc++-v3/libsupc++/eh_terminate.cc:48 7# 0x000055C047B28EC1 in /opt/apache-doris-3.0.6.2-bin-x64/be/lib/doris_be 8# 0x000055C047B29014 in /opt/apache-doris-3.0.6.2-bin-x64/be/lib/doris_be 9# orc::TypeImpl::getAttributeValue(std::__cxx11::basic_string, std::allocator > const&) const in /opt/apache-doris-3.0.6.2-bin-x64/be/lib/doris_be 10# doris::vectorized::OrcReader::get_schema_col_name_attribute(std::vector, std::allocator >, std::allocator, std::allocator > > >*, std::vector >*, std::__cxx11::basic_string, std::allocator >) at /home/zcp/repo_center/doris_release/doris/be/src/vec/exec/format/orc/vorc_reader.cpp:332 11# doris::vectorized::IcebergOrcReader::_gen_col_name_maps(doris::vectorized::OrcReader*) at ``` --- be/src/vec/exec/format/orc/vorc_reader.cpp | 28 ++- be/src/vec/exec/format/orc/vorc_reader.h | 11 +- be/src/vec/exec/format/parquet/schema_desc.h | 4 +- .../vec/exec/format/table/iceberg_reader.cpp | 190 ++++-------------- be/src/vec/exec/format/table/iceberg_reader.h | 29 +-- .../exec/format/table/table_format_reader.cpp | 103 ++++++++++ .../exec/format/table/table_format_reader.h | 41 ++++ .../table/transactional_hive_reader.cpp | 10 +- be/src/vec/exec/scan/vfile_scanner.cpp | 4 +- .../iceberg/run08.sql | 48 +++++ .../iceberg/iceberg_schema_change2.out | 64 ++++++ .../iceberg/iceberg_schema_change2.groovy | 66 ++++++ 12 files changed, 411 insertions(+), 187 deletions(-) create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run08.sql create mode 100644 regression-test/data/external_table_p0/iceberg/iceberg_schema_change2.out create mode 100644 regression-test/suites/external_table_p0/iceberg/iceberg_schema_change2.groovy diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp index acd979678fb4fc..88c073866db95d 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.cpp +++ b/be/src/vec/exec/format/orc/vorc_reader.cpp @@ -250,6 +250,10 @@ void OrcReader::_init_profile() { } Status OrcReader::_create_file_reader() { + if (_reader != nullptr) { + return Status::OK(); + } + if (_file_input_stream == nullptr) { _file_description.mtime = _scan_range.__isset.modification_time ? _scan_range.modification_time : 0; @@ -289,6 +293,7 @@ Status OrcReader::_create_file_reader() { Status OrcReader::init_reader( const std::vector* column_names, + const std::vector& missing_column_names, const std::unordered_map* colname_to_value_range, const VExprContextSPtrs& conjuncts, bool is_acid, const TupleDescriptor* tuple_descriptor, const RowDescriptor* row_descriptor, @@ -296,6 +301,7 @@ Status OrcReader::init_reader( const std::unordered_map* slot_id_to_filter_conjuncts, const bool hive_use_column_names) { _column_names = column_names; + _missing_column_names_set.insert(missing_column_names.begin(), missing_column_names.end()); _colname_to_value_range = colname_to_value_range; _lazy_read_ctx.conjuncts = conjuncts; _is_acid = is_acid; @@ -332,14 +338,21 @@ Status OrcReader::get_parsed_schema(std::vector* col_names, } Status OrcReader::get_schema_col_name_attribute(std::vector* col_names, - std::vector* col_attributes, - std::string attribute) { + std::vector* col_attributes, + const std::string& attribute, + bool* exist_attribute) { RETURN_IF_ERROR(_create_file_reader()); - auto& root_type = _is_acid ? _remove_acid(_reader->getType()) : _reader->getType(); + *exist_attribute = true; + const auto& root_type = _reader->getType(); for (int i = 0; i < root_type.getSubtypeCount(); ++i) { col_names->emplace_back(get_field_name_lower_case(&root_type, i)); + + if (!root_type.getSubtype(i)->hasAttributeKey(attribute)) { + *exist_attribute = false; + return Status::OK(); + } col_attributes->emplace_back( - std::stol(root_type.getSubtype(i)->getAttributeValue(attribute))); + std::stoi(root_type.getSubtype(i)->getAttributeValue(attribute))); } return Status::OK(); } @@ -355,8 +368,15 @@ Status OrcReader::_init_read_columns() { // TODO, should be removed in 2.2 or later _is_hive1_orc_or_use_idx = (is_hive1_orc || _is_hive1_orc_or_use_idx) && _scan_params.__isset.slot_name_to_schema_pos; + for (size_t i = 0; i < _column_names->size(); ++i) { auto& col_name = (*_column_names)[i]; + + if (_missing_column_names_set.contains(col_name)) { + _missing_cols.emplace_back(col_name); + continue; + } + if (_is_hive1_orc_or_use_idx) { auto iter = _scan_params.slot_name_to_schema_pos.find(col_name); if (iter != _scan_params.slot_name_to_schema_pos.end()) { diff --git a/be/src/vec/exec/format/orc/vorc_reader.h b/be/src/vec/exec/format/orc/vorc_reader.h index 551c000dca3b9c..77cc22b54ed30c 100644 --- a/be/src/vec/exec/format/orc/vorc_reader.h +++ b/be/src/vec/exec/format/orc/vorc_reader.h @@ -144,6 +144,7 @@ class OrcReader : public GenericReader { //If you want to read the file by index instead of column name, set hive_use_column_names to false. Status init_reader( const std::vector* column_names, + const std::vector& missing_column_names, const std::unordered_map* colname_to_value_range, const VExprContextSPtrs& conjuncts, bool is_acid, const TupleDescriptor* tuple_descriptor, const RowDescriptor* row_descriptor, @@ -184,8 +185,8 @@ class OrcReader : public GenericReader { std::vector* col_types) override; Status get_schema_col_name_attribute(std::vector* col_names, - std::vector* col_attributes, - std::string attribute); + std::vector* col_attributes, + const std::string& attribute, bool* exist_attribute); void set_table_col_to_file_col( std::unordered_map table_col_to_file_col) { _table_col_to_file_col = table_col_to_file_col; @@ -566,6 +567,10 @@ class OrcReader : public GenericReader { int64_t _range_size; const std::string& _ctz; const std::vector* _column_names; + // _missing_column_names_set: used in iceberg/hudi/paimon, the columns are dropped + // but added back(drop column a then add column a). Shouldn't read this column data in this case. + std::set _missing_column_names_set; + int32_t _offset_days = 0; cctz::time_zone _time_zone; @@ -590,7 +595,7 @@ class OrcReader : public GenericReader { OrcProfile _orc_profile; std::unique_ptr _batch; - std::unique_ptr _reader; + std::unique_ptr _reader = nullptr; std::unique_ptr _row_reader; std::unique_ptr _orc_filter; orc::RowReaderOptions _row_reader_options; diff --git a/be/src/vec/exec/format/parquet/schema_desc.h b/be/src/vec/exec/format/parquet/schema_desc.h index 2593da837c3da6..aed0da070008e9 100644 --- a/be/src/vec/exec/format/parquet/schema_desc.h +++ b/be/src/vec/exec/format/parquet/schema_desc.h @@ -71,7 +71,7 @@ class FieldDescriptor { std::unordered_map _name_to_field; // Used in from_thrift, marking the next schema position that should be parsed size_t _next_schema_pos; - std::unordered_map _field_id_name_mapping; + std::map _field_id_name_mapping; void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, FieldSchema* physical_field); @@ -135,6 +135,8 @@ class FieldDescriptor { bool has_parquet_field_id() const { return _field_id_name_mapping.size() > 0; } + std::map get_field_id_name_map() { return _field_id_name_mapping; } + const doris::Slice get_column_name_from_field_id(int32_t id) const; }; diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp index 7dea5d99617416..725ecc4dd35b50 100644 --- a/be/src/vec/exec/format/table/iceberg_reader.cpp +++ b/be/src/vec/exec/format/table/iceberg_reader.cpp @@ -120,32 +120,9 @@ Status IcebergTableReader::get_next_block(Block* block, size_t* read_rows, bool* } RETURN_IF_ERROR(_expand_block_if_need(block)); - // To support iceberg schema evolution. We change the column name in block to - // make it match with the column name in parquet file before reading data. and - // Set the name back to table column name before return this block. - if (_has_schema_change) { - for (int i = 0; i < block->columns(); i++) { - ColumnWithTypeAndName& col = block->get_by_position(i); - auto iter = _table_col_to_file_col.find(col.name); - if (iter != _table_col_to_file_col.end()) { - col.name = iter->second; - } - } - block->initialize_index_by_name(); - } - + RETURN_IF_ERROR(TableSchemaChangeHelper::get_next_block_before(block)); RETURN_IF_ERROR(_file_format_reader->get_next_block(block, read_rows, eof)); - // Set the name back to table column name before return this block. - if (_has_schema_change) { - for (int i = 0; i < block->columns(); i++) { - ColumnWithTypeAndName& col = block->get_by_position(i); - auto iter = _file_col_to_table_col.find(col.name); - if (iter != _file_col_to_table_col.end()) { - col.name = iter->second; - } - } - block->initialize_index_by_name(); - } + RETURN_IF_ERROR(TableSchemaChangeHelper::get_next_block_after(block)); if (_equality_delete_impl != nullptr) { RETURN_IF_ERROR(_equality_delete_impl->filter_data_block(block)); @@ -228,8 +205,9 @@ Status IcebergTableReader::_equality_delete_base( not_in_file_col_names, nullptr, {}, nullptr, nullptr, nullptr, nullptr, nullptr, false)); } else if (auto* orc_reader = typeid_cast(delete_reader.get())) { - RETURN_IF_ERROR(orc_reader->init_reader(&equality_delete_col_names, nullptr, {}, false, - {}, {}, nullptr, nullptr)); + RETURN_IF_ERROR(orc_reader->init_reader(&equality_delete_col_names, + not_in_file_col_names, nullptr, {}, false, {}, + {}, nullptr, nullptr)); } else { return Status::InternalError("Unsupported format of delete file"); } @@ -439,60 +417,6 @@ void IcebergTableReader::_sort_delete_rows(std::vector*>& d } } -/* - * Generate _all_required_col_names and _not_in_file_col_names. - * - * _all_required_col_names is all the columns required by user sql. - * If the column name has been modified after the data file was written, - * put the old name in data file to _all_required_col_names. - * - * _not_in_file_col_names is all the columns required by user sql but not in the data file. - * e.g. New columns added after this data file was written. - * The columns added with names used by old dropped columns should consider as a missing column, - * which should be in _not_in_file_col_names. - */ -void IcebergTableReader::_gen_file_col_names() { - _all_required_col_names.clear(); - _not_in_file_col_names.clear(); - for (int i = 0; i < _file_col_names.size(); ++i) { - auto name = _file_col_names[i]; - auto iter = _table_col_to_file_col.find(name); - if (iter == _table_col_to_file_col.end()) { - // If the user creates the iceberg table, directly append the parquet file that already exists, - // there is no 'iceberg.schema' field in the footer of parquet, the '_table_col_to_file_col' may be empty. - // Because we are ignoring case, so, it is converted to lowercase here - auto name_low = to_lower(name); - _all_required_col_names.emplace_back(name_low); - if (_has_iceberg_schema) { - _not_in_file_col_names.emplace_back(name); - } else { - _table_col_to_file_col.emplace(name, name_low); - _file_col_to_table_col.emplace(name_low, name); - if (name != name_low) { - _has_schema_change = true; - } - } - } else { - _all_required_col_names.emplace_back(iter->second); - } - } -} - -/* - * Generate _new_colname_to_value_range, by replacing the column name in - * _colname_to_value_range with column name in data file. - */ -void IcebergTableReader::_gen_new_colname_to_value_range() { - for (auto it = _colname_to_value_range->begin(); it != _colname_to_value_range->end(); it++) { - auto iter = _table_col_to_file_col.find(it->first); - if (iter == _table_col_to_file_col.end()) { - _new_colname_to_value_range.emplace(it->first, it->second); - } else { - _new_colname_to_value_range.emplace(iter->second, it->second); - } - } -} - void IcebergTableReader::_gen_position_delete_file_range(Block& block, DeleteFile* position_delete, size_t read_rows, bool file_path_column_dictionary_coded) { @@ -538,13 +462,9 @@ Status IcebergParquetReader::init_reader( const std::unordered_map* slot_id_to_filter_conjuncts) { _file_format = Fileformat::PARQUET; ParquetReader* parquet_reader = static_cast(_file_format_reader.get()); - _col_id_name_map = col_id_name_map; - _file_col_names = file_col_names; - _colname_to_value_range = colname_to_value_range; - FieldDescriptor field_desc = parquet_reader->get_file_metadata_schema(); - RETURN_IF_ERROR(_gen_col_name_maps(field_desc)); - _gen_file_col_names(); - _gen_new_colname_to_value_range(); + RETURN_IF_ERROR(TableSchemaChangeHelper::init_schema_info(file_col_names, col_id_name_map, + colname_to_value_range)); + parquet_reader->set_table_to_file_col_map(_table_col_to_file_col); parquet_reader->iceberg_sanitize(_all_required_col_names); RETURN_IF_ERROR(init_row_filters(_range, _io_ctx)); @@ -611,18 +531,14 @@ Status IcebergOrcReader::init_reader( const std::unordered_map* slot_id_to_filter_conjuncts) { _file_format = Fileformat::ORC; auto* orc_reader = static_cast(_file_format_reader.get()); - _col_id_name_map = col_id_name_map; - _file_col_names = file_col_names; - _colname_to_value_range = colname_to_value_range; - - RETURN_IF_ERROR(_gen_col_name_maps(orc_reader)); - _gen_file_col_names(); - _gen_new_colname_to_value_range(); + RETURN_IF_ERROR(TableSchemaChangeHelper::init_schema_info(file_col_names, col_id_name_map, + colname_to_value_range)); orc_reader->set_table_col_to_file_col(_table_col_to_file_col); RETURN_IF_ERROR(init_row_filters(_range, _io_ctx)); - return orc_reader->init_reader(&_all_required_col_names, &_new_colname_to_value_range, - conjuncts, false, tuple_descriptor, row_descriptor, - not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts); + return orc_reader->init_reader(&_all_required_col_names, _not_in_file_col_names, + &_new_colname_to_value_range, conjuncts, false, tuple_descriptor, + row_descriptor, not_single_slot_filter_conjuncts, + slot_id_to_filter_conjuncts); } Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete_range, @@ -630,8 +546,9 @@ Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete OrcReader orc_delete_reader(_profile, _state, _params, *delete_range, READ_DELETE_FILE_BATCH_SIZE, _state->timezone(), _io_ctx); std::unordered_map colname_to_value_range; - RETURN_IF_ERROR(orc_delete_reader.init_reader(&delete_file_col_names, &colname_to_value_range, - {}, false, {}, {}, nullptr, nullptr)); + RETURN_IF_ERROR(orc_delete_reader.init_reader(&delete_file_col_names, {}, + &colname_to_value_range, {}, false, {}, {}, + nullptr, nullptr)); std::unordered_map> partition_columns; @@ -652,61 +569,36 @@ Status IcebergOrcReader::_read_position_delete_file(const TFileRangeDesc* delete return Status::OK(); } -/* - * To support schema evolution, Iceberg write the column id to column name map to - * parquet file key_value_metadata. - * This function is to compare the table schema from FE (_col_id_name_map) with - * the schema in key_value_metadata for the current parquet file and generate two maps - * for future use: - * 1. table column name to parquet column name. - * 2. parquet column name to table column name. - * For example, parquet file has a column 'col1', - * after this file was written, iceberg changed the column name to 'col1_new'. - * The two maps would contain: - * 1. col1_new -> col1 - * 2. col1 -> col1_new - */ -Status IcebergParquetReader::_gen_col_name_maps(const FieldDescriptor& field_desc) { +// To support schema evolution, Iceberg write the column id to column name map to parquet file key_value_metadata. +Status IcebergParquetReader::get_file_col_id_to_name( + bool& exist_schema, std::map& file_col_id_to_name) { + auto* parquet_reader = static_cast(_file_format_reader.get()); + FieldDescriptor field_desc = parquet_reader->get_file_metadata_schema(); + if (field_desc.has_parquet_field_id()) { - for (const auto& pair : _col_id_name_map) { - auto name_slice = field_desc.get_column_name_from_field_id(pair.first); - if (name_slice.get_size() == 0) { - _has_schema_change = true; - } else { - auto name_string = name_slice.to_string(); - _table_col_to_file_col.emplace(pair.second, name_string); - _file_col_to_table_col.emplace(name_string, pair.second); - if (name_string != pair.second) { - _has_schema_change = true; - } - } - } + file_col_id_to_name = field_desc.get_field_id_name_map(); + } else { + //For early iceberg version, it doesn't write any schema information to Parquet file. + exist_schema = false; } + return Status::OK(); } -Status IcebergOrcReader::_gen_col_name_maps(OrcReader* orc_reader) { +//To support schema evolution, Iceberg write the column id to orc file attribute. +Status IcebergOrcReader::get_file_col_id_to_name( + bool& exist_schema, std::map& file_col_id_to_name) { + auto* orc_reader = static_cast(_file_format_reader.get()); + std::vector col_names; - std::vector col_ids; - RETURN_IF_ERROR( - orc_reader->get_schema_col_name_attribute(&col_names, &col_ids, ICEBERG_ORC_ATTRIBUTE)); - _has_iceberg_schema = true; - _table_col_to_file_col.clear(); - _file_col_to_table_col.clear(); - for (size_t i = 0; i < col_ids.size(); i++) { - auto col_id = col_ids[i]; - auto& file_col_name = col_names[i]; - - if (_col_id_name_map.find(col_id) == _col_id_name_map.end()) { - _has_schema_change = true; - continue; - } - auto& table_col_name = _col_id_name_map[col_id]; - _table_col_to_file_col.emplace(table_col_name, file_col_name); - _file_col_to_table_col.emplace(file_col_name, table_col_name); - if (table_col_name != file_col_name) { - _has_schema_change = true; - } + std::vector col_ids; + RETURN_IF_ERROR(orc_reader->get_schema_col_name_attribute( + &col_names, &col_ids, ICEBERG_ORC_ATTRIBUTE, &exist_schema)); + if (!exist_schema) { + return Status::OK(); + } + for (auto i = 0; i < col_names.size(); i++) { + file_col_id_to_name.emplace(col_ids[i], std::move(col_names[i])); } return Status::OK(); } diff --git a/be/src/vec/exec/format/table/iceberg_reader.h b/be/src/vec/exec/format/table/iceberg_reader.h index b057cb0657aa24..61964042386f66 100644 --- a/be/src/vec/exec/format/table/iceberg_reader.h +++ b/be/src/vec/exec/format/table/iceberg_reader.h @@ -67,7 +67,7 @@ class GenericReader; class ShardedKVCache; class VExprContext; -class IcebergTableReader : public TableFormatReader { +class IcebergTableReader : public TableFormatReader, public TableSchemaChangeHelper { public: struct PositionDeleteRange { std::vector data_file_path; @@ -118,9 +118,6 @@ class IcebergTableReader : public TableFormatReader { PositionDeleteRange _get_range(const ColumnString& file_path_column); - void _gen_file_col_names(); - - void _gen_new_colname_to_value_range(); static std::string _delet_file_cache_key(const std::string& path) { return "delete_" + path; } Status _position_delete_base(const std::string data_file_path, @@ -144,28 +141,12 @@ class IcebergTableReader : public TableFormatReader { ShardedKVCache* _kv_cache; IcebergProfile _iceberg_profile; std::vector _iceberg_delete_rows; - // col names from _file_slot_descs - std::vector _file_col_names; - // file column name to table column name map. For iceberg schema evolution. - std::unordered_map _file_col_to_table_col; - // table column name to file column name map. For iceberg schema evolution. - std::unordered_map _table_col_to_file_col; - const std::unordered_map* _colname_to_value_range; - // copy from _colname_to_value_range with new column name that is in parquet/orc file, to support schema evolution. - std::unordered_map _new_colname_to_value_range; - // column id to name map. Collect from FE slot descriptor. - std::unordered_map _col_id_name_map; - // col names in the parquet,orc file - std::vector _all_required_col_names; - // col names in table but not in parquet,orc file - std::vector _not_in_file_col_names; + // equality delete should read the primary columns std::vector _expand_col_names; std::vector _expand_columns; io::IOContext* _io_ctx; - bool _has_schema_change = false; - bool _has_iceberg_schema = false; // the table level row count for optimizing query like: // select count(*) from table; @@ -220,7 +201,8 @@ class IcebergParquetReader final : public IcebergTableReader { parquet_reader->set_delete_rows(&_iceberg_delete_rows); } - Status _gen_col_name_maps(const FieldDescriptor& field_desc); + Status get_file_col_id_to_name(bool& exist_schema, + std::map& file_col_id_to_name) final; protected: std::unique_ptr _create_equality_reader( @@ -258,7 +240,8 @@ class IcebergOrcReader final : public IcebergTableReader { const VExprContextSPtrs* not_single_slot_filter_conjuncts, const std::unordered_map* slot_id_to_filter_conjuncts); - Status _gen_col_name_maps(OrcReader* orc_reader); + Status get_file_col_id_to_name(bool& exist_schema, + std::map& file_col_id_to_name) final; protected: std::unique_ptr _create_equality_reader( diff --git a/be/src/vec/exec/format/table/table_format_reader.cpp b/be/src/vec/exec/format/table/table_format_reader.cpp index ea8111d81b3d04..8676287ffe6fde 100644 --- a/be/src/vec/exec/format/table/table_format_reader.cpp +++ b/be/src/vec/exec/format/table/table_format_reader.cpp @@ -22,4 +22,107 @@ namespace doris::vectorized { TableFormatReader::TableFormatReader(std::unique_ptr file_format_reader) : _file_format_reader(std::move(file_format_reader)) {} +Status TableSchemaChangeHelper::init_schema_info( + const std::vector& read_table_col_names, + const std::unordered_map& table_id_to_name, + const std::unordered_map* + table_col_name_to_value_range) { + bool exist_schema = true; + std::map file_id_to_name; + RETURN_IF_ERROR(get_file_col_id_to_name(exist_schema, file_id_to_name)); + if (!exist_schema) { + file_id_to_name.clear(); + for (const auto& [table_col_id, table_col_name] : table_id_to_name) { + file_id_to_name.emplace(table_col_id, table_col_name); + } + } + + /** This is to compare the table schema from FE (table_id_to_name) with + * the current file schema (file_id_to_name) , generate two maps for future use: + * 1. table column name to file column name. + * 2. file column name to table column name. + * For example, file has a column 'col1', + * after this file was written, iceberg changed the column name to 'col1_new'. + * The two maps would contain: + * 1. col1_new -> col1 + * 2. col1 -> col1_new + */ + for (const auto& [file_col_id, file_col_name] : file_id_to_name) { + if (table_id_to_name.find(file_col_id) == table_id_to_name.end()) { + continue; + } + + auto& table_col_name = table_id_to_name.at(file_col_id); + _table_col_to_file_col.emplace(table_col_name, file_col_name); + _file_col_to_table_col.emplace(file_col_name, table_col_name); + if (table_col_name != file_col_name) { + _has_schema_change = true; + } + } + + /** Generate _all_required_col_names and _not_in_file_col_names. + * + * _all_required_col_names is all the columns required by user sql. + * If the column name has been modified after the data file was written, + * put the old name in data file to _all_required_col_names. + * + * _not_in_file_col_names is all the columns required by user sql but not in the data file. + * e.g. New columns added after this data file was written. + * The columns added with names used by old dropped columns should consider as a missing column, + * which should be in _not_in_file_col_names. + */ + _all_required_col_names.clear(); + _not_in_file_col_names.clear(); + for (auto table_col_name : read_table_col_names) { + auto iter = _table_col_to_file_col.find(table_col_name); + if (iter == _table_col_to_file_col.end()) { + _all_required_col_names.emplace_back(table_col_name); + _not_in_file_col_names.emplace_back(table_col_name); + } else { + _all_required_col_names.emplace_back(iter->second); + } + } + + /** Generate _new_colname_to_value_range, by replacing the column name in + * _colname_to_value_range with column name in data file. + */ + for (auto& it : *table_col_name_to_value_range) { + auto iter = _table_col_to_file_col.find(it.first); + if (iter == _table_col_to_file_col.end()) { + _new_colname_to_value_range.emplace(it.first, it.second); + } else { + _new_colname_to_value_range.emplace(iter->second, it.second); + } + } + return Status::OK(); +} + +Status TableSchemaChangeHelper::get_next_block_before(Block* block) const { + if (_has_schema_change) { + for (int i = 0; i < block->columns(); i++) { + ColumnWithTypeAndName& col = block->get_by_position(i); + auto iter = _table_col_to_file_col.find(col.name); + if (iter != _table_col_to_file_col.end()) { + col.name = iter->second; + } + } + block->initialize_index_by_name(); + } + return Status::OK(); +} + +Status TableSchemaChangeHelper::get_next_block_after(Block* block) const { + if (_has_schema_change) { + for (int i = 0; i < block->columns(); i++) { + ColumnWithTypeAndName& col = block->get_by_position(i); + auto iter = _file_col_to_table_col.find(col.name); + if (iter != _file_col_to_table_col.end()) { + col.name = iter->second; + } + } + block->initialize_index_by_name(); + } + return Status::OK(); +} + } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/exec/format/table/table_format_reader.h b/be/src/vec/exec/format/table/table_format_reader.h index 5a102a7665e8f1..2f6b8742bae079 100644 --- a/be/src/vec/exec/format/table/table_format_reader.h +++ b/be/src/vec/exec/format/table/table_format_reader.h @@ -25,6 +25,7 @@ #include #include "common/status.h" +#include "exec/olap_common.h" #include "vec/exec/format/generic_reader.h" namespace doris { @@ -78,4 +79,44 @@ class TableFormatReader : public GenericReader { std::unique_ptr _file_format_reader; // parquet, orc }; +class TableSchemaChangeHelper { +public: + /** Get the mapping from the unique ID of the column in the current file to the file column name. + * Iceberg/Hudi/Paimon usually maintains field IDs to support schema changes. If you cannot obtain this + * information (maybe the old version does not have this information), you need to set `exist_schema` = `false`. + */ + virtual Status get_file_col_id_to_name(bool& exist_schema, + std::map& file_col_id_to_name) = 0; + + virtual ~TableSchemaChangeHelper() = default; + +protected: + /** table_id_to_name : table column unique id to table name map */ + Status init_schema_info(const std::vector& read_table_col_names, + const std::unordered_map& table_id_to_name, + const std::unordered_map* + table_col_name_to_value_range); + + /** To support schema evolution. We change the column name in block to + * make it match with the column name in file before reading data. and + * set the name back to table column name before return this block. + */ + Status get_next_block_before(Block* block) const; + + /** Set the name back to table column name before return this block.*/ + Status get_next_block_after(Block* block) const; + + // copy from _colname_to_value_range with new column name that is in parquet/orc file + std::unordered_map _new_colname_to_value_range; + // all the columns required by user sql. + std::vector _all_required_col_names; + // col names in table but not in parquet,orc file + std::vector _not_in_file_col_names; + bool _has_schema_change = false; + // file column name to table column name map + std::unordered_map _file_col_to_table_col; + // table column name to file column name map. + std::unordered_map _table_col_to_file_col; +}; + } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/table/transactional_hive_reader.cpp b/be/src/vec/exec/format/table/transactional_hive_reader.cpp index 8be11f6773a445..dc13dd51aaff7f 100644 --- a/be/src/vec/exec/format/table/transactional_hive_reader.cpp +++ b/be/src/vec/exec/format/table/transactional_hive_reader.cpp @@ -66,8 +66,8 @@ Status TransactionalHiveReader::init_reader( _col_names.insert(_col_names.end(), TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.begin(), TransactionalHive::READ_ROW_COLUMN_NAMES_LOWER_CASE.end()); Status status = orc_reader->init_reader( - &_col_names, colname_to_value_range, conjuncts, true, tuple_descriptor, row_descriptor, - not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts); + &_col_names, {}, colname_to_value_range, conjuncts, true, tuple_descriptor, + row_descriptor, not_single_slot_filter_conjuncts, slot_id_to_filter_conjuncts); return status; } @@ -129,9 +129,9 @@ Status TransactionalHiveReader::init_row_filters(const TFileRangeDesc& range, OrcReader delete_reader(_profile, _state, _params, delete_range, _MIN_BATCH_SIZE, _state->timezone(), _io_ctx, false); - RETURN_IF_ERROR( - delete_reader.init_reader(&TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE, - nullptr, {}, false, nullptr, nullptr, nullptr, nullptr)); + RETURN_IF_ERROR(delete_reader.init_reader( + &TransactionalHive::DELETE_ROW_COLUMN_NAMES_LOWER_CASE, {}, nullptr, {}, false, + nullptr, nullptr, nullptr, nullptr)); std::unordered_map> partition_columns; diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index a22777672b6b4f..7cbe6116b639d8 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -881,7 +881,7 @@ Status VFileScanner::_get_next_reader() { } else if (range.__isset.table_format_params && range.table_format_params.table_format_type == "paimon") { init_status = orc_reader->init_reader( - &_file_col_names, _colname_to_value_range, _push_down_conjuncts, false, + &_file_col_names, {}, _colname_to_value_range, _push_down_conjuncts, false, _real_tuple_desc, _default_val_row_desc.get(), &_not_single_slot_filter_conjuncts, &_slot_id_to_filter_conjuncts); std::unique_ptr paimon_reader = @@ -897,7 +897,7 @@ Status VFileScanner::_get_next_reader() { hive_orc_use_column_names = _state->query_options().hive_orc_use_column_names; } init_status = orc_reader->init_reader( - &_file_col_names, _colname_to_value_range, _push_down_conjuncts, false, + &_file_col_names, {}, _colname_to_value_range, _push_down_conjuncts, false, _real_tuple_desc, _default_val_row_desc.get(), &_not_single_slot_filter_conjuncts, &_slot_id_to_filter_conjuncts, hive_orc_use_column_names); diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run08.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run08.sql new file mode 100644 index 00000000000000..1a3d844ef6027e --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run08.sql @@ -0,0 +1,48 @@ + +use demo.test_db; + +CREATE TABLE sc_drop_add_orc ( + id BIGINT, + name STRING, + age INT +) +USING iceberg +PARTITIONED BY (id) +TBLPROPERTIES ('format'='orc'); + +INSERT INTO sc_drop_add_orc VALUES (1, 'Alice', 25); +INSERT INTO sc_drop_add_orc VALUES (2, 'Bob', 30); + +ALTER TABLE sc_drop_add_orc DROP COLUMN age; + +INSERT INTO sc_drop_add_orc (id, name) VALUES (3, 'Charlie'); +INSERT INTO sc_drop_add_orc (id, name) VALUES (4, 'David'); + +ALTER TABLE sc_drop_add_orc ADD COLUMN age INT; + +INSERT INTO sc_drop_add_orc VALUES (5, 'Eve', 28); +INSERT INTO sc_drop_add_orc VALUES (6, 'Frank', 35); + + + +CREATE TABLE sc_drop_add_parquet ( + id BIGINT, + name STRING, + age INT +) +USING iceberg +PARTITIONED BY (id) +TBLPROPERTIES ('format'='parquet'); + +INSERT INTO sc_drop_add_parquet VALUES (1, 'Alice', 25); +INSERT INTO sc_drop_add_parquet VALUES (2, 'Bob', 30); + +ALTER TABLE sc_drop_add_parquet DROP COLUMN age; + +INSERT INTO sc_drop_add_parquet (id, name) VALUES (3, 'Charlie'); +INSERT INTO sc_drop_add_parquet (id, name) VALUES (4, 'David'); + +ALTER TABLE sc_drop_add_parquet ADD COLUMN age INT; + +INSERT INTO sc_drop_add_parquet VALUES (5, 'Eve', 28); +INSERT INTO sc_drop_add_parquet VALUES (6, 'Frank', 35); \ No newline at end of file diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_schema_change2.out b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change2.out new file mode 100644 index 00000000000000..d68cde9a50eb02 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/iceberg_schema_change2.out @@ -0,0 +1,64 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !parquet_1 -- +1 Alice \N +2 Bob \N +3 Charlie \N +4 David \N +5 Eve 28 +6 Frank 35 + +-- !parquet_2 -- +1 Alice \N +2 Bob \N +3 Charlie \N +4 David \N + +-- !parquet_3 -- +5 Eve 28 +6 Frank 35 + +-- !parquet_4 -- +6 Frank 35 + +-- !parquet_5 -- +5 Eve 28 +6 Frank 35 + +-- !parquet_6 -- +5 Eve +6 Frank + +-- !parquet_7 -- +5 28 + +-- !orc_1 -- +1 Alice \N +2 Bob \N +3 Charlie \N +4 David \N +5 Eve 28 +6 Frank 35 + +-- !orc_2 -- +1 Alice \N +2 Bob \N +3 Charlie \N +4 David \N + +-- !orc_3 -- +5 Eve 28 +6 Frank 35 + +-- !orc_4 -- +6 Frank 35 + +-- !orc_5 -- +5 Eve 28 +6 Frank 35 + +-- !orc_6 -- +5 Eve +6 Frank + +-- !orc_7 -- +5 28 \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change2.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change2.groovy new file mode 100644 index 00000000000000..295d14b246e7e5 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_schema_change2.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("iceberg_schema_change2", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalog_name = "iceberg_schema_change2" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + logger.info("catalog " + catalog_name + " created") + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + sql """ use test_db;""" + + qt_parquet_1 """ select * from sc_drop_add_parquet order by id; """ + qt_parquet_2 """ select * from sc_drop_add_parquet where age is NULL order by id; """ + qt_parquet_3 """ select * from sc_drop_add_parquet where age is not NULL order by id; """ + qt_parquet_4 """ select * from sc_drop_add_parquet where age > 28 order by id; """ + qt_parquet_5 """ select * from sc_drop_add_parquet where age >= 28 order by id; """ + qt_parquet_6 """ select id, name from sc_drop_add_parquet where age >= 28 order by id; """ + qt_parquet_7 """ select id, age from sc_drop_add_parquet where name="Eve" order by id; """ + + + + qt_orc_1 """ select * from sc_drop_add_orc order by id; """ + qt_orc_2 """ select * from sc_drop_add_orc where age is NULL order by id; """ + qt_orc_3 """ select * from sc_drop_add_orc where age is not NULL order by id; """ + qt_orc_4 """ select * from sc_drop_add_orc where age > 28 order by id; """ + qt_orc_5 """ select * from sc_drop_add_orc where age >= 28 order by id; """ + qt_orc_6 """ select id, name from sc_drop_add_orc where age >= 28 order by id; """ + qt_orc_7 """ select id, age from sc_drop_add_orc where name="Eve" order by id; """ + +} \ No newline at end of file From 8429a02f9b3cbc77f0279a671e3f725c2e80e8aa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:10:21 +0800 Subject: [PATCH 401/572] branch-3.0: [fix](replication) force_olap_table_replication_allocation should not be set to masterOnly #53465 (#53537) Cherry-picked from #53465 Co-authored-by: camby <104178625@qq.com> --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 51240074645d45..f317f4071392a3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2530,7 +2530,7 @@ public class Config extends ConfigBase { + "This config is recommended to be used only in the test environment"}) public static int force_olap_table_replication_num = 0; - @ConfField(mutable = true, masterOnly = true, description = { + @ConfField(mutable = true, description = { "用于强制设定内表的副本分布，如果该参数不为空，则用户在建表或者创建分区时指定的副本数及副本标签将被忽略，而使用本参数设置的值。" + "该参数影响包括创建分区、修改表属性、动态分区等操作。该参数建议仅用于测试环境", "Used to force set the replica allocation of the internal table. If the config is not empty, " From faf35663b3494e420adc5382a3f050ba2381bf67 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:10:57 +0800 Subject: [PATCH 402/572] branch-3.0: [fix](resource) fix can't update BE s3 resource config #43025 (#54211) Cherry-picked from #43025 Co-authored-by: Xujian Duan <50550370+DarvenDuan@users.noreply.github.com> --- be/src/agent/task_worker_pool.cpp | 12 ++---------- be/src/io/fs/s3_file_system.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 350d0fcf8fbf23..dba0dd33df6acc 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -1480,15 +1480,7 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr DCHECK_EQ(existed_fs->type(), io::FileSystemType::S3) << param.id << ' ' << param.name; auto client = static_cast(existed_fs.get())->client_holder(); auto new_s3_conf = S3Conf::get_s3_conf(param.s3_storage_param); - S3ClientConf conf { - .endpoint {}, - .region {}, - .ak = std::move(new_s3_conf.client_conf.ak), - .sk = std::move(new_s3_conf.client_conf.sk), - .token = std::move(new_s3_conf.client_conf.token), - .bucket {}, - .provider = new_s3_conf.client_conf.provider, - }; + S3ClientConf conf = std::move(new_s3_conf.client_conf); st = client->reset(conf); fs = std::move(existed_fs); } @@ -1496,7 +1488,7 @@ void update_s3_resource(const TStorageResource& param, io::RemoteFileSystemSPtr if (!st.ok()) { LOG(WARNING) << "update s3 resource failed: " << st; } else { - LOG_INFO("successfully update hdfs resource") + LOG_INFO("successfully update s3 resource") .tag("resource_id", param.id) .tag("resource_name", param.name); put_storage_resource(param.id, {std::move(fs)}, param.version); diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 224368bf8d59a2..d74c539ca367d6 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -86,8 +86,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { S3ClientConf reset_conf; { std::shared_lock lock(_mtx); - if (conf.ak == _conf.ak && conf.sk == _conf.sk && conf.token == _conf.token && - conf.use_virtual_addressing == _conf.use_virtual_addressing) { + if (conf.get_hash() == _conf.get_hash()) { return Status::OK(); // Same conf } @@ -96,6 +95,9 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.sk = conf.sk; reset_conf.token = conf.token; reset_conf.bucket = conf.bucket; + reset_conf.connect_timeout_ms = conf.connect_timeout_ms; + reset_conf.max_connections = conf.max_connections; + reset_conf.request_timeout_ms = conf.request_timeout_ms; reset_conf.use_virtual_addressing = conf.use_virtual_addressing; reset_conf.role_arn = conf.role_arn; From 560444c41f89561f00fec994c77e11ff6d74c385 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Tue, 12 Aug 2025 11:11:36 +0800 Subject: [PATCH 403/572] branch-3.0: [fix](txn lazy commit) fix multi table load error with txn lazy commit #54322 (#54491) picked from #54322 --- cloud/src/meta-service/meta_service_txn.cpp | 1 + cloud/src/meta-service/txn_lazy_committer.cpp | 9 +- cloud/test/txn_lazy_commit_test.cpp | 89 +++++++++++++++++++ 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index b2326e395c8a2a..2f3f0d1554531e 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -1934,6 +1934,7 @@ void commit_txn_eventually( std::shared_ptr task = txn_lazy_committer->submit(instance_id, txn_id); TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::txn_lazy_committer_wait", &txn_id); std::pair ret = task->wait(); + TEST_SYNC_POINT_CALLBACK("commit_txn_eventually::task->wait", &ret); if (ret.first != MetaServiceCode::OK) { LOG(WARNING) << "txn lazy commit failed txn_id=" << txn_id << " code=" << ret.first << " msg=" << ret.second; diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 3ad57fb0b6a78a..c4e67b2ef017e0 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -45,7 +45,7 @@ void convert_tmp_rowsets( const std::string& instance_id, int64_t txn_id, std::shared_ptr txn_kv, MetaServiceCode& code, std::string& msg, int64_t db_id, std::vector>& tmp_rowsets_meta, - std::unordered_map& tablet_ids) { + std::map& tablet_ids) { std::stringstream ss; std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -323,8 +323,7 @@ void TxnLazyCommitTask::commit() { } // - std::unordered_map>> + std::map>> partition_to_tmp_rowset_metas; for (auto& [tmp_rowset_key, tmp_rowset_pb] : all_tmp_rowset_metas) { partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].emplace_back(); @@ -334,9 +333,9 @@ void TxnLazyCommitTask::commit() { tmp_rowset_pb; } - // tablet_id -> TabletIndexPB - std::unordered_map tablet_ids; for (auto& [partition_id, tmp_rowset_metas] : partition_to_tmp_rowset_metas) { + // tablet_id -> TabletIndexPB + std::map tablet_ids; for (size_t i = 0; i < tmp_rowset_metas.size(); i += config::txn_lazy_max_rowsets_per_batch) { size_t end = diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index e07485d8c0db8f..2e018839e4e1b7 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -2581,4 +2581,93 @@ TEST(TxnLazyCommitTest, RecycleIndexes) { sp->disable_processing(); } +TEST(TxnLazyCommitTest, CommitTxnEventuallyWithMultiTableTest) { + auto txn_kv = get_mem_txn_kv(); + int64_t db_id = 3132121; + int64_t table_id = 5452432; + int64_t index_id = 76763; + int64_t partition_id = 43432; + + int64_t table_id2 = 54524321231; + int64_t index_id2 = 543123; + int64_t partition_id2 = 214352; + bool commit_txn_eventually_finish_hit = false; + + auto sp = SyncPoint::get_instance(); + sp->set_call_back("commit_txn_eventually::task->wait", [&](auto&& args) { + auto [code, msg] = *try_any_cast*>(args[0]); + ASSERT_EQ(code, MetaServiceCode::OK); + commit_txn_eventually_finish_hit = true; + }); + sp->enable_processing(); + + auto meta_service = get_meta_service(txn_kv, true); + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label("test_label_multi_table_commit_txn"); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.add_table_ids(table_id2); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + int64_t txn_id = res.txn_id(); + + // mock rowset and tablet + int64_t tablet_id_base = 3131124; + for (int i = 0; i < 5; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id, + tablet_id_base + i); + auto tmp_rowset = create_rowset(txn_id, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + for (int i = 5; i < 10; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id2, index_id2, partition_id2, + tablet_id_base + i); + auto tmp_rowset = create_rowset(txn_id, tablet_id_base + i, index_id2, partition_id2); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + { + brpc::Controller cntl; + CommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.set_txn_id(txn_id); + req.set_is_2pc(false); + req.set_enable_txn_lazy_commit(true); + CommitTxnResponse res; + meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(commit_txn_eventually_finish_hit); + } + + { + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string mock_instance = "test_instance"; + for (int i = 0; i < 10; ++i) { + int64_t tablet_id = tablet_id_base + i; + check_tablet_idx_db_id(txn, db_id, tablet_id); + check_tmp_rowset_not_exist(txn, tablet_id, txn_id); + check_rowset_meta_exist(txn, tablet_id, 2); + } + } + + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + } // namespace doris::cloud From 4716900f119603e7181bc77b149e1eb20d67ac3a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:12:16 +0800 Subject: [PATCH 404/572] branch-3.0: [Opt](Clone) Use log debug instead DCHECK when get_peers_replica_backends reps is empty #54343 (#54529) Cherry-picked from #54343 Co-authored-by: abmdocrt --- be/src/olap/storage_engine.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index bcb9dd8453feb8..173d068114f568 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -1506,7 +1506,11 @@ bool StorageEngine::get_peers_replica_backends(int64_t tablet_id, std::vector lock(_peer_replica_infos_mutex); if (result.tablet_replica_infos.contains(tablet_id)) { std::vector reps = result.tablet_replica_infos[tablet_id]; - DCHECK_NE(reps.size(), 0); + if (reps.empty()) [[unlikely]] { + VLOG_DEBUG << "get_peers_replica_backends reps is empty, maybe this tablet is in " + "schema change. Go to FE to see more info. Tablet id: " + << tablet_id; + } for (const auto& rep : reps) { if (rep.replica_id != tablet->replica_id()) { TBackend backend; From 0b1c4e66a6953f1fc207156ba2b62d8c5e5776b2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:12:58 +0800 Subject: [PATCH 405/572] branch-3.0: [fix](cloud) Fix cloud balance warm up lack of latest rs information #54404 (#54504) Cherry-picked from #54404 Co-authored-by: deardeng --- be/src/cloud/cloud_internal_service.cpp | 6 + .../io/cache/block_file_cache_downloader.cpp | 3 +- .../cloud/catalog/CloudTabletRebalancer.java | 6 +- .../balance/test_balance_warm_up.groovy | 147 ++++++++++++++++++ 4 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 regression-test/suites/cloud_p0/balance/test_balance_warm_up.groovy diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 02dab747c33984..33482cd8850f3f 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -82,6 +82,12 @@ void CloudInternalServiceImpl::get_file_cache_meta_by_tablet_id( return; } CloudTabletSPtr tablet = std::move(res.value()); + auto st = tablet->sync_rowsets(); + if (!st) { + // just log failed, try it best + LOG(WARNING) << "failed to sync rowsets: " << tablet_id + << " err msg: " << st.to_string(); + } auto rowsets = tablet->get_snapshot_rowset(); std::for_each(rowsets.cbegin(), rowsets.cend(), [&](const RowsetSharedPtr& rowset) { std::string rowset_id = rowset->rowset_id().to_string(); diff --git a/be/src/io/cache/block_file_cache_downloader.cpp b/be/src/io/cache/block_file_cache_downloader.cpp index c7cc54764368c3..92bb91a7a3f47e 100644 --- a/be/src/io/cache/block_file_cache_downloader.cpp +++ b/be/src/io/cache/block_file_cache_downloader.cpp @@ -174,7 +174,8 @@ void FileCacheBlockDownloader::download_file_cache_block( std::ranges::for_each(metas, [&](const FileCacheBlockMeta& meta) { VLOG_DEBUG << "download_file_cache_block: start, tablet_id=" << meta.tablet_id() << ", rowset_id=" << meta.rowset_id() << ", segment_id=" << meta.segment_id() - << ", offset=" << meta.offset() << ", size=" << meta.size(); + << ", offset=" << meta.offset() << ", size=" << meta.size() + << ", type=" << meta.cache_type(); CloudTabletSPtr tablet; if (auto res = _engine.tablet_mgr().get_tablet(meta.tablet_id(), false); !res.has_value()) { LOG(INFO) << "failed to find tablet " << meta.tablet_id() << " : " << res.error(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java index e7f080ea329804..887490d95f636b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudTabletRebalancer.java @@ -830,15 +830,15 @@ private Map sendCheckWarmUpCacheAsyncRpc(List tabletIds, lo req.setTablets(tabletIds); TCheckWarmUpCacheAsyncResponse result = client.checkWarmUpCacheAsync(req); if (result.getStatus().getStatusCode() != TStatusCode.OK) { - LOG.warn("check pre cache status {} {}", result.getStatus().getStatusCode(), + LOG.warn("check pre tablets {} cache status {} {}", tabletIds, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs()); } else { - LOG.info("check pre cache succ status {} {}", result.getStatus().getStatusCode(), + LOG.info("check pre tablets {} cache succ status {} {}", tabletIds, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs()); } return result.getTaskDone(); } catch (Exception e) { - LOG.warn("send check pre cache rpc error. backend[{}]", destBackend.getId(), e); + LOG.warn("send check pre cache rpc error. tablets{} backend[{}]", tabletIds, destBackend.getId(), e); ok = false; } finally { if (ok) { diff --git a/regression-test/suites/cloud_p0/balance/test_balance_warm_up.groovy b/regression-test/suites/cloud_p0/balance/test_balance_warm_up.groovy new file mode 100644 index 00000000000000..f6afcfb2f99018 --- /dev/null +++ b/regression-test/suites/cloud_p0/balance/test_balance_warm_up.groovy @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + + +suite('test_balance_warm_up', 'docker') { + if (!isCloudMode()) { + return; + } + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + 'sys_log_verbose_modules=org', + 'heartbeat_interval_second=1', + 'rehash_tablet_after_be_dead_seconds=3600', + 'enable_cloud_warm_up_for_rebalance=true' + ] + options.beConfigs += [ + 'report_tablet_interval_seconds=1', + 'schedule_sync_tablets_interval_s=18000', + 'disable_auto_compaction=true', + 'sys_log_verbose_modules=*' + ] + options.setFeNum(1) + options.setBeNum(1) + options.cloudMode = true + options.enableDebugPoints() + + def testCase = { table -> + def ms = cluster.getAllMetaservices().get(0) + def msHttpPort = ms.host + ":" + ms.httpPort + sql """CREATE TABLE $table ( + `k1` int(11) NULL, + `v1` VARCHAR(2048) + ) + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ( + "replication_num"="1" + ); + """ + sql """ + insert into $table values (10, '1'), (20, '2') + """ + sql """ + insert into $table values (30, '3'), (40, '4') + """ + + // before add be + def beforeGetFromFe = getTabletAndBeHostFromFe(table) + def beforeGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends()) + // version 2 + def beforeCacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2) + logger.info("cache dir version 2 {}", beforeCacheDirVersion2) + // version 3 + def beforeCacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 3) + logger.info("cache dir version 3 {}", beforeCacheDirVersion3) + + def beforeMergedCacheDir = beforeCacheDirVersion2 + beforeCacheDirVersion3.collectEntries { host, hashFiles -> + [(host): beforeCacheDirVersion2[host] ? (beforeCacheDirVersion2[host] + hashFiles) : hashFiles] + } + logger.info("before fe tablets {}, be tablets {}, cache dir {}", beforeGetFromFe, beforeGetFromBe, beforeMergedCacheDir) + + def beforeWarmUpResult = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM $table""" + logger.info("before warm up result {}", beforeWarmUpResult) + + cluster.addBackend(1, "compute_cluster") + def oldBe = sql_return_maparray('show backends').get(0) + def newAddBe = sql_return_maparray('show backends').get(1) + // balance tablet + awaitUntil(500) { + def afterWarmUpResult = sql_return_maparray """ADMIN SHOW REPLICA DISTRIBUTION FROM $table""" + logger.info("after warm up result {}", afterWarmUpResult) + afterWarmUpResult.any { row -> + Integer.valueOf((String) row.ReplicaNum) == 1 + } + } + + // from be1 -> be2, warm up this tablet + // after add be + def afterGetFromFe = getTabletAndBeHostFromFe(table) + def afterGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends()) + // version 2 + def afterCacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2) + logger.info("after cache dir version 2 {}", afterCacheDirVersion2) + // version 3 + def afterCacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 3) + logger.info("after cache dir version 3 {}", afterCacheDirVersion3) + + def afterMergedCacheDir = afterCacheDirVersion2 + afterCacheDirVersion3.collectEntries { host, hashFiles -> + [(host): afterCacheDirVersion2[host] ? (afterCacheDirVersion2[host] + hashFiles) : hashFiles] + } + logger.info("after fe tablets {}, be tablets {}, cache dir {}", afterGetFromFe, afterGetFromBe, afterMergedCacheDir) + def newAddBeCacheDir = afterMergedCacheDir.get(newAddBe.Host) + logger.info("new add be cache dir {}", newAddBeCacheDir) + assert newAddBeCacheDir.size() != 0 + assert beforeMergedCacheDir[oldBe.Host].containsAll(afterMergedCacheDir[newAddBe.Host]) + + def be = cluster.getBeByBackendId(newAddBe.BackendId.toLong()) + def dataPath = new File("${be.path}/storage/file_cache") + logger.info("Checking file_cache directory: {}", dataPath.absolutePath) + logger.info("Directory exists: {}", dataPath.exists()) + + def subDirs = [] + + def collectDirs + collectDirs = { File dir -> + if (dir.exists()) { + dir.eachDir { subDir -> + logger.info("Found subdir: {}", subDir.name) + subDirs << subDir.name + collectDirs(subDir) + } + } + } + + collectDirs(dataPath) + logger.info("BE {} file_cache subdirs: {}", newAddBe.Host, subDirs) + + newAddBeCacheDir.each { hashFile -> + assertTrue(subDirs.any { subDir -> subDir.startsWith(hashFile) }, + "Expected cache file pattern ${hashFile} not found in BE ${newAddBe.Host}'s file_cache directory. " + + "Available subdirs: ${subDirs}") + } + } + + docker(options) { + testCase("test_balance_warm_up_tbl") + } +} From 9f5cd1a80db98160bad496a768e6f38bc910a1c8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 11:14:10 +0800 Subject: [PATCH 406/572] branch-3.0: [feat](cloud) change the type to optional in the alter storage vault command #54394 (#54475) Cherry-picked from #54394 Co-authored-by: lw112 <131352377+felixwluo@users.noreply.github.com> --- .../apache/doris/catalog/StorageVaultMgr.java | 21 ++++++++++++++++ .../commands/AlterStorageVaultCommand.java | 22 ++++++++++++++--- .../alter/test_alter_vault_type.groovy | 24 +++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index 165ec9ce2230b2..37730b077ecd5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -297,6 +297,27 @@ public Pair getDefaultStorageVault() { } } + public StorageVaultType getStorageVaultTypeByName(String vaultName) throws DdlException { + try { + Cloud.GetObjStoreInfoResponse resp = MetaServiceProxy.getInstance() + .getObjStoreInfo(Cloud.GetObjStoreInfoRequest.newBuilder().build()); + + for (Cloud.StorageVaultPB vault : resp.getStorageVaultList()) { + if (vault.getName().equals(vaultName)) { + if (vault.hasHdfsInfo()) { + return StorageVaultType.HDFS; + } else if (vault.hasObjInfo()) { + return StorageVaultType.S3; + } + } + } + return StorageVaultType.UNKNOWN; + } catch (RpcException e) { + LOG.warn("failed to get storage vault type due to RpcException: {}", e); + throw new DdlException(e.getMessage()); + } + } + @VisibleForTesting public void createHdfsVault(StorageVault vault) throws Exception { Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = buildAlterStorageVaultRequest(vault); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java index f877ef0da14afa..ac428935934cb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AlterStorageVaultCommand.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.StorageVault; import org.apache.doris.catalog.StorageVault.StorageVaultType; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; @@ -55,9 +56,24 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); } - StorageVault.StorageVaultType vaultType = StorageVaultType.fromString(properties.get(TYPE)); - if (vaultType == StorageVault.StorageVaultType.UNKNOWN) { - throw new AnalysisException("Unsupported Storage Vault type: " + type); + + StorageVault.StorageVaultType vaultType; + if (properties.containsKey(TYPE)) { + vaultType = StorageVaultType.fromString(properties.get(TYPE)); + if (vaultType == StorageVaultType.UNKNOWN) { + throw new AnalysisException("Unsupported Storage Vault type: " + type); + } + } else { + // auto-detect + try { + vaultType = Env.getCurrentEnv().getStorageVaultMgr().getStorageVaultTypeByName(name); + if (vaultType == StorageVaultType.UNKNOWN) { + throw new AnalysisException("Storage vault '" + name + "' does not exist or has unknown type. " + + "You can use `SHOW STORAGE VAULT` to get all available vaults."); + } + } catch (DdlException e) { + throw new AnalysisException("Failed to get storage vault type: " + e.getMessage()); + } } FeNameFormat.checkStorageVaultName(name); diff --git a/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy b/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy index a57207712e90cf..7babbeb4fe7023 100644 --- a/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy +++ b/regression-test/suites/vault_p0/alter/test_alter_vault_type.groovy @@ -95,4 +95,28 @@ suite("test_alter_vault_type", "nonConcurrent") { } catch (Exception e) { assertTrue(e.getMessage().contains("Access denied for user"), e.getMessage()) } + + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "s3.access_key" = "${getS3AK()}", + "s3.secret_key" = "${getS3SK()}" + ); + """ + + sql """ + ALTER STORAGE VAULT ${hdfsVaultName} + PROPERTIES ( + "hadoop.username" = "${getHmsUser()}" + ); + """ + + expectExceptionLike({ + sql """ + ALTER STORAGE VAULT non_existent_vault_${randomStr} + PROPERTIES ( + "s3.access_key" = "test_ak" + ); + """ + }, "does not exist") } \ No newline at end of file From 3c66edaea95bfe56e73aa536cdf79ec4c4d0be1d Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 12 Aug 2025 11:18:20 +0800 Subject: [PATCH 407/572] [opt](nereids) Improve sql parse performance by avoid parse twice when collect hint map (#52629) cherry-pick part code from https://github.com/apache/doris/pull/40202 pr: https://github.com/apache/doris/pull/40202 commitId: 81f3c484 ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/nereids/parser/NereidsParser.java | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java index 4ed71bbbc14673..c17f967aa0221b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java @@ -56,6 +56,7 @@ import java.lang.reflect.Method; import java.util.BitSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -228,7 +229,7 @@ private static Token readUntilNonComment(TokenSource tokenSource) { } private List parseSQLWithDialect(String sql, - SessionVariable sessionVariable) { + SessionVariable sessionVariable) { @Nullable Dialect sqlDialect = Dialect.getByName(sessionVariable.getSqlDialect()); if (sqlDialect == null) { return parseSQL(sql); @@ -244,7 +245,7 @@ private List parseSQLWithDialect(String sql, } } catch (Throwable throwable) { LOG.warn("Parse sql with dialect {} failed, plugin: {}, sql: {}.", - sqlDialect, plugin.getClass().getSimpleName(), sql, throwable); + sqlDialect, plugin.getClass().getSimpleName(), sql, throwable); } } @@ -280,7 +281,7 @@ public List> parseMultiple(String sql) { } public List> parseMultiple(String sql, - @Nullable LogicalPlanBuilder logicalPlanBuilder) { + @Nullable LogicalPlanBuilder logicalPlanBuilder) { return parse(sql, logicalPlanBuilder, DorisParser::multiStatements); } @@ -325,38 +326,31 @@ private T parse(String sql, Function parseFu } private T parse(String sql, @Nullable LogicalPlanBuilder logicalPlanBuilder, - Function parseFunction) { - ParserRuleContext tree = toAst(sql, parseFunction); + Function parseFunction) { + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, parseFunction); LogicalPlanBuilder realLogicalPlanBuilder = logicalPlanBuilder == null - ? new LogicalPlanBuilder(getHintMap(sql, DorisParser::selectHint)) : logicalPlanBuilder; + ? new LogicalPlanBuilder(getHintMap(sql, tokenStream, DorisParser::selectHint)) + : logicalPlanBuilder; return (T) realLogicalPlanBuilder.visit(tree); } public LogicalPlan parseForCreateView(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(tokenStream, DorisParser::singleStatement); LogicalPlanBuilder realLogicalPlanBuilder = new LogicalPlanBuilderForCreateView( - getHintMap(sql, DorisParser::selectHint)); + getHintMap(sql, tokenStream, DorisParser::selectHint)); return (LogicalPlan) realLogicalPlanBuilder.visit(tree); } - public Optional parseForSyncMv(String sql) { - ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); - LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv( - getHintMap(sql, DorisParser::selectHint)); - logicalPlanBuilderForSyncMv.visit(tree); - return logicalPlanBuilderForSyncMv.getQuerySql(); - } - /** get hint map */ - public static Map getHintMap(String sql, - Function parseFunction) { + public static Map getHintMap(String sql, CommonTokenStream hintTokenStream, + Function parseFunction) { // parse hint first round - DorisLexer hintLexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream hintTokenStream = new CommonTokenStream(hintLexer); - Map selectHintMap = Maps.newHashMap(); - Token hintToken = hintTokenStream.getTokenSource().nextToken(); + Iterator tokenIterator = hintTokenStream.getTokens().iterator(); + Token hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; while (hintToken != null && hintToken.getType() != DorisLexer.EOF) { if (hintToken.getChannel() == 2 && sql.charAt(hintToken.getStartIndex() + 2) == '+') { String hintSql = sql.substring(hintToken.getStartIndex() + 3, hintToken.getStopIndex() + 1); @@ -366,15 +360,19 @@ public static Map getHintMap(String sql, ParserRuleContext hintContext = parseFunction.apply(hintParser); selectHintMap.put(hintToken.getStartIndex(), hintContext); } - hintToken = hintTokenStream.getTokenSource().nextToken(); + hintToken = tokenIterator.hasNext() ? tokenIterator.next() : null; } return selectHintMap; } + public static ParserRuleContext toAst( + String sql, Function parseFunction) { + return toAst(parseAllTokens(sql), parseFunction); + } + /** toAst */ - public static ParserRuleContext toAst(String sql, Function parseFunction) { - DorisLexer lexer = new DorisLexer(new CaseInsensitiveStream(CharStreams.fromString(sql))); - CommonTokenStream tokenStream = new CommonTokenStream(lexer); + public static ParserRuleContext toAst( + CommonTokenStream tokenStream, Function parseFunction) { DorisParser parser = new DorisParser(tokenStream); parser.addParseListener(POST_PROCESSOR); @@ -405,9 +403,7 @@ public static ParserRuleContext toAst(String sql, Function parseForSyncMv(String sql) { + CommonTokenStream tokenStream = parseAllTokens(sql); + ParserRuleContext tree = toAst(sql, DorisParser::singleStatement); + LogicalPlanBuilderForSyncMv logicalPlanBuilderForSyncMv = new LogicalPlanBuilderForSyncMv( + getHintMap(sql, tokenStream, DorisParser::selectHint)); + logicalPlanBuilderForSyncMv.visit(tree); + return logicalPlanBuilderForSyncMv.getQuerySql(); + } } From 21e81540fd6ae6c7b7cb594a246b10203867c813 Mon Sep 17 00:00:00 2001 From: seawinde Date: Tue, 12 Aug 2025 11:18:48 +0800 Subject: [PATCH 408/572] [fix](test) Force meta sync to avoid stale meta data on follower fe when mv rewrite (#54296) (#54462) ### What problem does this PR solve? pr: https://github.com/apache/doris/pull/54296 commitId: d3833e4d2060a68cc1548e9b5b40920b36272782 Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../doris/regression/suite/Suite.groovy | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index e7d038a44eb48f..b8a1fb259ed146 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2062,6 +2062,7 @@ class Suite implements GroovyInterceptable { AS ${mv_sql} """ waitingMVTaskFinishedByMvName(db, table_name, mv_name) + sql """sync;""" } def create_async_mv = { db, mv_name, mv_sql -> @@ -2077,6 +2078,8 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) sql "analyze table ${db}.${mv_name} with sync;" + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" } def create_async_partition_mv = { db, mv_name, mv_sql, partition_col -> @@ -2093,6 +2096,8 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) sql "analyze table ${db}.${mv_name} with sync;" + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" } // mv not part in rewrite process @@ -2153,6 +2158,8 @@ class Suite implements GroovyInterceptable { is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen mv_rewrite_success_without_check_chosen(query_sql, mv_name, sync_cbo_rewrite) @@ -2179,6 +2186,8 @@ class Suite implements GroovyInterceptable { is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) @@ -2223,6 +2232,8 @@ class Suite implements GroovyInterceptable { is_partition_statistics_ready = true) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names + ", sync_cbo_rewrite = " +sync_cbo_rewrite + ", is_partition_statistics_ready = " + is_partition_statistics_ready) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!is_partition_statistics_ready) { // If partition statistics is no ready, degrade to without check cbo chosen mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite) @@ -2264,6 +2275,8 @@ class Suite implements GroovyInterceptable { void mv_rewrite_all_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()){ logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!sync_cbo_rewrite) { explain { sql("${query_sql}") @@ -2303,6 +2316,8 @@ class Suite implements GroovyInterceptable { void mv_rewrite_any_success_without_check_chosen(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!sync_cbo_rewrite) { explain { sql("${query_sql}") @@ -2341,6 +2356,8 @@ class Suite implements GroovyInterceptable { void mv_rewrite_success_without_check_chosen(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!sync_cbo_rewrite) { explain { sql("${query_sql}") @@ -2363,6 +2380,8 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite void mv_rewrite_fail(query_sql, mv_name, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_name = " + mv_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!sync_cbo_rewrite) { explain { sql("${query_sql}") @@ -2380,6 +2399,8 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite void mv_rewrite_all_fail(query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!sync_cbo_rewrite) { explain { sql("${query_sql}") @@ -2417,6 +2438,8 @@ class Suite implements GroovyInterceptable { // sync_cbo_rewrite is the bool value which control sync mv is use cbo based mv rewrite void mv_rewrite_any_fail (query_sql, mv_names, sync_cbo_rewrite = enable_sync_mv_cost_based_rewrite()) { logger.info("query_sql = " + query_sql + ", mv_names = " + mv_names) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" if (!sync_cbo_rewrite) { explain { sql("${query_sql}") @@ -2460,6 +2483,8 @@ class Suite implements GroovyInterceptable { """ def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" mv_rewrite_success(query_sql, mv_name, true) } @@ -2476,6 +2501,8 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" mv_rewrite_success_without_check_chosen(query_sql, mv_name, true) } @@ -2493,6 +2520,8 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" mv_rewrite_fail(query_sql, mv_name, true) } @@ -2508,6 +2537,8 @@ class Suite implements GroovyInterceptable { def job_name = getJobName(db, mv_name); waitingMTMVTaskFinished(job_name) + // force meta sync to avoid stale meta data on follower fe + sql """sync;""" } def token = context.config.metaServiceToken From d58eedebcc383483d103ba75bae3b0751405d209 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Tue, 12 Aug 2025 12:22:25 +0800 Subject: [PATCH 409/572] [improve](load) do not block delta writer if memtable memory is low (#42649) (#54517) cherry-pick #42649 --- be/src/olap/memtable_memory_limiter.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/memtable_memory_limiter.cpp b/be/src/olap/memtable_memory_limiter.cpp index 1f59f8fc341f14..cae597ccc9433f 100644 --- a/be/src/olap/memtable_memory_limiter.cpp +++ b/be/src/olap/memtable_memory_limiter.cpp @@ -153,7 +153,7 @@ void MemTableMemoryLimiter::handle_memtable_flush() { << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage); _flush_active_memtables(need_flush); } - } while (_hard_limit_reached()); + } while (_hard_limit_reached() && !_load_usage_low()); g_memtable_memory_limit_waiting_threads << -1; timer.stop(); int64_t time_ms = timer.elapsed_time() / 1000 / 1000; From f54a162ddf898cb0b4baed529a43c099b81ea4fa Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Tue, 12 Aug 2025 14:13:48 +0800 Subject: [PATCH 410/572] [branch-3.0] [fix](checker) Use cloud::blob_get/put to get tablet schema #54518 (#54592) pick: https://github.com/apache/doris/pull/54518 --- cloud/src/recycler/checker.cpp | 66 ++++--- cloud/test/recycler_test.cpp | 332 ++++++++++----------------------- 2 files changed, 134 insertions(+), 264 deletions(-) diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 96b17872386df9..828e6981b60acc 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -44,6 +44,8 @@ #include "common/logging.h" #include "common/util.h" #include "cpp/sync_point.h" +#include "meta-service/meta_service_schema.h" +#include "meta-store/blob_message.h" #include "meta-store/keys.h" #include "meta-store/txn_kv.h" #include "meta-store/txn_kv_error.h" @@ -571,25 +573,33 @@ int InstanceChecker::do_check() { TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to init txn, err=" << err; + check_ret = -1; return; } TabletIndexPB tablet_index; if (get_tablet_idx(txn_kv_.get(), instance_id_, rs_meta.tablet_id(), tablet_index) == -1) { LOG(WARNING) << "failed to get tablet index, tablet_id= " << rs_meta.tablet_id(); + check_ret = -1; return; } auto tablet_schema_key = meta_schema_key({instance_id_, tablet_index.index_id(), rs_meta.schema_version()}); - std::string tablet_schema_val; - err = txn->get(tablet_schema_key, &tablet_schema_val); - if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { - // rowset don't have tablet schema key means no index + ValueBuf tablet_schema_val; + err = cloud::blob_get(txn.get(), tablet_schema_key, &tablet_schema_val); + + if (err != TxnErrorCode::TXN_OK) { + check_ret = -1; + LOG(WARNING) << "failed to get schema, err=" << err; return; } + auto* schema = rs_meta.mutable_tablet_schema(); - schema->ParseFromString(tablet_schema_val); + if (!parse_schema_value(tablet_schema_val, schema)) { + LOG(WARNING) << "malformed schema value, key=" << hex(tablet_schema_key); + return; + } std::vector> index_ids; for (const auto& i : rs_meta.tablet_schema().index()) { @@ -597,8 +607,7 @@ int InstanceChecker::do_check() { index_ids.emplace_back(i.index_id(), i.index_suffix_name()); } } - std::string tablet_idx_key = meta_tablet_idx_key({instance_id_, rs_meta.tablet_id()}); - if (!key_exist(txn_kv_.get(), tablet_idx_key)) { + if (!index_ids.empty()) { for (int i = 0; i < rs_meta.num_segments(); ++i) { std::vector index_path_v; if (rs_meta.tablet_schema().inverted_index_storage_format() == @@ -617,16 +626,14 @@ int InstanceChecker::do_check() { inverted_index_path_v2(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), i)); } - if (!index_path_v.empty()) { - if (std::ranges::all_of(index_path_v, [&](const auto& idx_file_path) { - if (!tablet_files_cache.files.contains(idx_file_path)) { - LOG(INFO) << "loss index file: " << idx_file_path; - return false; - } - return true; - })) { - continue; - } + if (std::ranges::all_of(index_path_v, [&](const auto& idx_file_path) { + if (!tablet_files_cache.files.contains(idx_file_path)) { + LOG(INFO) << "loss index file: " << idx_file_path; + return false; + } + return true; + })) { + continue; } index_file_loss = true; data_loss = true; @@ -1273,10 +1280,6 @@ int InstanceChecker::check_inverted_index_file_storage_format_v1( return -1; } - for (size_t i = 0; i < rs_meta.num_segments(); i++) { - rowset_index_cache_v1.segment_ids.insert(i); - } - TabletIndexPB tablet_index; if (get_tablet_idx(txn_kv_.get(), instance_id_, rs_meta.tablet_id(), tablet_index) == -1) { @@ -1286,14 +1289,23 @@ int InstanceChecker::check_inverted_index_file_storage_format_v1( auto tablet_schema_key = meta_schema_key( {instance_id_, tablet_index.index_id(), rs_meta.schema_version()}); - std::string tablet_schema_val; - err = txn->get(tablet_schema_key, &tablet_schema_val); - if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { - // rowset don't have tablet schema key means no index - return 0; + ValueBuf tablet_schema_val; + err = cloud::blob_get(txn.get(), tablet_schema_key, &tablet_schema_val); + + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to get schema, err=" << err; + return -1; } + auto* schema = rs_meta.mutable_tablet_schema(); - schema->ParseFromString(tablet_schema_val); + if (!parse_schema_value(tablet_schema_val, schema)) { + LOG(WARNING) << "malformed schema value, key=" << hex(tablet_schema_key); + return -1; + } + + for (size_t i = 0; i < rs_meta.num_segments(); i++) { + rowset_index_cache_v1.segment_ids.insert(i); + } for (const auto& i : rs_meta.tablet_schema().index()) { if (i.has_index_type() && i.index_type() == IndexType::INVERTED) { diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index 0411e801905920..0d4f10e15f4a7d 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -39,6 +39,7 @@ #include "common/util.h" #include "cpp/sync_point.h" #include "meta-service/meta_service.h" +#include "meta-store/blob_message.h" #include "meta-store/keys.h" #include "meta-store/mem_txn_kv.h" #include "meta-store/txn_kv.h" @@ -86,37 +87,17 @@ std::vector index_v1_file_path = { "data/1753202846974/0200000000007864994f6aa97288842758c2e89b03e65682_0_1753202846943.idx", "data/1753202845724/020000000000786635407b55b72242ac167cf83cd4c598a2_0_1753202841593.idx", "data/1753202846984/020000000000788bdd40fcf18bcaa1bbd4058ef92606e79a_0_1753202846923.idx", - "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0_1753202846963.idx", - "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_0_1753202846903.idx", - "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_1_1753202846903.idx", - "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a2_1_1753202846963.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a0_0_1753202846963.idx", + "data/1753202846986/02000000000078e635407b55b72242ac167cf83cd4c598a1_0_1753202846963.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a0_0_1753202844931.idx", + "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a1_0_1753202844931.idx", "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202844931.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753222846410.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_0_1753202847011.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_1_1753202844931.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_1_1753222846410.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_1_1753202847011.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_2_1753202844931.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_2_1753222846410.idx", - "data/1753202847030/020000000000791335407b55b72242ac167cf83cd4c598a2_2_1753202847011.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65680_0_1753202843931.idx", + "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65681_0_1753202843931.idx", "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202843931.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753252846410.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_0_1753202847021.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_1_1753202843931.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_1_1753252846410.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_1_1753202847021.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_2_1753202843931.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_2_1753252846410.idx", - "data/1753202858558/0200000000007aed994f6aa97288842758c2e89b03e65682_2_1753202847021.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202824931.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1756202846410.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202847071.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_1_1753202824931.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_1_1756202846410.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_1_1753202847071.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_2_1753202824931.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_2_1756202846410.idx", - "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_2_1753202847071.idx"}; + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65680_0_1753202824931.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65681_0_1753202824931.idx", + "data/1753202858458/0200000000007afc994f6aa97288842758c2e89b03e65682_0_1753202824931.idx"}; // clang-format on doris::cloud::RecyclerThreadPoolGroup thread_group; @@ -303,92 +284,6 @@ static int create_tmp_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor, return 0; } -static int create_committed_rowset_with_tablet_schema( - TxnKv* txn_kv, StorageVaultAccessor* accessor, const std::string& resource_id, - int64_t tablet_id, int64_t version, int num_segments = 1, size_t num_inverted_indexes = 1, - bool use_inverted_index_storage_format_v1 = true) { - std::string val; - std::unique_ptr txn; - int64_t tablet_index_id = 123; - int64_t schema_version = 456; - - auto rowset_id = next_rowset_id(); - MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; - std::string rowset_meta_key = meta_rowset_key(key_info); - - doris::RowsetMetaCloudPB rowset_pb; - rowset_pb.set_rowset_id(0); // useless but required - rowset_pb.set_rowset_id_v2(rowset_id); - rowset_pb.set_num_segments(num_segments); - rowset_pb.set_tablet_id(tablet_id); - rowset_pb.set_resource_id(resource_id); - rowset_pb.set_creation_time(current_time); - rowset_pb.set_schema_version(schema_version); - rowset_pb.SerializeToString(&val); - - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } - txn->put(rowset_meta_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } - - TabletIndexPB tablet_index; - tablet_index.set_index_id(tablet_index_id); - tablet_index.set_tablet_id(tablet_id); - std::string tablet_index_key = meta_tablet_idx_key({instance_id, tablet_id}); - tablet_index.SerializeToString(&val); - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } - txn->put(tablet_index_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } - - if (num_inverted_indexes) { - doris::TabletSchemaCloudPB tablet_schema; - if (use_inverted_index_storage_format_v1) { - tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); - } else { - tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); - } - tablet_schema.set_schema_version(schema_version); - for (size_t i = 0; i < num_inverted_indexes; i++) { - auto index = tablet_schema.add_index(); - index->set_index_id(i); - index->set_index_type(IndexType::INVERTED); - } - std::string tablet_schema_key = - meta_schema_key({instance_id, tablet_index_id, schema_version}); - std::string val; - tablet_schema.SerializeToString(&val); - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } - txn->put(tablet_schema_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } - } - - for (int i = 0; i < num_segments; ++i) { - auto path = segment_path(tablet_id, rowset_id, i); - accessor->put_file(path, ""); - if (use_inverted_index_storage_format_v1) { - for (int j = 0; j < num_inverted_indexes; ++j) { - std::string path = inverted_index_path_v1(tablet_id, rowset_id, i, j, ""); - accessor->put_file(path, ""); - } - } else { - std::string path = inverted_index_path_v2(tablet_id, rowset_id, i); - accessor->put_file(path, ""); - } - } - return 0; -} - static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, StorageVaultAccessor* accessor, const std::string& resource_id, @@ -396,6 +291,7 @@ static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, int64_t version = 1) { std::string val; std::unique_ptr txn; + TxnErrorCode err; // Parse file path to extract tablet_id and rowset_id // Expected format: data/{tablet_id}/{rowset_id}_{segment_id}.{ext} @@ -427,7 +323,8 @@ static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, int64_t segment_id = stoll(segment_str); int64_t tablet_index_id = tablet_id + 10; // take the last 4 digits of tablet_id as the unique identifier - int64_t schema_version = std::atoll(path_parts[1].substr(path_parts[1].size() - 4).c_str()); + int64_t schema_version = + std::atoll(path_parts[1].substr(path_parts[1].size() - 4).c_str()) + version; // Create rowset meta data MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; @@ -443,13 +340,9 @@ static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, rowset_pb.set_schema_version(schema_version); rowset_pb.SerializeToString(&val); - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } + err = txn_kv->create_txn(&txn); + EXPECT_EQ(err, TxnErrorCode::TXN_OK) << err; txn->put(rowset_meta_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } // Create tablet index meta data TabletIndexPB tablet_index; @@ -457,47 +350,28 @@ static int create_committed_rowset_by_real_index_v2_file(TxnKv* txn_kv, tablet_index.set_tablet_id(tablet_id); std::string tablet_index_key = meta_tablet_idx_key({instance_id, tablet_id}); tablet_index.SerializeToString(&val); - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } txn->put(tablet_index_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } // Create tablet schema if dealing with index files if (extension == "idx") { std::string tablet_schema_key = meta_schema_key({instance_id, tablet_index_id, schema_version}); - std::string tablet_schema_val; - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } - doris::TabletSchemaCloudPB tablet_schema; - - if (txn->get(tablet_schema_key, &tablet_schema_val) == TxnErrorCode::TXN_KEY_NOT_FOUND) { - tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); - tablet_schema.set_schema_version(schema_version); - - auto index = tablet_schema.add_index(); - index->set_index_id(tablet_schema.index().size()); - index->set_index_type(IndexType::INVERTED); - - } else { - tablet_schema.ParseFromString(tablet_schema_val); - auto index = tablet_schema.add_index(); - index->set_index_id(tablet_schema.index().size()); - index->set_index_type(IndexType::INVERTED); - } - tablet_schema.SerializeToString(&val); + doris::TabletSchemaCloudPB tablet_schema; + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2); + tablet_schema.set_schema_version(schema_version); + auto index = tablet_schema.add_index(); + index->set_index_id(tablet_schema.index().size()); + index->set_index_type(IndexType::INVERTED); - txn->put(tablet_schema_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } + MetaServiceCode code; + std::string msg; + put_schema_kv(code, msg, txn.get(), tablet_schema_key, tablet_schema); } + err = txn->commit(); + EXPECT_EQ(err, TxnErrorCode::TXN_OK) << err; + std::string segment_path = file_path.substr(0, file_path.size() - 4) + ".dat"; accessor->put_file(segment_path, ""); accessor->put_file(file_path, ""); @@ -512,6 +386,7 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, size_t& version) { std::string val; std::unique_ptr txn; + TxnErrorCode err; // Parse file path to extract tablet_id and rowset_id // Expected format: data/{tablet_id}/{rowset_id}_{segment_id}_{index_id}{suffix}.idx @@ -554,7 +429,8 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, int segment_id = stoll(segment_str); int64_t index_id = std::stoll(index_id_str); int64_t tablet_index_id = tablet_id + 10; - int64_t schema_version = std::atoll(path_parts[1].substr(path_parts[1].size() - 4).c_str()); + int64_t schema_version = + std::atoll(path_parts[1].substr(path_parts[1].size() - 4).c_str()) + version; // Create rowset meta data MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; @@ -570,13 +446,9 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, rowset_pb.set_schema_version(schema_version); rowset_pb.SerializeToString(&val); - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } + err = txn_kv->create_txn(&txn); + EXPECT_EQ(err, TxnErrorCode::TXN_OK) << err; txn->put(rowset_meta_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } // Create tablet index meta data TabletIndexPB tablet_index; @@ -584,53 +456,31 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, tablet_index.set_tablet_id(tablet_id); std::string tablet_index_key = meta_tablet_idx_key({instance_id, tablet_id}); tablet_index.SerializeToString(&val); - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } + txn->put(tablet_index_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } // Create tablet schema if dealing with index files if (extension == "idx") { std::string tablet_schema_key = meta_schema_key({instance_id, tablet_index_id, schema_version}); - std::string tablet_schema_val; - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } - doris::TabletSchemaCloudPB tablet_schema; - if (txn->get(tablet_schema_key, &tablet_schema_val) == TxnErrorCode::TXN_KEY_NOT_FOUND) { - tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); - tablet_schema.set_schema_version(schema_version); - - auto index = tablet_schema.add_index(); - index->set_index_id(index_id); - index->set_index_type(IndexType::INVERTED); - if (!index_suffix.empty()) { - index->set_index_suffix_name(index_suffix); - } - - } else { - tablet_schema.ParseFromString(tablet_schema_val); + doris::TabletSchemaCloudPB tablet_schema; + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + tablet_schema.set_schema_version(schema_version); - auto* index = tablet_schema.add_index(); - index->set_index_id(index_id); - index->set_index_type(IndexType::INVERTED); - if (!index_suffix.empty()) { - index->set_index_suffix_name(index_suffix); - } + auto index = tablet_schema.add_index(); + index->set_index_id(index_id); + index->set_index_type(IndexType::INVERTED); + if (!index_suffix.empty()) { + index->set_index_suffix_name(index_suffix); } - tablet_schema.SerializeToString(&val); - txn->put(tablet_schema_key, val); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } + MetaServiceCode code; + std::string msg; + put_schema_kv(code, msg, txn.get(), tablet_schema_key, tablet_schema); } - + err = txn->commit(); + EXPECT_EQ(err, TxnErrorCode::TXN_OK) << err; std::string segment_path = fmt::format("data/{}/{}_{}.dat", tablet_id, rowset_id, segment_id); accessor->put_file(segment_path, ""); accessor->put_file(file_path, ""); @@ -640,10 +490,11 @@ static int create_committed_rowset_by_real_index_v1_file(TxnKv* txn_kv, static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor, const std::string& resource_id, int64_t tablet_id, - int64_t version, int num_segments = 1, + int64_t version, int64_t index_id, int num_segments = 1, int num_inverted_indexes = 1) { std::string key; std::string val; + int64_t schema_version = tablet_id + version + num_inverted_indexes + 1; auto rowset_id = next_rowset_id(); MetaRowsetKeyInfo key_info {instance_id, tablet_id, version}; @@ -656,15 +507,8 @@ static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor rowset_pb.set_tablet_id(tablet_id); rowset_pb.set_resource_id(resource_id); rowset_pb.set_creation_time(current_time); - if (num_inverted_indexes > 0) { - auto schema = rowset_pb.mutable_tablet_schema(); - schema->set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); - for (int i = 0; i < num_inverted_indexes; ++i) { - auto index = schema->add_index(); - index->set_index_id(i); - index->set_index_type(IndexType::INVERTED); - } - } + rowset_pb.set_schema_version(schema_version); + rowset_pb.set_index_id(index_id); rowset_pb.SerializeToString(&val); std::unique_ptr txn; @@ -672,6 +516,37 @@ static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor return -1; } txn->put(key, val); + + std::string key1; + std::string val1; + MetaTabletIdxKeyInfo key_info1 {instance_id, tablet_id}; + meta_tablet_idx_key(key_info1, &key1); + TabletIndexPB tablet_idx_pb; + tablet_idx_pb.set_db_id(db_id); + tablet_idx_pb.set_index_id(index_id); + tablet_idx_pb.set_tablet_id(tablet_id); + if (!tablet_idx_pb.SerializeToString(&val1)) { + return -1; + } + + txn->put(key1, val1); + + int64_t tablet_schema_version_cnt = tablet_id + version + 1; + + for (size_t i = 0; i < num_inverted_indexes; i++) { + std::string tablet_schema_key = + meta_schema_key({instance_id, index_id, ++tablet_schema_version_cnt}); + doris::TabletSchemaCloudPB tablet_schema; + tablet_schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1); + tablet_schema.set_schema_version(tablet_schema_version_cnt); + auto index = tablet_schema.add_index(); + index->set_index_id(i); + index->set_index_type(IndexType::INVERTED); + MetaServiceCode code; + std::string msg; + put_schema_kv(code, msg, txn.get(), tablet_schema_key, tablet_schema); + } + if (txn->commit() != TxnErrorCode::TXN_OK) { return -1; } @@ -680,24 +555,6 @@ static int create_committed_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor auto path = segment_path(tablet_id, rowset_id, i); accessor->put_file(path, ""); for (int j = 0; j < num_inverted_indexes; ++j) { - std::string key1; - std::string val1; - MetaTabletIdxKeyInfo key_info1 {instance_id, tablet_id}; - meta_tablet_idx_key(key_info1, &key1); - TabletIndexPB tablet_table; - tablet_table.set_db_id(db_id); - tablet_table.set_index_id(j); - tablet_table.set_tablet_id(tablet_id); - if (!tablet_table.SerializeToString(&val1)) { - return -1; - } - if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { - return -1; - } - txn->put(key1, val1); - if (txn->commit() != TxnErrorCode::TXN_OK) { - return -1; - } auto path = inverted_index_path_v1(tablet_id, rowset_id, i, j, ""); accessor->put_file(path, ""); } @@ -1480,7 +1337,8 @@ TEST(RecyclerTest, recycle_tablet) { i & 1); } for (int i = 0; i < 500; ++i) { - create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_tablet", tablet_id, i); + create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_tablet", tablet_id, i, + index_id); } ASSERT_EQ(create_partition_version_kv(txn_kv.get(), table_id, partition_id), 0); @@ -1567,7 +1425,8 @@ TEST(RecyclerTest, recycle_indexes) { create_tmp_rowset(txn_kv.get(), accessor.get(), tmp_rowset, j & 1); } for (int j = 0; j < 10; ++j) { - create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_indexes", tablet_id, j); + create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_indexes", tablet_id, j, + index_id); } } @@ -1683,7 +1542,7 @@ TEST(RecyclerTest, recycle_partitions) { } for (int j = 0; j < 10; ++j) { create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_partitions", - tablet_id, j); + tablet_id, j, index_id); } } } @@ -2635,7 +2494,8 @@ TEST(RecyclerTest, recycle_deleted_instance) { } for (int j = 0; j < 10; ++j) { // create meta key - create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_indexes", tablet_id, j); + create_committed_rowset(txn_kv.get(), accessor.get(), "recycle_indexes", tablet_id, j, + index_id); } } @@ -2822,12 +2682,12 @@ TEST(CheckerTest, normal_inverted_check) { auto accessor = checker.accessor_map_.begin()->second; for (int t = 10001; t <= 10100; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 1); } } for (int t = 10101; t <= 10200; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 5); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 5); } } ASSERT_EQ(checker.do_inverted_check(), 0); @@ -2872,7 +2732,7 @@ TEST(CheckerTest, DISABLED_abnormal_inverted_check) { auto accessor = checker.accessor_map_.begin()->second; for (int t = 10001; t <= 10100; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, 1); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 1); } } std::vector schemas; @@ -3388,12 +3248,12 @@ TEST(CheckerTest, normal) { auto accessor = checker.accessor_map_.begin()->second; for (int t = 10001; t <= 10100; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 1); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 1); } } for (int t = 10101; t <= 10200; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 5); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 5); } } ASSERT_EQ(checker.do_check(), 0); @@ -3414,14 +3274,12 @@ TEST(CheckerTest, abnormal) { auto accessor = checker.accessor_map_.begin()->second; for (int t = 10001; t <= 10100; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 1, - 0); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 1, 0); } } for (int t = 10101; t <= 10200; ++t) { for (int v = 0; v < 10; ++v) { - create_committed_rowset_with_tablet_schema(txn_kv.get(), accessor.get(), "1", t, v, 5, - 0); + create_committed_rowset(txn_kv.get(), accessor.get(), "1", t, v, t * 2, 5, 0); } } From 7a0e4acb2544e929e3420f132bbafd0ea26e28f7 Mon Sep 17 00:00:00 2001 From: Socrates Date: Wed, 13 Aug 2025 09:47:26 +0800 Subject: [PATCH 411/572] branch-3.0: [fix](hudi) fix quering hudi table with timestamp key (#53791) (#54595) bp: #53791 --- .../hudi/source/HudiCachedPartitionProcessor.java | 6 ++++++ .../datasource/hudi/source/HudiPartitionProcessor.java | 9 +++++---- .../hudi/test_hudi_partition_prune.out | 8 ++++++++ .../hudi/test_hudi_partition_prune.groovy | 10 +++++----- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index 62094b21c2ba1d..4b8c6fb5c29d27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -30,6 +30,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -148,6 +149,11 @@ public TablePartitionValues getPartitionValues(HMSExternalTable table, HoodieTab // (for example, if the Metastore is false for a Hudi table created with Flink), // we can still obtain the partition information through the HMS API. partitionNames = catalog.getClient().listPartitionNames(table.getDbName(), table.getName()); + // HMS stored Hudi partition paths may have double encoding issue (e.g., %3A + // becomes %253A), need to unescape first here. + partitionNames = partitionNames.stream() + .map(FileUtils::unescapePathName) + .collect(Collectors.toList()); if (partitionNames.size() == 0) { LOG.warn("Failed to get partitions from hms api, switch it from hudi api."); partitionNames = getAllPartitionNames(tableMetaClient); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java index 0ab9fef951a378..2e6846b0ca2a2b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiPartitionProcessor.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.hudi.source; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -96,8 +97,8 @@ public static List parsePartitionValues(List partitionColumns, S } else { partitionValue = partitionPath; } - // TODO: In hive, the specific characters like '=', '/' will be url encoded - return Collections.singletonList(partitionValue); + // In hive, the specific characters like '=', '/' will be url encoded + return Collections.singletonList(FileUtils.unescapePathName(partitionValue)); } else { // If the partition column size is not equal to the partition fragments size // and the partition column size > 1, we do not know how to map the partition @@ -117,9 +118,9 @@ public static List parsePartitionValues(List partitionColumns, S for (int i = 0; i < partitionFragments.length; i++) { String prefix = partitionColumns.get(i) + "="; if (partitionFragments[i].startsWith(prefix)) { - partitionValues.add(partitionFragments[i].substring(prefix.length())); + partitionValues.add(FileUtils.unescapePathName(partitionFragments[i].substring(prefix.length()))); } else { - partitionValues.add(partitionFragments[i]); + partitionValues.add(FileUtils.unescapePathName(partitionFragments[i])); } } return partitionValues; diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out b/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out index fd3eafa0255722..d3d4600a0e6857 100644 --- a/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out +++ b/regression-test/data/external_table_p2/hudi/test_hudi_partition_prune.out @@ -177,6 +177,10 @@ 1 Alice 2023-12-01 2 Bob 2023-12-01 +-- !one_partition_timestamp -- +1 Alice 2023-12-01T08:00 +2 Bob 2023-12-01T08:00 + -- !one_partition_1_1 -- 1 Alice 2024 2 Bob 2024 @@ -355,3 +359,7 @@ 1 Alice 2023-12-01 2 Bob 2023-12-01 +-- !one_partition_timestamp -- +1 Alice 2023-12-01T08:00 +2 Bob 2023-12-01T08:00 + diff --git a/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy b/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy index 063439d9a87915..629923da306681 100644 --- a/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy +++ b/regression-test/suites/external_table_p2/hudi/test_hudi_partition_prune.groovy @@ -320,11 +320,11 @@ suite("test_hudi_partition_prune", "p2,external,hudi,external_remote,external_re sql("${one_partition_date}") contains "partition=1/2" } - // qt_one_partition_timestamp one_partition_timestamp - // explain { - // sql("${one_partition_timestamp}") - // contains "partition=1/2" - // } + qt_one_partition_timestamp one_partition_timestamp + explain { + sql("${one_partition_timestamp}") + contains "partition=1/2" + } sql """drop catalog if exists ${catalog_name};""" From ac748c2fbcb2425e6b7654a1549b05e520b54865 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:49:44 +0800 Subject: [PATCH 412/572] branch-3.0: [Fix](Recycler) Fix recycler azure path fault #54291 (#54561) Cherry-picked from #54291 Co-authored-by: abmdocrt --- be/src/util/s3_util.cpp | 4 + cloud/src/recycler/s3_accessor.cpp | 6 +- cloud/test/util_test.cpp | 121 ++++++++++++++++++++++++++++- common/cpp/util.cpp | 55 +++++++++++++ common/cpp/util.h | 48 ++++++++++++ 5 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 common/cpp/util.cpp create mode 100644 common/cpp/util.h diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index d80e668c05c012..174b6f8a5e6d59 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -48,6 +48,7 @@ #include "cpp/aws_logger.h" #include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" +#include "cpp/util.h" #ifdef USE_AZURE #include "io/fs/azure_obj_storage_client.h" #endif @@ -242,6 +243,9 @@ std::shared_ptr S3ClientFactory::_create_azure_client( options.Retry.MaxRetries = config::max_s3_client_retry; options.PerRetryPolicies.emplace_back(std::make_unique()); + std::string normalized_uri = normalize_http_uri(uri); + VLOG_DEBUG << "uri:" << uri << ", normalized_uri:" << normalized_uri; + auto containerClient = std::make_shared( uri, cred, std::move(options)); LOG_INFO("create one azure client with {}", s3_conf.to_string()); diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 464beb58e2e6d8..3de8edfd048e1a 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -28,6 +28,7 @@ #include #include + #ifdef USE_AZURE #include #include @@ -47,6 +48,7 @@ #include "cpp/obj_retry_strategy.h" #include "cpp/s3_rate_limiter.h" #include "cpp/sync_point.h" +#include "cpp/util.h" #ifdef USE_AZURE #include "recycler/azure_obj_client.h" #endif @@ -335,6 +337,7 @@ int S3Accessor::init() { uri_ = "https://" + uri_; } } + uri_ = normalize_http_uri(uri_); // In Azure's HTTP requests, all policies in the vector are called in a chained manner following the HTTP pipeline approach. // Within the RetryPolicy, the nextPolicy is called multiple times inside a loop. // All policies in the PerRetryPolicies are downstream of the RetryPolicy. @@ -343,7 +346,7 @@ int S3Accessor::init() { auto container_client = std::make_shared( uri_, cred, std::move(options)); // uri format for debug: ${scheme}://${ak}.blob.core.windows.net/${bucket}/${prefix} - uri_ = uri_ + '/' + conf_.prefix; + uri_ = normalize_http_uri(uri_ + '/' + conf_.prefix); obj_client_ = std::make_shared(std::move(container_client)); return 0; #else @@ -357,6 +360,7 @@ int S3Accessor::init() { } else { uri_ = conf_.endpoint + '/' + conf_.bucket + '/' + conf_.prefix; } + uri_ = normalize_http_uri(uri_); // S3Conf::S3 Aws::Client::ClientConfiguration aws_config; diff --git a/cloud/test/util_test.cpp b/cloud/test/util_test.cpp index e0cd54acc8bf98..d8adfe6269b785 100644 --- a/cloud/test/util_test.cpp +++ b/cloud/test/util_test.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "recycler/util.h" +#include "cpp/util.h" #include #include @@ -331,4 +331,123 @@ TEST(UtilTest, test_split) { auto path = doris::get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';')); LOG(INFO) << "config:" << config::ca_cert_file_paths << " path:" << path; ASSERT_FALSE(path.empty()); +} + +TEST(UtilTest, test_normalize_http_uri) { + // ===== Basic functionality with HTTPS protocol ===== + EXPECT_EQ(doris::normalize_http_uri("https://example.com/path"), "https://example.com/path"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com//path"), "https://example.com/path"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com///path"), "https://example.com/path"); + + // ===== Basic functionality with HTTP protocol ===== + EXPECT_EQ(doris::normalize_http_uri("http://example.com/path"), "http://example.com/path"); + EXPECT_EQ(doris::normalize_http_uri("http://example.com//path"), "http://example.com/path"); + EXPECT_EQ(doris::normalize_http_uri("http://example.com///path"), "http://example.com/path"); + + // ===== Multiple consecutive slashes in different positions ===== + EXPECT_EQ(doris::normalize_http_uri("https://host.com//bucket//prefix"), + "https://host.com/bucket/prefix"); + EXPECT_EQ(doris::normalize_http_uri("https://host.com///bucket///prefix///"), + "https://host.com/bucket/prefix/"); + EXPECT_EQ(doris::normalize_http_uri("https://host.com////bucket////prefix////file"), + "https://host.com/bucket/prefix/file"); + + // ===== Azure blob storage specific URLs ===== + EXPECT_EQ(doris::normalize_http_uri("https://account.blob.core.windows.net//container"), + "https://account.blob.core.windows.net/container"); + EXPECT_EQ( + doris::normalize_http_uri("https://account.blob.core.windows.net///container//prefix"), + "https://account.blob.core.windows.net/container/prefix"); + EXPECT_EQ(doris::normalize_http_uri( + "https://account.blob.core.windows.net////container///prefix///file.txt"), + "https://account.blob.core.windows.net/container/prefix/file.txt"); + + // ===== URLs without protocol ===== + EXPECT_EQ(doris::normalize_http_uri("example.com//path"), "example.com/path"); + EXPECT_EQ(doris::normalize_http_uri("host.com///bucket//prefix"), "host.com/bucket/prefix"); + EXPECT_EQ(doris::normalize_http_uri("//path//to//file"), "/path/to/file"); + + // ===== Edge cases ===== + // Empty string + EXPECT_EQ(doris::normalize_http_uri(""), ""); + + // Only protocol + EXPECT_EQ(doris::normalize_http_uri("https://"), "https://"); + EXPECT_EQ(doris::normalize_http_uri("http://"), "http://"); + + // Only slashes + EXPECT_EQ(doris::normalize_http_uri("//"), "/"); + EXPECT_EQ(doris::normalize_http_uri("///"), "/"); + EXPECT_EQ(doris::normalize_http_uri("////"), "/"); + + // Single character paths + EXPECT_EQ(doris::normalize_http_uri("https://a"), "https://a"); + EXPECT_EQ(doris::normalize_http_uri("https://a/"), "https://a/"); + EXPECT_EQ(doris::normalize_http_uri("https://a//"), "https://a/"); + + // ===== Protocol preservation ===== + // Ensure protocol :// is never modified + EXPECT_EQ(doris::normalize_http_uri("https://example.com"), "https://example.com"); + EXPECT_EQ(doris::normalize_http_uri("http://example.com"), "http://example.com"); + + // Even with extra slashes after protocol + EXPECT_EQ(doris::normalize_http_uri("https:///example.com"), "https://example.com"); + EXPECT_EQ(doris::normalize_http_uri("http:///example.com"), "http://example.com"); + + // Mixed case protocol (though unusual) + EXPECT_EQ(doris::normalize_http_uri("HTTP://example.com//path"), "HTTP://example.com/path"); + EXPECT_EQ(doris::normalize_http_uri("HTTPS://example.com//path"), "HTTPS://example.com/path"); + + // ===== Trailing slashes ===== + EXPECT_EQ(doris::normalize_http_uri("https://example.com/path/"), "https://example.com/path/"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com/path//"), "https://example.com/path/"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com/path///"), + "https://example.com/path/"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com/path////"), + "https://example.com/path/"); + + // ===== Complex real-world scenarios ===== + // Simulating common configuration mistakes + EXPECT_EQ(doris::normalize_http_uri("https://endpoint.com///bucket//prefix//file.txt"), + "https://endpoint.com/bucket/prefix/file.txt"); + + // User configured endpoint with trailing slash + bucket with leading slash + EXPECT_EQ(doris::normalize_http_uri("https://endpoint.com///bucket"), + "https://endpoint.com/bucket"); + + // Multiple slashes everywhere + EXPECT_EQ( + doris::normalize_http_uri("https://host.com////bucket////prefix////subfolder////file"), + "https://host.com/bucket/prefix/subfolder/file"); + + // ===== Special characters in path ===== + EXPECT_EQ( + doris::normalize_http_uri("https://example.com//path-with-dash//file_with_underscore"), + "https://example.com/path-with-dash/file_with_underscore"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com//path.with.dots//file@special"), + "https://example.com/path.with.dots/file@special"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com//bucket123//prefix456//file789"), + "https://example.com/bucket123/prefix456/file789"); + + // ===== URLs with query parameters and fragments ===== + EXPECT_EQ(doris::normalize_http_uri("https://example.com//path?query=value"), + "https://example.com/path?query=value"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com//path#fragment"), + "https://example.com/path#fragment"); + EXPECT_EQ(doris::normalize_http_uri("https://example.com//path?query=value#fragment"), + "https://example.com/path?query=value#fragment"); +} + +TEST(UtilTest, test_long_normalize_http_uri) { + std::string longPath = "https://example.com"; + for (int i = 0; i < 100; i++) { + longPath += "//segment" + std::to_string(i); + } + + std::string expected = "https://example.com"; + for (int i = 0; i < 100; i++) { + expected += "/segment" + std::to_string(i); + } + + EXPECT_EQ(doris::normalize_http_uri(longPath), expected); } \ No newline at end of file diff --git a/common/cpp/util.cpp b/common/cpp/util.cpp new file mode 100644 index 00000000000000..5dae6ae70ce086 --- /dev/null +++ b/common/cpp/util.cpp @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Most code of this file is copied from rocksdb SyncPoint. +// https://github.com/facebook/rocksdb + +#include + +namespace doris { + +std::string normalize_http_uri(const std::string& uri) { + if (uri.empty()) { + return uri; + } + + // Find the end of protocol part (http:// or https://) + // Example: in "https://example.com", protocol_end will be 8 (position after "://") + size_t protocol_end = uri.find("://"); + if (protocol_end == std::string::npos) { + protocol_end = 0; // No protocol found, start from beginning + } else { + protocol_end += 3; // Skip past "://" + } + + // Keep protocol part (e.g., "https://") + std::string result = uri.substr(0, protocol_end); + + // Process the rest of URI to remove duplicate slashes + // Example: "//path//to///file" becomes "/path/to/file" + for (size_t i = protocol_end; i < uri.length(); i++) { + char current = uri[i]; + + // Add current character if it's not a slash, or if it's the first slash in sequence + // This prevents consecutive slashes like "//" or "///" from being added + if (current != '/' || result.empty() || result.back() != '/') { + result += current; + } + } + return result; +} +} // namespace doris diff --git a/common/cpp/util.h b/common/cpp/util.h new file mode 100644 index 00000000000000..2dc301bb0c4ee0 --- /dev/null +++ b/common/cpp/util.h @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +namespace doris { + /** + * Normalizes HTTP URI by removing duplicate slashes while preserving the protocol part. + * + * This function removes consecutive forward slashes from URIs while keeping the protocol + * section (http:// or https://) intact. It processes everything after the protocol to + * ensure clean URI formatting. + * + * @param uri The input URI string to be normalized + * @return A normalized URI string with duplicate slashes removed, or the original + * string if it's empty + * + * @example + * normalize_http_uri("https://example.com//path//to///file") + * returns "https://example.com/path/to/file" + * + * normalize_http_uri("http://host.com///bucket//prefix/") + * returns "http://host.com/bucket/prefix/" + * + * normalize_http_uri("endpoint.com//bucket///prefix") + * returns "endpoint.com/bucket/prefix" + * + * normalize_http_uri("https://account.blob.core.windows.net////container") + * returns "https://account.blob.core.windows.net/container" + */ + std::string normalize_http_uri(const std::string& uri); +} // namespace doris From 48c8fc7d9725caa632fe42bbfb2caddd188511b6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:50:34 +0800 Subject: [PATCH 413/572] branch-3.0: [improve](planner) select * unique_table limit n should use one instance #53948 (#54182) Cherry-picked from #53948 Co-authored-by: zhangstar333 --- be/src/pipeline/exec/scan_operator.cpp | 4 ++- .../org/apache/doris/planner/ScanNode.java | 27 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp index 94d6b6d4db78be..6ff5442f8278b0 100644 --- a/be/src/pipeline/exec/scan_operator.cpp +++ b/be/src/pipeline/exec/scan_operator.cpp @@ -1213,7 +1213,9 @@ Status ScanOperatorX::init(const TPlanNode& tnode, RuntimeState* // is checked in previous branch. if (query_options.enable_adaptive_pipeline_task_serial_read_on_limit) { DCHECK(query_options.__isset.adaptive_pipeline_task_serial_read_on_limit); - if (!tnode.__isset.conjuncts || tnode.conjuncts.empty()) { + if (!tnode.__isset.conjuncts || tnode.conjuncts.empty() || + (tnode.conjuncts.size() == 1 && tnode.__isset.olap_scan_node && + tnode.olap_scan_node.keyType == TKeysType::UNIQUE_KEYS)) { if (tnode.limit > 0 && tnode.limit <= query_options.adaptive_pipeline_task_serial_read_on_limit) { _should_run_serial = true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index b4033a0535ef3d..4a0a38c3adcf95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -38,6 +38,7 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PrimitiveType; @@ -757,8 +758,30 @@ public boolean shouldUseOneInstance(ConnectContext ctx) { // No connection context, typically for broker load. } - // For UniqueKey table, we will use multiple instance. - return hasLimit() && getLimit() <= adaptivePipelineTaskSerialReadOnLimit && conjuncts.isEmpty(); + if (hasLimit() && getLimit() <= adaptivePipelineTaskSerialReadOnLimit) { + if (conjuncts.isEmpty()) { + return true; + } else { + if (this instanceof OlapScanNode) { + OlapScanNode olapScanNode = (OlapScanNode) this; + if (olapScanNode.getOlapTable() != null + && olapScanNode.getOlapTable().getKeysType() == KeysType.UNIQUE_KEYS) { + // If the table is unique keys, we can check if the conjuncts only contains + // delete sign + if (conjuncts.size() == 1 && conjuncts.get(0) instanceof BinaryPredicate) { + BinaryPredicate binaryPredicate = (BinaryPredicate) conjuncts.get(0); + if (binaryPredicate.getOp() == BinaryPredicate.Operator.EQ + && binaryPredicate.getChild(0) instanceof SlotRef + && ((SlotRef) binaryPredicate.getChild(0)).getDesc().getColumn().getName() + .equals(Column.DELETE_SIGN)) { + return true; + } + } + } + } + } + } + return false; } // In cloud mode, meta read lock is not enough to keep a snapshot of the partition versions. From 597bb6962523bf7d5a3a38fba3330154719b84ce Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:51:11 +0800 Subject: [PATCH 414/572] branch-3.0: [enhance](job) timely rescheduling tasks to avoid write jitter #53853 (#54054) Cherry-picked from #53853 Co-authored-by: hui lai --- .../routineload/RoutineLoadTaskScheduler.java | 37 ++++-- .../test_routine_load_job_schedule.groovy | 118 ++++++++++++++++++ 2 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java index 040ca103004e34..28e03567765ba8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java @@ -24,6 +24,7 @@ import org.apache.doris.common.LoadException; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; @@ -228,23 +229,32 @@ private void scheduleOneTask(RoutineLoadTaskInfo routineLoadTaskInfo) throws Exc } } } catch (LoadException e) { - // submit task failed (such as TOO_MANY_TASKS error), but txn has already begun. - // Here we will still set the ExecuteStartTime of this task, which means - // we "assume" that this task has been successfully submitted. - // And this task will then be aborted because of a timeout. - // In this way, we can prevent the entire job from being paused due to submit errors, - // and we can also relieve the pressure on BE by waiting for the timeout period. - LOG.warn("failed to submit routine load task {} to BE: {}, error: {}", - DebugUtil.printId(routineLoadTaskInfo.getId()), - routineLoadTaskInfo.getBeId(), e.getMessage()); - routineLoadManager.getJob(routineLoadTaskInfo.getJobId()).setOtherMsg(e.getMessage()); - // fall through to set ExecuteStartTime + handleSubmitTaskFailure(routineLoadTaskInfo, e.getMessage()); + return; } // set the executeStartTimeMs of task routineLoadTaskInfo.setExecuteStartTimeMs(System.currentTimeMillis()); } + private void handleSubmitTaskFailure(RoutineLoadTaskInfo routineLoadTaskInfo, String errorMsg) { + LOG.warn("failed to submit routine load task {} to BE: {}, error: {}", + DebugUtil.printId(routineLoadTaskInfo.getId()), + routineLoadTaskInfo.getBeId(), errorMsg); + routineLoadTaskInfo.setBeId(-1); + RoutineLoadJob routineLoadJob = routineLoadManager.getJob(routineLoadTaskInfo.getJobId()); + routineLoadJob.setOtherMsg(errorMsg); + + // Check if this is a resource pressure error that should not be immediately rescheduled + if (errorMsg.contains("TOO_MANY_TASKS") || errorMsg.contains("MEM_LIMIT_EXCEEDED")) { + return; + } + + // for other errors (network issues, BE restart, etc.), reschedule immediately + RoutineLoadTaskInfo newTask = routineLoadJob.unprotectRenewTask(routineLoadTaskInfo); + addTaskInQueue(newTask); + } + private void updateBackendSlotIfNecessary() { long currentTime = System.currentTimeMillis(); if (lastBackendSlotUpdateTime == -1 @@ -287,6 +297,11 @@ private void submitTask(long beId, TRoutineLoadTask tTask) throws LoadException TStatus tStatus = client.submitRoutineLoadTask(Lists.newArrayList(tTask)); ok = true; + if (DebugPointUtil.isEnable("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED")) { + LOG.warn("debug point FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED, routine load task submit failed"); + throw new LoadException("debug point FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED"); + } + if (tStatus.getStatusCode() != TStatusCode.OK) { throw new LoadException("failed to submit task. error code: " + tStatus.getStatusCode() + ", msg: " + (tStatus.getErrorMsgsSize() > 0 ? tStatus.getErrorMsgs().get(0) : "NaN")); diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy new file mode 100644 index 00000000000000..c892d9dbde0bb2 --- /dev/null +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.kafka.clients.admin.AdminClient +import org.apache.kafka.clients.admin.NewTopic +import org.apache.kafka.clients.producer.KafkaProducer +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.clients.producer.ProducerConfig +import java.util.Collections + +suite("test_routine_load_job_schedule","nonConcurrent") { + def kafkaCsvTpoics = [ + "test_routine_load_job_schedule", + ] + String enabled = context.config.otherConfigs.get("enableKafkaTest") + String kafka_port = context.config.otherConfigs.get("kafka_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def kafka_broker = "${externalEnvIp}:${kafka_port}" + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + def props = new Properties() + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "${kafka_broker}".toString()) + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") + def producer = new KafkaProducer<>(props) + def adminClient = AdminClient.create(props) + def newTopic = new NewTopic(kafkaCsvTpoics[0], 5, (short)1) + def testData = [ + "1,test_data_1,2023-01-01,value1,2023-01-01 10:00:00,extra1", + "2,test_data_2,2023-01-02,value2,2023-01-02 11:00:00,extra2", + "3,test_data_3,2023-01-03,value3,2023-01-03 12:00:00,extra3", + "4,test_data_4,2023-01-04,value4,2023-01-04 13:00:00,extra4", + "5,test_data_5,2023-01-05,value5,2023-01-05 14:00:00,extra5" + ] + adminClient.createTopics(Collections.singletonList(newTopic)) + testData.eachWithIndex { line, index -> + logger.info("Sending data to kafka: ${line}") + def record = new ProducerRecord<>(newTopic.name(), index, null, line) + producer.send(record) + } + producer.close() + + def tableName = "test_routine_load_job_schedule" + def job = "test_routine_load_job_schedule" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(20) NULL, + `k2` string NULL, + `v1` date NULL, + `v2` string NULL, + `v3` datetime NULL, + `v4` string NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ("replication_allocation" = "tag.location.default: 1"); + """ + try { + GetDebugPoint().enableDebugPointForAllBEs("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED") + sql """ + CREATE ROUTINE LOAD ${job} ON ${tableName} + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${newTopic.name()}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING" + ); + """ + + sleep(5000) + GetDebugPoint().disableDebugPointForAllBEs("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED") + def count = 0 + def maxWaitCount = 60 + while (true) { + def state = sql "show routine load for ${job}" + def routineLoadState = state[0][8].toString() + def statistic = state[0][14].toString() + logger.info("Routine load state: ${routineLoadState}") + logger.info("Routine load statistic: ${statistic}") + def rowCount = sql "select count(*) from ${tableName}" + if (routineLoadState == "RUNNING" && rowCount[0][0] == 5) { + break + } + if (count > maxWaitCount) { + assertEquals(1, 2) + } + sleep(1000) + count++ + } + } catch (Exception e) { + logger.error("Test failed with exception: ${e.message}") + } finally { + GetDebugPoint().disableDebugPointForAllBEs("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED") + try { + sql "stop routine load for ${job}" + } catch (Exception e) { + logger.warn("Failed to stop routine load job: ${e.message}") + } + } + } +} \ No newline at end of file From 9018fc1272daf4d0661646117a98bbd30ac0dd5a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:55:58 +0800 Subject: [PATCH 415/572] branch-3.0: [fix](statistics)Fix drop expired stats. #54424 (#54441) Cherry-picked from #54424 Co-authored-by: James --- .../org/apache/doris/statistics/StatisticsCleaner.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java index a1c5fc4edc5979..acd90c8da14feb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java @@ -95,12 +95,11 @@ public synchronized void clear() { private void clearStats(OlapTable statsTbl, boolean isTableColumnStats) { ExpiredStats expiredStats; - long offset = 0; do { expiredStats = new ExpiredStats(); - offset = findExpiredStats(statsTbl, expiredStats, offset, isTableColumnStats); + findExpiredStats(statsTbl, expiredStats, 0, isTableColumnStats); deleteExpiredStats(expiredStats, statsTbl.getName(), isTableColumnStats); - } while (!expiredStats.isEmpty()); + } while (expiredStats.isFull()); } private void clearTableStats() { @@ -320,6 +319,9 @@ private long findExpiredStats(OlapTable statsTbl, ExpiredStats expiredStats, } } this.yieldForOtherTask(); + if (expiredStats.isFull()) { + LOG.info("expiredStats is full."); + } } return pos; } From afa30e5fbbe712679164bc805290be7be617dbba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:56:53 +0800 Subject: [PATCH 416/572] branch-3.0: [fix](function) Fix functionbuilder check #54255 (#54337) Cherry-picked from #54255 Co-authored-by: dwdwqfwe <149857218+dwdwqfwe@users.noreply.github.com> --- be/src/vec/functions/function.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/be/src/vec/functions/function.cpp b/be/src/vec/functions/function.cpp index 1fea4c70fc1753..cb5f1708f753b4 100644 --- a/be/src/vec/functions/function.cpp +++ b/be/src/vec/functions/function.cpp @@ -25,6 +25,7 @@ #include #include +#include "common/status.h" #include "vec/aggregate_functions/aggregate_function.h" #include "vec/columns/column.h" #include "vec/columns/column_const.h" @@ -259,9 +260,15 @@ void FunctionBuilderImpl::check_number_of_arguments(size_t number_of_arguments) size_t expected_number_of_arguments = get_number_of_arguments(); - CHECK_EQ(number_of_arguments, expected_number_of_arguments) << fmt::format( + DCHECK_EQ(number_of_arguments, expected_number_of_arguments) << fmt::format( "Number of arguments for function {} doesn't match: passed {} , should be {}", get_name(), number_of_arguments, expected_number_of_arguments); + if (number_of_arguments != expected_number_of_arguments) { + throw Exception( + ErrorCode::INVALID_ARGUMENT, + "Number of arguments for function {} doesn't match: passed {} , should be {}", + get_name(), number_of_arguments, expected_number_of_arguments); + } } DataTypePtr FunctionBuilderImpl::get_return_type_without_low_cardinality( From c5e61289137589cb529dd07731a9284584a8858d Mon Sep 17 00:00:00 2001 From: James Date: Wed, 13 Aug 2025 09:58:05 +0800 Subject: [PATCH 417/572] branch-3.0: [improve](statistics)Improve drop expired stats function. (#54290) (#54316) backport: https://github.com/apache/doris/pull/54290 --- .../doris/statistics/StatisticsCleaner.java | 13 +++-- .../statistics/StatisticsCleanerTest.java | 55 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java index acd90c8da14feb..a2fea570acb484 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java @@ -140,7 +140,7 @@ private void clearTableStats() { analysisManager.removeTableStats(id); Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(id)); } catch (Exception e) { - LOG.info(e); + LOG.info("Fail to remove table stats for table {}", id, e); } } } @@ -256,13 +256,18 @@ private void doDelete(String colName, List pred, String tblName) { } } - private long findExpiredStats(OlapTable statsTbl, ExpiredStats expiredStats, + protected long findExpiredStats(OlapTable statsTbl, ExpiredStats expiredStats, long offset, boolean isTableColumnStats) { long pos = offset; - while (pos < statsTbl.getRowCount() && !expiredStats.isFull()) { + while (!expiredStats.isFull()) { List rows = StatisticsRepository.fetchStatsFullName( StatisticConstants.FETCH_LIMIT, pos, isTableColumnStats); pos += StatisticConstants.FETCH_LIMIT; + if (rows.isEmpty()) { + LOG.info("Stats table {} has no more rows to fetch.", statsTbl.getName()); + break; + } + LOG.info("Process {} rows in stats table {}", rows.size(), statsTbl.getName()); for (ResultRow r : rows) { try { StatsId statsId = new StatsId(r); @@ -326,7 +331,7 @@ private long findExpiredStats(OlapTable statsTbl, ExpiredStats expiredStats, return pos; } - private static class ExpiredStats { + protected static class ExpiredStats { Set expiredCatalog = new HashSet<>(); Set expiredDatabase = new HashSet<>(); Set expiredTable = new HashSet<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java new file mode 100644 index 00000000000000..b06369acddfff1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.Config; +import org.apache.doris.statistics.StatisticsCleaner.ExpiredStats; + +import com.google.common.collect.Lists; +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class StatisticsCleanerTest { + + @Test + void testFindExpiredStats() { + StatisticsCleaner cleaner = new StatisticsCleaner(); + ExpiredStats stat = new ExpiredStats(); + OlapTable olapTable = new OlapTable(); + for (int i = 0; i <= Config.max_allowed_in_element_num_of_delete; i++) { + stat.expiredCatalog.add((long) i); + } + long expiredStats = cleaner.findExpiredStats(null, stat, 1, true); + Assertions.assertEquals(expiredStats, 1); + + new MockUp() { + @Mock + public List fetchStatsFullName(long limit, long offset, boolean isTableStats) { + return Lists.newArrayList(); + } + }; + stat.expiredCatalog.clear(); + expiredStats = cleaner.findExpiredStats(olapTable, stat, 0, true); + Assertions.assertEquals(expiredStats, StatisticConstants.FETCH_LIMIT); + } +} From eb39ad368f25cbe397c71724c9147d5162e3f643 Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Wed, 13 Aug 2025 10:01:23 +0800 Subject: [PATCH 418/572] [Enhancement-3.0](type) Support cast between time and datetime (#53734) --- be/src/vec/core/call_on_type_index.h | 3 + be/src/vec/functions/function_cast.h | 152 ++++++++++++------ be/src/vec/runtime/time_value.h | 46 ++++++ be/src/vec/runtime/vdatetime_value.h | 15 ++ be/test/runtime/time_value_test.cpp | 50 ++++++ .../apache/doris/catalog/PrimitiveType.java | 11 +- .../rules/expression/check/CheckCast.java | 3 +- .../test_cast_time_to_datetime.out | 16 ++ .../test_cast_time_to_datetime.groovy | 27 ++++ .../suites/nereids_syntax_p0/cast.groovy | 5 +- 10 files changed, 277 insertions(+), 51 deletions(-) create mode 100644 be/test/runtime/time_value_test.cpp create mode 100644 regression-test/data/correctness/test_cast_time_to_datetime.out create mode 100644 regression-test/suites/correctness/test_cast_time_to_datetime.groovy diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h index b651fe2ec47502..e64e2003cb5643 100644 --- a/be/src/vec/core/call_on_type_index.h +++ b/be/src/vec/core/call_on_type_index.h @@ -276,6 +276,9 @@ bool call_on_index_and_number_data_type(TypeIndex number, F&& f) { case TypeIndex::Float64: return f(TypePair, T>()); + case TypeIndex::TimeV2: + return f(TypePair()); + case TypeIndex::Decimal32: return f(TypePair, T>()); case TypeIndex::Decimal64: diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index 0e7db6e407b478..122587c4dc11e9 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -24,16 +24,13 @@ #include #include #include -#include -#include #include -#include #include #include +#include #include #include -#include #include #include #include @@ -91,22 +88,16 @@ #include "vec/data_types/data_type_time_v2.h" #include "vec/data_types/serde/data_type_serde.h" #include "vec/functions/function.h" -#include "vec/functions/function_convert_tz.h" #include "vec/functions/function_helpers.h" #include "vec/io/reader_buffer.h" +#include "vec/runtime/time_value.h" #include "vec/runtime/vdatetime_value.h" -#include "vec/utils/util.hpp" class DateLUTImpl; -namespace doris { -namespace vectorized { +namespace doris::vectorized { template class ColumnDecimal; -} // namespace vectorized -} // namespace doris - -namespace doris::vectorized { /** Type conversion functions. * toType - conversion in "natural way"; */ @@ -423,11 +414,58 @@ struct ConvertImpl { } } else { if constexpr (IsDateTimeV2Type) { - static_cast_set( - vec_to[i], - reinterpret_cast&>( - vec_from[i]) - .to_int64()); + if constexpr (std::is_same_v) { + // datetimev2 to timev2 + auto dtmv2 = binary_cast>( + col_from->get_data()[i]); + + const auto* type = assert_cast( + block.get_by_position(arguments[0]).type.get()); + auto scale = type->get_scale(); + const auto* to_type = assert_cast( + block.get_by_position(result).type.get()); + UInt32 to_scale = to_type->get_scale(); + + uint32_t hour = dtmv2.hour(); + uint32_t minute = dtmv2.minute(); + uint32_t second = dtmv2.second(); + uint32_t microseconds = dtmv2.microsecond(); + if (to_scale < scale) { // need to round + // e.g. scale reduce to 4, means we need to round the last 2 digits + // 999956: 56 > 100/2, then round up to 1000000 + uint32_t divisor = common::exp10_i64(6 - to_scale); + uint32_t remainder = microseconds % divisor; + microseconds = (microseconds / divisor) * divisor; + if (remainder >= divisor / 2) { + // do rounding up + microseconds += divisor; + } + } + + // carry on if microseconds >= 1000000 + if (microseconds >= 1000000) { + microseconds -= 1000000; + second += 1; + if (second >= 60) { + second -= 60; + minute += 1; + if (minute >= 60) { + minute -= 60; + hour += 1; + } + } + } + + auto time = TimeValue::limit_with_bound( + TimeValue::make_time(hour, minute, second, microseconds)); + col_to->get_data()[i] = time; + } else { + static_cast_set( + vec_to[i], + reinterpret_cast&>( + vec_from[i]) + .to_int64()); + } } else { static_cast_set(vec_to[i], reinterpret_cast&>( @@ -498,8 +536,8 @@ struct ConvertImplToTimeType { using FromFieldType = typename FromDataType::FieldType; using ToFieldType = typename ToDataType::FieldType; - static Status execute(Block& block, const ColumnNumbers& arguments, size_t result, - size_t /*input_rows_count*/) { + static Status execute(FunctionContext* context, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t /*input_rows_count*/) { const ColumnWithTypeAndName& named_from = block.get_by_position(arguments[0]); using ColVecFrom = @@ -527,29 +565,53 @@ struct ConvertImplToTimeType { col_null_map_to = ColumnUInt8::create(size, 0); auto& vec_null_map_to = col_null_map_to->get_data(); - UInt32 from_precision = 0; - UInt32 from_scale = 0; - UInt32 to_precision = NumberTraits::max_ascii_len(); - if constexpr (IsDecimalNumber) { - const auto& from_decimal_type = assert_cast(*named_from.type); - from_precision = from_decimal_type.get_precision(); - from_scale = from_decimal_type.get_scale(); + if constexpr (std::is_same_v) { + DateValueType current_date_value; + current_date_value.from_unixtime(context->state()->timestamp_ms() / 1000, + context->state()->timezone_obj()); + uint32_t scale = 0; + // Only DateTimeV2 has scale + if (std::is_same_v) { + scale = remove_nullable(block.get_by_position(result).type)->get_scale(); + } + // According to MySQL rules, when casting time type to date/datetime, + // the current date is added to the time + // So here we need to clear the time part + current_date_value.reset_time_part(); + for (size_t i = 0; i < size; ++i) { + auto& date_value = reinterpret_cast(vec_to[i]); + date_value = current_date_value; + int64_t microsecond = TimeValue::round_time(vec_from[i], scale); + // Only TimeV2 type needs microseconds + if constexpr (IsTimeV2Type) { + vec_null_map_to[i] = !date_value.template date_add_interval( + TimeInterval {MICROSECOND, microsecond, false}); + } else { + vec_null_map_to[i] = + !date_value.template date_add_interval(TimeInterval { + SECOND, microsecond / TimeValue::ONE_SECOND_MICROSECONDS, + false}); + } + + // DateType of VecDateTimeValue should cast to date + if constexpr (IsDateType) { + date_value.cast_to_date(); + } else if constexpr (IsDateTimeType) { + date_value.to_datetime(); + } + } + } else { + for (size_t i = 0; i < size; ++i) { + auto& date_value = reinterpret_cast(vec_to[i]); + vec_null_map_to[i] = !date_value.from_date_int64(int64_t(vec_from[i])); + // DateType of VecDateTimeValue should cast to date + if constexpr (IsDateType) { + date_value.cast_to_date(); + } else if constexpr (IsDateTimeType) { + date_value.to_datetime(); + } + } } - bool narrow_integral = to_precision < (from_precision - from_scale); - std::visit( - [&](auto narrow_integral) { - for (size_t i = 0; i < size; ++i) { - auto& date_value = reinterpret_cast(vec_to[i]); - vec_null_map_to[i] = !date_value.from_date_int64(int64_t(vec_from[i])); - // DateType of VecDateTimeValue should cast to date - if constexpr (IsDateType) { - date_value.cast_to_date(); - } else if constexpr (IsDateTimeType) { - date_value.to_datetime(); - } - } - }, - make_bool_variant(narrow_integral)); block.get_by_position(result).column = ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); } else { @@ -1687,10 +1749,10 @@ class FunctionConvertFromString : public IFunction { }; template -class FunctionConvertToTimeType : public IFunction { +class FunctionConvertFromDatelikeType : public IFunction { public: static constexpr auto name = Name::name; - static FunctionPtr create() { return std::make_shared(); } + static FunctionPtr create() { return std::make_shared(); } String get_name() const override { return name; } @@ -1715,7 +1777,7 @@ class FunctionConvertToTimeType : public IFunction { using RightDataType = typename Types::RightType; ret_status = ConvertImplToTimeType::execute( - block, arguments, result, input_rows_count); + context, block, arguments, result, input_rows_count); return true; }; @@ -1788,7 +1850,7 @@ class FunctionCast final : public IFunctionBase { check_and_get_data_type(from_type.get()) || check_and_get_data_type(from_type.get()) || check_and_get_data_type(from_type.get()))) { - function = FunctionConvertToTimeType::create(); + function = FunctionConvertFromDatelikeType::create(); } else { function = FunctionTo::Type::create(); } diff --git a/be/src/vec/runtime/time_value.h b/be/src/vec/runtime/time_value.h index d94e62b977c44d..8e1624b9780823 100644 --- a/be/src/vec/runtime/time_value.h +++ b/be/src/vec/runtime/time_value.h @@ -22,6 +22,7 @@ #include "runtime/define_primitive_type.h" #include "runtime/primitive_type.h" #include "util/date_func.h" +#include "vec/common/int_exp.h" #include "vec/data_types/data_type_time.h" namespace doris { @@ -34,10 +35,43 @@ class TimeValue { constexpr static int64_t ONE_HOUR_MICROSECONDS = 60 * ONE_MINUTE_MICROSECONDS; constexpr static int64_t ONE_MINUTE_SECONDS = 60; constexpr static int64_t ONE_HOUR_SECONDS = 60 * ONE_MINUTE_SECONDS; + constexpr static uint32_t MICROS_SCALE = 6; + constexpr static int64_t MAX_TIME = + 3024000LL * ONE_SECOND_MICROSECONDS - 1; // 840:00:00 - 1ms -> 838:59:59.999999 using TimeType = typename PrimitiveTypeTraits::CppType; using ColumnTime = vectorized::DataTypeTimeV2::ColumnType; + static int64_t round_time(TimeType value, uint32_t scale) { + auto time = (int64_t)value; + DCHECK(scale <= MICROS_SCALE); + int64_t factor = common::exp10_i64(6 - scale); + int64_t rounded_value = (time >= 0) ? (time + factor / 2) / factor * factor + : (time - factor / 2) / factor * factor; + return rounded_value; + } + + // Construct time based on hour/minute/second/microsecond + template + static TimeType make_time(int64_t hour, int64_t minute, int64_t second, int64_t microsecond = 0, + bool negative = false) { + if constexpr (CHECK) { + // the max time value is 838:59:59.999999 + if (std::abs(hour) > 838 || std::abs(minute) >= 60 || std::abs(second) >= 60 || + std::abs(microsecond) >= 1000000) [[unlikely]] { + throw Exception(ErrorCode::INVALID_ARGUMENT, + "Invalid time value: hour={}, minute={}, second={}, microsecond={}", + hour, minute, second, microsecond); + } + } + DCHECK(hour >= 0 && minute >= 0 && second >= 0 && microsecond >= 0) + << "Hour, minute, second and microsecond must be non-negative but got " << hour + << ":" << minute << ":" << second << "." << microsecond; + int64_t value = (hour * ONE_HOUR_MICROSECONDS) + (minute * ONE_MINUTE_MICROSECONDS) + + (second * ONE_SECOND_MICROSECONDS) + microsecond; + return static_cast(negative ? -value : value); + } + // refer to https://dev.mysql.com/doc/refman/5.7/en/time.html // the time value between '-838:59:59' and '838:59:59' /// TODO: Why is the time type stored as double? Can we directly use int64 and remove the time limit? @@ -67,6 +101,18 @@ class TimeValue { static int32_t second(TimeType time) { return (check_over_max_time(time) / ONE_SECOND_MICROSECONDS) % ONE_MINUTE_SECONDS; } + + // refer to https://dev.mysql.com/doc/refman/5.7/en/time.html + // the time value between '-838:59:59' and '838:59:59' + static TimeType limit_with_bound(TimeType time) { + if (time > MAX_TIME) { + return MAX_TIME; + } + if (time < -MAX_TIME) { + return -MAX_TIME; + } + return time; + } }; } // namespace doris diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 11dfb6e796bb0f..0e815dbf3e0b17 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -442,6 +442,12 @@ class VecDateTimeValue { // Now this type is a temp solution with little changes return _hour * SECOND_PER_HOUR + _minute * SECOND_PER_MINUTE + _second; } + void reset_time_part() { + _hour = 0; + _minute = 0; + _second = 0; + } + bool check_loss_accuracy_cast_to_date() { auto loss_accuracy = _hour != 0 || _minute != 0 || _second != 0; cast_to_date(); @@ -934,6 +940,15 @@ class DateV2Value { return hour() * SECOND_PER_HOUR + minute() * SECOND_PER_MINUTE + second(); } + void reset_time_part() { + if constexpr (is_datetime) { + date_v2_value_.hour_ = 0; + date_v2_value_.minute_ = 0; + date_v2_value_.second_ = 0; + date_v2_value_.microsecond_ = 0; + } + } + int64_t time_part_to_microsecond() const { return time_part_to_seconds() * 1000 * 1000 + microsecond(); } diff --git a/be/test/runtime/time_value_test.cpp b/be/test/runtime/time_value_test.cpp new file mode 100644 index 00000000000000..d38698131f23d9 --- /dev/null +++ b/be/test/runtime/time_value_test.cpp @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +namespace doris { + +TEST(TimeValueTest, make_time) { + int64_t hour = 1; + int64_t minute = 2; + int64_t second = 3; + TimeValue::TimeType time = TimeValue::make_time(hour, minute, second); + EXPECT_EQ(time, 3723000000); +} + +TEST(TimeValueTest, round_time) { + // 01:02:03.500000 -> 01:02:04.000000 + EXPECT_EQ(TimeValue::round_time(TimeValue::make_time(1, 2, 3, 500000), 0), + TimeValue::make_time(1, 2, 4)); + + // 01:02:03.499999 -> 01:01:03.000000 + EXPECT_EQ(TimeValue::round_time(TimeValue::make_time(1, 2, 3, 499999), 0), + TimeValue::make_time(1, 2, 3)); + + // -01:02:03.500000 -> -01:01:04.000000 + EXPECT_EQ(TimeValue::round_time(-TimeValue::make_time(1, 2, 3, 500000), 0), + -TimeValue::make_time(1, 2, 4)); + + // -01:02:03.499999 -> -01:01:03.000000 + EXPECT_EQ(TimeValue::round_time(-TimeValue::make_time(1, 2, 3, 499999), 0), + -TimeValue::make_time(1, 2, 3)); +} +} // namespace doris \ No newline at end of file diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java index fbd9bf6924ea39..3c1327e1e40b53 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java @@ -403,6 +403,7 @@ public static ImmutableSetMultimap getImplicitCast builder.put(DATETIMEV2, DATETIME); builder.put(DATETIMEV2, DATEV2); builder.put(DATETIMEV2, DATETIMEV2); + builder.put(DATETIMEV2, TIMEV2); builder.put(DATETIMEV2, DECIMALV2); builder.put(DATETIMEV2, DECIMAL32); builder.put(DATETIMEV2, DECIMAL64); @@ -620,13 +621,21 @@ public static ImmutableSetMultimap getImplicitCast builder.put(TIME, DOUBLE); builder.put(TIME, VARCHAR); builder.put(TIME, STRING); + builder.put(TIME, DATE); + builder.put(TIME, DATETIME); + builder.put(TIME, DATEV2); + builder.put(TIME, DATETIMEV2); - //TIMEV2 + // TIMEV2 builder.put(TIMEV2, TIME); builder.put(TIMEV2, TIMEV2); builder.put(TIMEV2, DOUBLE); builder.put(TIMEV2, VARCHAR); builder.put(TIMEV2, STRING); + builder.put(TIMEV2, DATE); + builder.put(TIMEV2, DATETIME); + builder.put(TIMEV2, DATEV2); + builder.put(TIMEV2, DATETIMEV2); implicitCastMap = builder.build(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index e76d7ef344d8ad..2f74144be08557 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -125,7 +125,8 @@ private static boolean checkPrimitiveType(DataType originalType, DataType target return false; } if (targetType.isTimeLikeType() && !(originalType.isIntegralType() - || originalType.isStringLikeType() || originalType.isFloatLikeType())) { + || originalType.isStringLikeType() || originalType.isFloatLikeType() + || originalType.isDateTimeType() || originalType.isDateTimeV2Type())) { return false; } return true; diff --git a/regression-test/data/correctness/test_cast_time_to_datetime.out b/regression-test/data/correctness/test_cast_time_to_datetime.out new file mode 100644 index 00000000000000..a498c5649f91cf --- /dev/null +++ b/regression-test/data/correctness/test_cast_time_to_datetime.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +12:12:12 + +-- !sql -- +12:12:12 + +-- !sql -- +12:12:12 + +-- !sql -- +12:12:12 + +-- !sql -- +36620 + diff --git a/regression-test/suites/correctness/test_cast_time_to_datetime.groovy b/regression-test/suites/correctness/test_cast_time_to_datetime.groovy new file mode 100644 index 00000000000000..bfdd40e07269b6 --- /dev/null +++ b/regression-test/suites/correctness/test_cast_time_to_datetime.groovy @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cast_time_to_datetime") { + qt_sql "select cast(cast('0000-01-01 12:12:12' as datetime) as time);" + qt_sql "select cast(cast('2000-02-03 12:12:12' as datetime) as time);" + qt_sql "select cast(cast('2000-02-03 12:12:12.123456' as datetime(6)) as time(4));" + qt_sql "select cast(cast(cast('2020-12-12 12:12:12' as time) as datetime) as time);" + qt_sql "select time_to_sec(cast('2002-05-30 10:10:20' as datetime));" + def res = sql "select date_format(cast(cast('2000-02-03 12:12:12.123456' as datetime(6)) as time(4)), '%b %e %Y %l:%i%p');" + // check final 7 char of res[0][0] is 12:12PM + assertEquals("12:12PM", res[0][0].substring(res[0][0].length() - 7)) +} \ No newline at end of file diff --git a/regression-test/suites/nereids_syntax_p0/cast.groovy b/regression-test/suites/nereids_syntax_p0/cast.groovy index 5354648801aaba..5e4377d0d2142c 100644 --- a/regression-test/suites/nereids_syntax_p0/cast.groovy +++ b/regression-test/suites/nereids_syntax_p0/cast.groovy @@ -244,10 +244,7 @@ suite("cast") { exception "cannot cast" } // datetime - test { - sql """select cast(k11 as time) ct from test order by ct;""" - exception "cannot cast" - } + sql """select cast(k11 as time) ct from test order by ct;""" sql "select cast(1 as signed)" sql "select cast(1 as signed int)" From fec60b10553c1d68e5dc7f57f4f3df105a46d77f Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 13 Aug 2025 11:00:48 +0800 Subject: [PATCH 419/572] [fix](warmup) ensure warmup only triggers if tablet exists on target backend (#53852) (#53850) ### What problem does this PR solve? Cherry-pick: #53852 Problem Summary: For event-driven warmup jobs, ensure warmup only triggers if tablet exists on the target backend. Because the source BE will cache the tablet location for a short time, after rebalancing, the old target BE will still receive warm up requests during that time. This PR prevents the old target BE to re-fetch the tablet cache when it no longer holds the tablet. For the new target BE, it will take a different path (`warm_up_cache_async`) to do the warm up. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/cloud/cloud_internal_service.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 33482cd8850f3f..39b51b06dd473a 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -171,7 +171,10 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c continue; } int64_t tablet_id = rs_meta.tablet_id(); - auto res = _engine.tablet_mgr().get_tablet(tablet_id); + auto res = _engine.tablet_mgr().get_tablet(tablet_id, /* warmup_data = */ false, + /* sync_delete_bitmap = */ true, + /* sync_stats = */ nullptr, + /* local_only = */ true); if (!res.has_value()) { LOG_WARNING("Warm up error ").tag("tablet_id", tablet_id).error(res.error()); continue; From 42b26c7842d2badcdd9a7db68d0ae4ea5cd0fcb9 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 13 Aug 2025 11:01:30 +0800 Subject: [PATCH 420/572] [log](load) suppress memtable memory limiter logs (#53277) (#54126) backport #53277 --- be/src/olap/memtable_memory_limiter.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/be/src/olap/memtable_memory_limiter.cpp b/be/src/olap/memtable_memory_limiter.cpp index cae597ccc9433f..dba30d042d73b9 100644 --- a/be/src/olap/memtable_memory_limiter.cpp +++ b/be/src/olap/memtable_memory_limiter.cpp @@ -158,7 +158,17 @@ void MemTableMemoryLimiter::handle_memtable_flush() { timer.stop(); int64_t time_ms = timer.elapsed_time() / 1000 / 1000; g_memtable_memory_limit_latency_ms << time_ms; - LOG(INFO) << "waited " << time_ms << " ms for memtable memory limit"; + if (time_ms > 0) { + LOG(INFO) << "waited " << PrettyPrinter::print(timer.elapsed_time(), TUnit::TIME_NS) + << " for memtable memory limit" + << ", " << GlobalMemoryArbitrator::process_memory_used_details_str() << ", " + << GlobalMemoryArbitrator::sys_mem_available_details_str() + << ", load mem: " << PrettyPrinter::print_bytes(_mem_tracker->consumption()) + << ", memtable writers num: " << _writers.size() + << ", active: " << PrettyPrinter::print_bytes(_active_mem_usage) + << ", queue: " << PrettyPrinter::print_bytes(_queue_mem_usage) + << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage); + } } void MemTableMemoryLimiter::_flush_active_memtables(int64_t need_flush) { From bd45f01a8af94a3d072086f29b0c930c4508a24a Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 13 Aug 2025 11:02:30 +0800 Subject: [PATCH 421/572] [opt](load) avoid over-flushing active memtables in memory limiter (#52906) (#53909) backport #52906 --- be/src/olap/memtable_memory_limiter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/be/src/olap/memtable_memory_limiter.cpp b/be/src/olap/memtable_memory_limiter.cpp index dba30d042d73b9..35700aa36d6d6a 100644 --- a/be/src/olap/memtable_memory_limiter.cpp +++ b/be/src/olap/memtable_memory_limiter.cpp @@ -39,6 +39,8 @@ bvar::Status g_memtable_flush_memory("mm_limiter_mem_flush", 0); bvar::Status g_memtable_load_memory("mm_limiter_mem_load", 0); bvar::Status g_load_hard_mem_limit("mm_limiter_limit_hard", 0); bvar::Status g_load_soft_mem_limit("mm_limiter_limit_soft", 0); +bvar::Adder g_memtable_memory_limit_flush_memtable_count("mm_limiter_flush_memtable_count"); +bvar::LatencyRecorder g_memtable_memory_limit_flush_size_bytes("mm_limiter_flush_size_bytes"); // Calculate the total memory limit of all load tasks on this BE static int64_t calc_process_max_load_memory(int64_t process_mem_limit) { @@ -113,7 +115,7 @@ int64_t MemTableMemoryLimiter::_need_flush() { int64_t limit2 = _sys_avail_mem_less_than_warning_water_mark(); int64_t limit3 = _process_used_mem_more_than_soft_mem_limit(); int64_t need_flush = std::max(limit1, std::max(limit2, limit3)); - return need_flush - _queue_mem_usage; + return need_flush - _queue_mem_usage - _flush_mem_usage; } void MemTableMemoryLimiter::handle_memtable_flush() { @@ -150,7 +152,8 @@ void MemTableMemoryLimiter::handle_memtable_flush() { << ", memtable writers num: " << _writers.size() << ", active: " << PrettyPrinter::print_bytes(_active_mem_usage) << ", queue: " << PrettyPrinter::print_bytes(_queue_mem_usage) - << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage); + << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage) + << ", need flush: " << PrettyPrinter::print_bytes(need_flush); _flush_active_memtables(need_flush); } } while (_hard_limit_reached() && !_load_usage_low()); @@ -219,6 +222,8 @@ void MemTableMemoryLimiter::_flush_active_memtables(int64_t need_flush) { } mem_flushed += mem; num_flushed += (mem > 0); + g_memtable_memory_limit_flush_memtable_count << 1; + g_memtable_memory_limit_flush_size_bytes << mem; } LOG(INFO) << "flushed " << num_flushed << " out of " << _active_writers.size() << " active writers, flushed size: " << PrettyPrinter::print_bytes(mem_flushed); From 0da0dc8dcd989ba403213ea97bc282fbc51996de Mon Sep 17 00:00:00 2001 From: shuke Date: Wed, 13 Aug 2025 11:43:01 +0800 Subject: [PATCH 422/572] [regression-test](fix) fix tpcds q24 query and result (#51497) (#54629) tpcds query 24 are not right before, fix it. --- .../tpcds_sf1_index/sql/q24_1.out | 7 - .../tpcds_sf1_index/sql/q24_2.out | 1 - .../sql/q24_1.out | 1169 +---------------- .../sql/q24_2.out | 155 +-- .../data/tpcds_sf100_p2/sql/q24_1.out | 1169 +---------------- .../data/tpcds_sf100_p2/sql/q24_2.out | 155 +-- .../data/tpcds_sf1_p1/sql/q24_1.out | 7 - .../data/tpcds_sf1_p1/sql/q24_2.out | 1 - .../data/tpcds_sf1_unique_p1/sql/q24_1.out | 7 - .../data/tpcds_sf1_unique_p1/sql/q24_2.out | 1 - .../tpcds_sf1_index/sql/q24_1.sql | 3 +- .../tpcds_sf1_index/sql/q24_2.sql | 3 +- .../sql/q24_1.sql | 3 +- .../sql/q24_2.sql | 3 +- .../suites/tpcds_sf100_p2/sql/q24_1.sql | 3 +- .../suites/tpcds_sf100_p2/sql/q24_2.sql | 3 +- .../suites/tpcds_sf1_p1/sql/q24_1.sql | 3 +- .../suites/tpcds_sf1_p1/sql/q24_2.sql | 3 +- .../suites/tpcds_sf1_unique_p1/sql/q24_1.sql | 3 +- .../suites/tpcds_sf1_unique_p1/sql/q24_2.sql | 3 +- 20 files changed, 132 insertions(+), 2570 deletions(-) diff --git a/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_1.out b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_1.out index b5b3781427f2e7..a87a3f6eb2f092 100644 --- a/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_1.out +++ b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_1.out @@ -1,10 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_1 -- - Tommy able 38118.08 -Holt Curtis able 8225.80 -Kunz Lee able 34631.52 -Littlefield Clarence able 127380.00 -Pettit Richard able 3930.52 -Townsend Franklin able 68983.20 -Winchester Margaret bar 14269.20 diff --git a/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_2.out b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_2.out index 3dd7690d52b3e7..04afb1f4cd59ca 100644 --- a/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_2.out +++ b/regression-test/data/inverted_index_p1/tpcds_sf1_index/sql/q24_2.out @@ -1,4 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_2 -- -Griffith Ray able 161564.48 diff --git a/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_1.out b/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_1.out index 2a91a6a701a3ed..239f109f79281a 100644 --- a/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_1.out +++ b/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_1.out @@ -1,1119 +1,56 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_1 -- - able 1021629.88 - anti 791301.84 - bar 1227497.84 - cally 5616234.34 - eing 860204.18 - ese 1932618.73 - n st 1800090.95 - ought 530419.60 - Aaron bar 37060.00 - Aaron ese 7108.56 - Aimee cally 28123.70 - Albert anti 39965.31 - Alberta ought 11466.00 - Aletha ese 35414.40 - Alex n st 46951.59 - Alexis cally 10015.72 - Allen able 144694.20 - Allen ese 57853.28 - Alonso cally 27210.00 - Amanda eing 27914.04 - Amber bar 4570.56 - Amber ought 3086.40 - Andrew bar 7524.00 - Andrew ese 7434.00 - Angela cally 4519.68 - Angela n st 21806.40 - Angie ought 53205.00 - Anthony able 41758.50 - Anthony cally 206319.48 - Anthony eing 21331.42 - Ardelle bar 66221.76 - Ashley cally 66980.20 - Ashley ese 40197.60 - Barbara able 25809.68 - Barbara cally 6164.40 - Barbra bar 65900.52 - Barry n st 72150.54 - Beatrice able 18787.58 - Benjamin bar 34466.24 - Billy cally 63817.80 - Billy ese 5772.00 - Bonnie bar 5153.28 - Bonnie ese 186461.28 - Bree cally 7988.64 - Carl bar 61009.76 - Carolyn able 56280.64 - Carrie bar 60318.72 - Carrie ese 28414.47 - Catherine ought 34860.00 - Cecil cally 41861.16 - Celeste bar 28300.80 - Charles bar 27927.52 - Charlotte cally 75533.55 - Christine ese 74597.76 - Christopher ese 21088.80 - Cindy n st 84084.00 - Claire able 2616.90 - Claire cally 6740.64 - Clarence n st 40026.84 - Clifford cally 24360.00 - Cody n st 15178.24 - Constance n st 19698.00 - Cristobal able 37630.18 - Curtis bar 88318.40 - Cynthia cally 19488.80 - Cynthia ese 5162.64 - Cynthia ought 99050.20 - Damaris able 104566.41 - Daniel cally 50320.80 - Daniel ese 33930.36 - David ese 67222.68 - David n st 14014.98 - Denise bar 17454.56 - Dennis n st 42812.22 - Diana anti 18198.18 - Dianna ese 3336.00 - Dolores eing 2646.16 - Donald ought 12663.60 - Donna able 93329.60 - Donna cally 14460.03 - Donna ese 29344.14 - Dora ese 4282.72 - Doretha ought 51451.20 - Doris cally 6845.08 - Dorothy ese 26849.28 - Douglas anti 2716.56 - Douglas bar 28371.60 - Earnest bar 22657.60 - Eduardo cally 12768.00 - Edward anti 48678.00 - Edwin ese 24665.55 - Elaine eing 4092.88 - Elizabeth ese 27740.16 - Ella n st 11840.64 - Emily cally 9080.20 - Emily ese 21020.58 - Errol ese 37747.20 - Eugene able 62410.32 - Eugene ese 106114.32 - Felix cally 21306.78 - Flora n st 3848.04 - Francine ought 89600.00 - Francis anti 195867.42 - Francis ese 10809.60 - Frank able 8321.04 - Franklin able 13519.52 - Franklin cally 80237.20 - Fred ese 55998.00 - Freda cally 3483.48 - Fredrick anti 9737.28 - Gabriela cally 32992.92 - Gary n st 8421969.36 - Gaston cally 108169.60 - George n st 8905.60 - Gladys ought 14548.80 - Grace cally 30564.96 - Gregory eing 241128.80 - Gregory n st 6870.15 - Gwendolyn ought 81058.00 - Harry bar 24641.28 - Hector eing 7917.80 - Helen ese 119687.82 - Henry cally 86119.79 - Herman bar 31543.52 - Herman ese 8660.80 - Hiroko ought 6572.00 - Hugh cally 11595.70 - Ila cally 84835.61 - Ivan anti 4397.19 - Jack ese 99417.60 - Jacques able 62339.16 - James anti 93096.57 - James cally 10163.11 - James ese 34504.32 - Jane cally 84292.80 - Janet bar 6049.28 - Janice bar 48906.00 - Janice ese 37263.36 - Jared ought 43809.00 - Jason ese 30310.08 - Jean cally 27843.40 - Jeffry anti 210522.90 - Jenna ought 2652.00 - Jennifer eing 2121956.10 - Jerry cally 30850.60 - Jessica cally 45254.88 - Jessica eing 124102.44 - Jo able 28828.80 - Joann n st 14108.60 - Joe n st 4296.60 - John bar 41023.08 - John cally 41636.00 - John ese 27941.04 - John n st 49222.74 - Jordan bar 56246.40 - Jose able 2966.04 - Joseph cally 12413.64 - Joseph ought 11917.40 - Josephine ese 24912.48 - Josh cally 33456.50 - Joyce ese 6328.32 - Julie bar 6085.28 - Julio able 7796.36 - Karen cally 62628.97 - Karen ese 7893.69 - Katherine bar 5614.40 - Kathleen cally 13156.30 - Kathleen eing 15915.90 - Kathleen ese 4410.00 - Keith bar 2971.80 - Kelvin eing 121847.00 - Kenneth able 4427.64 - Kenneth n st 14576.54 - Kristin ese 28851.00 - Lance cally 39247.20 - Latisha able 24376.80 - Laurie able 13091.40 - Laurie cally 67329.60 - Leon bar 64171.20 - Leroy able 29835.26 - Leticia n st 4607.90 - Linda cally 11880.00 - Lisa cally 30157.38 - Lissa cally 3093.20 - Lois ese 140388.15 - Lori ought 24931.20 - Louise able 53963.28 - Louise cally 26136.00 - Magaly cally 41644.56 - Manuel bar 16222.08 - Manuel ese 10214.40 - Maranda able 9328.80 - Marcella cally 12474.00 - Margery ese 67212.18 - Marie bar 42488.16 - Marti able 21468.85 - Martin cally 3450.92 - Marylou able 17305.08 - Maureen ese 20597.04 - Melinda eing 58556.08 - Melissa eing 15797.10 - Michael anti 81726.96 - Michael bar 71706.72 - Micheal n st 9108.12 - Michelle able 5785.78 - Mildred cally 32760.00 - Mira cally 179008.00 - Mitzi able 33790.40 - Nadia n st 31547.88 - Nancy bar 18152.00 - Nancy eing 17277.70 - Nathan eing 49694.92 - Nicholas bar 101031.48 - Nicole n st 26170.20 - Norma able 42642.60 - Pamela ese 7797.00 - Patrick cally 35389.86 - Patrick ought 3488.40 - Paul cally 63168.72 - Paul ese 42226.24 - Peter anti 116001.27 - Peter n st 44567.60 - Philip cally 12426.40 - Philip n st 101312.64 - Phyllis n st 46031.70 - Polly anti 62732.88 - Priscilla ese 12201.00 - Rachel bar 27285.12 - Ralph ese 8791.04 - Randall able 49756.98 - Randy ese 10580.64 - Rebekah ese 7211.61 - Rex cally 2692.80 - Reyna ese 9190.88 - Rhonda bar 19808.16 - Ricardo anti 13709.01 - Richard bar 15278.52 - Richard cally 70136.56 - Robert cally 7804.80 - Robert ese 3327.75 - Robert n st 25134.78 - Ronald ese 10637.48 - Rosemarie anti 62515.53 - Rosemary n st 7630.56 - Roxanna n st 87141.60 - Ryan n st 98939.82 - Sabrina eing 314305.20 - Sandra bar 133087.68 - Sandra eing 9588.92 - Sandra ese 13144.32 - Sara cally 24402.00 - Scott able 10369.45 - Scott cally 27062.64 - Scott n st 35694.75 - Sebastian bar 7528.96 - Sharon bar 3311.64 - Sharon cally 10535.14 - Shawn anti 10017.00 - Shawn ought 81820.00 - Steven ese 33776.82 - Stuart bar 9809.64 - Susan able 65456.16 - Takisha ese 4609.80 - Tamekia able 14260.48 - Tamela ese 676508.10 - Telma cally 29312.64 - Teresa able 34201.44 - Teresa cally 77541.42 - Teresa n st 17703.84 - Thelma ought 170473.60 - Theresa cally 6815.30 - Thomas able 174122.65 - Thomas bar 12982.32 - Thomas ese 68827.50 - Thomas n st 17602.20 - Timothy able 121340.94 - Timothy bar 57515.04 - Timothy eing 90578.40 - Timothy ese 29622.08 - Tina n st 22176.00 - Todd ought 57643.20 - Tommy cally 9978.40 - Toni n st 78906.24 - Travis eing 35255.00 - Tyrone able 10698.48 - Tyson bar 4542.60 - Una n st 7616.40 - Vicente able 12265.50 - Virginia anti 12540.99 - Virginia cally 25335.60 - Wade n st 8513.12 - Wanda able 138543.34 - Wanda cally 2988.09 - Wesley cally 124238.72 - William able 42572.04 - William cally 4660.70 - William eing 232656.60 - William ese 38062.50 - William ought 6177.00 - Winifred n st 15114.88 - Wm cally 106464.60 - Yolanda eing 98807.28 - Yvonne cally 22535.04 -Aaron Frank cally 2148943.86 -Abbott Joshua bar 24304.80 -Abel n st 170703.94 -Abrams bar 5359.08 -Acevedo bar 16197.12 -Ackerman bar 41156.64 -Acosta Rick ese 107171.10 -Adams eing 40609.80 -Adams Juanita bar 893396.24 -Ahmed cally 5557.20 -Alexander Henry able 445241.35 -Alfonso cally 20382.96 -Alfonso Evelyn cally 3861.89 -Allen Henry ought 20677.00 -Allen Mildred n st 51691.20 -Allen Roy eing 48015.00 -Amador Herminia bar 4868.64 -Anderson able 9606.24 -Anderson cally 6543.81 -Anderson eing 3311.00 -Anderson David able 92637.09 -Anderson John anti 23133.60 -Anderson Wm ese 28992.60 -Andrews Denise n st 18517.80 -Andrews Wayne ese 8992.80 -Armstrong anti 2589.30 -Armstrong Virginia bar 70974.96 -Arnold Bridgette ese 2565892.62 -Atwood Estela bar 3562.24 -Austin Alison n st 8415.88 -Ayala eing 7233.16 -Ayres cally 21481.68 -Bacon Debbie ese 4222.68 -Baker bar 26794.80 -Ball Jerry ese 150803.10 -Banks able 19002.23 -Barba Christopher cally 11222.00 -Barclay bar 5448.80 -Barclay ese 94435.20 -Barnes Annie cally 6747685.56 -Barnes Robert ese 61278.00 -Barnett cally 23383.50 -Bates Barry ese 4479097.50 -Baumann bar 14332.56 -Beall Roxie ese 6129003.20 -Bean bar 186505.92 -Beard ese 18230.94 -Beard Juanita ese 155382.08 -Beaudry Robert able 50239.80 -Beck ought 5210.20 -Becker anti 36276.66 -Benavides Leon bar 47808.00 -Bennett anti 3229.80 -Benson David cally 375103.41 -Bernal Carlos able 622876.50 -Berry William ese 15607.20 -Betts Don cally 363895.20 -Bishop ese 101154.72 -Blackburn Jacqueline cally 71278.28 -Blackwell Dina ought 113643.00 -Blake Martin bar 24740.64 -Blanco Justin bar 15023.40 -Blankenship Eric cally 1267043.05 -Bledsoe cally 24828.20 -Bower Cecilia bar 5806.08 -Bowers James cally 15628.44 -Boyer David able 1980217.80 -Boyles Polly ese 8523.48 -Braxton ought 60642.00 -Brim Martha n st 13412.70 -Britton ese 22272.00 -Broderick bar 41791.68 -Brogan ese 11886.00 -Brooks n st 53964.90 -Broussard n st 24611.84 -Broussard Matthew ese 9488.16 -Brown cally 98872.00 -Brown Gina able 5079947.40 -Brown Israel n st 5348173.50 -Brown Jaime ese 17076.80 -Brown Steve n st 6676.56 -Brown Steven anti 125148.24 -Browning able 52416.00 -Browning anti 102849.18 -Bryan anti 6442502.40 -Bryant Felix cally 231740.60 -Buckner Curtis cally 46865.60 -Burch n st 6673.92 -Burk n st 3434.20 -Burke Charles n st 136314.36 -Burns Latisha anti 14757.12 -Burns Richard bar 157651.98 -Burroughs cally 30775.14 -Burton cally 6068.16 -Bustos Eleanor ese 2881452.96 -Butler Justin cally 182620.00 -Butler Michael ese 1188184.20 -Byrne Jessica cally 88877.70 -Cabral ese 48888.00 -Cabrera Ruth cally 82587.36 -Cain Douglas eing 20880.86 -Caldwell anti 21275.10 -Callaway Joseph n st 83054.18 -Camarillo Scott bar 13077.76 -Cameron cally 30975.36 -Cantu Barbara ought 134985.20 -Caraballo Jerry ese 464478.80 -Carpenter Brenda n st 7070.70 -Carrier ese 5044.41 -Carson Vernell cally 346711.26 -Carter Brandon cally 108929.04 -Carter Glenna ought 20355.40 -Carter Michelle cally 35191.20 -Castro cally 59103.56 -Chalmers ese 40726.40 -Chambers James able 6918.08 -Champion Roberta n st 11756.64 -Chandler cally 41093.76 -Chandler Jane cally 2413479.75 -Chang Wayne bar 24130.32 -Chapman Luz anti 7527.24 -Chase Angela bar 47012.76 -Chatman cally 5179.90 -Chavarria Cynthia able 244530.00 -Chestnut Stephanie ese 18295.62 -Christie Anthony anti 104280.33 -Chun ese 7719.60 -Chun Maggie able 3554.33 -Chung bar 22796.76 -Clark able 46625.88 -Clark Sandra bar 49624.64 -Clemmons Normand bar 10583.04 -Clemons Judy bar 30235.52 -Cole Joseph able 61725.30 -Coleman bar 8401.80 -Coleman Justin ese 8038.92 -Collins Anna eing 919638.72 -Collins David eing 10494.44 -Conklin ese 24520.32 -Conklin Joanne eing 28163.52 -Contreras Leah ese 5544.00 -Corbett n st 253444.80 -Corcoran Fredrick ese 388122.80 -Cornett ese 37989.12 -Cortez Charles anti 139352.40 -Cottrell Angela cally 204918.98 -Cousins Marguerite ese 11476.08 -Coward Anthony anti 142960.86 -Craig Stephen able 79633.84 -Crandall Alejandro eing 9515.44 -Crane Rebeca cally 3010934.94 -Crawford ese 29127.60 -Crittenden able 9435.16 -Croft bar 695863.41 -Crooks Carlos ese 1878403.77 -Crum Christopher anti 59516.73 -Cruz Timothy cally 120143.40 -Cunningham n st 38394.72 -Cunningham Jade n st 8901.90 -Curran Dennis n st 681296.61 -Curry eing 7892.50 -Curtis Leslie n st 21190.47 -Damron cally 10434.60 -Danforth Maureen able 16235.94 -Daniels Ana n st 124173.42 -Daniels Travis bar 28566.72 -Darden bar 6758.76 -Davenport able 63633.92 -Davenport n st 43303.68 -Davidson Michael cally 3977846.32 -Davis able 26236.60 -Davis cally 20203.92 -Davis eing 16944.84 -Davis Gabriella able 78158.08 -Davis Lorraine n st 18585.42 -Davis Lydia ese 4213.44 -Davis Robert bar 21274.56 -Day able 50059.62 -Day Kimberly ese 421473.00 -Day Paul cally 3652.80 -Day Peter able 92552.34 -Dean Ashley ese 37882.20 -Delacruz able 41388.88 -Dempsey Nicole cally 12594.96 -Dennis Esther cally 2877574.70 -Devries bar 29203.20 -Devries cally 88190.52 -Diamond Michael ought 28760.00 -Dickens able 8009.82 -Doan Debra able 7685.44 -Doan Mack anti 2642382.00 -Dominguez Barbara able 3135666.60 -Dominguez Freddie cally 65339.04 -Dominquez bar 9233.04 -Dove n st 132248.55 -Dowell bar 8360.00 -Doyle Lorraine n st 348588.00 -Drew Lucia ought 23954.00 -Dumont Margaret cally 47174.60 -Dunlap Andrew ese 63625.59 -Dye n st 71496.81 -Early Angela ese 9045.12 -Ebert Emmitt cally 5245.18 -Edgar n st 10383.12 -Edwards Nancy bar 34176.00 -Elkins cally 29275.84 -Elliott Georgia cally 15710.20 -Elliott James bar 9092787.57 -Ellis ese 3078.18 -Erickson Jude ese 31923.99 -Espinosa Nicholas anti 39416.58 -Esposito Rosa ought 505245.60 -Estrella bar 15387.12 -Etheridge eing 53717.18 -Evans anti 3508.05 -Evans ese 42461.12 -Evans n st 7137.48 -Ezell cally 39956.40 -Felder Katheryn ought 25479.00 -Ferreira bar 57728.16 -Fields Judson bar 2066975.84 -Fields Susan ese 5115650.00 -Fisher Natalie ese 2780348.20 -Flood Michaela n st 26620.02 -Flores cally 5081.13 -Forbes Richard ought 143971.20 -Ford Melissa n st 3001999.44 -Forest ought 71836.80 -Fowler Leo ese 2102007.84 -Fox cally 70001.07 -Fox Hector cally 19510.56 -Francis Fred cally 14526.60 -Franco n st 43707.93 -Frank Monica n st 17820.00 -Frantz Roger ese 13962.24 -Fredrickson Luis n st 17793.60 -Freeman n st 4357.08 -Freeman Richard ought 7416313.20 -Friedman able 11080.16 -Frye Aaron able 3844665.00 -Fullerton Sherryl cally 4795.56 -Gallagher Stephen able 29841.00 -Galloway Lois cally 122700.60 -Gardner Samuel eing 2401304.40 -Garner cally 108309.89 -Garrison Jenny n st 69487.16 -Garza bar 45962.64 -Geiger Claudia n st 96340.64 -Gilbert Katie ese 3998.76 -Giles Joann able 16827.44 -Gilley Jennifer able 7508.80 -Gilliam ese 30143.52 -Gilmore Jordan able 16208.64 -Glaze cally 50991.00 -Glover Gwendolyn cally 18392.04 -Gomes ought 2702.80 -Gonzalez cally 22320.00 -Gonzalez Dorothy cally 58425.24 -Gonzalez Jack cally 3655.52 -Gonzalez Pamela cally 9231.00 -Gonzalez Scott ese 2894.40 -Goodwin Charles able 20700.68 -Goodwin Peggy bar 20176.00 -Gough Ralph able 20001.80 -Graham cally 24011.65 -Green cally 4177.08 -Green n st 30821.56 -Green Jennifer ese 5588.80 -Green Steve ought 562077.45 -Greene able 29694.60 -Grenier Antoinette ought 4696.00 -Griffin Amanda cally 4657194.59 -Griffin Gary ought 1373445.00 -Griffin Joseph cally 12689.82 -Griffin Pablo n st 894282.06 -Griffith Fred ought 22163.40 -Grossman Nisha anti 45501.33 -Guerra anti 5797.68 -Gunther John eing 6185091.00 -Guyton Lillian cally 79032.72 -Guzman cally 33297.60 -Hagen Erica able 49156.38 -Hall ese 3452.80 -Hall Adam cally 48798.72 -Hall Bobby n st 35620.20 -Hamilton cally 87362.78 -Hamm ese 4056.00 -Hansen bar 12015.36 -Hansen n st 13447.98 -Hansen Norman bar 121178.75 -Hardy Byron ese 37388.16 -Hargrove Naomi cally 46027.20 -Harmon Adele eing 5352.60 -Harmon So eing 441894.75 -Harrell Michael able 233962.14 -Harrington ought 7224.00 -Harris anti 33429.48 -Harris cally 24837.12 -Harris n st 12673.32 -Harris Lovetta bar 8668.80 -Harris Stephen bar 7267.68 -Hawkins ese 58339.20 -Hawkins Ernest bar 1435605.60 -Hayes n st 53306.88 -Heath cally 28232.40 -Heller ought 87658.80 -Heller Charles anti 45781.20 -Heller Charles bar 6131619.57 -Henderson John able 998444.70 -Henderson John n st 668883.16 -Herbert able 8722.98 -Hernandez n st 7707.92 -Hernandez Emely eing 61107.64 -Hernandez Emely ought 24039.60 -Hernandez Reba eing 106320.50 -Hernandez Teresa ese 983810.70 -Hickman ought 12437.80 -Higgs cally 4600.80 -Hilliard Bryan n st 919854.99 -Hines cally 160938.36 -Hines ese 4495.68 -Hines Vera cally 4782412.80 -Hoang Effie anti 72684.00 -Hobbs Paul n st 24285.24 -Hobbs Ryan ese 2651160.00 -Hoffmann Brian able 41617.81 -Hogue David n st 770013.03 -Holder Charles eing 727032.90 -Holder Thomas anti 2858943.60 -Hollis n st 107517.27 -Holt Michael cally 17134.00 -Hoover cally 14675.28 -Hoover Bonita able 7743.84 -House Steven bar 72610.40 -Houser able 16384.42 -Howard n st 14342.46 -Howard Melissa n st 8799.84 -Howe Joanne able 81104.88 -Howell Jo cally 3968.82 -Howell Simon cally 45021.20 -Hubbard ese 43761.60 -Hubbard Luther eing 32111.64 -Hudgens Robert ese 939328.65 -Hudson cally 30991.68 -Hughes ought 45692.40 -Hughey Christopher ese 3468744.00 -Hunter Deborah ese 1842750.00 -Hurst Donna cally 3074777.25 -Irwin Janet anti 119750.40 -Iverson Omega ese 87693.48 -Jacks eing 28833.42 -Jackson able 58616.74 -Jackson ought 21383.60 -Jackson Christina cally 3787.77 -Jackson Henry able 12994.80 -Jackson Lee ese 45813.60 -Jacobs ese 10943.76 -Jaime n st 74646.00 -James cally 23345.25 -Jankowski n st 95902.84 -Jarvis Marc ought 8213594.40 -Jenkins able 6839.04 -Jenkins Larry cally 14852.20 -Jennings Bernice ese 48620.67 -Jimenez Wilson n st 9841.44 -Jobe n st 62124.30 -Johns cally 32261.04 -Johns Willie cally 14069.03 -Johnson able 66851.72 -Johnson anti 15506.82 -Johnson cally 7854.00 -Johnson n st 50270.22 -Johnson Adrian bar 209701.80 -Johnson Charles ese 22802.72 -Johnson Eric ese 420887.20 -Johnson Kathleen able 3819.06 -Johnson Lena n st 110303.60 -Johnson Sung eing 2561.02 -Johnson Yvonne bar 61417.12 -Jones anti 169410.15 -Jones ese 35360.48 -Jones Allen eing 31689.90 -Jones Daniel anti 3418448.40 -Jones Georgina cally 224175.60 -Jones Jay eing 4633794.00 -Jones Melvin able 2198780.10 -Jones Nancy cally 79775.50 -Kane Christopher cally 300926.18 -Keegan Beth able 42725.34 -Keenan Gonzalo ought 5506065.45 -Keller ought 86184.00 -Keller Joseph anti 17403.75 -Kemp Jesse ese 10955.20 -Kendrick Michael cally 184052.88 -Kennedy Kenneth cally 2899269.60 -Kennedy Richard n st 5363.82 -Kerr April bar 3528.00 -Kim cally 8731.20 -Kimbrell Jeffrey anti 15286.11 -Kinard n st 23220.56 -Kirk Corey n st 14474.24 -Kirk Terry bar 30359.04 -Klein Anna bar 2401702.38 -Knight anti 79394.70 -Kohl Keren ese 39960.00 -Kraus Terence cally 7163.20 -Labelle ought 98786.20 -Lackey bar 59271.36 -Lackey Angelo ese 2110480.00 -Lacy Eric cally 573439.46 -Laflamme eing 38938.90 -Lambert bar 87494.40 -Lambert Sandra anti 14354.34 -Landry ese 83172.60 -Lane Brenda ese 401450.67 -Lane Brian eing 1030038.24 -Lanham cally 26125.00 -Lanham George cally 124003.44 -Lara anti 29092.14 -Larson Kevin cally 18333.00 -Larson Marjorie ese 3034040.00 -Lawler anti 22574.37 -Layton Summer able 422037.00 -Leavitt Zonia able 17585692.80 -Leblanc cally 125463.80 -Ledesma n st 18421.20 -Lee able 25108.16 -Lee cally 19934.03 -Lee Cathi ese 25424.00 -Lemke cally 7904.80 -Leon Rodney anti 85363.20 -Lester Karen cally 750804.89 -Lewis Bonnie n st 10598.91 -Lewis Earl anti 147558.60 -Lewis Eric eing 265300.20 -Lewis Lisa able 23458.50 -Lewis Magaret n st 3834643.12 -Lima Ronald eing 927907.20 -Little Jane eing 3995226.18 -Little Ron ese 186372.72 -Lloyd able 40974.08 -Long cally 30008.88 -Loomis Roger cally 19989.12 -Lopez n st 49536.96 -Lopez Justin anti 7825.44 -Lord able 12616.52 -Lovell Billy cally 326702.97 -Lugo Clayton ese 60500.00 -Lynch cally 26512.86 -Macdonald Margaret able 17592.96 -Maclean bar 26036.64 -Madrigal Justine n st 24091.20 -Maldonado n st 20285.76 -Malone Frank bar 58417.60 -Manning cally 23924.56 -Manning Sharon cally 30579.12 -Mansfield ese 42948.57 -Markley Wm cally 1417546.22 -Marquez Amy able 3336.34 -Marquez Billy ought 16613.20 -Marsh bar 7462.56 -Marsh cally 13115.08 -Marshall cally 67178.05 -Marshall Frank ese 2684.16 -Martin ese 19474.56 -Martin Shawn n st 7162.05 -Martinez ese 13896.57 -Martinez Anthony cally 179445.33 -Mason Jack able 1299311.65 -Mason Miguel anti 140775.39 -Massey Heather eing 1955725.20 -Masters Thomas able 3953129.40 -Mathis Linda cally 9325260.45 -May able 4583.04 -May Spring able 26280.80 -Mazza Brenda able 899464.50 -Mcallister bar 3679.08 -Mccann ese 54550.08 -Mccord Marian n st 33429.27 -Mccoy Antoine eing 56588.40 -Mcdaniel able 52594.08 -Mcdonald Brian cally 3119.16 -Mcdonald Eugena eing 29876.22 -Mckay Linda n st 814713.68 -Mckee eing 10148.16 -Mckee Leonard n st 13936.56 -Mcqueen Glenn eing 373804.20 -Medina Debra cally 26483.96 -Meeks cally 4895.88 -Mejia Marilyn n st 16134.80 -Melvin cally 46716.00 -Mendez cally 57428.36 -Mercier Carson ought 17404.80 -Messenger Edward cally 49432.80 -Meyer cally 70719.00 -Michel Francis bar 1054462.86 -Middleton bar 5052.48 -Middleton ese 178392.90 -Miles ese 57712.20 -Miller eing 17532.90 -Miller Anne n st 21757.68 -Miller Bradley able 8232008.40 -Miller Donna able 1342469.70 -Miller Herbert cally 42746.40 -Miner Andrew cally 75347.40 -Mitchell Jose able 64901.62 -Mize cally 4250.40 -Monaco ese 3074.40 -Monahan ought 12608.00 -Montgomery ese 28733.68 -Moore bar 24336.16 -Moore n st 39070.08 -Moore Bradley eing 100601.82 -Moore Ericka bar 1254099.00 -Morris Heather bar 7665.57 -Moses Nancy anti 73716.72 -Mosier Heather cally 45110.88 -Mueller ese 9286.20 -Mueller Chad n st 93095.73 -Mull cally 4857.92 -Mullins Thomas cally 36837.64 -Mundy Susan bar 300951.36 -Muniz Carl eing 28977.96 -Murphy anti 115932.60 -Murray Mac able 9618.56 -Myers bar 19296.00 -Nabors Robert ese 6523.44 -Navarro Jimmy n st 120245.58 -Navarro Sheila n st 174597.12 -Neal Shanna able 7503884.85 -Neff William ese 51180.48 -Nelson Debbie ought 264359.70 -Newton Brian ese 188250.00 -Newton Nathan cally 13388.76 -Nicholson ese 9449.28 -Nicholson Jacquelyn ese 24816.48 -Nixon Janet ought 144334.20 -Noe Alison bar 3650.88 -Norman Anne ese 9408.00 -Nunes ese 21717.15 -Ohara able 16990.48 -Oliver Barbara ese 16245.36 -Oneill Katie cally 595839.12 -Ontiveros bar 2667.60 -Oreilly anti 69946.80 -Oreilly Thelma cally 7483.40 -Orourke ese 21281.28 -Ortiz able 9698.92 -Osorio Ryan n st 5197.92 -Pace n st 46809.21 -Padgett Russell ese 77491.05 -Pape David anti 69404.58 -Park cally 4391.20 -Parker bar 3828.00 -Parks Lisa ese 2038944.60 -Parson cally 14081.98 -Pearson Rhonda n st 2702585.40 -Pelletier Eugene ese 4691.52 -Pelletier Judith ought 140346.00 -Pemberton Lisa n st 29359.00 -Pena bar 21115.68 -Perez cally 23360.40 -Perez ese 65494.12 -Perez Velma able 70686.00 -Perkins Virginia ese 4336.20 -Perry anti 50578.92 -Perry cally 4171.80 -Peterman Kevin bar 33362.52 -Peters Stephanie ese 3979.56 -Petty Maura able 22436.96 -Phillips Jean cally 14544.00 -Phillips Sharon able 5931783.00 -Pierce ese 146714.24 -Pierce n st 87769.88 -Pinckney n st 15238.08 -Pinto Jeffrey bar 643273.46 -Piper cally 57532.36 -Pittman Lawrence cally 24710.40 -Pitts able 5689.74 -Pollack Gloria able 188090.10 -Pool Peter ought 654223.50 -Porter Alexander n st 27885.48 -Posey cally 11318.60 -Post n st 22436.19 -Potter bar 16267.56 -Potts cally 22440.00 -Powell able 12975.20 -Powell ese 12996.06 -Prather anti 3791.97 -Priest bar 4740.12 -Proctor Gloria cally 5062.20 -Pruett David ese 17079.10 -Purdy Irene cally 2926.22 -Purnell Sylvia able 1577205.60 -Pyle Bessie able 69923.84 -Quezada n st 13651.47 -Quintanilla bar 5927.52 -Ragan Robert anti 9832071.60 -Ramirez Betty cally 3168.00 -Ramirez Freddie ese 18209.76 -Ramirez Theresa cally 9761.40 -Ramos Pat cally 175611.40 -Ramsey Bernice ese 42203.84 -Randall Eddie anti 3957120.00 -Rasmussen Gloria bar 36456.48 -Ratliff ese 11497.50 -Reed ese 16052.40 -Reed Glenda able 39105.92 -Reeves Joseph ese 17245.92 -Reich Brett anti 2753157.60 -Reilly Bonnie ese 5490.00 -Reno John ese 3482250.00 -Reyes ese 6646.20 -Rhodes eing 31218.88 -Rice Matthew ese 1450990.00 -Richard Edwin cally 86444.93 -Richards Judith anti 58810.50 -Richter Jesus bar 136090.79 -Ricketts Rebecca n st 21225.60 -Rivera bar 29740.56 -Rivera Charles n st 21869.76 -Roberson n st 16500.66 -Roberts Colleen bar 2265457.19 -Robinette n st 37324.32 -Robinson bar 22524.48 -Robinson Gerard ese 13796.64 -Roby ought 100915.20 -Rodriguez ese 24712.38 -Rodriguez n st 42044.94 -Rodriguez Brooke able 19909.50 -Roger Paul ese 3504.96 -Romero n st 39735.36 -Rooney Joseph cally 187633.08 -Rosales anti 16104.06 -Rose able 12495.28 -Ross cally 5142.40 -Rucker William cally 14057.47 -Ruiz Betty bar 6168.24 -Ruiz Mark ese 3482.01 -Rupp Whitney ese 13991.04 -Russell ese 23029.02 -Russell n st 21858.06 -Russell James ese 2042944.74 -Salinas Deborah cally 6192.00 -Sams ese 51979.20 -Sanchez ese 41336.40 -Sanchez Beth able 288614.70 -Sanders cally 24168.00 -Sanderson Judith bar 5636572.92 -Santana Donna able 211196.70 -Santiago cally 10988.12 -Santoro Juan able 10185.70 -Sapp ought 8312.00 -Sauer ought 2940.00 -Sawyer Jane able 26544.70 -Schaeffer n st 7354.20 -Schneider Kyung able 36267.40 -Schrader bar 10693.28 -Schreiner Evelyn n st 10798953.12 -Schulz cally 4544.76 -Scott able 10341.11 -Scott ese 97508.32 -Scott n st 29438.64 -Scott Yong able 3710.98 -Seaman bar 3609.24 -Sexton ese 113581.44 -Shaffer bar 28250.88 -Shannon Brian able 813126.60 -Shields Victoria able 6883.44 -Shrader Marc anti 9227196.00 -Shultz Richard ese 20109.60 -Silva Christine eing 2713095.00 -Simmons ought 82592.20 -Simpson ought 28206.60 -Slagle cally 57037.20 -Smart cally 36695.64 -Smith able 2821.00 -Smith bar 3360.00 -Smith cally 40224.12 -Smith eing 7014.70 -Smith ese 128457.92 -Smith n st 10752.00 -Smith Agnes cally 12882.60 -Smith Danny ese 23252.88 -Smith Henry n st 40632.48 -Smith Jeffrey able 24928.80 -Smith Katherine anti 2297700.00 -Smith Monique bar 3504.36 -Smith Patty bar 11843161.20 -Snow eing 9712.56 -Snyder n st 246563.10 -Somers cally 5304.85 -Somers n st 7498.26 -Sosa Roberto bar 519555.30 -Sotelo Eric able 26999.96 -Soto Julia ought 31250.00 -Soto Ralph able 11695.32 -Spellman Lloyd anti 27562.08 -Spence able 9152.26 -Spencer able 47351.64 -Stacy anti 24693.06 -Stahl cally 50300.80 -Starr ese 4381.28 -Steinberg Calvin n st 97402.56 -Stephens ese 26409.18 -Stephens Johnny anti 66624.81 -Stephens Maxine cally 88665.94 -Stevens able 15975.96 -Stewart bar 9029.04 -Stewart Kathryn n st 1137899.85 -Stiltner Mario cally 24521.40 -Story Susana ese 6685.56 -Stovall Hilda able 4193.28 -Strauss ought 43473.60 -Strickland Richard able 1543611.42 -Sutherland cally 3722.49 -Swanson Joshua cally 31469.76 -Tanner Donna ought 55252.60 -Tatum able 6948.24 -Taylor cally 10128.40 -Taylor eing 2237446.86 -Taylor Dinah cally 11781.12 -Taylor Karen able 27480.57 -Temple cally 19908.90 -Terry ese 29971.20 -Thomas bar 239757.92 -Thomas cally 88404.88 -Thomas ese 15883.20 -Thomas Rosie ought 24288.00 -Thompson Barbara able 7707.18 -Thompson Daniel ese 69885.27 -Thompson Jerri n st 1359909.99 -Thompson Susan eing 4610864.28 -Thorpe Santos eing 6846.84 -Tillman cally 3237.00 -Tomlinson bar 16080.00 -Torres Aracely bar 1896205.08 -Trahan n st 83328.30 -Trent cally 8068.80 -Tucker Leora n st 35724.04 -Turner eing 49184.96 -Underwood Kelly ese 4558.05 -Underwood Randy ese 2603052.80 -Vandiver Daniel bar 1704341.10 -Vasquez eing 157973.64 -Vasquez Lee ese 2295066.24 -Vaughn Nolan cally 105346.45 -Vaught Ronald ese 2650.56 -Vera Linda ese 55120.00 -Wade David anti 64010.31 -Walker cally 45480.50 -Walton able 5067.66 -Ward William cally 7723132.68 -Ware Wendell cally 18705.00 -Washburn bar 2818.80 -Watson bar 7596.16 -Watson ese 62181.60 -Watson Levi able 8736.00 -Watts Sharyn ese 2213500.00 -Weathers ese 54082.08 -Weathers Michelle bar 3057221.20 -Webber Samuel cally 19800.12 -Weeks ese 4811.52 -Weinberg Robert ese 42819.36 -Weiss Isela eing 6268914.63 -Wertz ought 26016.00 -Whelan Benjamin cally 3898.20 -White anti 62233.92 -White cally 27308.80 -White Jeanette cally 9309.24 -White Kenneth anti 27107.85 -White Sherry ese 7672.96 -White Theresa able 4468.94 -Whitley cally 3237.60 -Whyte Deborah able 9682.40 -Wilber Virginia n st 79142.49 -Wilkes cally 8092.17 -Wilkins ese 3169.32 -Williams able 143738.40 -Williams cally 20701.80 -Williams Abel cally 28217.28 -Williams Beatrice cally 123280.09 -Williams Charles eing 2441909.91 -Williams Eric ese 19896.96 -Williams Nikita anti 20700.75 -Wilson cally 14856.80 -Wilson eing 46839.10 -Wilson n st 36830.86 -Wilson Brenda n st 84638.40 -Wilson Dorothy n st 5118.96 -Wilson Janet n st 7045898.68 -Wilson Michelle able 1068691.47 -Wingate Katie cally 727612.02 -Winn bar 64561.60 -Winston bar 12228.48 -Wise bar 3738.00 -Wong ought 106310.40 -Wood Billy anti 5944881.60 -Woods Donald anti 158600.40 -Woods Stephanie cally 8570.52 -Woods Stephanie n st 5027.40 -Wooley able 14706.44 -Word ought 34108.80 -Wright able 124848.80 -Wright eing 82754.32 -Wright William ese 1507273.50 -Wu cally 234406.71 -Wynn Randolph ought 3310.20 -Yi anti 7048.44 -Yoon Norma cally 41775.60 -Yoon Norma eing 122074.04 -Young cally 7601.04 -Zimmerman Lakiesha bar 646580.55 + able 133.14 +Ashley Patrick ese 1681.99 +Atkins Rodney cally 182.91 +Bandy Wanda n st 1908.77 +Baylor Migdalia eing 1148.40 +Becker Edward able 1322.64 +Benton Keesha ought 305.55 +Brock Letha anti 510.48 +Brown Betty cally 6017.98 +Caraballo Rodrigo anti 303.03 +Cazares Jason ese 140.82 +Conway Soon cally 6977.18 +Cunningham Brenda ese 632.88 +Dixon Joseph bar 3089.46 +Edwards Juan able 553.80 +Erickson Kathy cally 501.80 +Figueroa Pablo eing 2090.01 +Frantz Geneva n st 350.32 +Freeman Randall cally 292.40 +Garrison Brian ese 318.06 +George Emilia cally 13669.51 +Goff John bar 4751.60 +Griffin Joseph cally 746.46 +Hancock Nancy bar 2182.54 +Hernandez Harold ese 2783.08 +Jones Anna ese 2398.27 +Jones Joseph cally 1688.54 +Love Rachel able 528.40 +Martin Sunshine ese 282.80 +Mathews Rebecca eing 194.74 +Mcbride John bar 1326.32 +Mccall Daniel bar 1728.54 +Mcleod Mark cally 1563.30 +Metz Dominic ese 1036.62 +Miller Dorothy cally 166.53 +Mills James n st 828.25 +Mitchell Mark ought 1625.26 +Moore Wesley eing 904.55 +Moran Kathleen able 1213.52 +Morris Marina cally 1412.80 +Murray Janet cally 2524.48 +Partin Eric ought 1739.10 +Phillips Sandra anti 194.02 +Roy John n st 662.48 +Santos Audrey eing 1840.32 +Schmidt William cally 1254.24 +Simmons Arlene bar 328.12 +Simmons Patsy cally 197.25 +Smith Daniel bar 4074.00 +Smith Ronald bar 1296.45 +Tan Ashley bar 826.56 +Webster Rose ought 2945.84 +Wright Hannah able 2525.67 diff --git a/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_2.out b/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_2.out index dd0227eea25823..aa16345d4d5fe2 100644 --- a/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_2.out +++ b/regression-test/data/tpcds_sf100_dup_without_key_p2/sql/q24_2.out @@ -1,155 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_2 -- - able 5445.96 - anti 123569.04 - bar 28714.88 - cally 406061.03 - eing 15095.08 - ese 372910.80 - n st 148248.93 - ought 23103.00 - Alfredo cally 11528.40 - Andrew cally 4621.98 - Annette ought 48585.20 - Anthony bar 6829.20 - Barbara cally 115192.00 - Beth cally 20044.80 - Bruce ese 21505.12 - Catherine ese 11488.68 - Charles bar 8155.80 - Charles cally 52558.80 - Clara cally 6422.24 - Cora ese 7393.26 - David cally 22160.52 - Deanna ese 22126.08 - Everett n st 17213.24 - Javier anti 15136.59 - Jennifer able 15426.32 - Jessica bar 5009.76 - Joe ese 46358.64 - Joel ought 17650.60 - John n st 6579.09 - Kevin ese 66556.80 - Kevin n st 4092.48 - Kristen cally 3275.00 - Kristin ese 9972.96 - Larry ese 17159.04 - Marc ese 41822.55 - Marie bar 21481.68 - Marissa able 8121.96 - Melissa cally 33291.20 - Melissa eing 5069.68 - Michael anti 25935.84 - Nicholas cally 27258.00 - Patrick n st 4195.84 - Ralph cally 27101.06 - Robert anti 54828.90 - Scott bar 21924.00 - Shari bar 7038.00 - Teresa ese 56511.84 - Timothy able 54182.31 - Timothy n st 12709.20 - Tyrone able 8113.14 - Tyson ought 5702.00 - Virgina cally 66348.00 - Wallace bar 86020.00 -Anderson Awilda cally 7342.72 -Bain Daniel n st 9507.08 -Baker cally 8985.60 -Baker Keith ought 615969.90 -Barger Marie ese 13458.84 -Barnhart Robert cally 6930043.18 -Bivins eing 77309.10 -Bradley Sue n st 135508.17 -Brown cally 67507.20 -Brown Jose ought 5832225.00 -Brown Steve eing 101260.50 -Caldwell Lester ese 147600.00 -Carver Larry bar 797865.42 -Cobb Michael ese 61207.68 -Collins bar 6753.60 -Cook Ashley bar 11495.40 -Cooper Efrain ese 30725.76 -Crawford Joanne able 12534.34 -Cruz bar 32302.40 -Cuevas Carolyn able 1774580.76 -Davies Walter anti 316872.00 -Davis bar 24991.20 -Davis Henry cally 18806.04 -Deaton bar 46466.16 -Diaz able 21324.29 -Elias Chrystal able 40709.76 -English Allen ese 226176.93 -Forte David bar 6046.08 -Garcia Danny ought 450949.50 -Gonzalez Esther ought 4176.00 -Gorman cally 59572.08 -Gray Brenda cally 19264.80 -Green Juan ese 941685.54 -Griffin bar 6004.68 -Hagen Larry ese 775882.00 -Hall bar 54864.00 -Haynes anti 41888.70 -Hinkle Katherine ese 504454.60 -Hitchcock bar 3055.32 -Howard Tammy ought 36859.80 -Janssen Joann n st 3194.40 -Johnson able 7903.98 -Jones n st 5360.25 -Jones Priscilla cally 31000.00 -Kidwell Jerry ought 4639.80 -Lacey Edward cally 25409.56 -Laird bar 25019.20 -Lindberg Robert cally 1314089.70 -Long Michael cally 67357.40 -Lopez n st 69360.72 -Loving ese 34842.40 -Mathis Shawn ese 2473119.42 -Mclaughlin Annette eing 66079.64 -Means cally 27119.84 -Miller Jodi able 1845210.84 -Mitchell bar 4044.60 -Morgan Gregory ese 22743.00 -Mundy Robert cally 38418.00 -Newman Brandon anti 49881.51 -Pena n st 26388.60 -Phelan anti 129004.26 -Phillips anti 7673.40 -Poe Loretta cally 6027123.26 -Pritchett Rochelle n st 1996916.67 -Pruitt Scot ese 30013.62 -Pryor Crystal anti 1349298.00 -Pryor Crystal ese 572639.76 -Ramos Kirsten n st 36206.94 -Reed ese 3120.18 -Rhodes Chad bar 41093.76 -Rhodes Charles eing 231287.76 -Riley Roy ese 9160.32 -Robert Linda ese 6075.52 -Rodriguez Debra bar 3948954.22 -Santos James anti 275097.60 -Scott Harold able 57461.60 -Sharpe Norman bar 63581.76 -Short ought 13336.40 -Smalley John n st 43408.26 -Smith bar 13601.28 -Smith Gloria eing 689298.39 -Stafford James anti 19467.00 -Sumner Veronica cally 2667.80 -Thompson Sabrina bar 4447984.05 -Tierney bar 27240.00 -Tomlin Faith bar 11748.48 -Torres Stella cally 3668.60 -Turner David ese 15949.44 -Vann cally 30198.72 -Velez Joe able 21086.80 -Ventura Anthony ese 2567648.55 -Waldron eing 32262.56 -Walker Alberto cally 212031.82 -Williams ese 135988.65 -Williams Inez cally 3602.50 -Winston Christina ese 519623.82 -Wolfe Dennis cally 8070092.80 -Wolff Paul anti 96933.27 -Wood Toni n st 5554.08 +Baker Letha ought 6995.34 +Kennedy Beatrice bar 215.68 +West William n st 375.42 diff --git a/regression-test/data/tpcds_sf100_p2/sql/q24_1.out b/regression-test/data/tpcds_sf100_p2/sql/q24_1.out index 2a91a6a701a3ed..239f109f79281a 100644 --- a/regression-test/data/tpcds_sf100_p2/sql/q24_1.out +++ b/regression-test/data/tpcds_sf100_p2/sql/q24_1.out @@ -1,1119 +1,56 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_1 -- - able 1021629.88 - anti 791301.84 - bar 1227497.84 - cally 5616234.34 - eing 860204.18 - ese 1932618.73 - n st 1800090.95 - ought 530419.60 - Aaron bar 37060.00 - Aaron ese 7108.56 - Aimee cally 28123.70 - Albert anti 39965.31 - Alberta ought 11466.00 - Aletha ese 35414.40 - Alex n st 46951.59 - Alexis cally 10015.72 - Allen able 144694.20 - Allen ese 57853.28 - Alonso cally 27210.00 - Amanda eing 27914.04 - Amber bar 4570.56 - Amber ought 3086.40 - Andrew bar 7524.00 - Andrew ese 7434.00 - Angela cally 4519.68 - Angela n st 21806.40 - Angie ought 53205.00 - Anthony able 41758.50 - Anthony cally 206319.48 - Anthony eing 21331.42 - Ardelle bar 66221.76 - Ashley cally 66980.20 - Ashley ese 40197.60 - Barbara able 25809.68 - Barbara cally 6164.40 - Barbra bar 65900.52 - Barry n st 72150.54 - Beatrice able 18787.58 - Benjamin bar 34466.24 - Billy cally 63817.80 - Billy ese 5772.00 - Bonnie bar 5153.28 - Bonnie ese 186461.28 - Bree cally 7988.64 - Carl bar 61009.76 - Carolyn able 56280.64 - Carrie bar 60318.72 - Carrie ese 28414.47 - Catherine ought 34860.00 - Cecil cally 41861.16 - Celeste bar 28300.80 - Charles bar 27927.52 - Charlotte cally 75533.55 - Christine ese 74597.76 - Christopher ese 21088.80 - Cindy n st 84084.00 - Claire able 2616.90 - Claire cally 6740.64 - Clarence n st 40026.84 - Clifford cally 24360.00 - Cody n st 15178.24 - Constance n st 19698.00 - Cristobal able 37630.18 - Curtis bar 88318.40 - Cynthia cally 19488.80 - Cynthia ese 5162.64 - Cynthia ought 99050.20 - Damaris able 104566.41 - Daniel cally 50320.80 - Daniel ese 33930.36 - David ese 67222.68 - David n st 14014.98 - Denise bar 17454.56 - Dennis n st 42812.22 - Diana anti 18198.18 - Dianna ese 3336.00 - Dolores eing 2646.16 - Donald ought 12663.60 - Donna able 93329.60 - Donna cally 14460.03 - Donna ese 29344.14 - Dora ese 4282.72 - Doretha ought 51451.20 - Doris cally 6845.08 - Dorothy ese 26849.28 - Douglas anti 2716.56 - Douglas bar 28371.60 - Earnest bar 22657.60 - Eduardo cally 12768.00 - Edward anti 48678.00 - Edwin ese 24665.55 - Elaine eing 4092.88 - Elizabeth ese 27740.16 - Ella n st 11840.64 - Emily cally 9080.20 - Emily ese 21020.58 - Errol ese 37747.20 - Eugene able 62410.32 - Eugene ese 106114.32 - Felix cally 21306.78 - Flora n st 3848.04 - Francine ought 89600.00 - Francis anti 195867.42 - Francis ese 10809.60 - Frank able 8321.04 - Franklin able 13519.52 - Franklin cally 80237.20 - Fred ese 55998.00 - Freda cally 3483.48 - Fredrick anti 9737.28 - Gabriela cally 32992.92 - Gary n st 8421969.36 - Gaston cally 108169.60 - George n st 8905.60 - Gladys ought 14548.80 - Grace cally 30564.96 - Gregory eing 241128.80 - Gregory n st 6870.15 - Gwendolyn ought 81058.00 - Harry bar 24641.28 - Hector eing 7917.80 - Helen ese 119687.82 - Henry cally 86119.79 - Herman bar 31543.52 - Herman ese 8660.80 - Hiroko ought 6572.00 - Hugh cally 11595.70 - Ila cally 84835.61 - Ivan anti 4397.19 - Jack ese 99417.60 - Jacques able 62339.16 - James anti 93096.57 - James cally 10163.11 - James ese 34504.32 - Jane cally 84292.80 - Janet bar 6049.28 - Janice bar 48906.00 - Janice ese 37263.36 - Jared ought 43809.00 - Jason ese 30310.08 - Jean cally 27843.40 - Jeffry anti 210522.90 - Jenna ought 2652.00 - Jennifer eing 2121956.10 - Jerry cally 30850.60 - Jessica cally 45254.88 - Jessica eing 124102.44 - Jo able 28828.80 - Joann n st 14108.60 - Joe n st 4296.60 - John bar 41023.08 - John cally 41636.00 - John ese 27941.04 - John n st 49222.74 - Jordan bar 56246.40 - Jose able 2966.04 - Joseph cally 12413.64 - Joseph ought 11917.40 - Josephine ese 24912.48 - Josh cally 33456.50 - Joyce ese 6328.32 - Julie bar 6085.28 - Julio able 7796.36 - Karen cally 62628.97 - Karen ese 7893.69 - Katherine bar 5614.40 - Kathleen cally 13156.30 - Kathleen eing 15915.90 - Kathleen ese 4410.00 - Keith bar 2971.80 - Kelvin eing 121847.00 - Kenneth able 4427.64 - Kenneth n st 14576.54 - Kristin ese 28851.00 - Lance cally 39247.20 - Latisha able 24376.80 - Laurie able 13091.40 - Laurie cally 67329.60 - Leon bar 64171.20 - Leroy able 29835.26 - Leticia n st 4607.90 - Linda cally 11880.00 - Lisa cally 30157.38 - Lissa cally 3093.20 - Lois ese 140388.15 - Lori ought 24931.20 - Louise able 53963.28 - Louise cally 26136.00 - Magaly cally 41644.56 - Manuel bar 16222.08 - Manuel ese 10214.40 - Maranda able 9328.80 - Marcella cally 12474.00 - Margery ese 67212.18 - Marie bar 42488.16 - Marti able 21468.85 - Martin cally 3450.92 - Marylou able 17305.08 - Maureen ese 20597.04 - Melinda eing 58556.08 - Melissa eing 15797.10 - Michael anti 81726.96 - Michael bar 71706.72 - Micheal n st 9108.12 - Michelle able 5785.78 - Mildred cally 32760.00 - Mira cally 179008.00 - Mitzi able 33790.40 - Nadia n st 31547.88 - Nancy bar 18152.00 - Nancy eing 17277.70 - Nathan eing 49694.92 - Nicholas bar 101031.48 - Nicole n st 26170.20 - Norma able 42642.60 - Pamela ese 7797.00 - Patrick cally 35389.86 - Patrick ought 3488.40 - Paul cally 63168.72 - Paul ese 42226.24 - Peter anti 116001.27 - Peter n st 44567.60 - Philip cally 12426.40 - Philip n st 101312.64 - Phyllis n st 46031.70 - Polly anti 62732.88 - Priscilla ese 12201.00 - Rachel bar 27285.12 - Ralph ese 8791.04 - Randall able 49756.98 - Randy ese 10580.64 - Rebekah ese 7211.61 - Rex cally 2692.80 - Reyna ese 9190.88 - Rhonda bar 19808.16 - Ricardo anti 13709.01 - Richard bar 15278.52 - Richard cally 70136.56 - Robert cally 7804.80 - Robert ese 3327.75 - Robert n st 25134.78 - Ronald ese 10637.48 - Rosemarie anti 62515.53 - Rosemary n st 7630.56 - Roxanna n st 87141.60 - Ryan n st 98939.82 - Sabrina eing 314305.20 - Sandra bar 133087.68 - Sandra eing 9588.92 - Sandra ese 13144.32 - Sara cally 24402.00 - Scott able 10369.45 - Scott cally 27062.64 - Scott n st 35694.75 - Sebastian bar 7528.96 - Sharon bar 3311.64 - Sharon cally 10535.14 - Shawn anti 10017.00 - Shawn ought 81820.00 - Steven ese 33776.82 - Stuart bar 9809.64 - Susan able 65456.16 - Takisha ese 4609.80 - Tamekia able 14260.48 - Tamela ese 676508.10 - Telma cally 29312.64 - Teresa able 34201.44 - Teresa cally 77541.42 - Teresa n st 17703.84 - Thelma ought 170473.60 - Theresa cally 6815.30 - Thomas able 174122.65 - Thomas bar 12982.32 - Thomas ese 68827.50 - Thomas n st 17602.20 - Timothy able 121340.94 - Timothy bar 57515.04 - Timothy eing 90578.40 - Timothy ese 29622.08 - Tina n st 22176.00 - Todd ought 57643.20 - Tommy cally 9978.40 - Toni n st 78906.24 - Travis eing 35255.00 - Tyrone able 10698.48 - Tyson bar 4542.60 - Una n st 7616.40 - Vicente able 12265.50 - Virginia anti 12540.99 - Virginia cally 25335.60 - Wade n st 8513.12 - Wanda able 138543.34 - Wanda cally 2988.09 - Wesley cally 124238.72 - William able 42572.04 - William cally 4660.70 - William eing 232656.60 - William ese 38062.50 - William ought 6177.00 - Winifred n st 15114.88 - Wm cally 106464.60 - Yolanda eing 98807.28 - Yvonne cally 22535.04 -Aaron Frank cally 2148943.86 -Abbott Joshua bar 24304.80 -Abel n st 170703.94 -Abrams bar 5359.08 -Acevedo bar 16197.12 -Ackerman bar 41156.64 -Acosta Rick ese 107171.10 -Adams eing 40609.80 -Adams Juanita bar 893396.24 -Ahmed cally 5557.20 -Alexander Henry able 445241.35 -Alfonso cally 20382.96 -Alfonso Evelyn cally 3861.89 -Allen Henry ought 20677.00 -Allen Mildred n st 51691.20 -Allen Roy eing 48015.00 -Amador Herminia bar 4868.64 -Anderson able 9606.24 -Anderson cally 6543.81 -Anderson eing 3311.00 -Anderson David able 92637.09 -Anderson John anti 23133.60 -Anderson Wm ese 28992.60 -Andrews Denise n st 18517.80 -Andrews Wayne ese 8992.80 -Armstrong anti 2589.30 -Armstrong Virginia bar 70974.96 -Arnold Bridgette ese 2565892.62 -Atwood Estela bar 3562.24 -Austin Alison n st 8415.88 -Ayala eing 7233.16 -Ayres cally 21481.68 -Bacon Debbie ese 4222.68 -Baker bar 26794.80 -Ball Jerry ese 150803.10 -Banks able 19002.23 -Barba Christopher cally 11222.00 -Barclay bar 5448.80 -Barclay ese 94435.20 -Barnes Annie cally 6747685.56 -Barnes Robert ese 61278.00 -Barnett cally 23383.50 -Bates Barry ese 4479097.50 -Baumann bar 14332.56 -Beall Roxie ese 6129003.20 -Bean bar 186505.92 -Beard ese 18230.94 -Beard Juanita ese 155382.08 -Beaudry Robert able 50239.80 -Beck ought 5210.20 -Becker anti 36276.66 -Benavides Leon bar 47808.00 -Bennett anti 3229.80 -Benson David cally 375103.41 -Bernal Carlos able 622876.50 -Berry William ese 15607.20 -Betts Don cally 363895.20 -Bishop ese 101154.72 -Blackburn Jacqueline cally 71278.28 -Blackwell Dina ought 113643.00 -Blake Martin bar 24740.64 -Blanco Justin bar 15023.40 -Blankenship Eric cally 1267043.05 -Bledsoe cally 24828.20 -Bower Cecilia bar 5806.08 -Bowers James cally 15628.44 -Boyer David able 1980217.80 -Boyles Polly ese 8523.48 -Braxton ought 60642.00 -Brim Martha n st 13412.70 -Britton ese 22272.00 -Broderick bar 41791.68 -Brogan ese 11886.00 -Brooks n st 53964.90 -Broussard n st 24611.84 -Broussard Matthew ese 9488.16 -Brown cally 98872.00 -Brown Gina able 5079947.40 -Brown Israel n st 5348173.50 -Brown Jaime ese 17076.80 -Brown Steve n st 6676.56 -Brown Steven anti 125148.24 -Browning able 52416.00 -Browning anti 102849.18 -Bryan anti 6442502.40 -Bryant Felix cally 231740.60 -Buckner Curtis cally 46865.60 -Burch n st 6673.92 -Burk n st 3434.20 -Burke Charles n st 136314.36 -Burns Latisha anti 14757.12 -Burns Richard bar 157651.98 -Burroughs cally 30775.14 -Burton cally 6068.16 -Bustos Eleanor ese 2881452.96 -Butler Justin cally 182620.00 -Butler Michael ese 1188184.20 -Byrne Jessica cally 88877.70 -Cabral ese 48888.00 -Cabrera Ruth cally 82587.36 -Cain Douglas eing 20880.86 -Caldwell anti 21275.10 -Callaway Joseph n st 83054.18 -Camarillo Scott bar 13077.76 -Cameron cally 30975.36 -Cantu Barbara ought 134985.20 -Caraballo Jerry ese 464478.80 -Carpenter Brenda n st 7070.70 -Carrier ese 5044.41 -Carson Vernell cally 346711.26 -Carter Brandon cally 108929.04 -Carter Glenna ought 20355.40 -Carter Michelle cally 35191.20 -Castro cally 59103.56 -Chalmers ese 40726.40 -Chambers James able 6918.08 -Champion Roberta n st 11756.64 -Chandler cally 41093.76 -Chandler Jane cally 2413479.75 -Chang Wayne bar 24130.32 -Chapman Luz anti 7527.24 -Chase Angela bar 47012.76 -Chatman cally 5179.90 -Chavarria Cynthia able 244530.00 -Chestnut Stephanie ese 18295.62 -Christie Anthony anti 104280.33 -Chun ese 7719.60 -Chun Maggie able 3554.33 -Chung bar 22796.76 -Clark able 46625.88 -Clark Sandra bar 49624.64 -Clemmons Normand bar 10583.04 -Clemons Judy bar 30235.52 -Cole Joseph able 61725.30 -Coleman bar 8401.80 -Coleman Justin ese 8038.92 -Collins Anna eing 919638.72 -Collins David eing 10494.44 -Conklin ese 24520.32 -Conklin Joanne eing 28163.52 -Contreras Leah ese 5544.00 -Corbett n st 253444.80 -Corcoran Fredrick ese 388122.80 -Cornett ese 37989.12 -Cortez Charles anti 139352.40 -Cottrell Angela cally 204918.98 -Cousins Marguerite ese 11476.08 -Coward Anthony anti 142960.86 -Craig Stephen able 79633.84 -Crandall Alejandro eing 9515.44 -Crane Rebeca cally 3010934.94 -Crawford ese 29127.60 -Crittenden able 9435.16 -Croft bar 695863.41 -Crooks Carlos ese 1878403.77 -Crum Christopher anti 59516.73 -Cruz Timothy cally 120143.40 -Cunningham n st 38394.72 -Cunningham Jade n st 8901.90 -Curran Dennis n st 681296.61 -Curry eing 7892.50 -Curtis Leslie n st 21190.47 -Damron cally 10434.60 -Danforth Maureen able 16235.94 -Daniels Ana n st 124173.42 -Daniels Travis bar 28566.72 -Darden bar 6758.76 -Davenport able 63633.92 -Davenport n st 43303.68 -Davidson Michael cally 3977846.32 -Davis able 26236.60 -Davis cally 20203.92 -Davis eing 16944.84 -Davis Gabriella able 78158.08 -Davis Lorraine n st 18585.42 -Davis Lydia ese 4213.44 -Davis Robert bar 21274.56 -Day able 50059.62 -Day Kimberly ese 421473.00 -Day Paul cally 3652.80 -Day Peter able 92552.34 -Dean Ashley ese 37882.20 -Delacruz able 41388.88 -Dempsey Nicole cally 12594.96 -Dennis Esther cally 2877574.70 -Devries bar 29203.20 -Devries cally 88190.52 -Diamond Michael ought 28760.00 -Dickens able 8009.82 -Doan Debra able 7685.44 -Doan Mack anti 2642382.00 -Dominguez Barbara able 3135666.60 -Dominguez Freddie cally 65339.04 -Dominquez bar 9233.04 -Dove n st 132248.55 -Dowell bar 8360.00 -Doyle Lorraine n st 348588.00 -Drew Lucia ought 23954.00 -Dumont Margaret cally 47174.60 -Dunlap Andrew ese 63625.59 -Dye n st 71496.81 -Early Angela ese 9045.12 -Ebert Emmitt cally 5245.18 -Edgar n st 10383.12 -Edwards Nancy bar 34176.00 -Elkins cally 29275.84 -Elliott Georgia cally 15710.20 -Elliott James bar 9092787.57 -Ellis ese 3078.18 -Erickson Jude ese 31923.99 -Espinosa Nicholas anti 39416.58 -Esposito Rosa ought 505245.60 -Estrella bar 15387.12 -Etheridge eing 53717.18 -Evans anti 3508.05 -Evans ese 42461.12 -Evans n st 7137.48 -Ezell cally 39956.40 -Felder Katheryn ought 25479.00 -Ferreira bar 57728.16 -Fields Judson bar 2066975.84 -Fields Susan ese 5115650.00 -Fisher Natalie ese 2780348.20 -Flood Michaela n st 26620.02 -Flores cally 5081.13 -Forbes Richard ought 143971.20 -Ford Melissa n st 3001999.44 -Forest ought 71836.80 -Fowler Leo ese 2102007.84 -Fox cally 70001.07 -Fox Hector cally 19510.56 -Francis Fred cally 14526.60 -Franco n st 43707.93 -Frank Monica n st 17820.00 -Frantz Roger ese 13962.24 -Fredrickson Luis n st 17793.60 -Freeman n st 4357.08 -Freeman Richard ought 7416313.20 -Friedman able 11080.16 -Frye Aaron able 3844665.00 -Fullerton Sherryl cally 4795.56 -Gallagher Stephen able 29841.00 -Galloway Lois cally 122700.60 -Gardner Samuel eing 2401304.40 -Garner cally 108309.89 -Garrison Jenny n st 69487.16 -Garza bar 45962.64 -Geiger Claudia n st 96340.64 -Gilbert Katie ese 3998.76 -Giles Joann able 16827.44 -Gilley Jennifer able 7508.80 -Gilliam ese 30143.52 -Gilmore Jordan able 16208.64 -Glaze cally 50991.00 -Glover Gwendolyn cally 18392.04 -Gomes ought 2702.80 -Gonzalez cally 22320.00 -Gonzalez Dorothy cally 58425.24 -Gonzalez Jack cally 3655.52 -Gonzalez Pamela cally 9231.00 -Gonzalez Scott ese 2894.40 -Goodwin Charles able 20700.68 -Goodwin Peggy bar 20176.00 -Gough Ralph able 20001.80 -Graham cally 24011.65 -Green cally 4177.08 -Green n st 30821.56 -Green Jennifer ese 5588.80 -Green Steve ought 562077.45 -Greene able 29694.60 -Grenier Antoinette ought 4696.00 -Griffin Amanda cally 4657194.59 -Griffin Gary ought 1373445.00 -Griffin Joseph cally 12689.82 -Griffin Pablo n st 894282.06 -Griffith Fred ought 22163.40 -Grossman Nisha anti 45501.33 -Guerra anti 5797.68 -Gunther John eing 6185091.00 -Guyton Lillian cally 79032.72 -Guzman cally 33297.60 -Hagen Erica able 49156.38 -Hall ese 3452.80 -Hall Adam cally 48798.72 -Hall Bobby n st 35620.20 -Hamilton cally 87362.78 -Hamm ese 4056.00 -Hansen bar 12015.36 -Hansen n st 13447.98 -Hansen Norman bar 121178.75 -Hardy Byron ese 37388.16 -Hargrove Naomi cally 46027.20 -Harmon Adele eing 5352.60 -Harmon So eing 441894.75 -Harrell Michael able 233962.14 -Harrington ought 7224.00 -Harris anti 33429.48 -Harris cally 24837.12 -Harris n st 12673.32 -Harris Lovetta bar 8668.80 -Harris Stephen bar 7267.68 -Hawkins ese 58339.20 -Hawkins Ernest bar 1435605.60 -Hayes n st 53306.88 -Heath cally 28232.40 -Heller ought 87658.80 -Heller Charles anti 45781.20 -Heller Charles bar 6131619.57 -Henderson John able 998444.70 -Henderson John n st 668883.16 -Herbert able 8722.98 -Hernandez n st 7707.92 -Hernandez Emely eing 61107.64 -Hernandez Emely ought 24039.60 -Hernandez Reba eing 106320.50 -Hernandez Teresa ese 983810.70 -Hickman ought 12437.80 -Higgs cally 4600.80 -Hilliard Bryan n st 919854.99 -Hines cally 160938.36 -Hines ese 4495.68 -Hines Vera cally 4782412.80 -Hoang Effie anti 72684.00 -Hobbs Paul n st 24285.24 -Hobbs Ryan ese 2651160.00 -Hoffmann Brian able 41617.81 -Hogue David n st 770013.03 -Holder Charles eing 727032.90 -Holder Thomas anti 2858943.60 -Hollis n st 107517.27 -Holt Michael cally 17134.00 -Hoover cally 14675.28 -Hoover Bonita able 7743.84 -House Steven bar 72610.40 -Houser able 16384.42 -Howard n st 14342.46 -Howard Melissa n st 8799.84 -Howe Joanne able 81104.88 -Howell Jo cally 3968.82 -Howell Simon cally 45021.20 -Hubbard ese 43761.60 -Hubbard Luther eing 32111.64 -Hudgens Robert ese 939328.65 -Hudson cally 30991.68 -Hughes ought 45692.40 -Hughey Christopher ese 3468744.00 -Hunter Deborah ese 1842750.00 -Hurst Donna cally 3074777.25 -Irwin Janet anti 119750.40 -Iverson Omega ese 87693.48 -Jacks eing 28833.42 -Jackson able 58616.74 -Jackson ought 21383.60 -Jackson Christina cally 3787.77 -Jackson Henry able 12994.80 -Jackson Lee ese 45813.60 -Jacobs ese 10943.76 -Jaime n st 74646.00 -James cally 23345.25 -Jankowski n st 95902.84 -Jarvis Marc ought 8213594.40 -Jenkins able 6839.04 -Jenkins Larry cally 14852.20 -Jennings Bernice ese 48620.67 -Jimenez Wilson n st 9841.44 -Jobe n st 62124.30 -Johns cally 32261.04 -Johns Willie cally 14069.03 -Johnson able 66851.72 -Johnson anti 15506.82 -Johnson cally 7854.00 -Johnson n st 50270.22 -Johnson Adrian bar 209701.80 -Johnson Charles ese 22802.72 -Johnson Eric ese 420887.20 -Johnson Kathleen able 3819.06 -Johnson Lena n st 110303.60 -Johnson Sung eing 2561.02 -Johnson Yvonne bar 61417.12 -Jones anti 169410.15 -Jones ese 35360.48 -Jones Allen eing 31689.90 -Jones Daniel anti 3418448.40 -Jones Georgina cally 224175.60 -Jones Jay eing 4633794.00 -Jones Melvin able 2198780.10 -Jones Nancy cally 79775.50 -Kane Christopher cally 300926.18 -Keegan Beth able 42725.34 -Keenan Gonzalo ought 5506065.45 -Keller ought 86184.00 -Keller Joseph anti 17403.75 -Kemp Jesse ese 10955.20 -Kendrick Michael cally 184052.88 -Kennedy Kenneth cally 2899269.60 -Kennedy Richard n st 5363.82 -Kerr April bar 3528.00 -Kim cally 8731.20 -Kimbrell Jeffrey anti 15286.11 -Kinard n st 23220.56 -Kirk Corey n st 14474.24 -Kirk Terry bar 30359.04 -Klein Anna bar 2401702.38 -Knight anti 79394.70 -Kohl Keren ese 39960.00 -Kraus Terence cally 7163.20 -Labelle ought 98786.20 -Lackey bar 59271.36 -Lackey Angelo ese 2110480.00 -Lacy Eric cally 573439.46 -Laflamme eing 38938.90 -Lambert bar 87494.40 -Lambert Sandra anti 14354.34 -Landry ese 83172.60 -Lane Brenda ese 401450.67 -Lane Brian eing 1030038.24 -Lanham cally 26125.00 -Lanham George cally 124003.44 -Lara anti 29092.14 -Larson Kevin cally 18333.00 -Larson Marjorie ese 3034040.00 -Lawler anti 22574.37 -Layton Summer able 422037.00 -Leavitt Zonia able 17585692.80 -Leblanc cally 125463.80 -Ledesma n st 18421.20 -Lee able 25108.16 -Lee cally 19934.03 -Lee Cathi ese 25424.00 -Lemke cally 7904.80 -Leon Rodney anti 85363.20 -Lester Karen cally 750804.89 -Lewis Bonnie n st 10598.91 -Lewis Earl anti 147558.60 -Lewis Eric eing 265300.20 -Lewis Lisa able 23458.50 -Lewis Magaret n st 3834643.12 -Lima Ronald eing 927907.20 -Little Jane eing 3995226.18 -Little Ron ese 186372.72 -Lloyd able 40974.08 -Long cally 30008.88 -Loomis Roger cally 19989.12 -Lopez n st 49536.96 -Lopez Justin anti 7825.44 -Lord able 12616.52 -Lovell Billy cally 326702.97 -Lugo Clayton ese 60500.00 -Lynch cally 26512.86 -Macdonald Margaret able 17592.96 -Maclean bar 26036.64 -Madrigal Justine n st 24091.20 -Maldonado n st 20285.76 -Malone Frank bar 58417.60 -Manning cally 23924.56 -Manning Sharon cally 30579.12 -Mansfield ese 42948.57 -Markley Wm cally 1417546.22 -Marquez Amy able 3336.34 -Marquez Billy ought 16613.20 -Marsh bar 7462.56 -Marsh cally 13115.08 -Marshall cally 67178.05 -Marshall Frank ese 2684.16 -Martin ese 19474.56 -Martin Shawn n st 7162.05 -Martinez ese 13896.57 -Martinez Anthony cally 179445.33 -Mason Jack able 1299311.65 -Mason Miguel anti 140775.39 -Massey Heather eing 1955725.20 -Masters Thomas able 3953129.40 -Mathis Linda cally 9325260.45 -May able 4583.04 -May Spring able 26280.80 -Mazza Brenda able 899464.50 -Mcallister bar 3679.08 -Mccann ese 54550.08 -Mccord Marian n st 33429.27 -Mccoy Antoine eing 56588.40 -Mcdaniel able 52594.08 -Mcdonald Brian cally 3119.16 -Mcdonald Eugena eing 29876.22 -Mckay Linda n st 814713.68 -Mckee eing 10148.16 -Mckee Leonard n st 13936.56 -Mcqueen Glenn eing 373804.20 -Medina Debra cally 26483.96 -Meeks cally 4895.88 -Mejia Marilyn n st 16134.80 -Melvin cally 46716.00 -Mendez cally 57428.36 -Mercier Carson ought 17404.80 -Messenger Edward cally 49432.80 -Meyer cally 70719.00 -Michel Francis bar 1054462.86 -Middleton bar 5052.48 -Middleton ese 178392.90 -Miles ese 57712.20 -Miller eing 17532.90 -Miller Anne n st 21757.68 -Miller Bradley able 8232008.40 -Miller Donna able 1342469.70 -Miller Herbert cally 42746.40 -Miner Andrew cally 75347.40 -Mitchell Jose able 64901.62 -Mize cally 4250.40 -Monaco ese 3074.40 -Monahan ought 12608.00 -Montgomery ese 28733.68 -Moore bar 24336.16 -Moore n st 39070.08 -Moore Bradley eing 100601.82 -Moore Ericka bar 1254099.00 -Morris Heather bar 7665.57 -Moses Nancy anti 73716.72 -Mosier Heather cally 45110.88 -Mueller ese 9286.20 -Mueller Chad n st 93095.73 -Mull cally 4857.92 -Mullins Thomas cally 36837.64 -Mundy Susan bar 300951.36 -Muniz Carl eing 28977.96 -Murphy anti 115932.60 -Murray Mac able 9618.56 -Myers bar 19296.00 -Nabors Robert ese 6523.44 -Navarro Jimmy n st 120245.58 -Navarro Sheila n st 174597.12 -Neal Shanna able 7503884.85 -Neff William ese 51180.48 -Nelson Debbie ought 264359.70 -Newton Brian ese 188250.00 -Newton Nathan cally 13388.76 -Nicholson ese 9449.28 -Nicholson Jacquelyn ese 24816.48 -Nixon Janet ought 144334.20 -Noe Alison bar 3650.88 -Norman Anne ese 9408.00 -Nunes ese 21717.15 -Ohara able 16990.48 -Oliver Barbara ese 16245.36 -Oneill Katie cally 595839.12 -Ontiveros bar 2667.60 -Oreilly anti 69946.80 -Oreilly Thelma cally 7483.40 -Orourke ese 21281.28 -Ortiz able 9698.92 -Osorio Ryan n st 5197.92 -Pace n st 46809.21 -Padgett Russell ese 77491.05 -Pape David anti 69404.58 -Park cally 4391.20 -Parker bar 3828.00 -Parks Lisa ese 2038944.60 -Parson cally 14081.98 -Pearson Rhonda n st 2702585.40 -Pelletier Eugene ese 4691.52 -Pelletier Judith ought 140346.00 -Pemberton Lisa n st 29359.00 -Pena bar 21115.68 -Perez cally 23360.40 -Perez ese 65494.12 -Perez Velma able 70686.00 -Perkins Virginia ese 4336.20 -Perry anti 50578.92 -Perry cally 4171.80 -Peterman Kevin bar 33362.52 -Peters Stephanie ese 3979.56 -Petty Maura able 22436.96 -Phillips Jean cally 14544.00 -Phillips Sharon able 5931783.00 -Pierce ese 146714.24 -Pierce n st 87769.88 -Pinckney n st 15238.08 -Pinto Jeffrey bar 643273.46 -Piper cally 57532.36 -Pittman Lawrence cally 24710.40 -Pitts able 5689.74 -Pollack Gloria able 188090.10 -Pool Peter ought 654223.50 -Porter Alexander n st 27885.48 -Posey cally 11318.60 -Post n st 22436.19 -Potter bar 16267.56 -Potts cally 22440.00 -Powell able 12975.20 -Powell ese 12996.06 -Prather anti 3791.97 -Priest bar 4740.12 -Proctor Gloria cally 5062.20 -Pruett David ese 17079.10 -Purdy Irene cally 2926.22 -Purnell Sylvia able 1577205.60 -Pyle Bessie able 69923.84 -Quezada n st 13651.47 -Quintanilla bar 5927.52 -Ragan Robert anti 9832071.60 -Ramirez Betty cally 3168.00 -Ramirez Freddie ese 18209.76 -Ramirez Theresa cally 9761.40 -Ramos Pat cally 175611.40 -Ramsey Bernice ese 42203.84 -Randall Eddie anti 3957120.00 -Rasmussen Gloria bar 36456.48 -Ratliff ese 11497.50 -Reed ese 16052.40 -Reed Glenda able 39105.92 -Reeves Joseph ese 17245.92 -Reich Brett anti 2753157.60 -Reilly Bonnie ese 5490.00 -Reno John ese 3482250.00 -Reyes ese 6646.20 -Rhodes eing 31218.88 -Rice Matthew ese 1450990.00 -Richard Edwin cally 86444.93 -Richards Judith anti 58810.50 -Richter Jesus bar 136090.79 -Ricketts Rebecca n st 21225.60 -Rivera bar 29740.56 -Rivera Charles n st 21869.76 -Roberson n st 16500.66 -Roberts Colleen bar 2265457.19 -Robinette n st 37324.32 -Robinson bar 22524.48 -Robinson Gerard ese 13796.64 -Roby ought 100915.20 -Rodriguez ese 24712.38 -Rodriguez n st 42044.94 -Rodriguez Brooke able 19909.50 -Roger Paul ese 3504.96 -Romero n st 39735.36 -Rooney Joseph cally 187633.08 -Rosales anti 16104.06 -Rose able 12495.28 -Ross cally 5142.40 -Rucker William cally 14057.47 -Ruiz Betty bar 6168.24 -Ruiz Mark ese 3482.01 -Rupp Whitney ese 13991.04 -Russell ese 23029.02 -Russell n st 21858.06 -Russell James ese 2042944.74 -Salinas Deborah cally 6192.00 -Sams ese 51979.20 -Sanchez ese 41336.40 -Sanchez Beth able 288614.70 -Sanders cally 24168.00 -Sanderson Judith bar 5636572.92 -Santana Donna able 211196.70 -Santiago cally 10988.12 -Santoro Juan able 10185.70 -Sapp ought 8312.00 -Sauer ought 2940.00 -Sawyer Jane able 26544.70 -Schaeffer n st 7354.20 -Schneider Kyung able 36267.40 -Schrader bar 10693.28 -Schreiner Evelyn n st 10798953.12 -Schulz cally 4544.76 -Scott able 10341.11 -Scott ese 97508.32 -Scott n st 29438.64 -Scott Yong able 3710.98 -Seaman bar 3609.24 -Sexton ese 113581.44 -Shaffer bar 28250.88 -Shannon Brian able 813126.60 -Shields Victoria able 6883.44 -Shrader Marc anti 9227196.00 -Shultz Richard ese 20109.60 -Silva Christine eing 2713095.00 -Simmons ought 82592.20 -Simpson ought 28206.60 -Slagle cally 57037.20 -Smart cally 36695.64 -Smith able 2821.00 -Smith bar 3360.00 -Smith cally 40224.12 -Smith eing 7014.70 -Smith ese 128457.92 -Smith n st 10752.00 -Smith Agnes cally 12882.60 -Smith Danny ese 23252.88 -Smith Henry n st 40632.48 -Smith Jeffrey able 24928.80 -Smith Katherine anti 2297700.00 -Smith Monique bar 3504.36 -Smith Patty bar 11843161.20 -Snow eing 9712.56 -Snyder n st 246563.10 -Somers cally 5304.85 -Somers n st 7498.26 -Sosa Roberto bar 519555.30 -Sotelo Eric able 26999.96 -Soto Julia ought 31250.00 -Soto Ralph able 11695.32 -Spellman Lloyd anti 27562.08 -Spence able 9152.26 -Spencer able 47351.64 -Stacy anti 24693.06 -Stahl cally 50300.80 -Starr ese 4381.28 -Steinberg Calvin n st 97402.56 -Stephens ese 26409.18 -Stephens Johnny anti 66624.81 -Stephens Maxine cally 88665.94 -Stevens able 15975.96 -Stewart bar 9029.04 -Stewart Kathryn n st 1137899.85 -Stiltner Mario cally 24521.40 -Story Susana ese 6685.56 -Stovall Hilda able 4193.28 -Strauss ought 43473.60 -Strickland Richard able 1543611.42 -Sutherland cally 3722.49 -Swanson Joshua cally 31469.76 -Tanner Donna ought 55252.60 -Tatum able 6948.24 -Taylor cally 10128.40 -Taylor eing 2237446.86 -Taylor Dinah cally 11781.12 -Taylor Karen able 27480.57 -Temple cally 19908.90 -Terry ese 29971.20 -Thomas bar 239757.92 -Thomas cally 88404.88 -Thomas ese 15883.20 -Thomas Rosie ought 24288.00 -Thompson Barbara able 7707.18 -Thompson Daniel ese 69885.27 -Thompson Jerri n st 1359909.99 -Thompson Susan eing 4610864.28 -Thorpe Santos eing 6846.84 -Tillman cally 3237.00 -Tomlinson bar 16080.00 -Torres Aracely bar 1896205.08 -Trahan n st 83328.30 -Trent cally 8068.80 -Tucker Leora n st 35724.04 -Turner eing 49184.96 -Underwood Kelly ese 4558.05 -Underwood Randy ese 2603052.80 -Vandiver Daniel bar 1704341.10 -Vasquez eing 157973.64 -Vasquez Lee ese 2295066.24 -Vaughn Nolan cally 105346.45 -Vaught Ronald ese 2650.56 -Vera Linda ese 55120.00 -Wade David anti 64010.31 -Walker cally 45480.50 -Walton able 5067.66 -Ward William cally 7723132.68 -Ware Wendell cally 18705.00 -Washburn bar 2818.80 -Watson bar 7596.16 -Watson ese 62181.60 -Watson Levi able 8736.00 -Watts Sharyn ese 2213500.00 -Weathers ese 54082.08 -Weathers Michelle bar 3057221.20 -Webber Samuel cally 19800.12 -Weeks ese 4811.52 -Weinberg Robert ese 42819.36 -Weiss Isela eing 6268914.63 -Wertz ought 26016.00 -Whelan Benjamin cally 3898.20 -White anti 62233.92 -White cally 27308.80 -White Jeanette cally 9309.24 -White Kenneth anti 27107.85 -White Sherry ese 7672.96 -White Theresa able 4468.94 -Whitley cally 3237.60 -Whyte Deborah able 9682.40 -Wilber Virginia n st 79142.49 -Wilkes cally 8092.17 -Wilkins ese 3169.32 -Williams able 143738.40 -Williams cally 20701.80 -Williams Abel cally 28217.28 -Williams Beatrice cally 123280.09 -Williams Charles eing 2441909.91 -Williams Eric ese 19896.96 -Williams Nikita anti 20700.75 -Wilson cally 14856.80 -Wilson eing 46839.10 -Wilson n st 36830.86 -Wilson Brenda n st 84638.40 -Wilson Dorothy n st 5118.96 -Wilson Janet n st 7045898.68 -Wilson Michelle able 1068691.47 -Wingate Katie cally 727612.02 -Winn bar 64561.60 -Winston bar 12228.48 -Wise bar 3738.00 -Wong ought 106310.40 -Wood Billy anti 5944881.60 -Woods Donald anti 158600.40 -Woods Stephanie cally 8570.52 -Woods Stephanie n st 5027.40 -Wooley able 14706.44 -Word ought 34108.80 -Wright able 124848.80 -Wright eing 82754.32 -Wright William ese 1507273.50 -Wu cally 234406.71 -Wynn Randolph ought 3310.20 -Yi anti 7048.44 -Yoon Norma cally 41775.60 -Yoon Norma eing 122074.04 -Young cally 7601.04 -Zimmerman Lakiesha bar 646580.55 + able 133.14 +Ashley Patrick ese 1681.99 +Atkins Rodney cally 182.91 +Bandy Wanda n st 1908.77 +Baylor Migdalia eing 1148.40 +Becker Edward able 1322.64 +Benton Keesha ought 305.55 +Brock Letha anti 510.48 +Brown Betty cally 6017.98 +Caraballo Rodrigo anti 303.03 +Cazares Jason ese 140.82 +Conway Soon cally 6977.18 +Cunningham Brenda ese 632.88 +Dixon Joseph bar 3089.46 +Edwards Juan able 553.80 +Erickson Kathy cally 501.80 +Figueroa Pablo eing 2090.01 +Frantz Geneva n st 350.32 +Freeman Randall cally 292.40 +Garrison Brian ese 318.06 +George Emilia cally 13669.51 +Goff John bar 4751.60 +Griffin Joseph cally 746.46 +Hancock Nancy bar 2182.54 +Hernandez Harold ese 2783.08 +Jones Anna ese 2398.27 +Jones Joseph cally 1688.54 +Love Rachel able 528.40 +Martin Sunshine ese 282.80 +Mathews Rebecca eing 194.74 +Mcbride John bar 1326.32 +Mccall Daniel bar 1728.54 +Mcleod Mark cally 1563.30 +Metz Dominic ese 1036.62 +Miller Dorothy cally 166.53 +Mills James n st 828.25 +Mitchell Mark ought 1625.26 +Moore Wesley eing 904.55 +Moran Kathleen able 1213.52 +Morris Marina cally 1412.80 +Murray Janet cally 2524.48 +Partin Eric ought 1739.10 +Phillips Sandra anti 194.02 +Roy John n st 662.48 +Santos Audrey eing 1840.32 +Schmidt William cally 1254.24 +Simmons Arlene bar 328.12 +Simmons Patsy cally 197.25 +Smith Daniel bar 4074.00 +Smith Ronald bar 1296.45 +Tan Ashley bar 826.56 +Webster Rose ought 2945.84 +Wright Hannah able 2525.67 diff --git a/regression-test/data/tpcds_sf100_p2/sql/q24_2.out b/regression-test/data/tpcds_sf100_p2/sql/q24_2.out index dd0227eea25823..aa16345d4d5fe2 100644 --- a/regression-test/data/tpcds_sf100_p2/sql/q24_2.out +++ b/regression-test/data/tpcds_sf100_p2/sql/q24_2.out @@ -1,155 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_2 -- - able 5445.96 - anti 123569.04 - bar 28714.88 - cally 406061.03 - eing 15095.08 - ese 372910.80 - n st 148248.93 - ought 23103.00 - Alfredo cally 11528.40 - Andrew cally 4621.98 - Annette ought 48585.20 - Anthony bar 6829.20 - Barbara cally 115192.00 - Beth cally 20044.80 - Bruce ese 21505.12 - Catherine ese 11488.68 - Charles bar 8155.80 - Charles cally 52558.80 - Clara cally 6422.24 - Cora ese 7393.26 - David cally 22160.52 - Deanna ese 22126.08 - Everett n st 17213.24 - Javier anti 15136.59 - Jennifer able 15426.32 - Jessica bar 5009.76 - Joe ese 46358.64 - Joel ought 17650.60 - John n st 6579.09 - Kevin ese 66556.80 - Kevin n st 4092.48 - Kristen cally 3275.00 - Kristin ese 9972.96 - Larry ese 17159.04 - Marc ese 41822.55 - Marie bar 21481.68 - Marissa able 8121.96 - Melissa cally 33291.20 - Melissa eing 5069.68 - Michael anti 25935.84 - Nicholas cally 27258.00 - Patrick n st 4195.84 - Ralph cally 27101.06 - Robert anti 54828.90 - Scott bar 21924.00 - Shari bar 7038.00 - Teresa ese 56511.84 - Timothy able 54182.31 - Timothy n st 12709.20 - Tyrone able 8113.14 - Tyson ought 5702.00 - Virgina cally 66348.00 - Wallace bar 86020.00 -Anderson Awilda cally 7342.72 -Bain Daniel n st 9507.08 -Baker cally 8985.60 -Baker Keith ought 615969.90 -Barger Marie ese 13458.84 -Barnhart Robert cally 6930043.18 -Bivins eing 77309.10 -Bradley Sue n st 135508.17 -Brown cally 67507.20 -Brown Jose ought 5832225.00 -Brown Steve eing 101260.50 -Caldwell Lester ese 147600.00 -Carver Larry bar 797865.42 -Cobb Michael ese 61207.68 -Collins bar 6753.60 -Cook Ashley bar 11495.40 -Cooper Efrain ese 30725.76 -Crawford Joanne able 12534.34 -Cruz bar 32302.40 -Cuevas Carolyn able 1774580.76 -Davies Walter anti 316872.00 -Davis bar 24991.20 -Davis Henry cally 18806.04 -Deaton bar 46466.16 -Diaz able 21324.29 -Elias Chrystal able 40709.76 -English Allen ese 226176.93 -Forte David bar 6046.08 -Garcia Danny ought 450949.50 -Gonzalez Esther ought 4176.00 -Gorman cally 59572.08 -Gray Brenda cally 19264.80 -Green Juan ese 941685.54 -Griffin bar 6004.68 -Hagen Larry ese 775882.00 -Hall bar 54864.00 -Haynes anti 41888.70 -Hinkle Katherine ese 504454.60 -Hitchcock bar 3055.32 -Howard Tammy ought 36859.80 -Janssen Joann n st 3194.40 -Johnson able 7903.98 -Jones n st 5360.25 -Jones Priscilla cally 31000.00 -Kidwell Jerry ought 4639.80 -Lacey Edward cally 25409.56 -Laird bar 25019.20 -Lindberg Robert cally 1314089.70 -Long Michael cally 67357.40 -Lopez n st 69360.72 -Loving ese 34842.40 -Mathis Shawn ese 2473119.42 -Mclaughlin Annette eing 66079.64 -Means cally 27119.84 -Miller Jodi able 1845210.84 -Mitchell bar 4044.60 -Morgan Gregory ese 22743.00 -Mundy Robert cally 38418.00 -Newman Brandon anti 49881.51 -Pena n st 26388.60 -Phelan anti 129004.26 -Phillips anti 7673.40 -Poe Loretta cally 6027123.26 -Pritchett Rochelle n st 1996916.67 -Pruitt Scot ese 30013.62 -Pryor Crystal anti 1349298.00 -Pryor Crystal ese 572639.76 -Ramos Kirsten n st 36206.94 -Reed ese 3120.18 -Rhodes Chad bar 41093.76 -Rhodes Charles eing 231287.76 -Riley Roy ese 9160.32 -Robert Linda ese 6075.52 -Rodriguez Debra bar 3948954.22 -Santos James anti 275097.60 -Scott Harold able 57461.60 -Sharpe Norman bar 63581.76 -Short ought 13336.40 -Smalley John n st 43408.26 -Smith bar 13601.28 -Smith Gloria eing 689298.39 -Stafford James anti 19467.00 -Sumner Veronica cally 2667.80 -Thompson Sabrina bar 4447984.05 -Tierney bar 27240.00 -Tomlin Faith bar 11748.48 -Torres Stella cally 3668.60 -Turner David ese 15949.44 -Vann cally 30198.72 -Velez Joe able 21086.80 -Ventura Anthony ese 2567648.55 -Waldron eing 32262.56 -Walker Alberto cally 212031.82 -Williams ese 135988.65 -Williams Inez cally 3602.50 -Winston Christina ese 519623.82 -Wolfe Dennis cally 8070092.80 -Wolff Paul anti 96933.27 -Wood Toni n st 5554.08 +Baker Letha ought 6995.34 +Kennedy Beatrice bar 215.68 +West William n st 375.42 diff --git a/regression-test/data/tpcds_sf1_p1/sql/q24_1.out b/regression-test/data/tpcds_sf1_p1/sql/q24_1.out index b5b3781427f2e7..a87a3f6eb2f092 100644 --- a/regression-test/data/tpcds_sf1_p1/sql/q24_1.out +++ b/regression-test/data/tpcds_sf1_p1/sql/q24_1.out @@ -1,10 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_1 -- - Tommy able 38118.08 -Holt Curtis able 8225.80 -Kunz Lee able 34631.52 -Littlefield Clarence able 127380.00 -Pettit Richard able 3930.52 -Townsend Franklin able 68983.20 -Winchester Margaret bar 14269.20 diff --git a/regression-test/data/tpcds_sf1_p1/sql/q24_2.out b/regression-test/data/tpcds_sf1_p1/sql/q24_2.out index 3dd7690d52b3e7..04afb1f4cd59ca 100644 --- a/regression-test/data/tpcds_sf1_p1/sql/q24_2.out +++ b/regression-test/data/tpcds_sf1_p1/sql/q24_2.out @@ -1,4 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_2 -- -Griffith Ray able 161564.48 diff --git a/regression-test/data/tpcds_sf1_unique_p1/sql/q24_1.out b/regression-test/data/tpcds_sf1_unique_p1/sql/q24_1.out index b5b3781427f2e7..a87a3f6eb2f092 100644 --- a/regression-test/data/tpcds_sf1_unique_p1/sql/q24_1.out +++ b/regression-test/data/tpcds_sf1_unique_p1/sql/q24_1.out @@ -1,10 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_1 -- - Tommy able 38118.08 -Holt Curtis able 8225.80 -Kunz Lee able 34631.52 -Littlefield Clarence able 127380.00 -Pettit Richard able 3930.52 -Townsend Franklin able 68983.20 -Winchester Margaret bar 14269.20 diff --git a/regression-test/data/tpcds_sf1_unique_p1/sql/q24_2.out b/regression-test/data/tpcds_sf1_unique_p1/sql/q24_2.out index 3dd7690d52b3e7..04afb1f4cd59ca 100644 --- a/regression-test/data/tpcds_sf1_unique_p1/sql/q24_2.out +++ b/regression-test/data/tpcds_sf1_unique_p1/sql/q24_2.out @@ -1,4 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !q24_2 -- -Griffith Ray able 161564.48 diff --git a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_1.sql b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_1.sql index 65571698e1ecc7..7dd33b66803098 100644 --- a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_1.sql +++ b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_1.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_2.sql b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_2.sql index db04507c9c98c8..90770946c0c420 100644 --- a/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_2.sql +++ b/regression-test/suites/inverted_index_p1/tpcds_sf1_index/sql/q24_2.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_1.sql b/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_1.sql index 71711c8579cf4c..d504d784da9237 100644 --- a/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_1.sql +++ b/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_1.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + and c_current_addr_sk = ca_address_sk + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_2.sql b/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_2.sql index b8f7099eb47dff..c4efb602430c20 100644 --- a/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_2.sql +++ b/regression-test/suites/tpcds_sf100_dup_without_key_p2/sql/q24_2.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf100_p2/sql/q24_1.sql b/regression-test/suites/tpcds_sf100_p2/sql/q24_1.sql index 71711c8579cf4c..89b4e0b9e4935a 100644 --- a/regression-test/suites/tpcds_sf100_p2/sql/q24_1.sql +++ b/regression-test/suites/tpcds_sf100_p2/sql/q24_1.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf100_p2/sql/q24_2.sql b/regression-test/suites/tpcds_sf100_p2/sql/q24_2.sql index b8f7099eb47dff..c4efb602430c20 100644 --- a/regression-test/suites/tpcds_sf100_p2/sql/q24_2.sql +++ b/regression-test/suites/tpcds_sf100_p2/sql/q24_2.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf1_p1/sql/q24_1.sql b/regression-test/suites/tpcds_sf1_p1/sql/q24_1.sql index 65571698e1ecc7..7dd33b66803098 100644 --- a/regression-test/suites/tpcds_sf1_p1/sql/q24_1.sql +++ b/regression-test/suites/tpcds_sf1_p1/sql/q24_1.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf1_p1/sql/q24_2.sql b/regression-test/suites/tpcds_sf1_p1/sql/q24_2.sql index db04507c9c98c8..90770946c0c420 100644 --- a/regression-test/suites/tpcds_sf1_p1/sql/q24_2.sql +++ b/regression-test/suites/tpcds_sf1_p1/sql/q24_2.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_1.sql b/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_1.sql index 65571698e1ecc7..7dd33b66803098 100644 --- a/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_1.sql +++ b/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_1.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size diff --git a/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_2.sql b/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_2.sql index db04507c9c98c8..90770946c0c420 100644 --- a/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_2.sql +++ b/regression-test/suites/tpcds_sf1_unique_p1/sql/q24_2.sql @@ -24,7 +24,8 @@ WITH AND (ss_customer_sk = c_customer_sk) AND (ss_item_sk = i_item_sk) AND (ss_store_sk = s_store_sk) - AND (c_birth_country = upper(ca_country)) + AND (c_current_addr_sk = ca_address_sk) + AND (c_birth_country <> upper(ca_country)) AND (s_zip = ca_zip) AND (s_market_id = 8) GROUP BY c_last_name, c_first_name, s_store_name, ca_state, s_state, i_color, i_current_price, i_manager_id, i_units, i_size From 653dd0674634c68943301416fd7f4250dfb04628 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 13 Aug 2025 16:16:29 +0800 Subject: [PATCH 423/572] branch-3.0: [feature](group by)Support group by with order. (#53037) (#53849) backport: https://github.com/apache/doris/pull/53037 --- .../org/apache/doris/nereids/DorisParser.g4 | 6 +- .../nereids/parser/GroupKeyWithOrder.java | 86 ++++++++++++++ .../nereids/parser/LogicalPlanBuilder.java | 45 ++++++-- .../nereids/parser/NereidsParserTest.java | 54 +++++++++ .../test_nereids_group_by_with_order.out | 106 ++++++++++++++++++ .../test_nereids_group_by_with_order.groovy | 89 +++++++++++++++ 6 files changed, 378 insertions(+), 8 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/parser/GroupKeyWithOrder.java create mode 100644 regression-test/data/nereids_syntax_p0/test_nereids_group_by_with_order.out create mode 100644 regression-test/suites/nereids_syntax_p0/test_nereids_group_by_with_order.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 4cc2eae2e1b85d..7b039284dc8ab5 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1192,6 +1192,10 @@ relationHint | HINT_START identifier (COMMA identifier)* HINT_END #commentRelationHint ; +expressionWithOrder + : expression ordering = (ASC | DESC)? + ; + aggClause : GROUP BY groupingElement ; @@ -1200,7 +1204,7 @@ groupingElement : ROLLUP LEFT_PAREN (expression (COMMA expression)*)? RIGHT_PAREN | CUBE LEFT_PAREN (expression (COMMA expression)*)? RIGHT_PAREN | GROUPING SETS LEFT_PAREN groupingSet (COMMA groupingSet)* RIGHT_PAREN - | expression (COMMA expression)* (WITH ROLLUP)? + | expressionWithOrder (COMMA expressionWithOrder)* (WITH ROLLUP)? ; groupingSet diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/GroupKeyWithOrder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/GroupKeyWithOrder.java new file mode 100644 index 00000000000000..f287825e12a365 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/GroupKeyWithOrder.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.parser; + +import org.apache.doris.nereids.trees.expressions.Expression; + +import java.util.Objects; + +/** + * Represents the group by expression with order of a statement. + */ +public class GroupKeyWithOrder { + + private final Expression expr; + + // Order is ascending. + private final boolean hasOrder; + + private final boolean isAsc; + + /** + * Constructor of GroupKeyWithOrder. + */ + public GroupKeyWithOrder(Expression expr, boolean hasOrder, boolean isAsc) { + this.expr = expr; + this.hasOrder = hasOrder; + this.isAsc = isAsc; + } + + public Expression getExpr() { + return expr; + } + + public boolean isAsc() { + return isAsc; + } + + public boolean hasOrder() { + return hasOrder; + } + + public GroupKeyWithOrder withExpression(Expression expr) { + return new GroupKeyWithOrder(expr, isAsc, hasOrder); + } + + public String toSql() { + return expr.toSql() + (hasOrder ? (isAsc ? " asc" : " desc") : ""); + } + + @Override + public String toString() { + return expr.toString() + (hasOrder ? (isAsc ? " asc" : " desc") : ""); + } + + @Override + public int hashCode() { + return Objects.hash(expr, isAsc, hasOrder); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + GroupKeyWithOrder that = (GroupKeyWithOrder) o; + return isAsc == that.isAsc() && hasOrder == that.hasOrder() && expr.equals(that.getExpr()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 39dc52a03fd1f1..36d69f1bcbe9d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -101,6 +101,7 @@ import org.apache.doris.nereids.DorisParser.ExistContext; import org.apache.doris.nereids.DorisParser.ExplainContext; import org.apache.doris.nereids.DorisParser.ExportContext; +import org.apache.doris.nereids.DorisParser.ExpressionWithOrderContext; import org.apache.doris.nereids.DorisParser.FixedPartitionDefContext; import org.apache.doris.nereids.DorisParser.FromClauseContext; import org.apache.doris.nereids.DorisParser.GroupingElementContext; @@ -2672,6 +2673,16 @@ public OrderKey visitSortItem(SortItemContext ctx) { }); } + @Override + public GroupKeyWithOrder visitExpressionWithOrder(ExpressionWithOrderContext ctx) { + return ParserUtils.withOrigin(ctx, () -> { + boolean hasOrder = ctx.ASC() != null || ctx.DESC() != null; + boolean isAsc = ctx.DESC() == null; + Expression expression = typedVisit(ctx.expression()); + return new GroupKeyWithOrder(expression, hasOrder, isAsc); + }); + } + private List visit(List contexts, Class clazz) { return contexts.stream() .map(this::visit) @@ -3164,8 +3175,10 @@ protected LogicalPlan withSelectQuerySpecification( // from -> where -> group by -> having -> select LogicalPlan filter = withFilter(inputRelation, whereClause); SelectColumnClauseContext selectColumnCtx = selectClause.selectColumnClause(); - LogicalPlan aggregate = withAggregate(filter, selectColumnCtx, aggClause); + List orderKeys = Lists.newArrayList(); + LogicalPlan aggregate = withAggregate(filter, selectColumnCtx, aggClause, orderKeys); boolean isDistinct = (selectClause.DISTINCT() != null); + LogicalPlan selectPlan; if (!(aggregate instanceof Aggregate) && havingClause.isPresent()) { // create a project node for pattern match of ProjectToGlobalAggregate rule // then ProjectToGlobalAggregate rule can insert agg node as LogicalHaving node's child @@ -3181,12 +3194,16 @@ protected LogicalPlan withSelectQuerySpecification( List projects = getNamedExpressions(selectColumnCtx.namedExpressionSeq()); project = new LogicalProject<>(projects, ImmutableList.of(), isDistinct, aggregate); } - return new LogicalHaving<>(ExpressionUtils.extractConjunctionToSet( + selectPlan = new LogicalHaving<>(ExpressionUtils.extractConjunctionToSet( getExpression((havingClause.get().booleanExpression()))), project); } else { LogicalPlan having = withHaving(aggregate, havingClause); - return withProjection(having, selectColumnCtx, aggClause, isDistinct); + selectPlan = withProjection(having, selectColumnCtx, aggClause, isDistinct); + } + if (!orderKeys.isEmpty()) { + selectPlan = new LogicalSort<>(orderKeys, selectPlan); } + return selectPlan; }); } @@ -3446,7 +3463,7 @@ private LogicalPlan withFilter(LogicalPlan input, Optional w } private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext selectCtx, - Optional aggCtx) { + Optional aggCtx, List orderKeys) { return input.optionalMap(aggCtx, () -> { GroupingElementContext groupingElementContext = aggCtx.get().groupingElement(); List namedExpressions = getNamedExpressions(selectCtx.namedExpressionSeq()); @@ -3460,13 +3477,27 @@ private LogicalPlan withAggregate(LogicalPlan input, SelectColumnClauseContext s List cubeExpressions = visit(groupingElementContext.expression(), Expression.class); List> groupingSets = ExpressionUtils.cubeToGroupingSets(cubeExpressions); return new LogicalRepeat<>(groupingSets, namedExpressions, input); - } else if (groupingElementContext.ROLLUP() != null) { + } else if (groupingElementContext.ROLLUP() != null && groupingElementContext.WITH() == null) { List rollupExpressions = visit(groupingElementContext.expression(), Expression.class); List> groupingSets = ExpressionUtils.rollupToGroupingSets(rollupExpressions); return new LogicalRepeat<>(groupingSets, namedExpressions, input); } else { - List groupByExpressions = visit(groupingElementContext.expression(), Expression.class); - return new LogicalAggregate<>(groupByExpressions, namedExpressions, input); + List groupKeyWithOrders = visit(groupingElementContext.expressionWithOrder(), + GroupKeyWithOrder.class); + ImmutableList groupByExpressions = groupKeyWithOrders.stream() + .map(GroupKeyWithOrder::getExpr) + .collect(ImmutableList.toImmutableList()); + if (groupKeyWithOrders.stream().anyMatch(GroupKeyWithOrder::hasOrder)) { + groupKeyWithOrders.stream() + .map(e -> new OrderKey(e.getExpr(), e.isAsc(), e.isAsc())) + .forEach(orderKeys::add); + } + if (groupingElementContext.ROLLUP() != null) { + List> groupingSets = ExpressionUtils.rollupToGroupingSets(groupByExpressions); + return new LogicalRepeat<>(groupingSets, namedExpressions, input); + } else { + return new LogicalAggregate<>(groupByExpressions, namedExpressions, input); + } } }); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index e7b3349c558363..27f19ca6999eee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -39,6 +39,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateType; import org.apache.doris.nereids.types.DecimalV2Type; @@ -696,4 +698,56 @@ public void testBlockSqlAst() { Assertions.fail(ex); } } + + private void checkQueryTopPlanClass(String sql, NereidsParser parser, Class clazz) { + if (clazz == null) { + Assertions.assertThrows(ParseException.class, () -> parser.parseSingle(sql)); + } else { + LogicalPlan logicalPlan = parser.parseSingle(sql); + Assertions.assertInstanceOf(clazz, logicalPlan.child(0)); + } + } + + @Test + public void testExpressionWithOrder() { + NereidsParser nereidsParser = new NereidsParser(); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a, b DESC", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a DESC, b", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a ASC, b", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a, b ASC", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a ASC, b ASC", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a DESC, b DESC", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a ASC, b DESC", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a DESC, b ASC", + nereidsParser, LogicalSort.class); + + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a, b DESC WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a DESC, b WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a ASC, b WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a, b ASC WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a ASC, b ASC WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a DESC, b DESC WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a ASC, b DESC WITH ROLLUP", + nereidsParser, LogicalSort.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a DESC, b ASC WITH ROLLUP", + nereidsParser, LogicalSort.class); + + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a, b", + nereidsParser, LogicalAggregate.class); + checkQueryTopPlanClass("SELECT a, b, sum(c) from test group by a, b WITH ROLLUP", + nereidsParser, LogicalRepeat.class); + } } diff --git a/regression-test/data/nereids_syntax_p0/test_nereids_group_by_with_order.out b/regression-test/data/nereids_syntax_p0/test_nereids_group_by_with_order.out new file mode 100644 index 00000000000000..95f3984452ce3b --- /dev/null +++ b/regression-test/data/nereids_syntax_p0/test_nereids_group_by_with_order.out @@ -0,0 +1,106 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1 -- +1 3 +2 8 +3 9 + +-- !select2 -- +3 9 +2 8 +1 3 + +-- !select3 -- +\N 20 +1 3 +2 8 +3 9 + +-- !select4 -- +3 9 +2 8 +1 3 +\N 20 + +-- !select5 -- +1 1 1 +1 0 \N +2 2 2 +3 3 3 +3 0 \N + +-- !select6 -- +1 1 1 +1 0 \N +2 2 2 +3 3 3 +3 0 \N + +-- !select7 -- +\N \N 1 +1 1 1 +1 0 \N +1 \N 1 +2 2 2 +2 \N 2 +3 3 3 +3 0 \N +3 \N 3 + +-- !select8 -- +\N \N 1 +1 1 1 +1 0 \N +1 \N 1 +2 2 2 +2 \N 2 +3 3 3 +3 0 \N +3 \N 3 + +-- !select9 -- +1 1 1 10 +1 0 \N 4 +2 2 2 18 +3 3 3 10 +3 0 \N 5 + +-- !select10 -- +3 0 \N 5 +3 3 3 10 +2 2 2 18 +1 0 \N 4 +1 1 1 10 + +-- !select11 -- +\N \N 1 47 +1 1 1 10 +1 0 \N 4 +1 \N 1 14 +2 2 2 18 +2 \N 2 18 +3 3 3 10 +3 0 \N 5 +3 \N 3 15 + +-- !select12 -- +\N \N 1 47 +1 1 1 10 +1 0 \N 4 +1 \N 1 14 +2 2 2 18 +2 \N 2 18 +3 3 3 10 +3 0 \N 5 +3 \N 3 15 + +-- !select13 -- +3 \N 3 15 +2 \N 2 18 +1 \N 1 14 +\N \N 1 47 +3 0 \N 5 +1 0 \N 4 +1 1 1 10 +2 2 2 18 +3 3 3 10 + diff --git a/regression-test/suites/nereids_syntax_p0/test_nereids_group_by_with_order.groovy b/regression-test/suites/nereids_syntax_p0/test_nereids_group_by_with_order.groovy new file mode 100644 index 00000000000000..8600ab35cacb69 --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/test_nereids_group_by_with_order.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_nereids_group_by_with_order") { + sql "SET enable_nereids_planner=true" + + sql "DROP TABLE IF EXISTS testGroupByWithOrder" + + sql """ + CREATE TABLE `testGroupByWithOrder` ( + `k1` bigint(20) NULL, + `k2` bigint(20) NULL, + `k3` bigint(20) NULL, + `k4` bigint(20) not null, + `k5` varchar(10), + `k6` varchar(10) + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k2`) BUCKETS 1 + PROPERTIES ('replication_num' = '1') + """ + + sql """ + INSERT INTO testGroupByWithOrder VALUES + (1, 1, 1, 3, 'a', 'b'), + (1, 1, 2, 3, 'a', 'c'), + (1, 1, 3, 4, 'a' , 'd'), + (1, 0, null, 4, 'b' , 'b'), + (2, 2, 2, 5, 'b', 'c'), + (2, 2, 4, 5, 'b' , 'd'), + (2, 2, 6, 4, 'c', 'b'), + (2, 2, null, 4, 'c', 'c'), + (3, 3, 3, 3, 'c', 'd'), + (3, 3, 6, 3, 'd', 'b'), + (3, 3, 9, 4, 'd', 'c'), + (3, 0, null, 5, 'd', 'd') + """ + + qt_select1 """select k1, sum(k2) from testGroupByWithOrder group by k1 asc""" + qt_select2 """select k1, sum(k2) from testGroupByWithOrder group by k1 desc""" + qt_select3 """select k1, sum(k2) from testGroupByWithOrder group by k1 asc with rollup""" + qt_select4 """select k1, sum(k2) from testGroupByWithOrder group by k1 desc with rollup""" + + qt_select5 """select k1, k2, min(k3) from testGroupByWithOrder group by k1 asc, k2 desc""" + qt_select6 """select k1, k2, min(k3) from testGroupByWithOrder group by k1, k2 desc""" + qt_select7 """select k1, k2, min(k3) from testGroupByWithOrder group by k1 asc, k2 desc with rollup""" + qt_select8 """select k1, k2, min(k3) from testGroupByWithOrder group by k1, k2 desc with rollup""" + + qt_select9 """select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1 asc, k2 desc having s > 2""" + qt_select10 """select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1 desc, k2 having s > 2""" + qt_select11 """select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1 asc, k2 desc with rollup having s > 2""" + qt_select12 """select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1, k2 desc with rollup having s > 2""" + qt_select13 """select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1, k2 desc with rollup having s > 2 order by k2, k1 desc""" + + explain { + sql("select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1, k2 having s > 2") + notContains "VSORT" + } + + explain { + sql("select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1 desc, k2 having s > 2") + contains "VSORT" + } + + explain { + sql("select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1, k2 with rollup having s > 2") + notContains "VSORT" + } + + explain { + sql("select k1, k2, min(k3) as m, sum(k4) as s from testGroupByWithOrder group by k1 desc, k2 with rollup having s > 2") + contains "VSORT" + } + +} From 803e0cad89e5c3484308f21fdb57e721596997ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 13 Aug 2025 16:22:48 +0800 Subject: [PATCH 424/572] branch-3.0: [test](inverted index) fix case segment count assert #54625 (#54644) Cherry-picked from #54625 Co-authored-by: Jack --- .../inverted_index_p1/show_data/test_show_index_data.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy b/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy index 9f7de0dc6c762e..bf946d57f625b9 100644 --- a/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy +++ b/regression-test/suites/inverted_index_p1/show_data/test_show_index_data.groovy @@ -129,7 +129,7 @@ suite("test_show_index_data", "p1") { for (String rowset in (List) tabletJson.rowsets) { beforeSegmentCount += Integer.parseInt(rowset.split(" ")[1]) } - assertEquals(beforeSegmentCount, 10) + assertTrue(beforeSegmentCount >= 10) } // trigger compactions for all tablets in ${tableName} @@ -170,7 +170,7 @@ suite("test_show_index_data", "p1") { logger.info("rowset is: " + rowset) afterSegmentCount += Integer.parseInt(rowset.split(" ")[1]) } - assertEquals(afterSegmentCount, 1) + assertTrue(afterSegmentCount >= 1) } } From 32eb7bbca8d35fcd57d655162833e9a4ed80b105 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Wed, 13 Aug 2025 22:29:19 +0800 Subject: [PATCH 425/572] branch-30: [Bug](ipv6) fix fe can't parse ipv4-mapped type to ipv6 (#54391) (#54686) ### What problem does this PR solve? Problem Summary: cherry-pick from master (#54391) --- .../apache/doris/analysis/IPv6Literal.java | 6 ++- .../expressions/literal/IPv6Literal.java | 23 ++++++++-- .../expressions/literal/IPV6LiteralTest.java | 43 +++++++++++++++++++ .../data/datatype_p0/ip/test_ip_crud.out | 15 +++++++ .../suites/datatype_p0/ip/test_ip_crud.groovy | 34 +++++++++++++++ 5 files changed, 117 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IPV6LiteralTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java index 7d719720442505..8648b2be867d22 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IPv6Literal.java @@ -41,6 +41,9 @@ public class IPv6Literal extends LiteralExpr { Pattern.compile("^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$"); private static final Pattern IPV6_COMPRESS_REGEX = Pattern.compile("^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::((([0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4})?)$"); + private static final String IPV4_PART = "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"; + private static final Pattern IPV6_MAPPED_REGEX = + Pattern.compile("^::[fF]{4}:(" + IPV4_PART + "\\.){3}" + IPV4_PART + "$"); @SerializedName("v") private String value; @@ -75,7 +78,8 @@ public void checkValueValid() throws AnalysisException { private void checkValueValid(String ipv6) throws AnalysisException { if (ipv6.length() > 39) { throw new AnalysisException("The length of IPv6 must not exceed 39. type: " + Type.IPV6); - } else if (!IPV6_STD_REGEX.matcher(ipv6).matches() && !IPV6_COMPRESS_REGEX.matcher(ipv6).matches()) { + } else if (!IPV6_STD_REGEX.matcher(ipv6).matches() && !IPV6_COMPRESS_REGEX.matcher(ipv6).matches() + && !IPV6_MAPPED_REGEX.matcher(ipv6).matches()) { throw new AnalysisException("Invalid IPv6 format: " + ipv6 + ". type: " + Type.IPV6); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java index 0ac12ae85e477a..a869541a41720f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IPv6Literal.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.IPv6Type; +import com.google.common.annotations.VisibleForTesting; + import java.util.regex.Pattern; /** @@ -33,6 +35,9 @@ public class IPv6Literal extends Literal { Pattern.compile("^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$"); private static final Pattern IPV6_COMPRESS_REGEX = Pattern.compile("^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::((([0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4})?)$"); + private static final String IPV4_PART = "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"; + private static final Pattern IPV6_MAPPED_REGEX = + Pattern.compile("^::[fF]{4}:(" + IPV4_PART + "\\.){3}" + IPV4_PART + "$"); private final String value; @@ -61,11 +66,23 @@ public LiteralExpr toLegacyLiteral() { } } + /** + * check IPv6 is valid + */ public void checkValueValid(String ipv6) throws AnalysisException { if (ipv6.length() > 39) { - throw new AnalysisException("The length of IPv6 must not exceed 39."); - } else if (!IPV6_STD_REGEX.matcher(ipv6).matches() && !IPV6_COMPRESS_REGEX.matcher(ipv6).matches()) { - throw new AnalysisException("Invalid IPv6 format."); + throw new AnalysisException("The length of IPv6 must not exceed 39: " + ipv6); + } else if (!isValidIPv6(ipv6)) { + throw new AnalysisException("Invalid IPv6 format: " + ipv6); + } + } + + @VisibleForTesting + public static boolean isValidIPv6(String ipv6) { + if (IPV6_STD_REGEX.matcher(ipv6).matches() || IPV6_COMPRESS_REGEX.matcher(ipv6).matches() + || IPV6_MAPPED_REGEX.matcher(ipv6).matches()) { + return true; } + return false; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IPV6LiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IPV6LiteralTest.java new file mode 100644 index 00000000000000..038608608c6ce5 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/IPV6LiteralTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class IPV6LiteralTest { + + @Test + public void testValidIPv6() { + Assertions.assertTrue(IPv6Literal.isValidIPv6("2001:0db8:85a3:0000:0000:8a2e:0370:7334")); + Assertions.assertTrue(IPv6Literal.isValidIPv6("fe80::1")); + Assertions.assertTrue(IPv6Literal.isValidIPv6("2001:db8::8a2e:370:7334")); + Assertions.assertTrue(IPv6Literal.isValidIPv6("::1")); + Assertions.assertTrue(IPv6Literal.isValidIPv6("::ffff:192.168.1.1")); + Assertions.assertTrue(IPv6Literal.isValidIPv6("::FFFF:10.0.0.255")); + } + + @Test + public void testInvalidIPv6() { + Assertions.assertFalse(IPv6Literal.isValidIPv6("2001:db8::8a2e::370")); + Assertions.assertFalse(IPv6Literal.isValidIPv6("2001:db8:xyz::1")); + Assertions.assertFalse(IPv6Literal.isValidIPv6("192.168.1.1")); + Assertions.assertFalse(IPv6Literal.isValidIPv6("::aaaa:192.168.1.1")); + Assertions.assertFalse(IPv6Literal.isValidIPv6("::FFFF:10.0.0.256")); + } +} diff --git a/regression-test/data/datatype_p0/ip/test_ip_crud.out b/regression-test/data/datatype_p0/ip/test_ip_crud.out index 07323da29e5bef..60297af92268a4 100644 --- a/regression-test/data/datatype_p0/ip/test_ip_crud.out +++ b/regression-test/data/datatype_p0/ip/test_ip_crud.out @@ -52,3 +52,18 @@ -- !sql15 -- 1 59.50.185.152 2a02:e980:83:5b09:ecb8:c669:b336:650e + +-- !sql16 -- +1 2025-08-05 1754386200 ::ffff:10.20.136.244 ::ffff:192.168.1.1 +2 2025-08-05 1754386200 ::ffff:11.20.136.244 ::ffff:192.168.1.2 +3 2025-08-05 1754386200 ::ffff:12.20.136.244 ::1 + +-- !sql17 -- +2 2025-08-05 1754386200 ::ffff:11.20.136.244 ::ffff:192.168.1.2 +3 2025-08-05 1754386200 ::ffff:12.20.136.244 ::1 + +-- !sql17 -- +3 2025-08-05 1754386200 ::ffff:12.20.136.244 ::1 + +-- !sql17 -- + diff --git a/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy b/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy index 255457fd2cf903..4cb5f1df9a0581 100644 --- a/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy +++ b/regression-test/suites/datatype_p0/ip/test_ip_crud.groovy @@ -98,4 +98,38 @@ suite("test_ip_crud") { qt_sql14 "select * from test_dup_ip_crud order by id" sql "delete from test_dup_ip_crud where ip_v6='2001:4888:1f:e891:161:26::'" qt_sql15 "select * from test_dup_ip_crud order by id" + + sql "DROP TABLE IF EXISTS log" + sql """ + CREATE TABLE IF NOT EXISTS log + ( + type INT, + day DATE NOT NULL, + timestamp BIGINT NOT NULL, + sip IPV6, + dip IPV6 + ) + DUPLICATE KEY (`type`, `day`, `timestamp`) + DISTRIBUTED BY HASH(`type`) BUCKETS 8 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql "INSERT INTO log VALUES (1, '2025-08-05', 1754386200, '::ffff:10.20.136.244', '::ffff:192.168.1.1');" + sql "INSERT INTO log VALUES (2, '2025-08-05', 1754386200, '::ffff:11.20.136.244', '::ffff:192.168.1.2');" + sql "INSERT INTO log VALUES (3, '2025-08-05', 1754386200, '::ffff:12.20.136.244', '::1');" + qt_sql16 "select * from log order by type;" + sql "delete from log where sip='::ffff:10.20.136.244';" + qt_sql17 "select * from log order by type;" + + sql "delete from log where sip='0000:0000:0000:0000:0000:FFFF:0B14:88F4';" + qt_sql17 "select * from log order by type;" + + sql "delete from log where dip='0000:0000:0000:0000:0000:0000:0000:0001';" + qt_sql17 "select * from log order by type;" + + sql "DROP TABLE test_unique_ip_crud" + sql "DROP TABLE test_agg_ip_crud" + sql "DROP TABLE test_dup_ip_crud" + sql "DROP TABLE log" } From 4ac93192aef5c3a61f84eb2f20e61ff6cc9a0b53 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 09:20:40 +0800 Subject: [PATCH 426/572] branch-3.0: [Fix](case) fix auto&dynamic partition case #54632 (#54712) Cherry-picked from #54632 Co-authored-by: zclllyybb --- .../partition_p0/auto_partition/test_auto_dynamic.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy b/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy index 2c94b41cbb141e..ad6bc3db160d05 100644 --- a/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy +++ b/regression-test/suites/partition_p0/auto_partition/test_auto_dynamic.groovy @@ -93,7 +93,7 @@ suite("test_auto_dynamic", "nonConcurrent") { ); """ def part_result = sql " show partitions from auto_dynamic " - assertEquals(part_result.size, 6) + assertEquals(part_result.size(), 6) sql " drop table if exists auto_dynamic " sql """ @@ -114,7 +114,7 @@ suite("test_auto_dynamic", "nonConcurrent") { ); """ part_result = sql " show partitions from auto_dynamic " - assertEquals(part_result.size, 1) + assertEquals(part_result.size(), 1) def skip_test = false test { @@ -137,7 +137,8 @@ suite("test_auto_dynamic", "nonConcurrent") { sleep(10000) part_result = sql " show partitions from auto_dynamic " log.info("${part_result}".toString()) - assertEquals(part_result.size, 3) + assertTrue(part_result.size() == 3 || part_result.size() == 4, + "The partition size should be 3 or 4, but got ${part_result.size()}") qt_sql_dynamic_auto "select * from auto_dynamic order by k0;" From 2a23e4f181e4c10a2bfe252e191478581dbb4a26 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Thu, 14 Aug 2025 09:29:30 +0800 Subject: [PATCH 427/572] branch-3.0: [opt](filecache) Let base compaction output rowsets write filecache when cache size is enough (#53801) (#54693) --- be/src/common/config.cpp | 6 + be/src/common/config.h | 1 + be/src/io/cache/block_file_cache.h | 6 + be/src/io/fs/broker_file_writer.h | 1 - be/src/io/fs/file_writer.h | 37 +++- be/src/io/fs/hdfs_file_writer.cpp | 8 +- be/src/io/fs/hdfs_file_writer.h | 6 - be/src/io/fs/local_file_writer.h | 3 - be/src/io/fs/s3_file_writer.cpp | 9 +- be/src/io/fs/s3_file_writer.h | 6 - be/src/io/fs/stream_sink_file_writer.h | 3 - be/src/olap/compaction.cpp | 2 + be/src/olap/rowset/rowset_writer_context.h | 6 +- be/test/olap/tablet_cooldown_test.cpp | 2 - ...test_filecache_with_base_compaction.groovy | 195 ++++++++++++++++++ 15 files changed, 252 insertions(+), 39 deletions(-) create mode 100644 regression-test/suites/compaction/test_filecache_with_base_compaction.groovy diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 491929a7a83362..0989fd38f18471 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1122,7 +1122,13 @@ DEFINE_mBool(enbale_dump_error_file, "false"); DEFINE_mInt64(file_cache_error_log_limit_bytes, "209715200"); // 200MB DEFINE_mInt64(cache_lock_wait_long_tail_threshold_us, "30000000"); DEFINE_mInt64(cache_lock_held_long_tail_threshold_us, "30000000"); + +// enable_file_cache_keep_base_compaction_output true means force base compaction output rowsets +// write to file cache, enable_file_cache_adaptive_write true means when file cache is enough, it +// will write to file cache; satisfying any of the two conditions will write to file cache. DEFINE_mBool(enable_file_cache_keep_base_compaction_output, "false"); +DEFINE_mBool(enable_file_cache_adaptive_write, "true"); + DEFINE_mInt64(file_cache_remove_block_qps_limit, "1000"); DEFINE_mInt64(file_cache_background_gc_interval_ms, "100"); DEFINE_mBool(enable_reader_dryrun_when_download_file_cache, "true"); diff --git a/be/src/common/config.h b/be/src/common/config.h index c664fd75f0d2d6..a36f185a2c2206 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1162,6 +1162,7 @@ DECLARE_mInt64(cache_lock_held_long_tail_threshold_us); // If your file cache is ample enough to accommodate all the data in your database, // enable this option; otherwise, it is recommended to leave it disabled. DECLARE_mBool(enable_file_cache_keep_base_compaction_output); +DECLARE_mBool(enable_file_cache_adaptive_write); DECLARE_mInt64(file_cache_remove_block_qps_limit); DECLARE_mInt64(file_cache_background_gc_interval_ms); DECLARE_mBool(enable_reader_dryrun_when_download_file_cache); diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index 7c046cc162783a..0e3d26ab60590f 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -359,6 +360,11 @@ class BlockFileCache { using QueryFileCacheContextHolderPtr = std::unique_ptr; QueryFileCacheContextHolderPtr get_query_context_holder(const TUniqueId& query_id); + int64_t approximate_available_cache_size() const { + return std::max( + _cache_capacity_metrics->get_value() - _cur_cache_size_metrics->get_value(), 0); + } + private: struct FileBlockCell { FileBlockSPtr file_block; diff --git a/be/src/io/fs/broker_file_writer.h b/be/src/io/fs/broker_file_writer.h index 173f1df108850f..6c46ee2d2faa13 100644 --- a/be/src/io/fs/broker_file_writer.h +++ b/be/src/io/fs/broker_file_writer.h @@ -49,7 +49,6 @@ class BrokerFileWriter final : public FileWriter { const Path& path() const override { return _path; } size_t bytes_appended() const override { return _cur_offset; } State state() const override { return _state; } - FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } private: Status _write(const uint8_t* buf, size_t buf_len, size_t* written_bytes); diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h index c388dcd40d8d3f..9ff06bf5a37b68 100644 --- a/be/src/io/fs/file_writer.h +++ b/be/src/io/fs/file_writer.h @@ -22,6 +22,9 @@ #include "common/status.h" #include "gutil/macros.h" +#include "io/cache/block_file_cache.h" +#include "io/cache/block_file_cache_factory.h" +#include "io/cache/file_cache_common.h" #include "io/fs/file_reader.h" #include "io/fs/file_reader_writer_fwd.h" #include "io/fs/path.h" @@ -43,8 +46,9 @@ struct FileWriterOptions { bool used_by_s3_committer = false; bool write_file_cache = false; bool is_cold_data = false; - bool sync_file_data = true; // Whether flush data into storage system - uint64_t file_cache_expiration = 0; // Absolute time + bool sync_file_data = true; // Whether flush data into storage system + uint64_t file_cache_expiration = 0; // Absolute time + uint64_t approximate_bytes_to_write = 0; // Approximate bytes to write, used for file cache }; struct AsyncCloseStatusPack { @@ -79,7 +83,34 @@ class FileWriter { virtual State state() const = 0; - virtual FileCacheAllocatorBuilder* cache_builder() const = 0; + FileCacheAllocatorBuilder* cache_builder() const { + return _cache_builder == nullptr ? nullptr : _cache_builder.get(); + } + +protected: + void init_cache_builder(const FileWriterOptions* opts, const Path& path) { + if (!config::enable_file_cache || opts == nullptr) { + return; + } + + io::UInt128Wrapper path_hash = BlockFileCache::hash(path.filename().native()); + BlockFileCache* file_cache_ptr = FileCacheFactory::instance()->get_by_path(path_hash); + + bool has_enough_file_cache_space = config::enable_file_cache_adaptive_write && + (opts->approximate_bytes_to_write > 0) && + (file_cache_ptr->approximate_available_cache_size() > + opts->approximate_bytes_to_write); + + if (opts->write_file_cache || has_enough_file_cache_space) { + _cache_builder = std::make_unique(FileCacheAllocatorBuilder { + opts ? opts->is_cold_data : false, opts ? opts->file_cache_expiration : 0, + path_hash, file_cache_ptr}); + } + return; + } + + std::unique_ptr _cache_builder = + nullptr; // nullptr if disable write file cache }; } // namespace doris::io diff --git a/be/src/io/fs/hdfs_file_writer.cpp b/be/src/io/fs/hdfs_file_writer.cpp index ff68d1c837ae92..2f3863179ee4f2 100644 --- a/be/src/io/fs/hdfs_file_writer.cpp +++ b/be/src/io/fs/hdfs_file_writer.cpp @@ -145,13 +145,7 @@ HdfsFileWriter::HdfsFileWriter(Path path, std::shared_ptr handler, _fs_name(std::move(fs_name)), _sync_file_data(opts ? opts->sync_file_data : true), _batch_buffer(MB * config::hdfs_write_batch_buffer_size_mb) { - if (config::enable_file_cache && opts != nullptr && opts->write_file_cache) { - _cache_builder = std::make_unique(FileCacheAllocatorBuilder { - opts ? opts->is_cold_data : false, opts ? opts->file_cache_expiration : 0, - BlockFileCache::hash(_path.filename().native()), - FileCacheFactory::instance()->get_by_path( - BlockFileCache::hash(_path.filename().native()))}); - } + init_cache_builder(opts, _path); hdfs_file_writer_total << 1; TEST_SYNC_POINT("HdfsFileWriter"); diff --git a/be/src/io/fs/hdfs_file_writer.h b/be/src/io/fs/hdfs_file_writer.h index 25835658f53eb2..fb6b778a2fff95 100644 --- a/be/src/io/fs/hdfs_file_writer.h +++ b/be/src/io/fs/hdfs_file_writer.h @@ -52,10 +52,6 @@ class HdfsFileWriter final : public FileWriter { Status close(bool non_block = false) override; - FileCacheAllocatorBuilder* cache_builder() const override { - return _cache_builder == nullptr ? nullptr : _cache_builder.get(); - } - private: Status _close_impl(); // Flush buffered data into HDFS client and write local file cache if enabled @@ -73,8 +69,6 @@ class HdfsFileWriter final : public FileWriter { std::string _fs_name; size_t _bytes_appended = 0; bool _sync_file_data; - std::unique_ptr - _cache_builder; // nullptr if disable write file cache class BatchBuffer { public: BatchBuffer(size_t capacity); diff --git a/be/src/io/fs/local_file_writer.h b/be/src/io/fs/local_file_writer.h index c27ebb931a8e99..66456b325b6afe 100644 --- a/be/src/io/fs/local_file_writer.h +++ b/be/src/io/fs/local_file_writer.h @@ -35,9 +35,6 @@ class LocalFileWriter final : public FileWriter { const Path& path() const override { return _path; } size_t bytes_appended() const override; State state() const override { return _state; } - - FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } - Status close(bool non_block = false) override; private: diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 5eabeea888d896..756f83c78da1fe 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -61,13 +61,8 @@ S3FileWriter::S3FileWriter(std::shared_ptr client, std::string s3_file_writer_total << 1; s3_file_being_written << 1; Aws::Http::SetCompliantRfc3986Encoding(true); - if (config::enable_file_cache && opts != nullptr && opts->write_file_cache) { - _cache_builder = std::make_unique(FileCacheAllocatorBuilder { - opts ? opts->is_cold_data : false, opts ? opts->file_cache_expiration : 0, - BlockFileCache::hash(_obj_storage_path_opts.path.filename().native()), - FileCacheFactory::instance()->get_by_path( - BlockFileCache::hash(_obj_storage_path_opts.path.filename().native()))}); - } + + init_cache_builder(opts, _obj_storage_path_opts.path); } S3FileWriter::~S3FileWriter() { diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 95ad52ddb67081..c02f7bc1c7f406 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -58,10 +58,6 @@ class S3FileWriter final : public FileWriter { size_t bytes_appended() const override { return _bytes_appended; } State state() const override { return _state; } - FileCacheAllocatorBuilder* cache_builder() const override { - return _cache_builder == nullptr ? nullptr : _cache_builder.get(); - } - const std::vector& completed_parts() const { return _completed_parts; } const std::string& key() const { return _obj_storage_path_opts.key; } @@ -103,8 +99,6 @@ class S3FileWriter final : public FileWriter { size_t _bytes_appended = 0; std::shared_ptr _pending_buf; - std::unique_ptr - _cache_builder; // nullptr if disable write file cache // S3 committer will start multipart uploading all files on BE side, // and then complete multipart upload these files on FE side. diff --git a/be/src/io/fs/stream_sink_file_writer.h b/be/src/io/fs/stream_sink_file_writer.h index 0950039077bd8c..f092319e7fa24c 100644 --- a/be/src/io/fs/stream_sink_file_writer.h +++ b/be/src/io/fs/stream_sink_file_writer.h @@ -53,9 +53,6 @@ class StreamSinkFileWriter final : public FileWriter { static Path dummy; return dummy; } - - FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } - Status close(bool non_block = false) override; private: diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index 77af2b30fe1b34..de0dcc79550cab 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1440,6 +1440,8 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext& (config::enable_file_cache_keep_base_compaction_output && compaction_type() == ReaderType::READER_BASE_COMPACTION); ctx.file_cache_ttl_sec = _tablet->ttl_seconds(); + ctx.approximate_bytes_to_write = _input_rowsets_total_size; + _output_rs_writer = DORIS_TRY(_tablet->create_rowset_writer(ctx, _is_vertical)); RETURN_IF_ERROR( _engine.meta_mgr().prepare_rowset(*_output_rs_writer->rowset_meta().get(), _uuid)); diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h index cb0fda83e60777..2862492d93cb7c 100644 --- a/be/src/olap/rowset/rowset_writer_context.h +++ b/be/src/olap/rowset/rowset_writer_context.h @@ -95,6 +95,7 @@ struct RowsetWriterContext { bool write_file_cache = false; bool is_hot_data = false; uint64_t file_cache_ttl_sec = 0; + uint64_t approximate_bytes_to_write = 0; /// end file cache opts // segcompaction for this RowsetWriter, disable it for some transient writers @@ -147,7 +148,10 @@ struct RowsetWriterContext { .is_cold_data = is_hot_data, .file_cache_expiration = file_cache_ttl_sec > 0 && newest_write_timestamp > 0 ? newest_write_timestamp + file_cache_ttl_sec - : 0}; + : 0, + .approximate_bytes_to_write = approximate_bytes_to_write, + }; + return opts; } }; diff --git a/be/test/olap/tablet_cooldown_test.cpp b/be/test/olap/tablet_cooldown_test.cpp index fbcbb4431315d9..fc167500f63e77 100644 --- a/be/test/olap/tablet_cooldown_test.cpp +++ b/be/test/olap/tablet_cooldown_test.cpp @@ -111,8 +111,6 @@ class FileWriterMock final : public io::FileWriter { const Path& path() const override { return _local_file_writer->path(); } - io::FileCacheAllocatorBuilder* cache_builder() const override { return nullptr; } - private: std::unique_ptr _local_file_writer; }; diff --git a/regression-test/suites/compaction/test_filecache_with_base_compaction.groovy b/regression-test/suites/compaction/test_filecache_with_base_compaction.groovy new file mode 100644 index 00000000000000..7c067afd76880c --- /dev/null +++ b/regression-test/suites/compaction/test_filecache_with_base_compaction.groovy @@ -0,0 +1,195 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.atomic.AtomicBoolean +import org.apache.doris.regression.suite.ClusterOptions +import org.codehaus.groovy.runtime.IOGroovyMethods +import org.apache.doris.regression.util.Http + +suite("test_filecache_with_base_compaction", "docker") { + def options = new ClusterOptions() + options.cloudMode = true + options.setFeNum(1) + options.setBeNum(1) + + options.beConfigs.add('enable_flush_file_cache_async=false') + options.beConfigs.add('file_cache_enter_disk_resource_limit_mode_percent=99') + options.beConfigs.add('enable_evict_file_cache_in_advance=false') + options.beConfigs.add('') + + def testTable = "test_filecache_with_base_compaction" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + def backendId_to_backendBrpcPort = [:] + + def triggerCumulativeCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def (code_1, out_1, err_1) = be_run_cumulative_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } + + def triggerBaseCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def (code_1, out_1, err_1) = be_run_base_compaction(be_host, be_http_port, tablet_id) + logger.info("Run compaction: code=" + code_1 + ", out=" + out_1 + ", err=" + err_1) + assertEquals(code_1, 0) + return out_1 + } + + def getTabletStatus = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/show?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get tablet status: =" + code + ", out=" + out) + assertEquals(code, 0) + def tabletStatus = parseJson(out.trim()) + return tabletStatus + } + + def waitForCompaction = { tablet -> + String tablet_id = tablet.TabletId + String trigger_backend_id = tablet.BackendId + def be_host = backendId_to_backendIP[trigger_backend_id] + def be_http_port = backendId_to_backendHttpPort[trigger_backend_id] + def running = true + do { + Thread.sleep(1000) + StringBuilder sb = new StringBuilder(); + sb.append("curl -X GET http://${be_host}:${be_http_port}") + sb.append("/api/compaction/run_status?tablet_id=") + sb.append(tablet_id) + + String command = sb.toString() + logger.info(command) + def process = command.execute() + def code = process.waitFor() + def out = process.getText() + logger.info("Get compaction status: code=" + code + ", out=" + out) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status + } while (running) + } + + docker(options) { + def fes = sql_return_maparray "show frontends" + logger.info("frontends: ${fes}") + def url = "jdbc:mysql://${fes[0].Host}:${fes[0].QueryPort}/" + logger.info("url: " + url) + + def result = sql 'SELECT DATABASE()' + + sql """ DROP TABLE IF EXISTS ${testTable} """ + + sql """ CREATE TABLE ${testTable} + ( + siteid INT DEFAULT '10', + citycode SMALLINT NOT NULL, + username VARCHAR(32) DEFAULT '', + pv BIGINT DEFAULT '0' + ) + DUPLICATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "disable_auto_compaction" = "true" + ) + """ + + // getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + getBackendIpHttpAndBrpcPort(backendId_to_backendIP, backendId_to_backendHttpPort, backendId_to_backendBrpcPort); + + def tablets = sql_return_maparray """ show tablets from ${testTable}; """ + logger.info("tablets: " + tablets) + assertEquals(1, tablets.size()) + def tablet = tablets[0] + String tablet_id = tablet.TabletId + + sql """ insert into ${testTable}(siteid, citycode, username, pv) values (1, 1, "xxx", 1); """ + sql """ delete from ${testTable} where siteid=2; """ + sql """ delete from ${testTable} where siteid=2; """ + sql """ delete from ${testTable} where siteid=2; """ + sql """ delete from ${testTable} where siteid=2; """ + sql """ delete from ${testTable} where siteid=2; """ + sql """ delete from ${testTable} where siteid=2; """ + sql """ delete from ${testTable} where siteid=2; """ + + sql """ sync """ + sql "select * from ${testTable}" + def tablet_status = getTabletStatus(tablet) + logger.info("tablet status: ${tablet_status}") + + triggerCumulativeCompaction(tablet) + waitForCompaction(tablet) + triggerCumulativeCompaction(tablet) + waitForCompaction(tablet) + triggerBaseCompaction(tablet) + waitForCompaction(tablet) + + tablet_status = getTabletStatus(tablet) + logger.info("tablet status: ${tablet_status}") + def base_compaction_finished = false + Set final_rowsets = new HashSet<>(); + for (int i = 0; i < 100; i++) { + tablet_status = getTabletStatus(tablet) + if (tablet_status["rowsets"].size() == 2) { + base_compaction_finished = true + final_rowsets.addAll(tablet_status["rowsets"]) + break + } + sleep(500) + } + assertTrue(base_compaction_finished) + + def be_host = backendId_to_backendIP[tablet.BackendId] + def be_http_port = backendId_to_backendHttpPort[tablet.BackendId] + + for (int i = 0; i < final_rowsets.size(); i++) { + def rowsetStr = final_rowsets[i] + def start_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[0].toInteger() + def end_version = rowsetStr.split(" ")[0].replace('[', '').replace(']', '').split("-")[1].toInteger() + def rowset_id = rowsetStr.split(" ")[4] + if (start_version == 0) { + continue + } + + logger.info("final rowset ${i}, start: ${start_version}, end: ${end_version}, id: ${rowset_id}") + def data = Http.GET("http://${be_host}:${be_http_port}/api/file_cache?op=list_cache&value=${rowset_id}_0.dat", true) + logger.info("file cache data: ${data}") + assertTrue(data.size() > 0) + } + } +} From 4d84afdb304e4aba3ec3154a08d49dbeba9a76c9 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Thu, 14 Aug 2025 10:00:53 +0800 Subject: [PATCH 428/572] [fix](errmsg) recover errmsg for -230 (#54697) It is intruduced by https://github.com/apache/doris/pull/40716 while it is dismissed by https://github.com/apache/doris/pull/52440. --- be/src/olap/rowset_version_mgr.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/be/src/olap/rowset_version_mgr.cpp b/be/src/olap/rowset_version_mgr.cpp index 731287dcb0049b..d222ccfbb02b60 100644 --- a/be/src/olap/rowset_version_mgr.cpp +++ b/be/src/olap/rowset_version_mgr.cpp @@ -73,7 +73,7 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch "version already has been merged. version_range={}, max_version={}, " "tablet_id={}", version_range.to_string(), _tablet_meta->max_version().second, tablet_id()); - return ResultError(Status::Error( + return ResultError(Status::Error( "missed versions is empty, version_range={}, max_version={}, tablet_id={}", version_range.to_string(), _tablet_meta->max_version().second, tablet_id())); } @@ -92,7 +92,10 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch return version_path; } if ((tablet_id != -1 && tablet_id == _tablet_meta->tablet_id()) || tablet_id == -2) { - return ResultError(Status::Error("version already merged")); + return ResultError(Status::Error( + "versions are already compacted, version_range={}, max_version={}, " + "tablet_id={}", + version_range.to_string(), _tablet_meta->max_version().second, tablet_id)); } }); return version_path; @@ -150,8 +153,8 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch } auto ret = _remote_capture_rowsets(version_range); if (!ret) { - auto st = Status::Error( - "version already merged, meet error during remote capturing rowsets, " + auto st = Status::Error( + "versions are already compacted, meet error during remote capturing rowsets, " "error={}, version_range={}", ret.error().to_string(), version_range.to_string()); return ResultError(std::move(st)); From ddd4a74d2a1757ab3184eeff890eb41e7b31270a Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 Aug 2025 11:44:31 +0800 Subject: [PATCH 429/572] branch-3.0: [improve](statistic)Reduce call of olap table getVisibleVersion for cloud version. (#53903) (#54661) backport: https://github.com/apache/doris/pull/53903 --- .../doris/statistics/AnalysisManager.java | 3 - .../statistics/StatisticsAutoCollector.java | 84 ++++++++++--------- .../statistics/StatisticsJobAppender.java | 58 +++++++------ .../doris/statistics/util/StatisticsUtil.java | 19 +++-- .../StatisticsAutoCollectorTest.java | 66 ++++++++++----- .../statistics/StatisticsJobAppenderTest.java | 78 ++++++++--------- .../statistics/util/StatisticsUtilTest.java | 43 +++++++--- 7 files changed, 196 insertions(+), 155 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index 0d6082ebb6fc90..e897604479cb4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -700,9 +700,6 @@ public void dropStats(DropStatsStmt dropStatsStmt) throws DdlException { public void dropStats(TableIf table, PartitionNames partitionNames) { try { TableStatsMeta tableStats = findTableStatsStatus(table.getId()); - if (tableStats == null) { - return; - } long catalogId = table.getDatabase().getCatalog().getId(); long dbId = table.getDatabase().getId(); long tableId = table.getId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index 2f287cca035fc2..8cd96bc4957966 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -27,7 +27,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.datasource.hive.HMSExternalTable; -import org.apache.doris.rpc.RpcException; +import org.apache.doris.persist.TableStatsDeletionLog; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; import org.apache.doris.statistics.AnalysisInfo.JobType; import org.apache.doris.statistics.AnalysisInfo.ScheduleType; @@ -139,23 +139,32 @@ protected Optional>>> fetchJobFrom } } - protected void processOneJob(TableIf table, Set> columns, - JobPriority priority) throws DdlException { - appendAllColumns(table, columns); - AnalysisMethod analysisMethod = table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes() + protected void processOneJob(TableIf table, Set> columns, JobPriority priority) { + AnalysisMethod analysisMethod = (StatisticsUtil.getHugeTableLowerBoundSizeInBytes() == 0 + || table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes()) ? AnalysisMethod.SAMPLE : AnalysisMethod.FULL; if (StatisticsUtil.enablePartitionAnalyze() && table.isPartitionedTable()) { analysisMethod = AnalysisMethod.FULL; } boolean isSampleAnalyze = analysisMethod.equals(AnalysisMethod.SAMPLE); OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table : null; + AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); + TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId()); + long rowCount = table.getRowCount(); + if (!readyToSample(table, rowCount, manager, tableStatsStatus, isSampleAnalyze)) { + return; + } + appendAllColumns(table, columns); + long olapTableVersion = StatisticsUtil.getOlapTableVersion(olapTable); columns = columns.stream() - .filter(c -> StatisticsUtil.needAnalyzeColumn(table, c) || StatisticsUtil.isLongTimeColumn(table, c)) + .filter(c -> StatisticsUtil.needAnalyzeColumn(table, c) + || StatisticsUtil.isLongTimeColumn(table, c, olapTableVersion)) .filter(c -> olapTable == null || StatisticsUtil.canCollectColumn( olapTable.getIndexMetaByIndexId(olapTable.getIndexIdByName(c.first)).getColumnByName(c.second), table, isSampleAnalyze, olapTable.getIndexIdByName(c.first))) .collect(Collectors.toSet()); - AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority, analysisMethod); + AnalysisInfo analyzeJob = createAnalyzeJobForTbl(table, columns, priority, analysisMethod, + rowCount, tableStatsStatus, olapTableVersion); if (analyzeJob == null) { return; } @@ -171,8 +180,34 @@ protected void processOneJob(TableIf table, Set> columns, } } + protected boolean readyToSample(TableIf table, long rowCount, AnalysisManager manager, + TableStatsMeta tableStatsStatus, boolean isSample) { + if (!isSample) { + return true; + } + OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table : null; + if (olapTable != null + && olapTable.getRowCountForIndex(olapTable.getBaseIndexId(), true) == TableIf.UNKNOWN_ROW_COUNT) { + LOG.info("Table {} row count is not fully reported, skip auto analyzing it.", olapTable.getName()); + return false; + } + // We don't auto analyze empty table to avoid all 0 stats. + // Because all 0 is more dangerous than unknown stats when row count report is delayed. + if (rowCount <= 0) { + LOG.info("Table {} is empty, remove its old stats and skip auto analyze it.", table.getName()); + // Remove the table's old stats if exists. + if (tableStatsStatus != null && !tableStatsStatus.isColumnsStatsEmpty()) { + manager.removeTableStats(table.getId()); + Env.getCurrentEnv().getEditLog().logDeleteTableStats(new TableStatsDeletionLog(table.getId())); + manager.dropStats(table, null); + } + return false; + } + return true; + } + // If partition changed (partition first loaded, partition dropped and so on), need re-analyze all columns. - protected void appendAllColumns(TableIf table, Set> columns) throws DdlException { + private void appendAllColumns(TableIf table, Set> columns) { if (!(table instanceof OlapTable)) { return; } @@ -197,28 +232,9 @@ protected boolean supportAutoAnalyze(TableIf tableIf) { && ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE); } - protected AnalysisInfo createAnalyzeJobForTbl( - TableIf table, Set> jobColumns, JobPriority priority, AnalysisMethod analysisMethod) { - AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); - TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId()); - if (table instanceof OlapTable && analysisMethod.equals(AnalysisMethod.SAMPLE)) { - OlapTable ot = (OlapTable) table; - if (ot.getRowCountForIndex(ot.getBaseIndexId(), true) == TableIf.UNKNOWN_ROW_COUNT) { - LOG.info("Table {} row count is not fully reported, skip auto analyzing this time.", ot.getName()); - return null; - } - } - // We don't auto analyze empty table to avoid all 0 stats. - // Because all 0 is more dangerous than unknown stats when row count report is delayed. - long rowCount = table.getRowCount(); - if (rowCount <= 0) { - LOG.info("Table {} is empty, remove its old stats and skip auto analyze it.", table.getName()); - // Remove the table's old stats if exists. - if (tableStatsStatus != null && !tableStatsStatus.isColumnsStatsEmpty()) { - manager.dropStats(table, null); - } - return null; - } + protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set> jobColumns, + JobPriority priority, AnalysisMethod analysisMethod, long rowCount, TableStatsMeta tableStatsStatus, + long version) { if (jobColumns == null || jobColumns.isEmpty()) { return null; } @@ -227,14 +243,6 @@ protected AnalysisInfo createAnalyzeJobForTbl( for (Pair pair : jobColumns) { stringJoiner.add(pair.toString()); } - long version = 0; - try { - if (table instanceof OlapTable) { - version = ((OlapTable) table).getVisibleVersion(); - } - } catch (RpcException e) { - LOG.warn("table {}, in cloud getVisibleVersion exception", table.getName(), e); - } return new AnalysisInfoBuilder() .setJobId(Env.getCurrentEnv().getNextId()) .setCatalogId(table.getDatabase().getCatalog().getId()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java index 7597c25633f53e..3ddd360fb5eaa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsJobAppender.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; import org.apache.doris.common.Pair; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.datasource.InternalCatalog; @@ -31,6 +32,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -144,10 +146,10 @@ protected void appendColumnsToJobs(Queue columnQueue, Map>> lowPriorityJobs, + protected int appendToLowJobs(Map>> lowPriorityJobs, Map>> veryLowPriorityJobs) { if (System.currentTimeMillis() - lastRoundFinishTime < lowJobIntervalMs) { - return; + return 0; } InternalCatalog catalog = Env.getCurrentInternalCatalog(); List sortedDbs = catalog.getDbIds().stream().sorted().collect(Collectors.toList()); @@ -173,35 +175,34 @@ protected void appendToLowJobs(Map>> lowPrio Set columns = t.getSchemaAllIndexes(false).stream() .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType())) .map(Column::getName).collect(Collectors.toSet()); - Set> columnIndexPairs = t.getColumnIndexPairs(columns) - .stream().filter(p -> StatisticsUtil.needAnalyzeColumn(t, p)) - .collect(Collectors.toSet()); TableName tableName = new TableName(t.getDatabase().getCatalog().getName(), t.getDatabase().getFullName(), t.getName()); - // Append to low job map first. - if (!columnIndexPairs.isEmpty()) { - boolean appended = doAppend(lowPriorityJobs, columnIndexPairs, tableName); - // If low job map is full, stop this iteration. - if (!appended) { - LOG.debug("Low Priority job map is full."); - return; - } - } else { - // Append to very low job map. - columnIndexPairs = t.getColumnIndexPairs(columns) - .stream().filter(p -> StatisticsUtil.isLongTimeColumn(t, p)) - .collect(Collectors.toSet()); - if (!columnIndexPairs.isEmpty()) { - boolean appended = doAppend(veryLowPriorityJobs, columnIndexPairs, tableName); - // If very low job map is full, simply ignore it and go to the next table. - if (!appended) { + boolean appended = false; + long version = Config.isCloudMode() ? 0 : StatisticsUtil.getOlapTableVersion((OlapTable) t); + for (Pair p : t.getColumnIndexPairs(columns)) { + // Append to low job map first. + if (StatisticsUtil.needAnalyzeColumn(t, p)) { + // If low job map is full, stop this iteration. + if (!doAppend(lowPriorityJobs, p, tableName)) { + LOG.debug("Low Priority job map is full."); + return processed; + } + appended = true; + } else if (StatisticsUtil.isLongTimeColumn(t, p, version)) { + // If very low job map is full, simply ignore it and go to the next column. + if (!doAppend(veryLowPriorityJobs, p, tableName)) { LOG.debug("Very low Priority job map is full."); + } else { + appended = true; } } } currentTableId = t.getId(); - if (++processed >= TABLE_BATCH_SIZE) { - return; + if (appended) { + processed++; + } + if (processed >= TABLE_BATCH_SIZE) { + return processed; } } } @@ -212,6 +213,7 @@ protected void appendToLowJobs(Map>> lowPrio currentDbId = 0; currentTableId = 0; lastRoundFinishTime = System.currentTimeMillis(); + return processed; } protected List
sortTables(List
tables) { @@ -223,16 +225,18 @@ protected List
sortTables(List
tables) { @VisibleForTesting public boolean doAppend(Map>> jobMap, - Set> columnIndexPairs, + Pair columnIndexPair, TableName tableName) { synchronized (jobMap) { if (!jobMap.containsKey(tableName) && jobMap.size() >= JOB_MAP_SIZE) { return false; } if (jobMap.containsKey(tableName)) { - jobMap.get(tableName).addAll(columnIndexPairs); + jobMap.get(tableName).add(columnIndexPair); } else { - jobMap.put(tableName, columnIndexPairs); + Set> columnSet = Sets.newHashSet(); + columnSet.add(columnIndexPair); + jobMap.put(tableName, columnSet); } } return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java index 14865aa467b4af..9889a4a746c0d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java @@ -1165,7 +1165,7 @@ public static boolean needAnalyzePartition(OlapTable table, TableStatsMeta table } // This function return true means the column hasn't been analyzed for longer than the configured time. - public static boolean isLongTimeColumn(TableIf table, Pair column) { + public static boolean isLongTimeColumn(TableIf table, Pair column, long version) { if (column == null) { return false; } @@ -1198,15 +1198,20 @@ public static boolean isLongTimeColumn(TableIf table, Pair colum } // For olap table, if the table visible version and row count doesn't change since last analyze, // we don't need to analyze it because its data is not changed. - OlapTable olapTable = (OlapTable) table; - long version = 0; + return version != columnStats.tableVersion + || table.getRowCount() != columnStats.rowCount; + } + + public static long getOlapTableVersion(OlapTable olapTable) { + if (olapTable == null) { + return 0; + } try { - version = ((OlapTable) table).getVisibleVersion(); + return olapTable.getVisibleVersion(); } catch (RpcException e) { - LOG.warn("in cloud getVisibleVersion exception", e); + LOG.warn("table {}, in cloud getVisibleVersion exception", olapTable.getName(), e); + return 0; } - return version != columnStats.tableVersion - || olapTable.getRowCount() != columnStats.rowCount; } public static boolean canCollect() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index 305eb201dc0882..01778cf8f977d4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -19,11 +19,14 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; @@ -31,10 +34,12 @@ import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod; +import com.google.common.collect.Sets; import mockit.Mock; import mockit.MockUp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; import java.util.HashSet; @@ -147,30 +152,45 @@ public DLAType getDlaType() { @Test public void testCreateAnalyzeJobForTbl() { StatisticsAutoCollector collector = new StatisticsAutoCollector(); - OlapTable table = new OlapTable(); - new MockUp() { - @Mock - public long getDataSize(boolean singleReplica) { - return 100; - } + OlapTable table = Mockito.mock(OlapTable.class); + Database db = Mockito.mock(Database.class); + InternalCatalog catalog = Mockito.mock(InternalCatalog.class); + Mockito.when(table.getDatabase()).thenReturn(db); + Mockito.when(db.getCatalog()).thenReturn(catalog); + Mockito.when(db.getId()).thenReturn(100L); + Mockito.when(catalog.getId()).thenReturn(10L); + + + Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null, null, AnalysisMethod.SAMPLE, 100, null, 10)); + + Set> jobColumns = Sets.newHashSet(); + jobColumns.add(Pair.of("a", "b")); + jobColumns.add(Pair.of("c", "d")); + AnalysisInfo analyzeJobForTbl = collector.createAnalyzeJobForTbl(table, jobColumns, JobPriority.HIGH, AnalysisMethod.SAMPLE, 100, + null, 10); + Assertions.assertEquals("[a:b,c:d]", analyzeJobForTbl.colName); + Assertions.assertEquals(JobPriority.HIGH, analyzeJobForTbl.priority); + Assertions.assertEquals(AnalysisMethod.SAMPLE, analyzeJobForTbl.analysisMethod); + Assertions.assertEquals(100, analyzeJobForTbl.rowCount); + Assertions.assertEquals(10, analyzeJobForTbl.tableVersion); + } - @Mock - public long getRowCountForIndex(long indexId, boolean strict) { - return -1; - } + @Test + public void testReadyToSample() { + StatisticsAutoCollector collector = new StatisticsAutoCollector(); + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getName()).thenReturn("table"); + Mockito.when(table.getRowCountForIndex(Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(TableIf.UNKNOWN_ROW_COUNT); + // not sample + Assertions.assertTrue(collector.readyToSample(table, 100, null, null, false)); + // not fully reported. + Assertions.assertFalse(collector.readyToSample(table, 100, null, null, true)); + + Mockito.when(table.getRowCountForIndex(Mockito.anyLong(), Mockito.anyBoolean())).thenReturn(100L); + // Row count is 0 + Assertions.assertFalse(collector.readyToSample(table, 0, null, null, true)); + // ready to sample + Assertions.assertTrue(collector.readyToSample(table, 100, null, null, true)); - @Mock - public boolean isPartitionedTable() { - return false; - } - }; - Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null, null, AnalysisMethod.SAMPLE)); - new MockUp() { - @Mock - public long getRowCountForIndex(long indexId, boolean strict) { - return 100; - } - }; - Assertions.assertNull(collector.createAnalyzeJobForTbl(table, null, null, AnalysisMethod.SAMPLE)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java index 94c50c91ee3f19..aa7f3fc06d4b3c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsJobAppenderTest.java @@ -34,7 +34,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import mockit.Mock; import mockit.MockUp; import org.junit.jupiter.api.Assertions; @@ -192,30 +191,29 @@ public boolean needAnalyzeColumn(TableIf table, Pair column) { StatisticsJobAppender appender = new StatisticsJobAppender(); appender.appendToLowJobs(testLowMap, testVeryLowMap); Assertions.assertEquals(100, testLowMap.size()); + Assertions.assertEquals(0, testVeryLowMap.size()); testLowMap.clear(); appender.appendToLowJobs(testLowMap, testVeryLowMap); Assertions.assertEquals(40, testLowMap.size()); - - for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) { - Database db = new Database(id++, "testDb" + i); - testCatalog.unprotectCreateDb(db); - Column column1 = new Column("placeholder", PrimitiveType.INT); - List schema = new ArrayList<>(); - schema.add(column1); - OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); - OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); - db.createTableWithLock(table1, true, false); - db.createTableWithLock(table2, true, false); - } - + Assertions.assertEquals(0, testVeryLowMap.size()); testLowMap.clear(); - appender.setLastRoundFinishTime(0); + // Less than 1 minutes since last iteration. appender.appendToLowJobs(testLowMap, testVeryLowMap); + Assertions.assertEquals(0, testLowMap.size()); + Assertions.assertEquals(0, testVeryLowMap.size()); + + testLowMap.clear(); appender.setLastRoundFinishTime(0); - appender.appendToLowJobs(testLowMap, testVeryLowMap); + int processed = appender.appendToLowJobs(testLowMap, testVeryLowMap); + Assertions.assertEquals(100, testLowMap.size()); + Assertions.assertEquals(0, testVeryLowMap.size()); + Assertions.assertEquals(100, processed); appender.setLastRoundFinishTime(0); - appender.appendToLowJobs(testLowMap, testVeryLowMap); + processed = appender.appendToLowJobs(testLowMap, testVeryLowMap); + Assertions.assertEquals(100, testLowMap.size()); + Assertions.assertEquals(0, testVeryLowMap.size()); Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testLowMap.size()); + Assertions.assertEquals(0, processed); } @Test @@ -260,7 +258,7 @@ public boolean needAnalyzeColumn(TableIf table, Pair column) { } @Mock - public boolean isLongTimeColumn(TableIf table, Pair column) { + public boolean isLongTimeColumn(TableIf table, Pair column, long version) { return true; } }; @@ -268,33 +266,28 @@ public boolean isLongTimeColumn(TableIf table, Pair column) { Map>> testLowMap = new HashMap<>(); Map>> testVeryLowMap = new HashMap<>(); StatisticsJobAppender appender = new StatisticsJobAppender(); - appender.appendToLowJobs(testLowMap, testVeryLowMap); + int processed = appender.appendToLowJobs(testLowMap, testVeryLowMap); + Assertions.assertEquals(0, testLowMap.size()); Assertions.assertEquals(100, testVeryLowMap.size()); + Assertions.assertEquals(100, processed); testVeryLowMap.clear(); - appender.appendToLowJobs(testLowMap, testVeryLowMap); + processed = appender.appendToLowJobs(testLowMap, testVeryLowMap); + Assertions.assertEquals(0, testLowMap.size()); Assertions.assertEquals(40, testVeryLowMap.size()); - - for (int i = 0; i < StatisticsJobAppender.JOB_MAP_SIZE; i++) { - Database db = new Database(id++, "testDb" + i); - testCatalog.unprotectCreateDb(db); - Column column1 = new Column("placeholder", PrimitiveType.INT); - List schema = new ArrayList<>(); - schema.add(column1); - OlapTable table1 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); - OlapTable table2 = new OlapTable(id++, "testTable" + id + "_1", schema, null, null, null); - db.createTableWithLock(table1, true, false); - db.createTableWithLock(table2, true, false); - } + Assertions.assertEquals(40, processed); testLowMap.clear(); appender.setLastRoundFinishTime(0); - appender.appendToLowJobs(testLowMap, testVeryLowMap); - appender.setLastRoundFinishTime(0); - appender.appendToLowJobs(testLowMap, testVeryLowMap); + processed = appender.appendToLowJobs(testLowMap, testVeryLowMap); + Assertions.assertEquals(0, testLowMap.size()); + Assertions.assertEquals(100, testVeryLowMap.size()); + Assertions.assertEquals(100, processed); + appender.setLastRoundFinishTime(0); - appender.appendToLowJobs(testLowMap, testVeryLowMap); + processed = appender.appendToLowJobs(testLowMap, testVeryLowMap); Assertions.assertEquals(0, testLowMap.size()); - Assertions.assertEquals(StatisticsJobAppender.JOB_MAP_SIZE, testVeryLowMap.size()); + Assertions.assertEquals(100, testVeryLowMap.size()); + Assertions.assertEquals(0, processed); } @Test @@ -348,23 +341,19 @@ public int getAutoAnalyzeTableWidthThreshold() { @Test public void testDoAppend() { Map>> jobMap = Maps.newHashMap(); - Set> columnIndexPairs1 = Sets.newHashSet(); - Set> columnIndexPairs2 = Sets.newHashSet(); TableName tableName1 = new TableName("catalog1", "db1", "table1"); TableName tableName2 = new TableName("catalog2", "db2", "table2"); Pair pair1 = Pair.of("index1", "col1"); - columnIndexPairs1.add(pair1); StatisticsJobAppender appender = new StatisticsJobAppender(); - Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs1, tableName1)); + Assertions.assertTrue(appender.doAppend(jobMap, pair1, tableName1)); Assertions.assertEquals(1, jobMap.size()); Assertions.assertTrue(jobMap.containsKey(tableName1)); Assertions.assertEquals(1, jobMap.get(tableName1).size()); Assertions.assertTrue(jobMap.get(tableName1).contains(pair1)); Pair pair2 = Pair.of("index2", "col2"); - columnIndexPairs1.add(pair2); - Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs1, tableName1)); + Assertions.assertTrue(appender.doAppend(jobMap, pair2, tableName1)); Assertions.assertEquals(1, jobMap.size()); Assertions.assertTrue(jobMap.containsKey(tableName1)); Assertions.assertEquals(2, jobMap.get(tableName1).size()); @@ -372,8 +361,7 @@ public void testDoAppend() { Assertions.assertTrue(jobMap.get(tableName1).contains(pair2)); Pair pair3 = Pair.of("index3", "col3"); - columnIndexPairs2.add(pair3); - Assertions.assertTrue(appender.doAppend(jobMap, columnIndexPairs2, tableName2)); + Assertions.assertTrue(appender.doAppend(jobMap, pair3, tableName2)); Assertions.assertEquals(2, jobMap.size()); Assertions.assertTrue(jobMap.containsKey(tableName2)); Assertions.assertEquals(1, jobMap.get(tableName2).size()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index 0467f9b89dd234..3ad3b6c977b5ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -35,12 +35,15 @@ import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergHadoopExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.AnalysisManager; import org.apache.doris.statistics.ColStatsMeta; import org.apache.doris.statistics.ResultRow; @@ -48,10 +51,13 @@ import org.apache.doris.thrift.TStorageType; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import mockit.Mock; import mockit.MockUp; +import org.apache.iceberg.CatalogProperties; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.nio.charset.StandardCharsets; import java.time.LocalTime; @@ -377,7 +383,7 @@ void testLongTimeNoAnalyze() { OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); // Test column is null - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, null)); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, null, 0)); // Test table auto analyze is disabled. new MockUp() { @@ -386,7 +392,7 @@ public boolean autoAnalyzeEnabled() { return false; } }; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 0)); new MockUp() { @Mock public boolean autoAnalyzeEnabled() { @@ -402,8 +408,11 @@ public boolean autoAnalyzeEnabled() { } }; IcebergExternalDatabase icebergDatabase = new IcebergExternalDatabase(null, 1L, "", ""); - IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null, icebergDatabase); - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable, Pair.of("index", column.getName()))); + Map props = Maps.newHashMap(); + props.put(CatalogProperties.WAREHOUSE_LOCATION, "s3://tmp"); + IcebergExternalCatalog catalog = new IcebergHadoopExternalCatalog(0, "iceberg_ctl", "", props, ""); + IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", catalog, icebergDatabase); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable, Pair.of("index", column.getName()), 0)); // Test table stats meta is null. new MockUp() { @@ -412,7 +421,7 @@ public TableStatsMeta findTableStatsStatus(long tblId) { return null; } }; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 0)); // Test column stats meta is null TableStatsMeta tableMeta = new TableStatsMeta(); @@ -428,7 +437,7 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { return null; } }; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 0)); new MockUp() { @Mock public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { @@ -438,16 +447,16 @@ public ColStatsMeta findColumnStatsMeta(String indexName, String colName) { // Test table stats is user injected tableMeta.userInjected = true; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 0)); tableMeta.userInjected = false; // Test Config.auto_analyze_interval_seconds == 0 Config.auto_analyze_interval_seconds = 0; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 0)); // Test column analyzed within the time interval Config.auto_analyze_interval_seconds = 86400; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 0)); // Test column hasn't analyzed for longer than time interval, but version and row count doesn't change new MockUp() { @@ -474,7 +483,7 @@ public long fetchRowCount() { } }; Config.auto_analyze_interval_seconds = 1; - Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 10)); // Test column hasn't analyzed for longer than time interval, and version change new MockUp() { @@ -488,7 +497,7 @@ public long fetchRowCount() { return 100; } }; - Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 11)); // Test column hasn't analyzed for longer than time interval, and row count change new MockUp() { @@ -502,7 +511,7 @@ public long fetchRowCount() { return 101; } }; - Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()))); + Assertions.assertTrue(StatisticsUtil.isLongTimeColumn(table, Pair.of("index", column.getName()), 10)); } @Test @@ -558,4 +567,14 @@ public boolean isUniqKeyMergeOnWrite() { Assertions.assertTrue(StatisticsUtil.canCollectColumn(column, table, true, 1)); } + + @Test + public void testGetOlapTableVersion() throws RpcException { + Assertions.assertEquals(0, StatisticsUtil.getOlapTableVersion(null)); + OlapTable ot = Mockito.mock(OlapTable.class); + Mockito.when(ot.getVisibleVersion()).thenReturn(100L); + Assertions.assertEquals(100, StatisticsUtil.getOlapTableVersion(ot)); + Mockito.when(ot.getVisibleVersion()).thenThrow(new RpcException("", "")); + Assertions.assertEquals(0, StatisticsUtil.getOlapTableVersion(ot)); + } } From 27f74f716be94680e5795b858c28ec32a22391cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:45:49 +0800 Subject: [PATCH 430/572] branch-3.0: [fix](load)fix load json format boolean type to int column. #54397 (#54640) Cherry-picked from #54397 Co-authored-by: daidai --- .../vec/exec/format/json/new_json_reader.cpp | 11 +++++++ .../load_p0/stream_load/test_json_load.out | 6 ++++ .../stream_load/test_read_boolean_to_int.json | 6 ++++ .../load_p0/stream_load/test_json_load.groovy | 29 +++++++++++++++++++ 4 files changed, 52 insertions(+) create mode 100644 regression-test/data/load_p0/stream_load/test_read_boolean_to_int.json diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp b/be/src/vec/exec/format/json/new_json_reader.cpp index fb1b04e867c44b..21c5fa7feabe0a 100644 --- a/be/src/vec/exec/format/json/new_json_reader.cpp +++ b/be/src/vec/exec/format/json/new_json_reader.cpp @@ -1683,6 +1683,17 @@ Status NewJsonReader::_simdjson_write_data_to_column(simdjson::ondemand::value& RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice, _serde_options)); + } else if (value.type() == simdjson::ondemand::json_type::boolean) { + const char* str_value = nullptr; + // insert "1"/"0" , not "true"/"false". + if (value.get_bool()) { + str_value = (char*)"1"; + } else { + str_value = (char*)"0"; + } + Slice slice {str_value, 1}; + RETURN_IF_ERROR(data_serde->deserialize_one_cell_from_json(*data_column_ptr, slice, + _serde_options)); } else { // Maybe we can `switch (value->GetType()) case: kNumberType`. // Note that `if (value->IsInt())`, but column is FloatColumn. diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out index 3ef9ecb5be941f..b2cf149e12ba33 100644 --- a/regression-test/data/load_p0/stream_load/test_json_load.out +++ b/regression-test/data/load_p0/stream_load/test_json_load.out @@ -267,3 +267,9 @@ test k2_value 12347 {"k1":12347,"k3":"33333","k4":[22222]} {"k1":12347,"k3":"33333","k4":[22222]} 33333 12348 {"k1":12348,"k3":"33333","k5":{"k51":1024,"xxxx":[11111]}} {"k1":12348,"k3":"33333","k5":{"k51":1024,"xxxx":[11111]}} 33333 +-- !select31 -- +1 1 1 1 1.00 +2 0 0 0 0.00 +3 100 100 100 100.00 +4 \N \N \N \N + diff --git a/regression-test/data/load_p0/stream_load/test_read_boolean_to_int.json b/regression-test/data/load_p0/stream_load/test_read_boolean_to_int.json new file mode 100644 index 00000000000000..62ff27be0bfdbf --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_read_boolean_to_int.json @@ -0,0 +1,6 @@ +[ +{"id":1,"k1":true,"k2":true,"k3":true,"k4":true}, +{"id":2,"k1":false,"k2":false,"k3":false,"k4":false}, +{"id":3,"k1":100,"k2":100,"k3":100,"k4":100}, +{"id":4,"k1":null,"k2":null,"k3":null,"k4":null} +] \ No newline at end of file diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index 8bbd6d844e914b..3be34c7989a29b 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -959,4 +959,33 @@ suite("test_json_load", "p0,nonConcurrent") { } finally { // try_sql("DROP TABLE IF EXISTS ${testTable}") } + + // try to load `boolean` => `tinyint, int , string, decimal` + try { + sql "DROP TABLE IF EXISTS ${testTable}" + sql """CREATE TABLE IF NOT EXISTS ${testTable} + ( + `id` int, + `k1` tinyint NULL, + `k2` int NULL, + `k3` string NULL, + `k4` decimal(10,2) NULL + ) + DUPLICATE KEY(`id`) + COMMENT '' + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + + load_json_data.call("${testTable}", "${testTable}_case31", 'true', '', 'json', '', '', + '', '', '', 'test_read_boolean_to_int.json') + + sql "sync" + qt_select31 "select * from ${testTable} order by id" + + } finally { + // try_sql("DROP TABLE IF EXISTS ${testTable}") + } } From 8accbc30beb1c55eabb8fe272385eae9aab4ede8 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 14 Aug 2025 11:52:18 +0800 Subject: [PATCH 431/572] branch-3.0: [fix](prepare statement)Use client prepare statement when placeholders are more than 65536. (#54568) (#54668) https://github.com/apache/doris/pull/54568 --- .../nereids/trees/plans/commands/PrepareCommand.java | 6 ++++++ .../suites/prepared_stmt_p0/prepared_stmt.groovy | 12 +++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java index 42274c5eaf8d4d..b418e7d211ac72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/PrepareCommand.java @@ -18,6 +18,8 @@ package org.apache.doris.nereids.trees.plans.commands; import org.apache.doris.analysis.StmtType; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; import org.apache.doris.mysql.MysqlCommand; import org.apache.doris.nereids.trees.expressions.Placeholder; import org.apache.doris.nereids.trees.plans.PlanType; @@ -43,6 +45,7 @@ public class PrepareCommand extends Command { private final List placeholders = new ArrayList<>(); private final LogicalPlan logicalPlan; + private final int maxPlaceholderCount = 65536; private final String name; @@ -99,6 +102,9 @@ public List getLabels() { @Override public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { List labels = getLabels(); + if (labels.size() >= maxPlaceholderCount) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_PS_MANY_PARAM); + } // register prepareStmt if (LOG.isDebugEnabled()) { LOG.debug("add prepared statement {}, isBinaryProtocol {}", diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index 353b4b4552fff3..80058e72894ebd 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -63,7 +63,17 @@ suite("test_prepared_stmt", "nonConcurrent") { sql "set enable_fallback_to_original_planner = false" sql """set global enable_server_side_prepared_statement = true""" - def stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1" + int count = 65536; + StringBuilder sb = new StringBuilder(); + sb.append("?"); + for (int i = 1; i < count; i++) { + sb.append(", ?"); + } + String sqlWithTooManyPlaceholder = sb.toString(); + def stmt_read = prepareStatement "select * from ${tableName} where k1 in ${sqlWithTooManyPlaceholder}" + assertEquals(com.mysql.cj.jdbc.ClientPreparedStatement, stmt_read.class) + + stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1" assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt_read.class) stmt_read.setInt(1, 1231) qe_select0 stmt_read From 43b104da10e34e166582498fc8822a364a5d7d1c Mon Sep 17 00:00:00 2001 From: minghong Date: Thu, 14 Aug 2025 11:54:15 +0800 Subject: [PATCH 432/572] branch-3.0 [fix](nereids) fix bug in StatsCalculator.disableJoinReorderIfStatsInvalid when rowCount is not aviable #53834 (#54427) ### What problem does this PR solve? pick #53834 Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../apache/doris/nereids/stats/StatsCalculator.java | 10 ++++++++-- .../suites/nereids_rules_p0/mv/ssb/mv_ssb_test.groovy | 8 +++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index a4f06cb9568eb6..1314d81b1b8392 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -258,8 +258,14 @@ public static Optional disableJoinReorderIfStatsInvalid(List Date: Thu, 14 Aug 2025 11:55:11 +0800 Subject: [PATCH 433/572] branch-3.0: [Bug](fix) Try to prevent the bug of brpc issue 2146 (#47432) #53804 (#53940) Cherry-picked from #53804 Co-authored-by: HappenLee --- be/src/util/brpc_client_cache.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/be/src/util/brpc_client_cache.h b/be/src/util/brpc_client_cache.h index 24bd284f302fb9..a8aea36ee0cd83 100644 --- a/be/src/util/brpc_client_cache.h +++ b/be/src/util/brpc_client_cache.h @@ -144,6 +144,8 @@ class BrpcClientCache { } else if (_connection_group != "") { options.connection_group = _connection_group; } + // Add random connection id to connection_group to make sure use new socket + options.connection_group += std::to_string(_connection_id.fetch_add(1)); options.connect_timeout_ms = 2000; options.timeout_ms = 2000; options.max_retry = 10; @@ -227,6 +229,9 @@ class BrpcClientCache { const std::string _protocol; const std::string _connection_type; const std::string _connection_group; + // use to generate unique connection id for each connection + // to prevent the connection problem of brpc: https://github.com/apache/brpc/issues/2146 + std::atomic _connection_id {0}; }; using InternalServiceClientCache = BrpcClientCache; From b119a5b1b748135146e4520471cee9827a9f6523 Mon Sep 17 00:00:00 2001 From: wangbo Date: Thu, 14 Aug 2025 11:58:28 +0800 Subject: [PATCH 434/572] [branch-3.0]disable workload group memory check limit by default (#54344) --- .../java/org/apache/doris/common/Config.java | 2 +- .../workload_manager_p0/test_curd_wlg.groovy | 46 ------------------- 2 files changed, 1 insertion(+), 47 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index f317f4071392a3..1a33cc1d72f3f3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1921,7 +1921,7 @@ public class Config extends ConfigBase { public static boolean enable_workload_group = true; @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL) - public static boolean enable_wg_memory_sum_limit = true; + public static boolean enable_wg_memory_sum_limit = false; @ConfField(mutable = true) public static boolean enable_query_queue = true; diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy index e5e13d3df830fa..a6200133d234d8 100644 --- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy +++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy @@ -138,13 +138,6 @@ suite("test_crud_wlg") { sql "drop workload group test_drop_wg" qt_show_del_wg_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num,tag from information_schema.workload_groups where name in ('normal','test_group','test_drop_wg') order by name;" - // test memory_limit - test { - sql "alter workload group test_group properties ( 'memory_limit'='100%' );" - - exception "cannot be greater than" - } - sql "alter workload group test_group properties ( 'memory_limit'='11%' );" qt_mem_limit_1 """ select count(1) from ${table_name} """ qt_mem_limit_2 "select name,cpu_share,memory_limit,enable_memory_overcommit,max_concurrency,max_queue_size,queue_timeout,cpu_hard_limit,scan_thread_num from information_schema.workload_groups where name in ('normal','test_group') order by name;" @@ -233,30 +226,6 @@ suite("test_crud_wlg") { exception "The allowed cpu_share value is -1 or a positive integer" } - // failed for mem_limit - test { - sql "create workload group if not exists test_group2 " + - "properties ( " + - " 'cpu_share'='10', " + - " 'memory_limit'='200%', " + - " 'enable_memory_overcommit'='true' " + - ");" - - exception "cannot be greater than" - } - - test { - sql "create workload group if not exists test_group2 " + - "properties ( " + - " 'cpu_share'='10', " + - " 'memory_limit'='99%', " + - " 'enable_memory_overcommit'='true' " + - ");" - - exception "cannot be greater than" - } - - // failed for mem_overcommit test { sql "create workload group if not exists test_group2 " + @@ -504,31 +473,16 @@ suite("test_crud_wlg") { // test workload group's tag property, memory_limit sql "create workload group if not exists tag1_mem_wg1 properties ( 'memory_limit'='50%', 'tag'='mem_tag1');" - test { - sql "create workload group if not exists tag1_mem_wg2 properties ( 'memory_limit'='60%', 'tag'='mem_tag1');" - exception "cannot be greater than 100.0%" - } - sql "create workload group if not exists tag1_mem_wg2 properties ('memory_limit'='49%', 'tag'='mem_tag1');" sql "create workload group if not exists tag1_mem_wg3 properties ( 'memory_limit'='2%');" - test { - sql "alter workload group tag1_mem_wg3 properties ( 'tag'='mem_tag1' );" - exception "cannot be greater than 100.0%" - } - sql "alter workload group tag1_mem_wg3 properties ( 'memory_limit'='1%' );" sql "alter workload group tag1_mem_wg3 properties ( 'tag'='mem_tag1' );" sql "create workload group tag1_mem_wg4 properties('memory_limit'='-1','tag'='mem_tag1');" - test { - sql "alter workload group tag1_mem_wg4 properties ( 'memory_limit'='1%' );" - exception "cannot be greater than 100.0%" - } - qt_show_wg_tag "select name,MEMORY_LIMIT,CPU_HARD_LIMIT,TAG from information_schema.workload_groups where name in('tag1_wg1','tag1_wg2','tag2_wg1','tag1_wg3','tag1_mem_wg1','tag1_mem_wg2','tag1_mem_wg3') order by tag,name;" From 2a519b2dd70c936a9270f0b1f29030873f559885 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 11:59:29 +0800 Subject: [PATCH 435/572] branch-3.0: [fix](regression) fix test_reset_capacity undefined global variable #54573 (#54646) Cherry-picked from #54573 Co-authored-by: zhengyu --- .../suites/cloud_p0/cache/http/test_reset_capacity.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy b/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy index 4c9608e512cdff..a5274fcf04b2f7 100644 --- a/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy +++ b/regression-test/suites/cloud_p0/cache/http/test_reset_capacity.groovy @@ -286,7 +286,7 @@ suite("test_reset_capacity") { continue } def i = line.indexOf(' ') - ttl_cache_size = line.substring(i).toLong() + def ttl_cache_size = line.substring(i).toLong() logger.info("current ttl_cache_size " + ttl_cache_size); assertTrue(ttl_cache_size > 1073741824) flag1 = true From d757b0417af97b48f385aa73cd83c55ade5cef12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:02:10 +0800 Subject: [PATCH 436/572] branch-3.0: [fix](nereids) fix rule count on index #53825 (#53877) Cherry-picked from #53825 Co-authored-by: Sun Chenyang --- .../rules/implementation/AggregateStrategies.java | 6 +++--- .../inverted_index_p0/test_count_on_index.groovy | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index ace11cd3eb3bed..badb2d95f4a5c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -582,7 +582,7 @@ private LogicalAggregate pushdownCountOnIndex( List argumentsOfAggregateFunction = normalizeArguments(agg.getAggregateFunctions(), project); - if (!onlyContainsSlot(argumentsOfAggregateFunction)) { + if (!onlyContainsSlotOrLiteral(argumentsOfAggregateFunction)) { return agg; } @@ -614,9 +614,9 @@ private List normalizeArguments(Set aggregateFunc return arguments; } - private boolean onlyContainsSlot(List arguments) { + private boolean onlyContainsSlotOrLiteral(List arguments) { return arguments.stream().allMatch(argument -> { - if (argument instanceof SlotReference) { + if (argument instanceof SlotReference || argument instanceof Literal) { return true; } return false; diff --git a/regression-test/suites/inverted_index_p0/test_count_on_index.groovy b/regression-test/suites/inverted_index_p0/test_count_on_index.groovy index 6e5a33002f0a72..9823124df50589 100644 --- a/regression-test/suites/inverted_index_p0/test_count_on_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_count_on_index.groovy @@ -405,6 +405,17 @@ suite("test_count_on_index_httplogs", "p0") { contains "pushAggOp=NONE" } + explain { + sql("select COUNT(23) from ${tableName6} where value1 > 20 and value2 > 5") + contains "pushAggOp=COUNT_ON_INDEX" + } + + sql """ set disable_nereids_rules='COUNT_LITERAL_REWRITE'; """ + explain { + sql("select COUNT(23) from ${tableName6} where value1 > 20 and value2 > 5") + contains "pushAggOp=COUNT_ON_INDEX" + } + } finally { //try_sql("DROP TABLE IF EXISTS ${testTable}") } From c1ab85e0d5da9e005cafcb457af70ead48f7c8c4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:03:14 +0800 Subject: [PATCH 437/572] branch-3.0: [opt](nereids) use one fragment to execute point query #53541 (#53573) Cherry-picked from #53541 Co-authored-by: 924060929 --- .../nereids/properties/ChildOutputPropertyDeriver.java | 4 ++++ .../org/apache/doris/qe/ShortCircuitQueryContext.java | 6 ++++-- .../suites/point_query_p0/test_point_query.groovy | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java index 66d30483b4e9db..7c0bf04f5678d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java @@ -168,6 +168,10 @@ public PhysicalProperties visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, PlanC @Override public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanContext context) { + // make sure only one fragment when use point query + if (context.getStatementContext().isShortCircuitQuery() && olapScan.getSelectedTabletIds().size() == 1) { + return PhysicalProperties.GATHER; + } return new PhysicalProperties(olapScan.getDistributionSpec()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShortCircuitQueryContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShortCircuitQueryContext.java index 7452b57b4004b1..057368fc3f8d70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShortCircuitQueryContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShortCircuitQueryContext.java @@ -42,6 +42,7 @@ public class ShortCircuitQueryContext { // Cached for better CPU performance, since serialize DescriptorTable and // outputExprs are heavy work + public final Planner planner; public final ByteString serializedDescTable; public final ByteString serializedOutputExpr; public final ByteString serializedQueryOptions; @@ -79,13 +80,14 @@ List getReturnTypes() { } public ShortCircuitQueryContext(Planner planner, Queriable analzyedQuery) throws TException { + this.planner = planner; this.serializedDescTable = ByteString.copyFrom( new TSerializer().serialize(planner.getDescTable().toThrift())); TQueryOptions options = planner.getQueryOptions() != null ? planner.getQueryOptions() : new TQueryOptions(); this.serializedQueryOptions = ByteString.copyFrom( new TSerializer().serialize(options)); List exprs = new ArrayList<>(); - OlapScanNode olapScanNode = (OlapScanNode) planner.getFragments().get(1).getPlanRoot(); + OlapScanNode olapScanNode = (OlapScanNode) planner.getScanNodes().get(0); if (olapScanNode.getProjectList() != null) { // project on scan node exprs.addAll(olapScanNode.getProjectList().stream() @@ -99,7 +101,7 @@ public ShortCircuitQueryContext(Planner planner, Queriable analzyedQuery) throws serializedOutputExpr = ByteString.copyFrom( new TSerializer().serialize(exprList)); this.cacheID = UUID.randomUUID(); - this.scanNode = ((OlapScanNode) planner.getScanNodes().get(0)); + this.scanNode = olapScanNode; this.tbl = this.scanNode.getOlapTable(); this.schemaVersion = this.tbl.getBaseSchemaVersion(); this.analzyedQuery = analzyedQuery; diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy index 04afed2ed74fee..21c4306f02d9ea 100644 --- a/regression-test/suites/point_query_p0/test_point_query.groovy +++ b/regression-test/suites/point_query_p0/test_point_query.groovy @@ -445,6 +445,16 @@ suite("test_point_query", "nonConcurrent") { sql "set enable_short_circuit_query = true" qt_sql "select length(loc3) from table_with_chars where col1 = 10" + def ensure_one_fragment = { + sql "set enable_nereids_planner=true" + explain { + sql "select * from table_with_chars where col1 = 10" + check { explainStr -> + assertEquals(1, explainStr.count("PLAN FRAGMENT")) + } + } + }() + // test variant type sql "DROP TABLE IF EXISTS test_with_variant" sql """ From cb91dd9ee342752805052fc09259945de4846ee1 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Thu, 14 Aug 2025 12:04:04 +0800 Subject: [PATCH 438/572] branch-3.0:[fix](mtmv) release read lock when align mvmv's partition (#53069) (#54657) pick: https://github.com/apache/doris/pull/53069 Co-authored-by: Lijia Liu Co-authored-by: liutang123 --- .../doris/job/extensions/mtmv/MTMVTask.java | 18 +++++++++++++++++- .../apache/doris/mtmv/MTMVPartitionUtil.java | 16 ++++++++++------ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 198ba21891832b..f8d21311fcf104 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -17,6 +17,7 @@ package org.apache.doris.job.extensions.mtmv; +import org.apache.doris.analysis.PartitionKeyDesc; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; @@ -27,6 +28,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DebugPointUtil; @@ -194,6 +196,7 @@ public void run() throws JobException { tableIfs.sort(Comparator.comparing(TableIf::getId)); MTMVRefreshContext context; + Pair, List> syncPartitions = null; // lock table order by id to avoid deadlock MetaLockUtils.readLockTables(tableIfs); try { @@ -203,8 +206,21 @@ public void run() throws JobException { checkColumnTypeIfChange(mtmv, ctx); } if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { - MTMVPartitionUtil.alignMvPartition(mtmv); + syncPartitions = MTMVPartitionUtil.alignMvPartition(mtmv); } + } finally { + MetaLockUtils.readUnlockTables(tableIfs); + } + if (syncPartitions != null) { + for (String pName : syncPartitions.first) { + MTMVPartitionUtil.dropPartition(mtmv, pName); + } + for (PartitionKeyDesc partitionKeyDesc : syncPartitions.second) { + MTMVPartitionUtil.addPartition(mtmv, partitionKeyDesc); + } + } + MetaLockUtils.readLockTables(tableIfs); + try { context = MTMVRefreshContext.buildContext(mtmv); this.needRefreshPartitions = calculateNeedRefreshPartitions(context); } finally { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 8da00cdeeb3421..c67f43974e0978 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -33,6 +33,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.rpc.RpcException; @@ -46,6 +47,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -113,24 +115,26 @@ public static boolean isMTMVPartitionSync(MTMVRefreshContext refreshContext, Str * @throws DdlException * @throws AnalysisException */ - public static void alignMvPartition(MTMV mtmv) - throws DdlException, AnalysisException { + public static Pair, List> alignMvPartition(MTMV mtmv) throws AnalysisException { Map mtmvPartitionDescs = mtmv.generateMvPartitionDescs(); Set relatedPartitionDescs = generateRelatedPartitionDescs(mtmv.getMvPartitionInfo(), mtmv.getMvProperties()).keySet(); + List partitionsToDrop = new ArrayList<>(); + List partitionsToAdd = new ArrayList<>(); // drop partition of mtmv for (Entry entry : mtmvPartitionDescs.entrySet()) { if (!relatedPartitionDescs.contains(entry.getValue())) { - dropPartition(mtmv, entry.getKey()); + partitionsToDrop.add(entry.getKey()); } } // add partition for mtmv HashSet mtmvPartitionDescsSet = Sets.newHashSet(mtmvPartitionDescs.values()); for (PartitionKeyDesc desc : relatedPartitionDescs) { if (!mtmvPartitionDescsSet.contains(desc)) { - addPartition(mtmv, desc); + partitionsToAdd.add(desc); } } + return Pair.of(partitionsToDrop, partitionsToAdd); } /** @@ -365,7 +369,7 @@ public static String generatePartitionName(PartitionKeyDesc desc) { * @param mtmv * @param partitionName */ - private static void dropPartition(MTMV mtmv, String partitionName) throws DdlException { + public static void dropPartition(MTMV mtmv, String partitionName) throws DdlException { if (!mtmv.writeLockIfExist()) { return; } @@ -386,7 +390,7 @@ private static void dropPartition(MTMV mtmv, String partitionName) throws DdlExc * @param oldPartitionKeyDesc * @throws DdlException */ - private static void addPartition(MTMV mtmv, PartitionKeyDesc oldPartitionKeyDesc) + public static void addPartition(MTMV mtmv, PartitionKeyDesc oldPartitionKeyDesc) throws DdlException { Map partitionProperties = Maps.newHashMap(); SinglePartitionDesc singlePartitionDesc = new SinglePartitionDesc(true, From ea758e432b66e519f8a5a9903be4d2ed2d1530c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:04:49 +0800 Subject: [PATCH 439/572] branch-3.0: [fix](nereids) when ColumnStatistics.avgSizeByte is NaN, use default size 1. #54150 (#54272) Cherry-picked from #54150 Co-authored-by: minghong --- .../apache/doris/statistics/Statistics.java | 6 ++- .../doris/statistics/StatisticsTest.java | 53 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java index 20da97df0205b8..858810a9013e03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/Statistics.java @@ -167,7 +167,11 @@ public double computeTupleSize(List slots) { for (Slot slot : slots) { ColumnStatistic s = expressionToColumnStats.get(slot); if (s != null) { - tempSize += Math.max(1, Math.min(CharacterType.DEFAULT_WIDTH, s.avgSizeByte)); + double avgSize = s.avgSizeByte; + if (!Double.isFinite(avgSize)) { + avgSize = 1; + } + tempSize += Math.max(1, Math.min(CharacterType.DEFAULT_WIDTH, avgSize)); } } tupleSize = Math.max(1, tempSize); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsTest.java new file mode 100644 index 00000000000000..d6519aaed56bbd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsTest.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.statistics; + +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.IntegerType; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class StatisticsTest { + @Test + public void testAvgSizeAbnormal() { + SlotReference slot = SlotReference.of("a", IntegerType.INSTANCE); + ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(); + colBuilder.setAvgSizeByte(Double.NaN); + Statistics stats = new Statistics(1, 1, ImmutableMap.of(slot, colBuilder.build())); + double tupleSize = stats.computeTupleSize(ImmutableList.of(slot)); + Assertions.assertEquals(1, tupleSize); + + colBuilder.setAvgSizeByte(Double.POSITIVE_INFINITY); + stats = new Statistics(1, 1, ImmutableMap.of(slot, colBuilder.build())); + tupleSize = stats.computeTupleSize(ImmutableList.of(slot)); + Assertions.assertEquals(1, tupleSize); + + colBuilder.setAvgSizeByte(Double.NEGATIVE_INFINITY); + stats = new Statistics(1, 1, ImmutableMap.of(slot, colBuilder.build())); + tupleSize = stats.computeTupleSize(ImmutableList.of(slot)); + Assertions.assertEquals(1, tupleSize); + + colBuilder.setAvgSizeByte(-1.0); + stats = new Statistics(1, 1, ImmutableMap.of(slot, colBuilder.build())); + tupleSize = stats.computeTupleSize(ImmutableList.of(slot)); + Assertions.assertEquals(1, tupleSize); + } +} From 592415b39796dcef9c97832f62c26b7c65a99a9c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 12:13:56 +0800 Subject: [PATCH 440/572] branch-3.0: [fix](test)fix some docker regression test #54701 (#54724) Cherry-picked from #54701 Co-authored-by: koarz --- .../suites/cloud_p0/multi_cluster/test_tvf.groovy | 9 +++------ .../info_schema_db/test_backend_configuration.groovy | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/regression-test/suites/cloud_p0/multi_cluster/test_tvf.groovy b/regression-test/suites/cloud_p0/multi_cluster/test_tvf.groovy index 09b19fa364b84c..f43c3406c81392 100644 --- a/regression-test/suites/cloud_p0/multi_cluster/test_tvf.groovy +++ b/regression-test/suites/cloud_p0/multi_cluster/test_tvf.groovy @@ -29,11 +29,8 @@ suite('test_tvf', 'multi_cluster,docker') { for (def i = 0; i < 100; i++) { def ret = sql """select * from numbers("number" = "100")""" assertEquals(ret.size(), 100) - test { - // current cloud not implement it - sql """select START_VERSION,END_VERSION from information_schema.rowsets""" - exception "_get_all_rowsets is not implemented" - } + ret = sql """select START_VERSION,END_VERSION from information_schema.rowsets""" + assertTrue(ret.size() > 0) } } @@ -74,7 +71,7 @@ suite('test_tvf', 'multi_cluster,docker') { // use old clusterName, has been droped test { sql """select * from numbers("number" = "100")""" - exception "in cloud maybe this cluster has been dropped" + exception "Can not find compute group" } // switch to old cluster sql """use @${currentCluster.cluster}""" diff --git a/regression-test/suites/external_table_p0/info_schema_db/test_backend_configuration.groovy b/regression-test/suites/external_table_p0/info_schema_db/test_backend_configuration.groovy index 704135cb8882bd..a9321d9b1a2ed3 100644 --- a/regression-test/suites/external_table_p0/info_schema_db/test_backend_configuration.groovy +++ b/regression-test/suites/external_table_p0/info_schema_db/test_backend_configuration.groovy @@ -27,12 +27,12 @@ suite("test_backend_configuration", "docker, p0, external_table,information_sche assertTrue(res.size() == 3) sql """ - select CONFIG_NAME, CONFIG_TYPE, CONFIG_VALUE, IS_MUTABLE from information_schema.backend_configuration where CONFIGURATION = "disable_auto_compaction"; + select * from information_schema.backend_configuration where CONFIG_NAME = "disable_auto_compaction"; """ assertTrue(res.size() == 3) res = sql """ - select CONFIG_NAME, CONFIG_TYPE, CONFIG_VALUE, IS_MUTABLE from information_schema.backend_configuration where CONFIGURATION = "LZ4_HC_compression_level"; + select * from information_schema.backend_configuration where CONFIG_NAME = "LZ4_HC_compression_level"; """ assertTrue(res.size() == 3) } From ac4d676a951867b3a74e017ce862fb829b20ecf1 Mon Sep 17 00:00:00 2001 From: amory Date: Thu, 14 Aug 2025 12:58:18 +0800 Subject: [PATCH 441/572] branch-3.0 [cherry-pick](variant) Rename session-var for flatten nested (#54599) backport : https://github.com/apache/doris/pull/54413 --- .../apache/doris/datasource/InternalCatalog.java | 10 ++++++---- .../org/apache/doris/qe/SessionVariable.java | 16 ++++++++-------- .../load.groovy | 2 ++ .../load.groovy | 1 + .../suites/variant_p0/delete_update.groovy | 4 ++-- regression-test/suites/variant_p0/nested.groovy | 2 +- regression-test/suites/variant_p0/nested2.groovy | 2 +- .../test_double_write_when_schema_change.groovy | 2 +- 8 files changed, 22 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 3be70a8c90be77..83ef30b0d0e808 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2645,20 +2645,22 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx boolean variantEnableFlattenNested = false; try { variantEnableFlattenNested = PropertyAnalyzer.analyzeVariantFlattenNested(properties); - // only if session variable: disable_variant_flatten_nested = false and + // only if session variable: enable_variant_flatten_nested = true and // table property: variant_enable_flatten_nested = true // we can enable variant flatten nested otherwise throw error - if (ctx != null && !ctx.getSessionVariable().getDisableVariantFlattenNested() + if (ctx != null && ctx.getSessionVariable().getEnableVariantFlattenNested() && variantEnableFlattenNested) { olapTable.setVariantEnableFlattenNested(variantEnableFlattenNested); } else if (variantEnableFlattenNested) { throw new DdlException("If you want to enable variant flatten nested, " - + "please set session variable: disable_variant_flatten_nested = false"); + + "please set session variable: enable_variant_flatten_nested = true"); + } else { + // keep table property: variant_enable_flatten_nested = false + olapTable.setVariantEnableFlattenNested(false); } } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } - olapTable.setVariantEnableFlattenNested(variantEnableFlattenNested); // get storage format TStorageFormat storageFormat = TStorageFormat.V2; // default is segment v2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 50a70b67844a0e..e05ca844f58960 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -681,9 +681,9 @@ public class SessionVariable implements Serializable, Writable { public static final String DISABLE_INVERTED_INDEX_V1_FOR_VARIANT = "disable_inverted_index_v1_for_variant"; - // disable variant flatten nested as session variable, default is true, + // enable variant flatten nested as session variable, default is false, // which means disable variant flatten nested when create table - public static final String DISABLE_VARIANT_FLATTEN_NESTED = "disable_variant_flatten_nested"; + public static final String ENABLE_VARIANT_FLATTEN_NESTED = "enable_variant_flatten_nested"; // CLOUD_VARIABLES_BEGIN public static final String CLOUD_CLUSTER = "cloud_cluster"; @@ -1284,8 +1284,8 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = DISABLE_INVERTED_INDEX_V1_FOR_VARIANT, needForward = true) private boolean disableInvertedIndexV1ForVaraint = true; - @VariableMgr.VarAttr(name = DISABLE_VARIANT_FLATTEN_NESTED, needForward = true) - private boolean disableVariantFlattenNested = true; + @VariableMgr.VarAttr(name = ENABLE_VARIANT_FLATTEN_NESTED, needForward = true) + private boolean enableVariantFlattenNested = false; public int getBeNumberForTest() { return beNumberForTest; @@ -4794,12 +4794,12 @@ public boolean getDisableInvertedIndexV1ForVaraint() { return disableInvertedIndexV1ForVaraint; } - public void setDisableVariantFlattenNested(boolean disableVariantFlattenNested) { - this.disableVariantFlattenNested = disableVariantFlattenNested; + public void setEnableVariantFlattenNested(boolean enableVariantFlattenNested) { + this.enableVariantFlattenNested = enableVariantFlattenNested; } - public boolean getDisableVariantFlattenNested() { - return disableVariantFlattenNested; + public boolean getEnableVariantFlattenNested() { + return enableVariantFlattenNested; } public void checkSqlConvertorFeatures(String features) { diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy index 316ebabbb55a60..ba499801cd0fc3 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy @@ -149,6 +149,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" + sql """ set enable_variant_flatten_nested = true """ table_name = "github_events" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( @@ -197,6 +198,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ qt_sql """select cast(v["payload"]["pull_request"]["additions"] as int) from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1;""" qt_sql """select * from github_events where cast(v["repo"]["name"] as string) = 'xpressengine/xe-core' order by 1 limit 10""" sql """select * from github_events order by k limit 10""" + sql """ set enable_variant_flatten_nested = true """ sql """ CREATE TABLE IF NOT EXISTS github_events2 ( k bigint, diff --git a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy index d4843eee09450c..9db13756749750 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_upgrade_p2/load.groovy @@ -73,6 +73,7 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" + sql """ set enable_variant_flatten_nested = true """ table_name = "github_events" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy index dcae6c628bf6e4..f7c27b55dc8237 100644 --- a/regression-test/suites/variant_p0/delete_update.groovy +++ b/regression-test/suites/variant_p0/delete_update.groovy @@ -21,7 +21,7 @@ suite("regression_test_variant_delete_and_update", "variant_type"){ // MOR def table_name = "var_delete_update" sql "DROP TABLE IF EXISTS ${table_name}" - sql """ set disable_variant_flatten_nested = false """ + sql """ set enable_variant_flatten_nested = true """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, @@ -171,4 +171,4 @@ suite("regression_test_variant_delete_and_update", "variant_type"){ sql "sync" qt_sql """ select * from ${tableName} order by id;""" -} \ No newline at end of file +} diff --git a/regression-test/suites/variant_p0/nested.groovy b/regression-test/suites/variant_p0/nested.groovy index 30559859f6c726..7453a4d893558d 100644 --- a/regression-test/suites/variant_p0/nested.groovy +++ b/regression-test/suites/variant_p0/nested.groovy @@ -24,7 +24,7 @@ suite("regression_test_variant_nested", "p0"){ def table_name = "var_nested" sql "DROP TABLE IF EXISTS ${table_name}" - sql "set disable_variant_flatten_nested = false" + sql "set enable_variant_flatten_nested = true" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, diff --git a/regression-test/suites/variant_p0/nested2.groovy b/regression-test/suites/variant_p0/nested2.groovy index 099b1c903f15d8..7ab313c8303b15 100644 --- a/regression-test/suites/variant_p0/nested2.groovy +++ b/regression-test/suites/variant_p0/nested2.groovy @@ -24,7 +24,7 @@ suite("variant_nested_type_conflict", "p0"){ sql "DROP TABLE IF EXISTS ${table_name}" sql """set describe_extend_variant_column = true""" - sql """ set disable_variant_flatten_nested = false """ + sql """ set enable_variant_flatten_nested = true """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, diff --git a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy index a8b78bdd258582..3bad6181251234 100644 --- a/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy +++ b/regression-test/suites/variant_p0/schema_change/test_double_write_when_schema_change.groovy @@ -57,7 +57,7 @@ suite("double_write_schema_change_with_variant", "nonConcurrent") { def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" - sql "set disable_variant_flatten_nested = false" + sql "set enable_variant_flatten_nested = true" sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, From 9a25102ebd99b7e68f1279b26fce1d8bf8c741af Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 14 Aug 2025 14:09:43 +0800 Subject: [PATCH 442/572] branch-3.0: [Fix](case) fix missing output block for test_cloud_multi_segments_re_calc_in_publish (#54732) --- .../test_cloud_multi_segments_re_calc_in_publish.out | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out b/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out index f9e767c3d20f3e..908ea49d7ddced 100644 --- a/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out +++ b/regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out @@ -10,3 +10,14 @@ -- !dup_key_count -- 0 +-- !sql -- +77777 77777 77777 +88888 88888 88888 +99999 99999 99999 + +-- !sql -- +3 + +-- !dup_key_count -- +0 + From 3b51acb612e7a6280212273d613fac4050091af2 Mon Sep 17 00:00:00 2001 From: Luwei Date: Thu, 14 Aug 2025 14:10:55 +0800 Subject: [PATCH 443/572] [regression-test](multi-az) remove redundant case (#54709) --- .../cloud/multi_cluster/vcluster/vcluster.out | 7 - .../multi_cluster/vcluster/vcluster.groovy | 173 ------------------ 2 files changed, 180 deletions(-) delete mode 100644 regression-test/data/cloud/multi_cluster/vcluster/vcluster.out delete mode 100644 regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy diff --git a/regression-test/data/cloud/multi_cluster/vcluster/vcluster.out b/regression-test/data/cloud/multi_cluster/vcluster/vcluster.out deleted file mode 100644 index 369f2e3af7889b..00000000000000 --- a/regression-test/data/cloud/multi_cluster/vcluster/vcluster.out +++ /dev/null @@ -1,7 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !all11 -- -20 - --- !all12 -- -11 - diff --git a/regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy b/regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy deleted file mode 100644 index 390e65d4b39da4..00000000000000 --- a/regression-test/suites/cloud/multi_cluster/vcluster/vcluster.groovy +++ /dev/null @@ -1,173 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import groovy.json.JsonOutput - -suite("vcluster") { - def token = context.config.metaServiceToken - def instance_id = context.config.multiClusterInstance - String tableName = "test_all_vcluster" - - List ipList = new ArrayList<>() - List hbPortList = new ArrayList<>() - List httpPortList = new ArrayList<>() - List beUniqueIdList = new ArrayList<>() - List bePortList = new ArrayList<>() - - String[] bes = context.config.multiClusterBes.split(','); - println("the value is " + context.config.multiClusterBes); - for(String values : bes) { - println("the value is " + values); - String[] beInfo = values.split(':'); - ipList.add(beInfo[0]); - hbPortList.add(beInfo[1]); - httpPortList.add(beInfo[2]); - beUniqueIdList.add(beInfo[3]); - } - - println("the ip is " + ipList); - println("the heartbeat port is " + hbPortList); - println("the http port is " + httpPortList); - println("the be unique id is " + beUniqueIdList); - - for (unique_id : beUniqueIdList) { - resp = get_cluster.call(unique_id); - for (cluster : resp) { - log.info("lw test drop : ${cluster.type} ".toString()) - if (cluster.type == "COMPUTE" || cluster.type == "VIRTUAL") { - drop_cluster.call(cluster.cluster_name, cluster.cluster_id); - } - } - } - wait_cluster_change() - - List> result = sql "show clusters" - assertTrue(result.size() == 0); - - add_cluster.call(beUniqueIdList[0], ipList[0], hbPortList[0], - "regression_cluster_name0", "regression_cluster_id0"); - add_cluster.call(beUniqueIdList[1], ipList[1], hbPortList[1], - "regression_cluster_name1", "regression_cluster_id1"); - add_vcluster.call("regression_vcluster_name0", "regression_vcluster_id0", - "regression_cluster_name1", "regression_cluster_name0"); - wait_cluster_change() - - result = sql "show clusters" - assertTrue(result.size() == 3); - - for (row : result) { - println row - } - - try { - sql """ use @regression_vcluster_name0 """ - sql """ drop table if exists ${tableName} """ - - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `k1` int(11) NULL, - `k2` tinyint(4) NULL, - `k3` smallint(6) NULL, - `k4` bigint(20) NULL, - `k5` largeint(40) NULL, - `k6` float NULL, - `k7` double NULL, - `k8` decimal(9, 0) NULL, - `k9` char(10) NULL, - `k10` varchar(1024) NULL, - `k11` text NULL, - `k12` date NULL, - `k13` datetime NULL - ) ENGINE=OLAP - DISTRIBUTED BY HASH(`k1`) BUCKETS 3 - ; - """ - - sql """ set enable_profile = true """ - - before_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); - log.info("before_cluster0_load_rows : ${before_cluster0_load_rows}".toString()) - before_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); - log.info("before_cluster0_flush : ${before_cluster0_flush}".toString()) - - before_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); - log.info("before_cluster1_load_rows : ${before_cluster1_load_rows}".toString()) - before_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); - log.info("before_cluster1_flush : ${before_cluster1_flush}".toString()) - - txnId = -1; - streamLoad { - table "${tableName}" - - set 'column_separator', ',' - set 'cloud_cluster', 'regression_vcluster_name0' - - file 'all_types.csv' - time 10000 // limit inflight 10s - - check { loadResult, exception, startTime, endTime -> - if (exception != null) { - throw exception - } - log.info("Stream load result: ${loadResult}".toString()) - def json = parseJson(loadResult) - assertEquals("success", json.Status.toLowerCase()) - assertEquals(20, json.NumberTotalRows) - assertEquals(0, json.NumberFilteredRows) - txnId = json.TxnId - } - } - sql "sync" - order_qt_all11 "SELECT count(*) FROM ${tableName}" // 20 - order_qt_all12 "SELECT count(*) FROM ${tableName} where k1 <= 10" // 11 - - after_cluster0_load_rows = get_be_metric(ipList[0], httpPortList[0], "load_rows"); - log.info("after_cluster0_load_rows : ${after_cluster0_load_rows}".toString()) - after_cluster0_flush = get_be_metric(ipList[0], httpPortList[0], "memtable_flush_total"); - log.info("after_cluster0_flush : ${after_cluster0_flush}".toString()) - - after_cluster1_load_rows = get_be_metric(ipList[1], httpPortList[1], "load_rows"); - log.info("after_cluster1_load_rows : ${after_cluster1_load_rows}".toString()) - after_cluster1_flush = get_be_metric(ipList[1], httpPortList[1], "memtable_flush_total"); - log.info("after_cluster1_flush : ${after_cluster1_flush}".toString()) - - assertTrue(before_cluster0_load_rows == after_cluster0_load_rows) - assertTrue(before_cluster0_flush == after_cluster0_flush) - - assertTrue(before_cluster1_load_rows < after_cluster1_load_rows) - assertTrue(before_cluster1_flush < after_cluster1_flush) - - // fill bePortList - for (int i = 0; i < ipList.size(); ++i) { - result = sql """show backends""" - for (row : result) { - println row - println row[2] - if (ipList[i] == row[1] && hbPortList[i] == row[2]) { - bePortList.add(row[5]); - } - } - } - - set = [ipList[1] + ":" +bePortList[1]] as Set - sql """ select count(k2) AS theCount, k3 from test_all_vcluster group by k3 order by theCount limit 1 """ - checkProfileNew.call(set) - //checkProfileNew1.call("e329bc41f42c49f5-9326cb8429dacc06") - } finally { - sql """ drop table if exists ${tableName} """ - } -} From 2661d4f7e8055d34bbf3f4a3a64fee52ed50c17f Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Thu, 14 Aug 2025 17:15:22 +0800 Subject: [PATCH 444/572] [fix](inverted index) create empty idx file when creating a index on variant-type column (#53814) (#54151) pick from master #53814 --- .../segment_v2/inverted_index_file_writer.cpp | 4 ++- be/test/io/fs/s3_file_writer_test.cpp | 25 +++++++++++++++++++ .../test_variant_empty_index_file.out | 4 +-- .../test_variant_empty_index_file.groovy | 15 ++++++----- 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 08f52920471285..9c9b1d67f68fd4 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -23,6 +23,7 @@ #include #include "common/status.h" +#include "io/fs/s3_file_writer.h" #include "io/fs/stream_sink_file_writer.h" #include "olap/rowset/segment_v2/inverted_index_desc.h" #include "olap/rowset/segment_v2/inverted_index_fs_directory.h" @@ -126,7 +127,8 @@ Status InvertedIndexFileWriter::close() { _closed = true; if (_indices_dirs.empty()) { // An empty file must still be created even if there are no indexes to write - if (dynamic_cast(_idx_v2_writer.get()) != nullptr) { + if (dynamic_cast(_idx_v2_writer.get()) != nullptr || + dynamic_cast(_idx_v2_writer.get()) != nullptr) { return _idx_v2_writer->close(); } return Status::OK(); diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 0662565ec6adac..d537decff4eb04 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -57,6 +57,7 @@ #include "io/fs/s3_file_system.h" #include "io/fs/s3_obj_storage_client.h" #include "io/io_common.h" +#include "olap/rowset/segment_v2/inverted_index_file_writer.h" #include "runtime/exec_env.h" #include "util/slice.h" #include "util/threadpool.h" @@ -1470,4 +1471,28 @@ TEST_F(S3FileWriterTest, write_buffer_boundary) { // clang-format on } +TEST_F(S3FileWriterTest, test_empty_file) { + std::vector paths; + paths.emplace_back(std::string("tmp_dir"), 1024000000); + auto tmp_file_dirs = std::make_unique(paths); + EXPECT_TRUE(tmp_file_dirs->init().ok()); + ExecEnv::GetInstance()->set_tmp_file_dir(std::move(tmp_file_dirs)); + doris::io::FileWriterOptions opts; + io::FileWriterPtr file_writer; + auto st = s3_fs->create_file("test_empty_file.idx", &file_writer, &opts); + EXPECT_TRUE(st.ok()) << st; + auto holder = std::make_shared(S3ClientConf {}); + auto mock_client = std::make_shared(); + holder->_client = mock_client; + dynamic_cast(file_writer.get())->_obj_client = holder; + auto fs = io::global_local_filesystem(); + std::string index_path = "/tmp/empty_index_file_test"; + std::string rowset_id = "1234567890"; + int64_t seg_id = 1234567890; + auto index_file_writer = std::make_unique( + fs, index_path, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2, + std::move(file_writer)); + EXPECT_TRUE(index_file_writer->close().ok()); +} + } // namespace doris diff --git a/regression-test/data/inverted_index_p0/test_variant_empty_index_file.out b/regression-test/data/inverted_index_p0/test_variant_empty_index_file.out index e6e8da3fd6ca66..d06eb4ef1e37d6 100644 --- a/regression-test/data/inverted_index_p0/test_variant_empty_index_file.out +++ b/regression-test/data/inverted_index_p0/test_variant_empty_index_file.out @@ -1,4 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -1 "abcd" +-- !sql9 -- +1 \N diff --git a/regression-test/suites/inverted_index_p0/test_variant_empty_index_file.groovy b/regression-test/suites/inverted_index_p0/test_variant_empty_index_file.groovy index dc98eed4802dad..50fc8fe2826212 100644 --- a/regression-test/suites/inverted_index_p0/test_variant_empty_index_file.groovy +++ b/regression-test/suites/inverted_index_p0/test_variant_empty_index_file.groovy @@ -35,8 +35,9 @@ suite("test_variant_empty_index_file", "p0") { """ sql """ set enable_memtable_on_sink_node = true """ - sql """ insert into ${tableName} values (1, 'abcd') """ - + sql """ insert into ${tableName} values (1, NULL) """ + qt_sql9 "select * from ${tableName}" + sql "sync" def tablets = sql_return_maparray """ show tablets from ${tableName}; """ def backendId_to_backendIP = [:] @@ -47,12 +48,10 @@ suite("test_variant_empty_index_file", "p0") { String backend_id = tablets[0].BackendId String ip = backendId_to_backendIP.get(backend_id) String port = backendId_to_backendHttpPort.get(backend_id) - if (!isCloudMode()) { - def (code, out, err) = http_client("GET", String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s", ip, port, tablet_id)) - logger.info("Run show_nested_index_file_on_tablet: code=" + code + ", out=" + out + ", err=" + err) - assertEquals("E-6004", parseJson(out.trim()).status) - assertTrue(out.contains(" is empty")) - } + def (code, out, err) = http_client("GET", String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s", ip, port, tablet_id)) + logger.info("Run show_nested_index_file_on_tablet: code=" + code + ", out=" + out + ", err=" + err) + assertEquals("E-6004", parseJson(out.trim()).status) + assertTrue(out.contains(" is empty")) try { sql """ select /*+ SET_VAR(enable_match_without_inverted_index = 0) */ * from ${tableName} where v match 'abcd'; """ From 4436a88e362ae7e891bbc00c4efc9f91dc77fa27 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 22:42:15 +0800 Subject: [PATCH 445/572] branch-3.0: [chore](case) add information for cases #54706 (#54747) Cherry-picked from #54706 Co-authored-by: Yongqiang YANG --- .../cold_data_compaction.groovy | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy b/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy index c80e39b8a9d654..d7bc2c2625c2f5 100644 --- a/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy +++ b/regression-test/suites/cold_heat_separation/cold_data_compaction.groovy @@ -103,10 +103,25 @@ suite("test_cold_data_compaction", "nonConcurrent") { }) String tabletId = sql_return_maparray("show tablets from t_recycle_in_s3")[0].TabletId + + // get be http ip and port for tabletId + def tablets = sql_return_maparray("show tablets from t_recycle_in_s3") + def tablet = tablets.find { it.TabletId == tabletId } + def backendId = tablet.BackendId + def bes = sql_return_maparray("show backends") + def injectBe = bes.find { it.BackendId == backendId } + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) + // check number of remote files def filesBeforeCompaction = getS3Client().listObjects( new ListObjectsRequest().withBucketName(getS3BucketName()).withPrefix(s3Prefix + "/data/${tabletId}")).getObjectSummaries() + // logout files + logger.info("Files in S3 before compaction:") + filesBeforeCompaction.each { file -> + logger.info(" - ${file.getKey()} (size: ${file.getSize()})") + } // 5 RowSets + 1 meta assertEquals(6, filesBeforeCompaction.size()) From dff704042195206d16ecf7afedf9fa859b329586 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 09:54:49 +0800 Subject: [PATCH 446/572] branch-3.0: [Fix](case) Fix case `test_cloud_mow_correctness_inject` due to config name change #54738 (#54798) Cherry-picked from #54738 Co-authored-by: bobhan1 --- .../cloud/test_cloud_mow_correctness_inject.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_correctness_inject.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_correctness_inject.groovy index fa447e131d90bf..157622f6fce86d 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_correctness_inject.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_correctness_inject.groovy @@ -62,7 +62,7 @@ suite("test_cloud_mow_correctness_inject", "nonConcurrent") { delete_bitmap_lock_expiration_seconds : 10, calculate_delete_bitmap_task_timeout_seconds : 2, mow_calculate_delete_bitmap_retry_times : 3, - enable_schema_change_retry_in_cloud_mode : false // turn off to shorten the test's time consumption + enable_schema_change_retry : false // turn off to shorten the test's time consumption ] setFeConfigTemporary(customFeConfig) { From 3df50c6e05c2a4c1f1eb5fbab730e6f29806cadc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 09:57:35 +0800 Subject: [PATCH 447/572] branch-3.0: [Fix](case) Fix case `test_cloud_mow_new_tablet_compaction` #54736 (#54797) Cherry-picked from #54736 Co-authored-by: bobhan1 --- .../cloud/test_cloud_mow_new_tablet_compaction.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_new_tablet_compaction.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_new_tablet_compaction.groovy index ea510c03176d9c..acbaac4f163144 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_new_tablet_compaction.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_new_tablet_compaction.groovy @@ -119,9 +119,15 @@ suite("test_cloud_mow_new_tablet_compaction", "nonConcurrent") { logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assert code == 0 def compactJson = parseJson(out.trim()) - assert "success" != compactJson.status.toLowerCase() + assert "success" == compactJson.status.toLowerCase() } + Thread.sleep(1000) + + def (code, out, err) = be_show_tablet_status(tabletBackend.Host, tabletBackend.HttpPort, newTabletId) + assert code == 0 + assert !out.contains("\"last cumulative failure time\": \"1970-01-01") + GetDebugPoint().disableDebugPointForAllBEs("CloudSchemaChangeJob::_process_delete_bitmap.after.capture_without_lock") // wait for sc to finish waitForSchemaChangeDone { From fabd1f2d76de8796a05ea4828d422e0a7d72266a Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 15 Aug 2025 09:58:37 +0800 Subject: [PATCH 448/572] branch-3.0: [fix](load) update scanned rows and loaded bytes progressively #54606 (#54790) Backport #54606 ### What problem does this PR solve? Issue Number: DORIS-20541 Related PR: #29802 Problem Summary: Fix an issue where the scanned rows and loaded bytes metrics were not updated progressively in the FE. Although the BE periodically reports execution status, the FE was ignoring these reports due to a change introduced in PR #29802, which skips processing reports without the isDone flag. This fix ensures that intermediate execution reports are processed, allowing progressive updates of scanned rows and loaded bytes during query execution. --- be/src/vec/sink/writer/vtablet_writer.cpp | 6 + be/src/vec/sink/writer/vtablet_writer_v2.cpp | 5 + .../doris/load/loadv2/LoadStatistic.java | 17 ++- .../java/org/apache/doris/qe/Coordinator.java | 38 ++++--- ...st_s3_load_progressive_scanned_rows.groovy | 107 ++++++++++++++++++ 5 files changed, 153 insertions(+), 20 deletions(-) create mode 100644 regression-test/suites/fault_injection_p0/test_s3_load_progressive_scanned_rows.groovy diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp index 4a5057b7b2d6c7..14e9d3206169a3 100644 --- a/be/src/vec/sink/writer/vtablet_writer.cpp +++ b/be/src/vec/sink/writer/vtablet_writer.cpp @@ -1557,6 +1557,12 @@ Status VTabletWriter::close(Status exec_status) { _do_try_close(_state, exec_status); TEST_INJECTION_POINT("VOlapTableSink::close"); + DBUG_EXECUTE_IF("VTabletWriter.close.sleep", { + auto sleep_sec = DebugPoints::instance()->get_debug_param_or_default( + "VTabletWriter.close.sleep", "sleep_sec", 1); + std::this_thread::sleep_for(std::chrono::seconds(sleep_sec)); + }); + // If _close_status is not ok, all nodes have been canceled in try_close. if (_close_status.ok()) { auto status = Status::OK(); diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp index f432595efa5516..c9e63349270aa4 100644 --- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp @@ -594,6 +594,11 @@ Status VTabletWriterV2::close(Status exec_status) { status = _send_new_partition_batch(); } + DBUG_EXECUTE_IF("VTabletWriterV2.close.sleep", { + auto sleep_sec = DebugPoints::instance()->get_debug_param_or_default( + "VTabletWriterV2.close.sleep", "sleep_sec", 1); + std::this_thread::sleep_for(std::chrono::seconds(sleep_sec)); + }); DBUG_EXECUTE_IF("VTabletWriterV2.close.cancel", { status = Status::InternalError("load cancel"); }); if (status.ok()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java index 0c65aa27851e4e..43c67098bfd1f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java @@ -25,6 +25,8 @@ import com.google.common.collect.Maps; import com.google.common.collect.Table; import com.google.gson.Gson; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.List; @@ -32,6 +34,8 @@ import java.util.Set; public class LoadStatistic { + private static final Logger LOG = LogManager.getLogger(LoadStatistic.class); + // number of rows processed on BE, this number will be updated periodically by query report. // A load job may has several load tasks(queries), and each task has several fragments. // each fragment will report independently. @@ -78,15 +82,24 @@ public synchronized void removeLoad(TUniqueId loadId) { public synchronized void updateLoadProgress(long backendId, TUniqueId loadId, TUniqueId fragmentId, long rows, long bytes, boolean isDone) { if (counterTbl.contains(loadId, fragmentId)) { - counterTbl.put(loadId, fragmentId, rows); + if (counterTbl.get(loadId, fragmentId) < rows) { + counterTbl.put(loadId, fragmentId, rows); + } } if (loadBytes.contains(loadId, fragmentId)) { - loadBytes.put(loadId, fragmentId, bytes); + if (loadBytes.get(loadId, fragmentId) < bytes) { + loadBytes.put(loadId, fragmentId, bytes); + } } if (isDone && unfinishedBackendIds.containsKey(loadId)) { unfinishedBackendIds.get(loadId).remove(backendId); } + + LOG.debug("updateLoadProgress: loadId={}, fragmentId={}, backendId={}, " + + "rows={}, bytes={}, isDone={}, scannedRows={}, loadBytes={}", + DebugUtil.printId(loadId), DebugUtil.printId(fragmentId), backendId, + rows, bytes, isDone, getScannedRows(), getLoadBytes()); } public synchronized long getScannedRows() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index cc6dd424ecbad8..14aa624239af5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -2376,8 +2376,28 @@ private void updateScanRangeNumByScanRange(TScanRangeParams param) { // update job progress from BE public void updateFragmentExecStatus(TReportExecStatusParams params) { + if (params.isSetLoadedRows() && jobId != -1) { + if (params.isSetFragmentInstanceReports()) { + for (TFragmentInstanceReport report : params.getFragmentInstanceReports()) { + Env.getCurrentEnv().getLoadManager().updateJobProgress( + jobId, params.getBackendId(), params.getQueryId(), report.getFragmentInstanceId(), + report.getLoadedRows(), report.getLoadedBytes(), params.isDone()); + Env.getCurrentEnv().getProgressManager().updateProgress(String.valueOf(jobId), + params.getQueryId(), report.getFragmentInstanceId(), report.getNumFinishedRange()); + } + } else { + Env.getCurrentEnv().getLoadManager().updateJobProgress( + jobId, params.getBackendId(), params.getQueryId(), params.getFragmentInstanceId(), + params.getLoadedRows(), params.getLoadedBytes(), params.isDone()); + Env.getCurrentEnv().getProgressManager().updateProgress(String.valueOf(jobId), + params.getQueryId(), params.getFragmentInstanceId(), params.getFinishedScanRanges()); + } + } + PipelineExecContext ctx = pipelineExecContexts.get(Pair.of(params.getFragmentId(), params.getBackendId())); if (ctx == null || !ctx.updatePipelineStatus(params)) { + LOG.debug("Fragment {} is not done, ignore report status: {}", + params.getFragmentId(), params.toString()); return; } @@ -2443,24 +2463,6 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { } fragmentsDoneLatch.markedCountDown(params.getFragmentId(), params.getBackendId()); } - - if (params.isSetLoadedRows() && jobId != -1) { - if (params.isSetFragmentInstanceReports()) { - for (TFragmentInstanceReport report : params.getFragmentInstanceReports()) { - Env.getCurrentEnv().getLoadManager().updateJobProgress( - jobId, params.getBackendId(), params.getQueryId(), report.getFragmentInstanceId(), - report.getLoadedRows(), report.getLoadedBytes(), params.isDone()); - Env.getCurrentEnv().getProgressManager().updateProgress(String.valueOf(jobId), - params.getQueryId(), report.getFragmentInstanceId(), report.getNumFinishedRange()); - } - } else { - Env.getCurrentEnv().getLoadManager().updateJobProgress( - jobId, params.getBackendId(), params.getQueryId(), params.getFragmentInstanceId(), - params.getLoadedRows(), params.getLoadedBytes(), params.isDone()); - Env.getCurrentEnv().getProgressManager().updateProgress(String.valueOf(jobId), - params.getQueryId(), params.getFragmentInstanceId(), params.getFinishedScanRanges()); - } - } } /* diff --git a/regression-test/suites/fault_injection_p0/test_s3_load_progressive_scanned_rows.groovy b/regression-test/suites/fault_injection_p0/test_s3_load_progressive_scanned_rows.groovy new file mode 100644 index 00000000000000..590000dae92826 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_s3_load_progressive_scanned_rows.groovy @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods +import org.apache.doris.regression.util.Http + +suite("test_s3_load_progressive_scanned_rows", "nonConcurrent,p0") { + def tableName = "segcompaction_correctness_test" + def create_table_sql = """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `col_0` BIGINT NOT NULL,`col_1` VARCHAR(20),`col_2` VARCHAR(20),`col_3` VARCHAR(20),`col_4` VARCHAR(20), + `col_5` VARCHAR(20),`col_6` VARCHAR(20),`col_7` VARCHAR(20),`col_8` VARCHAR(20),`col_9` VARCHAR(20), + `col_10` VARCHAR(20),`col_11` VARCHAR(20),`col_12` VARCHAR(20),`col_13` VARCHAR(20),`col_14` VARCHAR(20), + `col_15` VARCHAR(20),`col_16` VARCHAR(20),`col_17` VARCHAR(20),`col_18` VARCHAR(20),`col_19` VARCHAR(20), + `col_20` VARCHAR(20),`col_21` VARCHAR(20),`col_22` VARCHAR(20),`col_23` VARCHAR(20),`col_24` VARCHAR(20), + `col_25` VARCHAR(20),`col_26` VARCHAR(20),`col_27` VARCHAR(20),`col_28` VARCHAR(20),`col_29` VARCHAR(20), + `col_30` VARCHAR(20),`col_31` VARCHAR(20),`col_32` VARCHAR(20),`col_33` VARCHAR(20),`col_34` VARCHAR(20), + `col_35` VARCHAR(20),`col_36` VARCHAR(20),`col_37` VARCHAR(20),`col_38` VARCHAR(20),`col_39` VARCHAR(20), + `col_40` VARCHAR(20),`col_41` VARCHAR(20),`col_42` VARCHAR(20),`col_43` VARCHAR(20),`col_44` VARCHAR(20), + `col_45` VARCHAR(20),`col_46` VARCHAR(20),`col_47` VARCHAR(20),`col_48` VARCHAR(20),`col_49` VARCHAR(20) + ) + DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1 + PROPERTIES ( "replication_num" = "1" ); + """ + def columns = "col_0, col_1, col_2, col_3, col_4, col_5, col_6, col_7, col_8, col_9, col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, col_18, col_19, col_20, col_21, col_22, col_23, col_24, col_25, col_26, col_27, col_28, col_29, col_30, col_31, col_32, col_33, col_34, col_35, col_36, col_37, col_38, col_39, col_40, col_41, col_42, col_43, col_44, col_45, col_46, col_47, col_48, col_49" + def runLoadWithSleep = { + String ak = getS3AK() + String sk = getS3SK() + String endpoint = getS3Endpoint() + String region = getS3Region() + String bucket = getS3BucketName() + try { + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql "${create_table_sql}" + + def uuid = UUID.randomUUID().toString().replace("-", "0") + String columns_str = ("$columns" != "") ? "($columns)" : ""; + + sql """ + LOAD LABEL $uuid ( + DATA INFILE("s3://$bucket/regression/segcompaction/segcompaction.orc") + INTO TABLE ${tableName} + FORMAT AS "ORC" + $columns_str + ) + WITH S3 ( + "AWS_ACCESS_KEY" = "$ak", + "AWS_SECRET_KEY" = "$sk", + "AWS_ENDPOINT" = "$endpoint", + "AWS_REGION" = "$region", + "provider" = "${getS3Provider()}" + ) + """ + + def max_try_milli_secs = 120000 + def scannedRows = 0 + def loadBytes = 0 + String [][] result = '' + while (max_try_milli_secs > 0) { + result = sql """ show load where label="$uuid" order by createtime desc limit 1; """ + logger.info("SHOW LOAD result: ${result}") + + scannedRows = (result =~ /"ScannedRows":(\d+)/)[0][1] as long + loadBytes = (result =~ /"LoadBytes":(\d+)/)[0][1] as long + + if (scannedRows > 0 && loadBytes > 0) { + break; + } + Thread.sleep(1000) + max_try_milli_secs -= 1000 + if(max_try_milli_secs <= 0) { + assertTrue(1 == 2, "load Timeout: $uuid") + } + } + try_sql(""" cancel load where label="$uuid"; """) + assertTrue(scannedRows >= 0, "ScannedRows should be >= 0 but was ${scannedRows}") + assertTrue(loadBytes >= 0, "LoadBytes should be >= 0 but was ${loadBytes}") + } finally { + try_sql("DROP TABLE IF EXISTS ${tableName}") + } + } + + try { + GetDebugPoint().enableDebugPointForAllBEs("VTabletWriter.close.sleep", [sleep_sec:150]); + GetDebugPoint().enableDebugPointForAllBEs("VTabletWriterV2.close.sleep", [sleep_sec:150]); + runLoadWithSleep() + } finally { + GetDebugPoint().disableDebugPointForAllBEs("VTabletWriter.close.sleep") + GetDebugPoint().disableDebugPointForAllBEs("VTabletWriterV2.close.sleep") + } +} + From 03ffdd2dc6c8336ffea0656ff86c5fa9749f3757 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 15 Aug 2025 09:59:27 +0800 Subject: [PATCH 449/572] branch-3.0: [Fix](case) Fix some cases (#54695) (#54756) pick https://github.com/apache/doris/pull/54695 --- .../pipeline/cloud_p0/conf/regression-conf-custom.groovy | 1 - .../test_partial_update_2pc_schema_change.groovy | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy index 3630f4e50d1916..475daad620d976 100644 --- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy +++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy @@ -42,7 +42,6 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th "test_spark_load," + "test_index_lowercase_fault_injection," + "test_index_compaction_failure_injection," + - "test_partial_update_2pc_schema_change," + // mow 2pc "test_query_sys_rowsets," + // rowsets sys table "test_unique_table_debug_data," + // disable auto compaction "test_insert," + // txn insert diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy index d506d0ee766e58..58fe50c73c2ce4 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy @@ -35,7 +35,9 @@ import org.apache.http.client.methods.CloseableHttpResponse import org.apache.http.util.EntityUtils suite("test_partial_update_2pc_schema_change", "p0") { - + if (isCloudMode()) { + return + } String db = context.config.getDbNameByFile(context.file) sql "select 1;" // to create database def user = context.config.jdbcUser From f41baed9e211f237b31706ed7787a9e250a3db0f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:01:29 +0800 Subject: [PATCH 450/572] branch-3.0: [regression-test](variant) fix test_variant_index_format_v1 in cloud mode #54730 (#54767) Cherry-picked from #54730 Co-authored-by: lihangyu --- .../inverted_index_p2/test_variant_index_format_v1.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy index b6b6d5aeef510d..be0c76ab588c8d 100644 --- a/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy +++ b/regression-test/suites/inverted_index_p2/test_variant_index_format_v1.groovy @@ -16,6 +16,9 @@ // under the License. suite("test_variant_index_format_v1", "p2, nonConcurrent") { + if (isCloudMode()) { + return; + } def calc_file_crc_on_tablet = { ip, port, tablet -> return curl("GET", String.format("http://%s:%s/api/calc_crc?tablet_id=%s", ip, port, tablet)) } @@ -61,9 +64,6 @@ suite("test_variant_index_format_v1", "p2, nonConcurrent") { def table_name = "github_events" sql """DROP TABLE IF EXISTS ${table_name}""" setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) { - if (isCloudMode()) { - return; - } sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, From 8ba8466397e1c665ebe2561ee76255e48be63092 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:17:49 +0800 Subject: [PATCH 451/572] branch-3.0: [test](doris-compose) add test readme #54726 (#54774) Cherry-picked from #54726 Co-authored-by: yujun --- docker/runtime/doris-compose/Readme.md | 16 +++++- docker/runtime/doris-compose/requirements.txt | 7 ++- regression-test/README.md | 10 ++++ .../doris/regression/suite/Suite.groovy | 5 ++ .../regression/suite/SuiteCluster.groovy | 20 +++++++ .../suites/demo_p0/docker_action.groovy | 56 +++++++++++++------ 6 files changed, 91 insertions(+), 23 deletions(-) diff --git a/docker/runtime/doris-compose/Readme.md b/docker/runtime/doris-compose/Readme.md index 34bab4578724a6..f304069c3c1f79 100644 --- a/docker/runtime/doris-compose/Readme.md +++ b/docker/runtime/doris-compose/Readme.md @@ -117,7 +117,7 @@ So if multiple users use different `LOCAL_DORIS_PATH`, their clusters may have d ### Create a cluster or recreate its containers ```shell -python docker/runtime/doris-compose/doris-compose.py up +python docker/runtime/doris-compose/doris-compose.py up --add-fe-num --add-be-num [--fe-id --be-id ] ... @@ -176,11 +176,23 @@ Otherwise it will just list summary of each clusters. There are more options about doris-compose. Just try ```shell -python docker/runtime/doris-compose/doris-compose.py -h +python docker/runtime/doris-compose/doris-compose.py -h ``` +### Docker suite in regression test + +Regression test support running a suite in a docker doris cluster. + +See the example [demo_p0/docker_action.groovy](https://github.com/apache/doris/blob/master/regression-test/suites/demo_p0/docker_action.groovy). + +The docker suite can specify fe num and be num, and add/drop/start/stop/restart the fe and be. + +Before run a docker suite, read the annotation in `demo_p0/docker_action.groovy` carefully. + ### Generate regression custom conf file +provide a command for let the regression test connect to a docker cluster. + ```shell python docker/runtime/doris-compose/doris-compose.py config [-q] [--connect-follow-fe] ``` diff --git a/docker/runtime/doris-compose/requirements.txt b/docker/runtime/doris-compose/requirements.txt index 46eebbd0a3f0dd..e519f99af032ea 100644 --- a/docker/runtime/doris-compose/requirements.txt +++ b/docker/runtime/doris-compose/requirements.txt @@ -15,9 +15,6 @@ # specific language governing permissions and limitations # under the License. -# if install docker failed, specific pyyaml version and docker version -#pyyaml==5.3.1 -#docker==6.1.3 docker docker-compose filelock @@ -26,3 +23,7 @@ prettytable pymysql python-dateutil requests<=2.31.0 + +# NOTICE: if install docker failed, specific pyyaml version and docker version +#pyyaml==5.3.1 +#docker==6.1.3 diff --git a/regression-test/README.md b/regression-test/README.md index fb7bdde2ee27c8..e28865d8e141bc 100644 --- a/regression-test/README.md +++ b/regression-test/README.md @@ -84,6 +84,16 @@ under the License. 8. Cases injected should be marked as nonConcurrent and ensured injection to be removed after running the case. +9. Docker case run in a docker cluster. The docker cluster is new created and independent, not contains history data, not affect other cluster. + + Docker case can add/drop/start/stop/restart fe and be, and specify fe and be num. + + Example will see [demo_p0/docker_action.groovy](https://github.com/apache/doris/blob/master/regression-test/suites/demo_p0/docker_action.groovy) + + Read the annotation carefully in the example file. + + Also read the [doris-compose](https://github.com/apache/doris/tree/master/docker/runtime/doris-compose) readme. + ## Compatibility case Refers to the resources or rules created on the initial cluster during FE testing or upgrade testing, which can still be used normally after the cluster restart or upgrade, such as permissions, UDF, etc. diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index b8a1fb259ed146..b5a3f78dd9da6c 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -297,6 +297,7 @@ class Suite implements GroovyInterceptable { // more explaination can see example file: demo_p0/docker_action.groovy public void docker(ClusterOptions options = new ClusterOptions(), Closure actionSupplier) throws Exception { if (context.config.excludeDockerTest) { + logger.info("do not run the docker suite {}, because regression config excludeDockerTest=true", name) return } @@ -314,9 +315,13 @@ class Suite implements GroovyInterceptable { } } else { if (options.cloudMode == true && context.config.runMode == RunMode.NOT_CLOUD) { + logger.info("do not run the docker suite {}, because the suite's ClusterOptions.cloudMode=true " + + "but regression test is local mode", name) return } if (options.cloudMode == false && context.config.runMode == RunMode.CLOUD) { + logger.info("do not run the docker suite {}, because the suite's ClusterOptions.cloudMode=false " + + "but regression test is cloud mode", name) return } dockerImpl(options, options.cloudMode, actionSupplier) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy index aa7d3e7afe7865..e18f6f8061c2f1 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy @@ -119,6 +119,7 @@ class ListHeader { class ServerNode { + // all node index start from 1, not 0 int index String host int httpPort @@ -558,48 +559,57 @@ class SuiteCluster { int START_WAIT_TIMEOUT = 120 int STOP_WAIT_TIMEOUT = 60 + // indices start from 1, not 0 // if not specific fe indices, then start all frontends void startFrontends(int... indices) { runFrontendsCmd(START_WAIT_TIMEOUT + 5, "start --wait-timeout ${START_WAIT_TIMEOUT}".toString(), indices) } + // indices start from 1, not 0 // if not specific be indices, then start all backends void startBackends(int... indices) { runBackendsCmd(START_WAIT_TIMEOUT + 5, "start --wait-timeout ${START_WAIT_TIMEOUT}".toString(), indices) } + // indices start from 1, not 0 // if not specific fe indices, then stop all frontends void stopFrontends(int... indices) { runFrontendsCmd(STOP_WAIT_TIMEOUT + 5, "stop --wait-timeout ${STOP_WAIT_TIMEOUT}".toString(), indices) waitHbChanged() } + // indices start from 1, not 0 // if not specific be indices, then stop all backends void stopBackends(int... indices) { runBackendsCmd(STOP_WAIT_TIMEOUT + 5, "stop --wait-timeout ${STOP_WAIT_TIMEOUT}".toString(), indices) waitHbChanged() } + // indices start from 1, not 0 // if not specific fe indices, then restart all frontends void restartFrontends(int... indices) { runFrontendsCmd(START_WAIT_TIMEOUT + 5, "restart --wait-timeout ${START_WAIT_TIMEOUT}".toString(), indices) } + // indices start from 1, not 0 // if not specific be indices, then restart all backends void restartBackends(int... indices) { runBackendsCmd(START_WAIT_TIMEOUT + 5, "restart --wait-timeout ${START_WAIT_TIMEOUT}".toString(), indices) } + // indices start from 1, not 0 // if not specific ms indices, then restart all ms void restartMs(int... indices) { runMsCmd(START_WAIT_TIMEOUT + 5, "restart --wait-timeout ${START_WAIT_TIMEOUT}".toString(), indices) } + // indices start from 1, not 0 // if not specific recycler indices, then restart all recyclers void restartRecyclers(int... indices) { runRecyclerCmd(START_WAIT_TIMEOUT + 5, "restart --wait-timeout ${START_WAIT_TIMEOUT}".toString(), indices) } + // indices start from 1, not 0 // if not specific fe indices, then drop all frontends void dropFrontends(boolean clean, int... indices) { def cmd = 'down' @@ -609,6 +619,7 @@ class SuiteCluster { runFrontendsCmd(60, cmd, indices) } + // indices start from 1, not 0 // if not specific be indices, then decommission all backends void decommissionBackends(boolean clean, int... indices) { def cmd = 'down' @@ -618,6 +629,7 @@ class SuiteCluster { runBackendsCmd(300, cmd, indices) } + // indices start from 1, not 0 // if not specific be indices, then drop force all backends void dropForceBackends(boolean clean, int... indices) { def cmd = 'down --drop-force' @@ -627,6 +639,7 @@ class SuiteCluster { runBackendsCmd(60, cmd, indices) } + // index start from 1, not 0 void checkFeIsAlive(int index, boolean isAlive) { def fe = getFeByIndex(index) assert fe != null : 'frontend with index ' + index + ' not exists!' @@ -634,6 +647,7 @@ class SuiteCluster { : 'frontend with index ' + index + ' dead') } + // index start from 1, not 0 void checkBeIsAlive(int index, boolean isAlive) { def be = getBeByIndex(index) assert be != null : 'backend with index ' + index + ' not exists!' @@ -641,6 +655,7 @@ class SuiteCluster { : 'backend with index ' + index + ' dead') } + // index start from 1, not 0 void checkFeIsExists(int index, boolean isExists) { def fe = getFeByIndex(index) if (isExists) { @@ -650,6 +665,7 @@ class SuiteCluster { } } + // index start from 1, not 0 void checkBeIsExists(int index, boolean isExists) { def be = getBeByIndex(index) if (isExists) { @@ -669,21 +685,25 @@ class SuiteCluster { Thread.sleep(7000) } + // indices start from 1, not 0 private void runFrontendsCmd(int timeoutSecond, String op, int... indices) { def cmd = op + ' ' + name + ' --fe-id ' + indices.join(' ') runCmd(cmd, timeoutSecond) } + // indices start from 1, not 0 private void runBackendsCmd(int timeoutSecond, String op, int... indices) { def cmd = op + ' ' + name + ' --be-id ' + indices.join(' ') runCmd(cmd, timeoutSecond) } + // indices start from 1, not 0 private void runMsCmd(int timeoutSecond, String op, int... indices) { def cmd = op + ' ' + name + ' --ms-id ' + indices.join(' ') runCmd(cmd, timeoutSecond) } + // indices start from 1, not 0 private void runRecyclerCmd(int timeoutSecond, String op, int... indices) { def cmd = op + ' ' + name + ' --recycle-id ' + indices.join(' ') runCmd(cmd, timeoutSecond) diff --git a/regression-test/suites/demo_p0/docker_action.groovy b/regression-test/suites/demo_p0/docker_action.groovy index 7e111b4828595d..b0569cb930d88d 100644 --- a/regression-test/suites/demo_p0/docker_action.groovy +++ b/regression-test/suites/demo_p0/docker_action.groovy @@ -17,32 +17,51 @@ import org.apache.doris.regression.suite.ClusterOptions +// Every docker suite will connect to a docker cluster. +// The docker cluster is new created and independent, not contains history data, +// not affect the external doris cluster, not affect other docker cluster. + // Run docker suite steps: -// 1. Read 'docker/runtime/doris-compose/Readme.md', make sure you can setup a doris docker cluster; -// 2. update regression-conf-custom.groovy with config: +// 1. Before run docker regreesion test, make sure you can setup a doris docker cluster. +// Read readme in [doris-compose](https://github.com/apache/doris/tree/master/docker/runtime/doris-compose)' +// to setup a docker doris cluster; +// 2. Then run the docker suite, and setup regression-conf-custom.groovy with following config: // image = "xxxx" // your doris docker image // excludeDockerTest = false // do run docker suite, default is true // dockerEndDeleteFiles = false // after run docker suite, whether delete contains's log and data in directory '/tmp/doris/' -// When run docker suite, then no need an external doris cluster. +// When run docker suite, the regression test no need to connect to an external doris cluster, +// but can still connect to one just like run a non-docker suite. // But whether run a docker suite, need more check. -// Firstly, get the pipeline's run mode (cloud or not_cloud): -// If there's an external doris cluster, then fetch pipeline's runMode from it. -// If there's no external doris cluster, then set pipeline's runMode with command args. -// for example: sh run-regression-test.sh --run docker_action -runMode=cloud/not_cloud -// Secondly, compare ClusterOptions.cloudMode and pipeline's runMode -// If ClusterOptions.cloudMode = null then let ClusterOptions.cloudMode = pipeline's cloudMode, and run docker suite. -// if ClusterOptions.cloudMode = true or false, if cloudMode == pipeline's cloudMode or pipeline's cloudMode is unknown, -// then run docker suite, otherwise don't run docker suite. +// Firstly, get the regression test's run mode (cloud or not_cloud): +// a) If the regression test connect to an external doris cluster, +// then will use the external doris cluster's runMode(cloud or not_cloud) as the regression runMode. +// b) If there's no external doris cluster, then user can set the regression runMode with command arg `-runMode`. +// for example: +// `sh run-regression-test.sh --run -d demo_p0 -s docker_action -runMode=cloud/not_cloud` +// what's more, if the docker suite not contains 'isCloudMode()', then no need specify the command arg `-runMode`. +// for exmaple, if the regression not connect to an external doris cluster, then command: +// `sh run-regression-test.sh --run -d demo_p0 -s docker_action` +// will run both cloud case and not_cloud case. +// Secondly, compare ClusterOptions.cloudMode and the regression's runMode +// a) If ClusterOptions.cloudMode = null then let ClusterOptions.cloudMode = the regression's cloudMode, and run docker suite. +// b) if ClusterOptions.cloudMode = true or false, and regreesion runMode equals equals the suite's cloudMode +// or regression's cloudMode is unknown, then run docker suite. +// +// +// By default, after run a docker suite, whether run succ or fail, the suite's relate docker cluster will auto destroy. +// If user don't want to destroy the docker cluster after the test, then user can specify with arg `-noKillDocker` +// for exmaple: +// `sh run-regression-test.sh --run -d demo_p0 -s docker_action -noKillDocker` +// will run 3 docker cluster, and the last docker cluster will not destroy . -// NOTICE: +// NOTICE, for code: // 1. Need add 'docker' to suite's group, and don't add 'nonConcurrent' to it; -// 2. In docker closure: -// a. remove function dockerAwaitUntil(...), should use 'Awaitility.await()...until(f)' directly or use 'awaitUntil(...)'; -// 3. No need to use code ` if (isCloudMode()) { return } ` in docker suites, -// instead should use `ClusterOptions.cloudMode = true/false` is enough. -// Because when run docker suite without an external doris cluster, if suite use code `isCloudMode()`, it need specific -runMode=cloud/not_cloud. -// On the contrary, `ClusterOptions.cloudMode = true/false` no need specific -runMode=cloud/not_cloud when no external doris cluster exists. +// 2. No need to use code ` if (isCloudMode()) { return } ` in docker suites, +// instead should use `ClusterOptions.cloudMode = true/false` is enough. +// Because when run docker suite without an external doris cluster, if suite use code `isCloudMode()`, it need specific -runMode=cloud/not_cloud. +// On the contrary, `ClusterOptions.cloudMode = true/false` no need specific -runMode=cloud/not_cloud when no external doris cluster exists. +// 3. For more options and functions usage, read the file `suite/SuiteCluster.groovy` in regression framework. suite('docker_action', 'docker') { // run a new docker @@ -52,6 +71,7 @@ suite('docker_action', 'docker') { cluster.checkBeIsAlive(2, true) + // fe and be's index start from 1, not 0. // stop backend 2, 3 cluster.stopBackends(2, 3) From 7658dd962e51823bb202a68b82b2dbe2b0ad9b43 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:18:28 +0800 Subject: [PATCH 452/572] branch-3.0: [fix](test) increase timeout in test_broker_load_func #54783 (#54810) Cherry-picked from #54783 Co-authored-by: Kaijie Chen --- .../external_table_p0/broker_load/test_broker_load_func.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/external_table_p0/broker_load/test_broker_load_func.groovy b/regression-test/suites/external_table_p0/broker_load/test_broker_load_func.groovy index 862c37d28fa1d1..ed8ff2161cfc9f 100644 --- a/regression-test/suites/external_table_p0/broker_load/test_broker_load_func.groovy +++ b/regression-test/suites/external_table_p0/broker_load/test_broker_load_func.groovy @@ -59,7 +59,7 @@ suite("test_broker_load_func", "p0,external,hive,external_docker,external_docker """ def check_load_result = {checklabel, testTablex -> - def max_try_milli_secs = 120000 + def max_try_milli_secs = 600000 while(max_try_milli_secs) { def result = sql "show load where label = '${checklabel}'" if(result[0][2] == "FINISHED") { From 8ae386e68dcfec90e003e3d97a72153e7bfa8876 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Fri, 15 Aug 2025 10:20:46 +0800 Subject: [PATCH 453/572] =?UTF-8?q?branch-3.0:[enhance](auth)Remove=20rest?= =?UTF-8?q?rictions=20on=20user=20creation=20and=20other=20operati?= =?UTF-8?q?=E2=80=A6=20(#53819)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ons when enabling ranger/LDAP (#50139) pick: https://github.com/apache/doris/pull/50139 --- .../java/org/apache/doris/analysis/CreateUserStmt.java | 8 -------- .../main/java/org/apache/doris/analysis/DropUserStmt.java | 7 ------- 2 files changed, 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java index 74ca252779e61f..0ab1f1ff30b30a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateUserStmt.java @@ -19,13 +19,10 @@ import org.apache.doris.catalog.Env; import org.apache.doris.cluster.ClusterNamespace; -import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; -import org.apache.doris.mysql.authenticate.AuthenticateType; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.mysql.privilege.Role; import org.apache.doris.qe.ConnectContext; @@ -146,11 +143,6 @@ public String getComment() { public void analyze(Analyzer analyzer) throws UserException { super.analyze(analyzer); - if (Config.access_controller_type.equalsIgnoreCase("ranger-doris") - && AuthenticateType.getAuthTypeConfig() == AuthenticateType.LDAP) { - throw new AnalysisException("Create user is prohibited when Ranger and LDAP are enabled at same time."); - } - userIdent.analyze(); if (userIdent.isRootUser()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java index 5f9872e42ad7cc..39012f866d3315 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropUserStmt.java @@ -19,11 +19,9 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; -import org.apache.doris.mysql.authenticate.AuthenticateType; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; @@ -56,11 +54,6 @@ public UserIdentity getUserIdentity() { public void analyze(Analyzer analyzer) throws AnalysisException, UserException { super.analyze(analyzer); - if (Config.access_controller_type.equalsIgnoreCase("ranger-doris") - && AuthenticateType.getAuthTypeConfig() == AuthenticateType.LDAP) { - throw new AnalysisException("Drop user is prohibited when Ranger and LDAP are enabled at same time."); - } - userIdent.analyze(); if (userIdent.isSystemUser()) { From f5129dcb2a9119e5b747b8d5cfce9b1c88f51e3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 10:26:53 +0800 Subject: [PATCH 454/572] branch-3.0: [chore](case) do not use global var in groovy #54700 (#54805) Cherry-picked from #54700 Co-authored-by: Yongqiang YANG --- ...test_schema_change_with_compaction11.groovy | 18 +++++++++--------- ...t_sc_fail_release_delete_bitmap_lock.groovy | 4 ++-- ...test_fix_tablet_stat_fault_injection.groovy | 6 +++--- .../load_p0/stream_load/test_json_load.groovy | 2 +- ...rtition_mod_distribution_key_restart.groovy | 2 +- ...t_partial_update_conflict_be_restart.groovy | 7 +++---- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy index 0e765e46b7b82c..310c10b4dbcb55 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy @@ -109,7 +109,7 @@ suite('test_schema_change_with_compaction11', 'docker') { def newTabletId = array[1].TabletId trigger_and_wait_compaction("date", "base") logger.info("run compaction:" + newTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, newTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, newTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) @@ -120,11 +120,11 @@ suite('test_schema_change_with_compaction11', 'docker') { (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - running = true + def running = true do { Thread.sleep(100) - tabletId = array[0].TabletId - (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) + def currentTabletId = array[0].TabletId + (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, currentTabletId) logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def compactionStatus = parseJson(out.trim()) @@ -133,9 +133,9 @@ suite('test_schema_change_with_compaction11', 'docker') { } while (running) // new tablet cannot do cu compaction - tabletId = array[1].TabletId - logger.info("run compaction:" + tabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) + def newTabletIdForCu = array[1].TabletId + logger.info("run compaction:" + newTabletIdForCu) + (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, newTabletIdForCu) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) @@ -162,7 +162,7 @@ suite('test_schema_change_with_compaction11', 'docker') { assertEquals(count[0][0], 2556); // check rowsets logger.info("run show:" + originTabletId) - (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_show_tablet_status(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run show: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("[0-1]")) assertTrue(out.contains("[2-7]")) @@ -186,7 +186,7 @@ suite('test_schema_change_with_compaction11', 'docker') { // wait for all compactions done - boolean running = true + def running = true while (running) { Thread.sleep(100) (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, newTabletId) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_sc_fail_release_delete_bitmap_lock.groovy b/regression-test/suites/fault_injection_p0/cloud/test_sc_fail_release_delete_bitmap_lock.groovy index 60d995b323807a..ffebceae152f25 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_sc_fail_release_delete_bitmap_lock.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_sc_fail_release_delete_bitmap_lock.groovy @@ -122,9 +122,9 @@ suite("test_sc_fail_release_delete_bitmap_lock", "nonConcurrent") { assert res[0].State == "CANCELLED" assert res[0].Msg.contains("[DELETE_BITMAP_LOCK_ERROR]test update delete bitmap failed") GetDebugPoint().disableDebugPointForAllBEs("CloudMetaMgr::test_update_delete_bitmap_fail") - now = System.currentTimeMillis() + def now = System.currentTimeMillis() do_insert_into() - time_cost = System.currentTimeMillis() - now + def time_cost = System.currentTimeMillis() - now log.info("time_cost(ms): ${time_cost}") assertTrue(time_cost < 10000, "wait time should less than 10s") } diff --git a/regression-test/suites/fault_injection_p0/test_fix_tablet_stat_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_fix_tablet_stat_fault_injection.groovy index 5703bbd29c4af5..cc248522a27d2e 100644 --- a/regression-test/suites/fault_injection_p0/test_fix_tablet_stat_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_fix_tablet_stat_fault_injection.groovy @@ -71,7 +71,7 @@ suite("test_fix_tablet_stat_fault_injection", "nonConcurrent") { int rowsetCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId - (code, out, err) = curl("GET", tablet.CompactionStatus) + def (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status after insert data: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def tabletJson = parseJson(out.trim()) @@ -88,7 +88,7 @@ suite("test_fix_tablet_stat_fault_injection", "nonConcurrent") { rowsetCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId - (code, out, err) = curl("GET", tablet.CompactionStatus) + def (code, out, err) = curl("GET", tablet.CompactionStatus) logger.info("Show tablets status after full compaction: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def tabletJson = parseJson(out.trim()) @@ -109,7 +109,7 @@ suite("test_fix_tablet_stat_fault_injection", "nonConcurrent") { rowsetCount = 0 for (def tablet in tablets) { String tablet_id = tablet.TabletId - (code, out, err) = curl("GET", tablet.CompactionStatus) + def (code, out, err) = curl("GET", tablet.CompactionStatus) //logger.info("Show tablets status after fix stats: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def tabletJson = parseJson(out.trim()) diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy index 3be34c7989a29b..172fd9fe7a972b 100644 --- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy +++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy @@ -230,7 +230,7 @@ suite("test_json_load", "p0,nonConcurrent") { def check_load_result = {checklabel, testTablex -> def max_try_milli_secs = 10000 while(max_try_milli_secs) { - result = sql "show load where label = '${checklabel}'" + def result = sql "show load where label = '${checklabel}'" if(result[0][2] == "FINISHED") { sql "sync" qt_select "select * from ${testTablex} order by id" diff --git a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key_restart.groovy b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key_restart.groovy index d31b26758e01fc..e40a5769649adb 100644 --- a/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key_restart.groovy +++ b/regression-test/suites/partition_p0/dynamic_partition/test_dynamic_partition_mod_distribution_key_restart.groovy @@ -63,7 +63,7 @@ suite("test_dynamic_partition_mod_distribution_key_restart", "docker") { sql """ ADMIN SET FRONTEND CONFIG ('dynamic_partition_check_interval_seconds' = '1') """ sql """ alter table ${tableName} set('dynamic_partition.end'='5') """ - result = sql "show partitions from ${tableName}" + def result = sql "show partitions from ${tableName}" for (def retry = 0; retry < 10; retry++) { // at most wait 120s if (result.size() == 9) { break; diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_conflict_be_restart.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_conflict_be_restart.groovy index 642363f909866d..aaf8a3e0bd01cb 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_conflict_be_restart.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_conflict_be_restart.groovy @@ -30,7 +30,6 @@ import org.apache.http.protocol.HttpContext import org.apache.http.HttpRequest import org.apache.http.impl.client.LaxRedirectStrategy import org.apache.http.client.methods.RequestBuilder -import org.apache.http.entity.StringEntity import org.apache.http.client.methods.CloseableHttpResponse import org.apache.http.util.EntityUtils import org.apache.doris.regression.suite.ClusterOptions @@ -69,9 +68,9 @@ suite("test_partial_update_conflict_be_restart", 'docker') { log.info("http_stream execute 2pc: ${command}") def process = command.execute() - code = process.waitFor() - out = process.text - json2pc = parseJson(out) + def code = process.waitFor() + def out = process.text + def json2pc = parseJson(out) log.info("http_stream 2pc result: ${out}".toString()) assertEquals(code, 0) assertEquals("success", json2pc.status.toLowerCase()) From 9ce5fcc74a78d87d7dc1e9e8852c1dcaaacea4ad Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Fri, 15 Aug 2025 13:09:13 +0800 Subject: [PATCH 455/572] branch-3.0: [test](cases) fix some wrong cases (#54631) (#54770) pick https://github.com/apache/doris/pull/54631 --- .../auto_partition/ddl/create_list_part_data_table.sql | 2 +- .../auto_partition/ddl/create_range_part_data_table.sql | 2 +- .../auto_partition/ddl/small_data_high_concurrrent_load.sql | 2 +- .../auto_partition/ddl/stream_load_list_test_table.sql | 2 +- .../auto_partition/ddl/stream_load_range_test_table.sql | 2 +- .../auto_partition/ddl/two_streamload_table1.sql | 2 +- .../auto_partition/ddl/two_streamload_table2.sql | 2 +- .../same_data/stress_test_same_date_range.groovy | 6 +++--- .../two_stream_load/stress_test_two_stream_load.groovy | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/create_list_part_data_table.sql b/regression-test/suites/partition_p2/auto_partition/ddl/create_list_part_data_table.sql index 221129306030d1..78f92d8aa47312 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/create_list_part_data_table.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/create_list_part_data_table.sql @@ -40,7 +40,7 @@ CREATE TABLE `test2`( `col39` decimal(9, 3), `col40` char(128) ) UNIQUE KEY(`col1`) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/create_range_part_data_table.sql b/regression-test/suites/partition_p2/auto_partition/ddl/create_range_part_data_table.sql index 556e5e3fe87643..44be0f23a62c81 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/create_range_part_data_table.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/create_range_part_data_table.sql @@ -40,7 +40,7 @@ CREATE TABLE `test1`( `col39` decimal(9, 3), `col40` char(128) ) UNIQUE KEY(`col1`) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/small_data_high_concurrrent_load.sql b/regression-test/suites/partition_p2/auto_partition/ddl/small_data_high_concurrrent_load.sql index c3f7895dfa1738..081300d9cd72ba 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/small_data_high_concurrrent_load.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/small_data_high_concurrrent_load.sql @@ -7,7 +7,7 @@ CREATE TABLE `small_data_high_concurrent_load_range`( auto partition by range (date_trunc(`col1`, 'day')) ( ) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_list_test_table.sql b/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_list_test_table.sql index 3866d2a81f108d..17b5dffab06456 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_list_test_table.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_list_test_table.sql @@ -43,7 +43,7 @@ CREATE TABLE `stream_load_list_test_table`( AUTO PARTITION BY list(`col1`) ( ) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_range_test_table.sql b/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_range_test_table.sql index 481006919458f6..59d09279afaab2 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_range_test_table.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/stream_load_range_test_table.sql @@ -43,7 +43,7 @@ CREATE TABLE `stream_load_range_test_table`( auto partition by range (date_trunc(`col1`, 'day')) ( ) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table1.sql b/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table1.sql index 486c6df5117b14..1848e493c97034 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table1.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table1.sql @@ -8,7 +8,7 @@ CREATE TABLE `two_streamload_list1`( AUTO PARTITION BY list(`col1`) ( ) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table2.sql b/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table2.sql index ed36d3cf58b508..fd96b4253fd4cd 100644 --- a/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table2.sql +++ b/regression-test/suites/partition_p2/auto_partition/ddl/two_streamload_table2.sql @@ -8,7 +8,7 @@ CREATE TABLE `two_streamload_list2`( AUTO PARTITION BY list(`col1`) ( ) -DISTRIBUTED BY HASH(`col1`) BUCKETS 10 +DISTRIBUTED BY HASH(`col1`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); \ No newline at end of file diff --git a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy index cbece2071e324b..926b98fed3c5f8 100644 --- a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy +++ b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy @@ -163,12 +163,12 @@ suite("stress_test_same_date_range", "p2,nonConcurrent") { def row_count_range = sql """select count(*) from ${tb_name2};""" def partition_res_range = sql """show partitions from ${tb_name2};""" - assertTrue(row_count_range[0][0] == partition_res_range.size) + assertEquals(row_count_range[0][0], partition_res_range.size()) def part_context = [] - for (int i = 0; i < partition_res_range.size; i++) { + for (int i = 0; i < partition_res_range.size(); i++) { part_context.add(partition_res_range[i][6]) } def part_context_unique = part_context.clone().unique() - assertTrue(part_context.size == part_context_unique.size) + assertEquals(part_context.size(), part_context_unique.size()) } diff --git a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy index 212c8adfce48d3..f0804cfc99aa27 100644 --- a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy +++ b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy @@ -169,7 +169,7 @@ suite("stress_test_two_stream_load", "p2,nonConcurrent") { def row_count_range = sql """select count(*) from ${tb_name2};""" def partition_res_range = sql """show partitions from ${tb_name2};""" - assertTrue(row_count_range[0][0] == partition_res_range.size) + assertTrue(row_count_range[0][0] == partition_res_range.size()) data_delete("range") doris_dbgen_create_data(database_name, tb_name4, "list") @@ -192,7 +192,7 @@ suite("stress_test_two_stream_load", "p2,nonConcurrent") { def row_count_list = sql """select count(*) from ${tb_name3};""" def partition_res_list = sql """show partitions from ${tb_name3};""" - assertTrue(row_count_list[0][0] == partition_res_list.size) + assertTrue(row_count_list[0][0] == partition_res_list.size()) data_delete("list") } From febc6e3ee6362014adb088c34d099e5734d6b54f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:02:13 +0800 Subject: [PATCH 456/572] branch-3.0: [fix](case) make some cases stable #54763 (#54852) Cherry-picked from #54763 Co-authored-by: Sun Chenyang --- .../test_single_compaction_fault_injection.groovy | 4 ++++ .../fault_injection_p0/test_variant_bloom_filter.groovy | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy index 5399002ba58b82..a8f3da94470de1 100644 --- a/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy +++ b/regression-test/suites/compaction/test_single_compaction_fault_injection.groovy @@ -18,6 +18,10 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_single_compaction_fault_injection", "p2, nonConcurrent") { + if (isCloudMode()) { + logger.info("Skip test in cloud mode") + return + } def tableName = "test_single_compaction" def backendId_to_backendIP = [:] diff --git a/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy b/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy index 5c0d1f2591b1e5..6a8efb1efd28a0 100644 --- a/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy +++ b/regression-test/suites/fault_injection_p0/test_variant_bloom_filter.groovy @@ -59,7 +59,7 @@ suite("test_variant_bloom_filter", "nonConcurrent") { ) DUPLICATE KEY(`k`) DISTRIBUTED BY HASH(k) BUCKETS 1 - properties("replication_num" = "1", "disable_auto_compaction" = "false", "bloom_filter_columns" = "v"); + properties("replication_num" = "1", "disable_auto_compaction" = "true", "bloom_filter_columns" = "v"); """ load_json_data.call(index_table, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") load_json_data.call(index_table, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""") @@ -73,6 +73,8 @@ suite("test_variant_bloom_filter", "nonConcurrent") { def tablets = sql_return_maparray """ show tablets from ${index_table}; """ + sql """ select count() from ${index_table}; """ + for (def tablet in tablets) { int beforeSegmentCount = 0 String tablet_id = tablet.TabletId @@ -90,6 +92,8 @@ suite("test_variant_bloom_filter", "nonConcurrent") { // trigger compactions for all tablets in ${tableName} trigger_and_wait_compaction(index_table, "full") + sql """ select count() from ${index_table}; """ + for (def tablet in tablets) { int afterSegmentCount = 0 String tablet_id = tablet.TabletId From 0cfc31f9ecf340fd445221346fa0661a5018492f Mon Sep 17 00:00:00 2001 From: amory Date: Fri, 15 Aug 2025 19:03:27 +0800 Subject: [PATCH 457/572] branch-3.0[fix](cases)fix all session var for variant (#54833) --- regression-test/suites/variant_p0/nested/load.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/variant_p0/nested/load.groovy b/regression-test/suites/variant_p0/nested/load.groovy index 0fee39d4b3d8aa..b52ce79ff8bfdd 100644 --- a/regression-test/suites/variant_p0/nested/load.groovy +++ b/regression-test/suites/variant_p0/nested/load.groovy @@ -26,7 +26,7 @@ suite("variant_nested_type_load", "p0"){ sql """set describe_extend_variant_column = true""" // set disable_variant_flatten_nested = true to disable variant flatten nested which is default behavior - sql """ set disable_variant_flatten_nested = true """ + sql """ set enable_variant_flatten_nested = false """ test { sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( @@ -42,7 +42,7 @@ suite("variant_nested_type_load", "p0"){ // set disable_variant_flatten_nested = false to enable variant flatten nested - sql """ set disable_variant_flatten_nested = false """ + sql """ set enable_variant_flatten_nested = true """ sql """ CREATE TABLE IF NOT EXISTS ${table_name} ( k bigint, From af876dde844b2840f8bcb003f8b630d6867b0fd9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:04:31 +0800 Subject: [PATCH 458/572] branch-3.0: [fix](case) remove global variable declarations in test_disable_move_memtable #54773 (#54808) Cherry-picked from #54773 Co-authored-by: Xin Liao --- .../suites/fault_injection_p0/test_disable_move_memtable.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy b/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy index 2945d2f4365098..19b721315d1833 100644 --- a/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy +++ b/regression-test/suites/fault_injection_p0/test_disable_move_memtable.groovy @@ -230,7 +230,7 @@ suite("test_disable_move_memtable", "nonConcurrent") { def check_load_result = {checklabel, testTablex, res -> def max_try_milli_secs = 10000 while(max_try_milli_secs) { - result = sql "show load where label = '${checklabel}'" + def result = sql "show load where label = '${checklabel}'" log.info("result: ${result}") if(result[0][2].toString() == "${res}".toString()) { break From cf0f45945f4f4c86101b942912c82733a23187e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:05:27 +0800 Subject: [PATCH 459/572] branch-3.0: [fix](variant) building index on the variant column is prohibited #54777 (#54855) Cherry-picked from #54777 Co-authored-by: Sun Chenyang --- .../apache/doris/alter/SchemaChangeHandler.java | 14 ++++++++++++++ .../load.groovy | 6 ++---- .../suites/variant_p0/with_index/var_index.groovy | 6 ++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 075a46b756187c..553c0065a25a06 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2116,6 +2116,20 @@ public int getAsInt() { throw new DdlException("BUILD INDEX operation failed: No need to do it in cloud mode."); } + for (Column column : olapTable.getBaseSchema()) { + if (!column.getType().isVariantType()) { + continue; + } + // variant type column can not support for building index + for (String indexColumn : index.getColumns()) { + if (column.getName().equalsIgnoreCase(indexColumn)) { + throw new DdlException("BUILD INDEX operation failed: The " + + indexDef.getIndexName() + " index can not be built on the " + + indexColumn + " column, because it is a variant type column."); + } + } + } + if (indexDef.getPartitionNames().isEmpty()) { indexOnPartitions.put(index.getIndexId(), olapTable.getPartitionNames()); } else { diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy index ba499801cd0fc3..1aac1d1d94dbc0 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy @@ -179,11 +179,9 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""") // BUILD INDEX - try { + test { sql """ BUILD INDEX idx_var ON github_events""" - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("The idx_var index can not be built on the v column, because it is a variant type column")) + exception "The idx_var index can not be built on the v column, because it is a variant type column" } // // add bloom filter at the end of loading data diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy b/regression-test/suites/variant_p0/with_index/var_index.groovy index b846043b30c47a..45bda42537173c 100644 --- a/regression-test/suites/variant_p0/with_index/var_index.groovy +++ b/regression-test/suites/variant_p0/with_index/var_index.groovy @@ -139,11 +139,9 @@ suite("regression_test_variant_var_index", "p0, nonConcurrent"){ properties("replication_num" = "1", "disable_auto_compaction" = "true", "inverted_index_storage_format" = "V1"); """ sql """ALTER TABLE var_index ADD INDEX idx_var(v) USING INVERTED""" - try { + test { sql """ build index idx_var on var_index""" - } catch (Exception e) { - log.info(e.getMessage()) - assertTrue(e.getMessage().contains("The idx_var index can not be built on the v column, because it is a variant type column")) + exception "The idx_var index can not be built on the v column, because it is a variant type column" } } From 250c7b9ba2ee2607013158d4678f9e1add42ba73 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 19:58:12 +0800 Subject: [PATCH 460/572] branch-3.0: [Fix](p0) Fix `test_cumu_compaction_with_delete` #54651 (#54743) Cherry-picked from #54651 Co-authored-by: bobhan1 --- .../compaction/test_cumu_compaction_with_delete.groovy | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy index dead69e9729bb0..48ae2b0a4fd052 100644 --- a/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy +++ b/regression-test/suites/compaction/test_cumu_compaction_with_delete.groovy @@ -51,11 +51,10 @@ suite("test_cumu_compaction_with_delete", "nonConcurrent") { CREATE TABLE ${tableName} ( `user_id` INT NOT NULL, `value` INT NOT NULL) - UNIQUE KEY(`user_id`) + DUPLICATE KEY(`user_id`) DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 - PROPERTIES ("replication_allocation" = "tag.location.default: 1", - "enable_mow_light_delete" = "true")""" + PROPERTIES ("replication_allocation" = "tag.location.default: 1")""" for(int i = 1; i <= 100; ++i){ sql """ INSERT INTO ${tableName} VALUES (1,1)""" @@ -95,11 +94,10 @@ suite("test_cumu_compaction_with_delete", "nonConcurrent") { CREATE TABLE ${tableName} ( `user_id` INT NOT NULL, `value` INT NOT NULL) - UNIQUE KEY(`user_id`) + DUPLICATE KEY(`user_id`) DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 - PROPERTIES ("replication_allocation" = "tag.location.default: 1", - "enable_mow_light_delete" = "true")""" + PROPERTIES ("replication_allocation" = "tag.location.default: 1");""" for(int i = 1; i <= 100; ++i){ sql """ INSERT INTO ${tableName} VALUES (1,1)""" From 0224143ce847ba1e3733b02d3b8b49a29d5b86a1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:03:49 +0800 Subject: [PATCH 461/572] branch-3.0: [Fix](compaction) Remove incorrect DCHECK #54650 (#54742) Cherry-picked from #54650 Co-authored-by: bobhan1 --- be/src/cloud/cloud_tablet.cpp | 7 ------- be/src/olap/compaction.cpp | 8 -------- 2 files changed, 15 deletions(-) diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp index f96f2a9deb7291..c2b3e0600dfc19 100644 --- a/be/src/cloud/cloud_tablet.cpp +++ b/be/src/cloud/cloud_tablet.cpp @@ -1085,13 +1085,6 @@ Status CloudTablet::calc_delete_bitmap_for_compaction( RETURN_IF_ERROR(check_rowid_conversion(output_rowset, *location_map)); } int64_t t5 = MonotonicMicros(); - if (missed_rows) { - DCHECK_EQ(missed_rows->size(), missed_rows_size); - if (missed_rows->size() != missed_rows_size) { - LOG(WARNING) << "missed rows don't match, before: " << missed_rows_size - << " after: " << missed_rows->size(); - } - } // 3. store delete bitmap auto st = _engine.meta_mgr().update_delete_bitmap(*this, -1, initiator, diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index de0dcc79550cab..2ad9eeed2a26b5 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -1190,14 +1190,6 @@ Status CompactionMixin::modify_rowsets() { missed_rows.get(), location_map.get(), *_tablet->tablet_meta()->delete_bitmap(), &output_rowset_delete_bitmap); - if (missed_rows) { - DCHECK_EQ(missed_rows->size(), missed_rows_size); - if (missed_rows->size() != missed_rows_size) { - LOG(WARNING) << "missed rows don't match, before: " << missed_rows_size - << " after: " << missed_rows->size(); - } - } - if (location_map) { RETURN_IF_ERROR(tablet()->check_rowid_conversion(_output_rowset, *location_map)); } From bcc52a4a178efee58260a564a80455ebccdb810f Mon Sep 17 00:00:00 2001 From: koarz <3577087577@qq.com> Date: Fri, 15 Aug 2025 21:06:50 +0800 Subject: [PATCH 462/572] branch-3.0: [fix](regression-test)fix metric checker ignore empty label values (#54236) (#54843) pick: https://github.com/apache/doris/pull/54236 --- .../org/apache/doris/regression/util/PromethuesChecker.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy index 8a45052fb7fda1..cf6b5bb509d5f6 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/PromethuesChecker.groovy @@ -28,7 +28,7 @@ class PromethuesChecker { static boolean regexp(String s) { if (s == null) return false - s =~ /^[a-zA-Z_][a-zA-Z0-9_]*(\{[a-zA-Z_][a-zA-Z0-9_]*="[^"]+"(,[a-zA-Z_][a-zA-Z0-9_]*="[^"]+")*\})?\s+-?([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)$/ + s =~ /^[a-zA-Z_][a-zA-Z0-9_]*(\{[a-zA-Z_][a-zA-Z0-9_]*="[^"]*"(,[a-zA-Z_][a-zA-Z0-9_]*="[^"]*")*\})?\s+-?([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)$/ } static boolean check(String str) { From 3734fcdd17b4a691070a382267fe50502d6b2357 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Fri, 15 Aug 2025 21:07:53 +0800 Subject: [PATCH 463/572] branch-3.0: [fix](warmup) refresh tablet location cache and retry on error #54755 (#54757) backport #54755 --- be/src/cloud/cloud_internal_service.cpp | 7 ++- be/src/cloud/cloud_warm_up_manager.cpp | 68 +++++++++++++++++-------- be/src/cloud/cloud_warm_up_manager.h | 7 ++- gensrc/proto/internal_service.proto | 3 ++ 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/be/src/cloud/cloud_internal_service.cpp b/be/src/cloud/cloud_internal_service.cpp index 39b51b06dd473a..9705aca7fdc45f 100644 --- a/be/src/cloud/cloud_internal_service.cpp +++ b/be/src/cloud/cloud_internal_service.cpp @@ -171,12 +171,17 @@ void CloudInternalServiceImpl::warm_up_rowset(google::protobuf::RpcController* c continue; } int64_t tablet_id = rs_meta.tablet_id(); + bool local_only = !(request->has_skip_existence_check() && request->skip_existence_check()); auto res = _engine.tablet_mgr().get_tablet(tablet_id, /* warmup_data = */ false, /* sync_delete_bitmap = */ true, /* sync_stats = */ nullptr, - /* local_only = */ true); + /* local_only = */ local_only); if (!res.has_value()) { LOG_WARNING("Warm up error ").tag("tablet_id", tablet_id).error(res.error()); + if (res.error().msg().find("local_only=true") != std::string::npos) { + res.error().set_code(ErrorCode::TABLE_NOT_FOUND); + } + res.error().to_protobuf(response->mutable_status()); continue; } auto tablet = res.value(); diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 57e0a9ad27568c..5b23f8ebe88244 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -415,30 +415,39 @@ Status CloudWarmUpManager::set_event(int64_t job_id, TWarmUpEventType::type even return st; } -std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_id) { +std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_id, bool bypass_cache, + bool& cache_hit) { std::vector replicas; std::vector cancelled_jobs; std::lock_guard lock(_mtx); + cache_hit = false; for (auto& [job_id, cache] : _tablet_replica_cache) { - auto it = cache.find(tablet_id); - if (it != cache.end()) { - // check ttl expire - auto now = std::chrono::steady_clock::now(); - auto sec = std::chrono::duration_cast(now - it->second.first); - if (sec.count() < config::warmup_tablet_replica_info_cache_ttl_sec) { - replicas.push_back(it->second.second); - LOG(INFO) << "get_replica_info: cache hit, tablet_id=" << tablet_id - << ", job_id=" << job_id; - continue; - } else { - LOG(INFO) << "get_replica_info: cache expired, tablet_id=" << tablet_id - << ", job_id=" << job_id; - cache.erase(it); + if (!bypass_cache) { + auto it = cache.find(tablet_id); + if (it != cache.end()) { + // check ttl expire + auto now = std::chrono::steady_clock::now(); + auto sec = std::chrono::duration_cast(now - it->second.first); + if (sec.count() < config::warmup_tablet_replica_info_cache_ttl_sec) { + replicas.push_back(it->second.second); + LOG(INFO) << "get_replica_info: cache hit, tablet_id=" << tablet_id + << ", job_id=" << job_id; + cache_hit = true; + continue; + } else { + LOG(INFO) << "get_replica_info: cache expired, tablet_id=" << tablet_id + << ", job_id=" << job_id; + cache.erase(it); + } } + LOG(INFO) << "get_replica_info: cache miss, tablet_id=" << tablet_id + << ", job_id=" << job_id; } - LOG(INFO) << "get_replica_info: cache miss, tablet_id=" << tablet_id - << ", job_id=" << job_id; + if (!cache_hit) { + // We are trying to save one retry by refresh all the remaining caches + bypass_cache = true; + } ClusterInfo* cluster_info = ExecEnv::GetInstance()->cluster_info(); if (cluster_info == nullptr) { LOG(WARNING) << "get_replica_info: have not get FE Master heartbeat yet, job_id=" @@ -506,20 +515,37 @@ std::vector CloudWarmUpManager::get_replica_info(int64_t tablet_id } void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta) { - auto replicas = get_replica_info(rs_meta.tablet_id()); + bool cache_hit = false; + auto replicas = get_replica_info(rs_meta.tablet_id(), false, cache_hit); if (replicas.empty()) { LOG(INFO) << "There is no need to warmup tablet=" << rs_meta.tablet_id() << ", skipping rowset=" << rs_meta.rowset_id().to_string(); return; } + Status st = _do_warm_up_rowset(rs_meta, replicas, !cache_hit); + if (cache_hit && !st.ok() && st.is()) { + replicas = get_replica_info(rs_meta.tablet_id(), true, cache_hit); + st = _do_warm_up_rowset(rs_meta, replicas, true); + } + if (!st.ok()) { + LOG(WARNING) << "Failed to warm up rowset, tablet_id=" << rs_meta.tablet_id() + << ", rowset_id=" << rs_meta.rowset_id().to_string() << ", status=" << st; + } +} + +Status CloudWarmUpManager::_do_warm_up_rowset(RowsetMeta& rs_meta, + std::vector& replicas, + bool skip_existence_check) { int64_t now_ts = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); g_file_cache_warm_up_rowset_last_call_unix_ts.set_value(now_ts); + auto ret_st = Status::OK(); PWarmUpRowsetRequest request; request.add_rowset_metas()->CopyFrom(rs_meta.get_rowset_pb()); request.set_unix_ts_us(now_ts); + request.set_skip_existence_check(skip_existence_check); for (auto& replica : replicas) { // send sync request std::string host = replica.host; @@ -531,7 +557,7 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta) { if (!status.ok()) { LOG(WARNING) << "failed to get ip from host " << replica.host << ": " << status.to_string(); - return; + continue; } } std::string brpc_addr = get_host_port(host, replica.brpc_port); @@ -587,12 +613,14 @@ void CloudWarmUpManager::warm_up_rowset(RowsetMeta& rs_meta) { PWarmUpRowsetResponse response; brpc_stub->warm_up_rowset(&cntl, &request, &response, nullptr); } + return ret_st; } void CloudWarmUpManager::recycle_cache(int64_t tablet_id, const std::vector& rowsets) { LOG(INFO) << "recycle_cache: tablet_id=" << tablet_id << ", num_rowsets=" << rowsets.size(); - auto replicas = get_replica_info(tablet_id); + bool cache_hit = false; + auto replicas = get_replica_info(tablet_id, false, cache_hit); if (replicas.empty()) { return; } diff --git a/be/src/cloud/cloud_warm_up_manager.h b/be/src/cloud/cloud_warm_up_manager.h index 6feef0e9d421ea..02ad4a7b6e0aa0 100644 --- a/be/src/cloud/cloud_warm_up_manager.h +++ b/be/src/cloud/cloud_warm_up_manager.h @@ -78,7 +78,12 @@ class CloudWarmUpManager { private: void handle_jobs(); - std::vector get_replica_info(int64_t tablet_id); + + Status _do_warm_up_rowset(RowsetMeta& rs_meta, std::vector& replicas, + bool skip_existence_check); + + std::vector get_replica_info(int64_t tablet_id, bool bypass_cache, + bool& cache_hit); void submit_download_tasks(io::Path path, int64_t file_size, io::FileSystemSPtr file_system, int64_t expiration_time, diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto index 108f7d7613812d..5fdce934879841 100644 --- a/gensrc/proto/internal_service.proto +++ b/gensrc/proto/internal_service.proto @@ -840,9 +840,12 @@ message PGetFileCacheMetaResponse { message PWarmUpRowsetRequest { repeated RowsetMetaPB rowset_metas = 1; optional int64 unix_ts_us = 2; + optional int64 sync_wait_timeout_ms = 3; + optional bool skip_existence_check = 4; } message PWarmUpRowsetResponse { + optional PStatus status = 1; } message RecycleCacheMeta { From 8967e49e60154f34c0f950746da62526dae2799d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:17:52 +0800 Subject: [PATCH 464/572] branch-3.0: [regression](audit) throw Exception if test failed #53223 (#53279) Cherry-picked from #53223 Co-authored-by: yagagagaga --- regression-test/suites/audit/test_audit_log_behavior.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/regression-test/suites/audit/test_audit_log_behavior.groovy b/regression-test/suites/audit/test_audit_log_behavior.groovy index 8b31c18f40dcee..f14c01209669cf 100644 --- a/regression-test/suites/audit/test_audit_log_behavior.groovy +++ b/regression-test/suites/audit/test_audit_log_behavior.groovy @@ -91,8 +91,7 @@ suite("test_audit_log_behavior") { def res = sql "${query}" while (res.isEmpty()) { if (retry-- < 0) { - logger.warn("It has retried a few but still failed, you need to check it") - return + throw new RuntimeException("It has retried a few but still failed, you need to check it") } sleep(1000) res = sql "${query}" From b26cba1247c32ed97a7ad73a0afad97094880fb5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:21:05 +0800 Subject: [PATCH 465/572] branch-3.0: [fix](config) dynamic set FE masterOnly config will lost after change MASTER #54409 (#54601) Cherry-picked from #54409 Co-authored-by: camby <104178625@qq.com> --- .../apache/doris/analysis/AdminSetConfigStmt.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java index 458116924d7ee5..b197c6c2bca113 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java @@ -49,11 +49,13 @@ public AdminSetConfigStmt(NodeType type, Map configs, boolean ap } this.applyToAll = applyToAll; - // we have to analyze configs here to determine whether to forward it to master - for (String key : this.configs.keySet()) { - if (ConfigBase.checkIsMasterOnly(key)) { - redirectStatus = RedirectStatus.FORWARD_NO_SYNC; - this.applyToAll = false; + if (!this.applyToAll) { + // we have to analyze configs here to determine whether to forward it to master + for (String key : this.configs.keySet()) { + if (ConfigBase.checkIsMasterOnly(key)) { + redirectStatus = RedirectStatus.FORWARD_NO_SYNC; + break; + } } } } From 94ef7380036a5ed82d0c134e33dd42b9a04f68cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:22:02 +0800 Subject: [PATCH 466/572] branch-3.0: [case](auth)Fix test_dml_stream_load_auth #54682 (#54836) Cherry-picked from #54682 Co-authored-by: zhangdong --- .../test_dml_stream_load_auth.groovy | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy index 8531a193da7b4e..c0e7748fa3e70e 100644 --- a/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy +++ b/regression-test/suites/auth_call/test_dml_stream_load_auth.groovy @@ -70,7 +70,7 @@ suite("test_dml_stream_load_auth","p0,auth_call") { proc.consumeProcessOutput(sout, serr) proc.waitForOrKill(7200000) logger.info("std out: " + sout + "std err: " + serr) - assertTrue(sout.toString().indexOf("Success") == -1) + assertTrue(sout.toString().indexOf("denied") != -1) sql """grant load_priv on ${dbName}.${tableName} to ${user}""" @@ -81,31 +81,22 @@ suite("test_dml_stream_load_auth","p0,auth_call") { proc.consumeProcessOutput(sout, serr) proc.waitForOrKill(7200000) logger.info("std out: " + sout + "std err: " + serr) - assertTrue(sout.toString().indexOf("Success") != -1) - - int pos1 = sout.indexOf("TxnId") - int pos2 = sout.indexOf(",", pos1) - int pos3 = sout.indexOf(":", pos1) - def tsc_id = sout.substring(pos3+2, pos2) + assertTrue(sout.toString().indexOf("denied") == -1) connect(user, "${pwd}", context.config.jdbcUrl) { test { - sql """SHOW TRANSACTION FROM ${dbName} WHERE ID=${tsc_id};""" + sql """SHOW TRANSACTION FROM ${dbName} WHERE ID=111;""" exception "denied" } } - def res = sql """select count() from ${dbName}.${tableName}""" - assertTrue(res[0][0] == 3) - - def stream_res = sql """SHOW STREAM LOAD FROM ${dbName};""" - logger.info("stream_res: " + stream_res) - sql """grant admin_priv on *.*.* to ${user}""" connect(user, "${pwd}", context.config.jdbcUrl) { - def transaction_res = sql """SHOW TRANSACTION FROM ${dbName} WHERE ID=${tsc_id};""" - assertTrue(transaction_res.size() == 1) + test { + sql """SHOW TRANSACTION FROM ${dbName} WHERE ID=111;""" + exception "exist" + } } sql """drop database if exists ${dbName}""" From 3dbd773620240c36252c5df2ef3c13a5d5f2b347 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:57:28 +0800 Subject: [PATCH 467/572] branch-3.0: [fix](regression)Fix test_analyze_mv #54727 (#54880) Cherry-picked from #54727 Co-authored-by: James --- .../transaction/DatabaseTransactionMgr.java | 4 +++ .../suites/statistics/test_analyze_mv.groovy | 28 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 119571b7b29f47..3db8edb865911f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -2264,6 +2264,10 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat } replica.updateVersionWithFailed(newVersion, lastFailedVersion, lastSuccessVersion); if (newVersion == Partition.PARTITION_INIT_VERSION + 1) { + if (LOG.isDebugEnabled()) { + LOG.debug("{}.{}, index {} first loaded, set reported to false.", + db.getName(), table.getName(), index.getId()); + } index.setRowCountReported(false); } long beId = replica.getBackendIdWithoutException(); diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy index 508aa538a83639..33179586e88650 100644 --- a/regression-test/suites/statistics/test_analyze_mv.groovy +++ b/regression-test/suites/statistics/test_analyze_mv.groovy @@ -699,8 +699,10 @@ suite("test_analyze_mv") { logger.info("show frontends result master: " + result) for (int i = 0; i < 120; i++) { result_row = sql """show index stats mvTestDup mv3""" + def result_row1 = sql """show index stats mvTestDup mvTestDup""" logger.info("mv3 stats: " + result_row) - if (result_row[0][4] == "0") { + logger.info("mvTestDup stats: " + result_row1) + if (result_row[0][4] == "0" && result_row1[0][4] == "0") { break; } Thread.sleep(5000) @@ -711,6 +713,12 @@ suite("test_analyze_mv") { assertEquals("mv3", result_row[0][1]) assertEquals("0", result_row[0][3]) assertEquals("0", result_row[0][4]) + result_row = sql """show index stats mvTestDup mvTestDup""" + assertEquals(1, result_row.size()) + assertEquals("mvTestDup", result_row[0][0]) + assertEquals("mvTestDup", result_row[0][1]) + assertEquals("0", result_row[0][3]) + assertEquals("0", result_row[0][4]) } // ** Embedded test for skip auto analyze when table is empty again @@ -718,7 +726,6 @@ suite("test_analyze_mv") { sql """analyze table mvTestDup properties ("use.auto.analyzer" = "true")""" empty_test = sql """show auto analyze mvTestDup""" assertEquals(0, empty_test.size()) - empty_test = sql """show column stats mvTestDup""" for (int i = 0; i < 100; i++) { empty_test = sql """show column stats mvTestDup""" if (empty_test.size() == 0) { @@ -731,11 +738,18 @@ suite("test_analyze_mv") { // ** End of embedded test sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);""" - result_row = sql """show index stats mvTestDup mv3""" - assertEquals(1, result_row.size()) - assertEquals("mvTestDup", result_row[0][0]) - assertEquals("mv3", result_row[0][1]) - assertEquals("-1", result_row[0][4]) + connect(context.config.jdbcUser, context.config.jdbcPassword, url) { + sql """use test_analyze_mv""" + for (int i = 0; i < 10; i++) { + result_row = sql """show index stats mvTestDup mvTestDup""" + if (result_row[0][4] == "-1") { + break; + } + logger.info("row count for mvTestDup is not -1: " + result_row) + Thread.sleep(1000) + } + assertEquals("-1", result_row[0][4]) + } // Test alter column stats sql """drop stats mvTestDup""" From 0ea772c5466736639ab2b05e4de0e197e8494547 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 15 Aug 2025 21:58:22 +0800 Subject: [PATCH 468/572] =?UTF-8?q?branch-3.0:=20[feature](regression-fram?= =?UTF-8?q?ework)=20print=20current=20pipeline=20tasks=20every=20mi?= =?UTF-8?q?=E2=80=A6=20#54850=20(#54860)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #54850 Co-authored-by: shuke --- .../plugin_query_timeout_debugger.groovy | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 regression-test/plugins/plugin_query_timeout_debugger.groovy diff --git a/regression-test/plugins/plugin_query_timeout_debugger.groovy b/regression-test/plugins/plugin_query_timeout_debugger.groovy new file mode 100644 index 00000000000000..5d863b9a754a9d --- /dev/null +++ b/regression-test/plugins/plugin_query_timeout_debugger.groovy @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.sql.DriverManager +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import org.apache.doris.regression.suite.Suite +import org.apache.doris.regression.util.JdbcUtils +import org.slf4j.Logger + +// make sure PluginQueryTimeoutDebugger quit gracefully +class PluginQueryTimeoutDebuggerHolder { + static final PluginQueryTimeoutDebugger staticResource = new PluginQueryTimeoutDebugger() + + static { + Runtime.runtime.addShutdownHook { + staticResource?.stopWorker() + } + } +} + +PluginQueryTimeoutDebugger.jdbcUrl = context.config.jdbcUrl +PluginQueryTimeoutDebugger.jdbcUser = context.config.jdbcUser +PluginQueryTimeoutDebugger.jdbcPassword = context.config.jdbcPassword +PluginQueryTimeoutDebugger.logger = logger +PluginQueryTimeoutDebuggerHolder.staticResource.startWorker() + +/** + * print pipeline tasks every 1 minutes to help debugging query timeout. + * be list refreshed every 5 minutes. + */ +class PluginQueryTimeoutDebugger { + static private final String HostColumnName = "Host" + static private final String PortColumnName = "HttpPort" + static private final int HTTP_TIMEOUT = 5000 + static private final long BACKEND_REFRESH_INTERVAL = 5 * 60 * 1000 + + static public String jdbcUrl + static public String jdbcUser + static public String jdbcPassword + static public Logger logger + + private ScheduledExecutorService scheduler + private List backendUrls = [] + private long lastBackendRefreshTime = 0 + + // catch all exceptions in timer function. + private void startWorker() { + if (scheduler?.isShutdown() == false) { + logger.warn("worker already started") + return + } + + scheduler = Executors.newSingleThreadScheduledExecutor { r -> + Thread thread = new Thread(r) + thread.setName("query-timeout-debugger-thread") + thread.setDaemon(true) + return thread + } + scheduler.scheduleAtFixedRate({ + try { + work() + } catch (Exception e) { + logger.warn("work exception: ${e.getMessage()}", e) + } + }, 0, 1, TimeUnit.MINUTES) + + logger.info("worker started with scheduler") + } + + private void stopWorker() { + logger.info("stop worker") + if (scheduler != null) { + scheduler.shutdown() + if (!scheduler.awaitTermination(3, TimeUnit.SECONDS)) { + scheduler.shutdownNow() + logger.warn("worker scheduler forced to stop") + } + } + } + + private void work() { + initBackendsUrls() + for (String url : backendUrls) { + logger.info("${url} pipeline tasks: ${curl(url)}") + } + } + + void initBackendsUrls() { + // refreshed every BACKEND_REFRESH_INTERVAL. + long now = System.currentTimeMillis() + if (!backendUrls.isEmpty() && (now - lastBackendRefreshTime < BACKEND_REFRESH_INTERVAL)) { + return + } + lastBackendRefreshTime = now + + List urls = [] + DriverManager.getConnection(jdbcUrl, jdbcUser, jdbcPassword).withCloseable { conn -> + def (result, meta) = JdbcUtils.executeToList(conn, "show backends") + int hostIndex = -1 + int portIndex = -1 + + for (int i = 0; i < meta.getColumnCount(); i++) { + if (meta.getColumnLabel(i+1) == HostColumnName) { + hostIndex = i + } else if (meta.getColumnLabel(i+1) == PortColumnName) { + portIndex = i + } + } + + if (hostIndex != -1 && portIndex != -1) { + for (int i = 0; i < result.size(); i++) { + urls.add(String.format("http://%s:%s/api/running_pipeline_tasks/180", result.get(i).get(hostIndex), result.get(i).get(portIndex))) + } + backendUrls = urls + } + } + + logger.info("backends: ${backendUrls}") + } + + String curl(String urlStr) { + HttpURLConnection connection = null + try { + URL url = new URL(urlStr) + connection = url.openConnection() as HttpURLConnection + connection.requestMethod = "GET" + connection.connectTimeout = HTTP_TIMEOUT + connection.readTimeout = HTTP_TIMEOUT + + if (connection.responseCode == 200) { + return connection.inputStream.text + } else { + throw new Exception("curl ${urlStr} failed, code: ${connection.responseCode}") + } + } finally { + connection?.disconnect() + } + } +} From c86516696e58eeaa98c8057639e5af421795c2e5 Mon Sep 17 00:00:00 2001 From: Socrates Date: Sat, 16 Aug 2025 10:15:04 +0800 Subject: [PATCH 469/572] branch-3.0: [fix](iceberg) fix insert iceberg transform partition table problems and add some cases (#54557) (#54796) bp: #54557 --- .../writer/iceberg/partition_transformers.h | 15 +- .../iceberg/run19.sql | 365 ++++++++++++++++++ .../iceberg/source/IcebergScanNode.java | 3 + .../test_iceberg_transform_partitions.out | 124 ++++++ .../test_iceberg_transform_partitions.groovy | 205 ++++++++++ ..._iceberg_write_transform_partitions.groovy | 70 ++++ 6 files changed, 773 insertions(+), 9 deletions(-) create mode 100644 docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run19.sql create mode 100644 regression-test/data/external_table_p0/iceberg/test_iceberg_transform_partitions.out create mode 100644 regression-test/suites/external_table_p0/iceberg/test_iceberg_transform_partitions.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy diff --git a/be/src/vec/sink/writer/iceberg/partition_transformers.h b/be/src/vec/sink/writer/iceberg/partition_transformers.h index 84ee3029cdd18f..9ceb21e0c5ffcd 100644 --- a/be/src/vec/sink/writer/iceberg/partition_transformers.h +++ b/be/src/vec/sink/writer/iceberg/partition_transformers.h @@ -347,11 +347,10 @@ class DecimalTruncatePartitionColumnTransform : public PartitionColumnTransform auto col_res = ColumnDecimal::create(vec_src.size(), decimal_col->get_scale()); auto& vec_res = col_res->get_data(); - const typename T::NativeType* __restrict p_in = - reinterpret_cast(vec_src.data()); - const typename T::NativeType* end_in = + const auto* __restrict p_in = reinterpret_cast(vec_src.data()); + const auto* end_in = reinterpret_cast(vec_src.data()) + vec_src.size(); - typename T::NativeType* __restrict p_out = reinterpret_cast(vec_res.data()); + auto* __restrict p_out = reinterpret_cast(vec_res.data()); while (p_in < end_in) { typename T::NativeType remainder = ((*p_in % _width) + _width) % _width; @@ -535,13 +534,11 @@ class DecimalBucketPartitionColumnTransform : public PartitionColumnTransform { auto col_res = ColumnInt32::create(); ColumnInt32::Container& out_data = col_res->get_data(); out_data.resize(in_data.size()); - auto& vec_res = col_res->get_data(); - const typename T::NativeType* __restrict p_in = - reinterpret_cast(in_data.data()); - const typename T::NativeType* end_in = + const auto* __restrict p_in = reinterpret_cast(in_data.data()); + const auto* end_in = reinterpret_cast(in_data.data()) + in_data.size(); - typename T::NativeType* __restrict p_out = reinterpret_cast(vec_res.data()); + Int32* __restrict p_out = out_data.data(); while (p_in < end_in) { std::string buffer = BitUtil::IntToByteBuffer(*p_in); diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run19.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run19.sql new file mode 100644 index 00000000000000..8bd75b2809f55d --- /dev/null +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run19.sql @@ -0,0 +1,365 @@ +create database if not exists transform_partition_db; + +use transform_partition_db; + +-- set time zone for deterministic timestamp partitioning +SET TIME ZONE '+08:00'; + +-- ============================================= +-- Bucket partition coverage across many types +-- ============================================= +-- Bucket by INT but empty +CREATE TABLE bucket_int_empty ( + id BIGINT, + name STRING, + partition_key INT +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +-- Bucket by INT +CREATE TABLE bucket_int_4 ( + id BIGINT, + name STRING, + partition_key INT +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_int_4_copy AS +SELECT * FROM bucket_int_4; + +INSERT INTO + bucket_int_4 +VALUES (1, 'n100', -100), + (2, 'n1', -1), + (3, 'z', 0), + (4, 'p1', 1), + (5, 'p2', 2), + (6, 'p9', 9), + (7, 'p16', 16), + (8, 'null', NULL); + +-- Bucket by BIGINT +CREATE TABLE bucket_bigint_4 ( + id BIGINT, + name STRING, + partition_key BIGINT +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_bigint_4_copy AS +SELECT * FROM bucket_bigint_4; + +INSERT INTO + bucket_bigint_4 +VALUES ( + 1, + 'minish', + -9223372036854775808 + ), + (2, 'large-', -1234567890123), + (3, 'neg1', -1), + (4, 'zero', 0), + (5, 'pos1', 1), + (6, 'large+', 1234567890123), + ( + 7, + 'maxish', + 9223372036854775807 + ), + (8, 'null', NULL); + +-- Bucket by STRING +CREATE TABLE bucket_string_4 ( + id BIGINT, + name STRING, + partition_key STRING +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_string_4_copy AS +SELECT * FROM bucket_string_4; + +INSERT INTO + bucket_string_4 +VALUES (1, 'empty', ''), + (2, 'a', 'a'), + (3, 'abc', 'abc'), + (4, 'unicode', '北'), + (5, 'emoji', '😊'), + (6, 'space', ' '), + ( + 7, + 'long', + 'this is a relatively long string for hashing' + ), + (8, 'null', NULL); + +-- Bucket by DATE +CREATE TABLE bucket_date_4 ( + id BIGINT, + name STRING, + partition_key DATE +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_date_4_copy AS +SELECT * FROM bucket_date_4; + +INSERT INTO + bucket_date_4 +VALUES (1, 'd1', DATE '1970-01-01'), + (2, 'd2', DATE '1999-12-31'), + (3, 'd3', DATE '2000-01-01'), + (4, 'd4', DATE '2024-02-29'), + (5, 'd5', DATE '2024-03-01'), + (6, 'd6', DATE '2038-01-19'), + (7, 'null', NULL); + +-- Bucket by TIMESTAMP (with time zone semantics) +CREATE TABLE bucket_timestamp_4 ( + id BIGINT, + name STRING, + partition_key TIMESTAMP +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_timestamp_4_copy AS +SELECT * FROM bucket_timestamp_4; + +INSERT INTO + bucket_timestamp_4 +VALUES ( + 1, + 't1', + TIMESTAMP '2024-01-15 08:00:00' + ), + ( + 2, + 't2', + TIMESTAMP '2024-01-15 09:00:00' + ), + ( + 3, + 't3', + TIMESTAMP '2024-06-30 23:59:59' + ), + ( + 4, + 't4', + TIMESTAMP '1970-01-01 00:00:00' + ), + ( + 5, + 't5', + TIMESTAMP '2030-12-31 23:59:59' + ), + (6, 'null', NULL); + +-- Bucket by TIMESTAMP_NTZ +CREATE TABLE bucket_timestamp_ntz_4 ( + id BIGINT, + name STRING, + partition_key TIMESTAMP_NTZ +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_timestamp_ntz_4_copy AS +SELECT * FROM bucket_timestamp_ntz_4; + +INSERT INTO + bucket_timestamp_ntz_4 +VALUES ( + 1, + 'ntz1', + TIMESTAMP_NTZ '2024-01-15 08:00:00' + ), + ( + 2, + 'ntz2', + TIMESTAMP_NTZ '2024-01-15 09:00:00' + ), + ( + 3, + 'ntz3', + TIMESTAMP_NTZ '2024-06-30 23:59:59' + ), + ( + 4, + 'ntz4', + TIMESTAMP_NTZ '1970-01-01 00:00:00' + ), + ( + 5, + 'ntz5', + TIMESTAMP_NTZ '2030-12-31 23:59:59' + ), + (6, 'null', NULL); + +-- Bucket by DECIMAL +CREATE TABLE bucket_decimal_4 ( + id BIGINT, + name STRING, + partition_key DECIMAL(10, 2) +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_decimal_4_copy AS +SELECT * FROM bucket_decimal_4; + +INSERT INTO + bucket_decimal_4 +VALUES (1, 'p1', 0.00), + (2, 'p2', 1.00), + (3, 'p3', 10.50), + (4, 'n1', -1.25), + (5, 'n2', -10.50), + (6, 'big', 9999999.99), + (7, 'null', NULL); + +-- Bucket by BINARY +CREATE TABLE bucket_binary_4 ( + id BIGINT, + name STRING, + partition_key BINARY +) USING ICEBERG PARTITIONED BY (bucket (4, partition_key)); + +CREATE TABLE bucket_binary_4_copy AS +SELECT * FROM bucket_binary_4; + +INSERT INTO + bucket_binary_4 +VALUES (1, 'b1', CAST('' AS BINARY)), + (2, 'b2', CAST('a' AS BINARY)), + ( + 3, + 'b3', + CAST('abc' AS BINARY) + ), + (4, 'b4', CAST('你好' AS BINARY)), + ( + 5, + 'b5', + CAST('01010101' AS BINARY) + ), + (6, 'null', NULL); + +-- ============================================= +-- Truncate partition coverage for supported types +-- ============================================= + +-- Truncate STRING to length 3 +CREATE TABLE truncate_string_3 ( + id BIGINT, + name STRING, + partition_key STRING +) USING ICEBERG PARTITIONED BY ( + truncate (3, partition_key) +); + +CREATE TABLE truncate_string_3_copy AS +SELECT * FROM truncate_string_3; + +INSERT INTO + truncate_string_3 +VALUES (1, 'empty', ''), + (2, 'short', 'a'), + (3, 'two', 'ab'), + (4, 'three', 'abc'), + (5, 'long', 'abcdef'), + (6, 'unicode', '你好世界'), + (7, 'space', ' ab'), + (8, 'null', NULL); + +-- Truncate BINARY to length 4 bytes +CREATE TABLE truncate_binary_4 ( + id BIGINT, + name STRING, + partition_key BINARY +) USING ICEBERG PARTITIONED BY ( + truncate (4, partition_key) +); + +CREATE TABLE truncate_binary_4_copy AS +SELECT * FROM truncate_binary_4; + +INSERT INTO + truncate_binary_4 +VALUES (1, 'b0', CAST('' AS BINARY)), + (2, 'b1', CAST('a' AS BINARY)), + ( + 3, + 'b2', + CAST('abcd' AS BINARY) + ), + ( + 4, + 'b3', + CAST('abcdef' AS BINARY) + ), + (5, 'b4', CAST('你好' AS BINARY)), + (6, 'null', NULL); + +-- Truncate INT by width 10 +CREATE TABLE truncate_int_10 ( + id BIGINT, + name STRING, + partition_key INT +) USING ICEBERG PARTITIONED BY ( + truncate (10, partition_key) +); + +CREATE TABLE truncate_int_10_copy AS +SELECT * FROM truncate_int_10; + +INSERT INTO + truncate_int_10 +VALUES (1, 'n23', -23), + (2, 'n1', -1), + (3, 'z', 0), + (4, 'p7', 7), + (5, 'p10', 10), + (6, 'p19', 19), + (7, 'p20', 20), + (8, 'p999', 999), + (9, 'null', NULL); + +-- Truncate BIGINT by width 100 +CREATE TABLE truncate_bigint_100 ( + id BIGINT, + name STRING, + partition_key BIGINT +) USING ICEBERG PARTITIONED BY ( + truncate (100, partition_key) +); + +CREATE TABLE truncate_bigint_100_copy AS +SELECT * FROM truncate_bigint_100; + +INSERT INTO + truncate_bigint_100 +VALUES (1, 'n1001', -1001), + (2, 'n1', -1), + (3, 'z', 0), + (4, 'p7', 7), + (5, 'p100', 100), + (6, 'p199', 199), + (7, 'p200', 200), + (8, 'p10101', 10101), + (9, 'null', NULL); + +-- Truncate DECIMAL(10,2) by width 10 +CREATE TABLE truncate_decimal_10 ( + id BIGINT, + name STRING, + partition_key DECIMAL(10, 2) +) USING ICEBERG PARTITIONED BY ( + truncate (10, partition_key) +); + +CREATE TABLE truncate_decimal_10_copy AS +SELECT * FROM truncate_decimal_10; + +INSERT INTO + truncate_decimal_10 +VALUES (1, 'z', 0.00), + (2, 'p1', 1.23), + (3, 'p9', 9.99), + (4, 'p10', 10.00), + (5, 'p19', 19.99), + (6, 'p20', 20.00), + (7, 'n1', -1.23), + (8, 'n20', -20.00), + (9, 'big', 9999999.99), + (10, 'null', NULL); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 215a94f110f816..3a20a64599e18f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -459,6 +459,9 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { } private void assignCountToSplits(List splits, long totalCount) { + if (splits.isEmpty()) { + return; + } int size = splits.size(); long countPerSplit = totalCount / size; for (int i = 0; i < size - 1; i++) { diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_transform_partitions.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_transform_partitions.out new file mode 100644 index 00000000000000..b62eb88826691d --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_transform_partitions.out @@ -0,0 +1,124 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bucket_int_empty_cnt1 -- +0 + +-- !bucket_int_empty_select1 -- + +-- !bucket_int_4_select1 -- +4 p1 1 + +-- !bucket_int_4_select2 -- +5 p2 2 +7 p16 16 + +-- !bucket_int_4_cnt1 -- +1 + +-- !bucket_bigint_4_cnt1 -- +1 + +-- !bucket_bigint_4_cnt2 -- +2 + +-- !bucket_bigint_4_select1 -- +5 pos1 1 + +-- !bucket_string_4_cnt1 -- +1 + +-- !bucket_string_4_cnt2 -- +2 + +-- !bucket_string_4_select1 -- +3 abc abc + +-- !bucket_date_4_cnt1 -- +1 + +-- !bucket_date_4_cnt2 -- +2 + +-- !bucket_date_4_select1 -- +4 d4 2024-02-29 + +-- !bucket_timestamp_4_cnt1 -- +1 + +-- !bucket_timestamp_4_cnt2 -- +2 + +-- !bucket_timestamp_4_select1 -- +1 t1 2024-01-15T08:00 + +-- !bucket_timestamp_ntz_4_cnt1 -- +1 + +-- !bucket_timestamp_ntz_4_cnt2 -- +2 + +-- !bucket_timestamp_ntz_4_select1 -- +1 ntz1 2024-01-15T08:00 + +-- !bucket_decimal_4_cnt1 -- +1 + +-- !bucket_decimal_4_cnt2 -- +2 + +-- !bucket_decimal_4_select1 -- +3 p3 10.50 + +-- !bucket_binary_4_cnt1 -- +1 + +-- !bucket_binary_4_cnt2 -- +2 + +-- !bucket_binary_4_select1 -- +3 b3 abc + +-- !truncate_string_3_cnt1 -- +1 + +-- !truncate_string_3_cnt2 -- +2 + +-- !truncate_string_3_select1 -- +5 long abcdef + +-- !truncate_binary_4_cnt1 -- +1 + +-- !truncate_binary_4_cnt2 -- +2 + +-- !truncate_binary_4_select1 -- +4 b3 abcdef + +-- !truncate_int_10_cnt1 -- +1 + +-- !truncate_int_10_cnt2 -- +2 + +-- !truncate_int_10_select1 -- +6 p19 19 + +-- !truncate_bigint_100_cnt1 -- +1 + +-- !truncate_bigint_100_cnt2 -- +2 + +-- !truncate_bigint_100_select1 -- +6 p199 199 + +-- !truncate_decimal_10_cnt1 -- +1 + +-- !truncate_decimal_10_cnt2 -- +2 + +-- !truncate_decimal_10_select1 -- +5 p19 19.99 + diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_transform_partitions.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_transform_partitions.groovy new file mode 100644 index 00000000000000..f6c6d778f46f59 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_transform_partitions.groovy @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_transform_partitions", "p0,external,doris,external_docker,external_docker_doris") { + + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalog_name = "test_iceberg_transform_partitions" + String db_name = "transform_partition_db" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + sql """switch ${catalog_name}""" + sql """use ${db_name}""" + + def test_iceberg_transform_partitions = { + // Bucket by INT (empty table) + qt_bucket_int_empty_cnt1 """ + select count(*) from bucket_int_empty; + """ + qt_bucket_int_empty_select1 """ + select * from bucket_int_empty order by id; + """ + + // Bucket by INT + qt_bucket_int_4_select1 """ + select * from bucket_int_4 where partition_key = 1 order by id; + """ + qt_bucket_int_4_select2 """ + select * from bucket_int_4 where partition_key in (2, 16) order by id; + """ + qt_bucket_int_4_cnt1 """ + select count(*) from bucket_int_4 where partition_key = -100; + """ + + // Bucket by BIGINT + qt_bucket_bigint_4_cnt1 """ + select count(*) from bucket_bigint_4 where partition_key = 1; + """ + qt_bucket_bigint_4_cnt2 """ + select count(*) from bucket_bigint_4 where partition_key in (-1, 1234567890123); + """ + qt_bucket_bigint_4_select1 """ + select * from bucket_bigint_4 where partition_key = 1 order by id; + """ + + // Bucket by STRING + qt_bucket_string_4_cnt1 """ + select count(*) from bucket_string_4 where partition_key = 'abc'; + """ + qt_bucket_string_4_cnt2 """ + select count(*) from bucket_string_4 where partition_key in ('', '😊'); + """ + qt_bucket_string_4_select1 """ + select * from bucket_string_4 where partition_key = 'abc' order by id; + """ + + // Bucket by DATE + qt_bucket_date_4_cnt1 """ + select count(*) from bucket_date_4 where partition_key = DATE '2024-02-29'; + """ + qt_bucket_date_4_cnt2 """ + select count(*) from bucket_date_4 where partition_key in (DATE '1970-01-01', DATE '1999-12-31'); + """ + qt_bucket_date_4_select1 """ + select * from bucket_date_4 where partition_key = DATE '2024-02-29' order by id; + """ + + // Bucket by TIMESTAMP + qt_bucket_timestamp_4_cnt1 """ + select count(*) from bucket_timestamp_4 where partition_key = TIMESTAMP '2024-01-15 08:00:00'; + """ + qt_bucket_timestamp_4_cnt2 """ + select count(*) from bucket_timestamp_4 where partition_key in (TIMESTAMP '2024-06-30 23:59:59', TIMESTAMP '2030-12-31 23:59:59'); + """ + qt_bucket_timestamp_4_select1 """ + select * from bucket_timestamp_4 where partition_key = TIMESTAMP '2024-01-15 08:00:00' order by id; + """ + + // Bucket by TIMESTAMP_NTZ + qt_bucket_timestamp_ntz_4_cnt1 """ + select count(*) from bucket_timestamp_ntz_4 where partition_key = '2024-01-15 08:00:00'; + """ + qt_bucket_timestamp_ntz_4_cnt2 """ + select count(*) from bucket_timestamp_ntz_4 where partition_key in ('2024-06-30 23:59:59', '2030-12-31 23:59:59'); + """ + qt_bucket_timestamp_ntz_4_select1 """ + select * from bucket_timestamp_ntz_4 where partition_key = '2024-01-15 08:00:00' order by id; + """ + + // Bucket by DECIMAL + qt_bucket_decimal_4_cnt1 """ + select count(*) from bucket_decimal_4 where partition_key = 10.50; + """ + qt_bucket_decimal_4_cnt2 """ + select count(*) from bucket_decimal_4 where partition_key in (-1.25, 9999999.99); + """ + qt_bucket_decimal_4_select1 """ + select * from bucket_decimal_4 where partition_key = 10.50 order by id; + """ + + // Bucket by BINARY + qt_bucket_binary_4_cnt1 """ + select count(*) from bucket_binary_4 where partition_key = 'abc'; + """ + qt_bucket_binary_4_cnt2 """ + select count(*) from bucket_binary_4 where partition_key in ('', '你好'); + """ + qt_bucket_binary_4_select1 """ + select * from bucket_binary_4 where partition_key = 'abc' order by id; + """ + + // Truncate STRING(3) + qt_truncate_string_3_cnt1 """ + select count(*) from truncate_string_3 where partition_key = 'abcdef'; + """ + qt_truncate_string_3_cnt2 """ + select count(*) from truncate_string_3 where partition_key in ('abc', 'abcdef'); + """ + qt_truncate_string_3_select1 """ + select * from truncate_string_3 where partition_key = 'abcdef' order by id; + """ + + // Truncate BINARY(4) + qt_truncate_binary_4_cnt1 """ + select count(*) from truncate_binary_4 where partition_key = 'abcdef'; + """ + qt_truncate_binary_4_cnt2 """ + select count(*) from truncate_binary_4 where partition_key in ('abcd', 'abcdef'); + """ + qt_truncate_binary_4_select1 """ + select * from truncate_binary_4 where partition_key = 'abcdef' order by id; + """ + + // Truncate INT(10) + qt_truncate_int_10_cnt1 """ + select count(*) from truncate_int_10 where partition_key = 19; + """ + qt_truncate_int_10_cnt2 """ + select count(*) from truncate_int_10 where partition_key in (7, 19); + """ + qt_truncate_int_10_select1 """ + select * from truncate_int_10 where partition_key = 19 order by id; + """ + + // Truncate BIGINT(100) + qt_truncate_bigint_100_cnt1 """ + select count(*) from truncate_bigint_100 where partition_key = 199; + """ + qt_truncate_bigint_100_cnt2 """ + select count(*) from truncate_bigint_100 where partition_key in (7, 199); + """ + qt_truncate_bigint_100_select1 """ + select * from truncate_bigint_100 where partition_key = 199 order by id; + """ + + // Truncate DECIMAL(10,2) width 10 + qt_truncate_decimal_10_cnt1 """ + select count(*) from truncate_decimal_10 where partition_key = 19.99; + """ + qt_truncate_decimal_10_cnt2 """ + select count(*) from truncate_decimal_10 where partition_key in (9.99, 19.99); + """ + qt_truncate_decimal_10_select1 """ + select * from truncate_decimal_10 where partition_key = 19.99 order by id; + """ + } + + try { + sql """ set time_zone = 'Asia/Shanghai'; """ + test_iceberg_transform_partitions() + } finally { + sql """ unset variable time_zone; """ + } +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy new file mode 100644 index 00000000000000..2662de4dfdaeff --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_transform_partitions.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_transform_partitions", "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test.") + return + } + + String catalog_name = "test_iceberg_write_transform_partitions" + String db_name = "transform_partition_db" + String rest_port = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minio_port = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + sql """drop catalog if exists ${catalog_name}""" + sql """ + CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='iceberg', + 'iceberg.catalog.type'='rest', + 'uri' = 'http://${externalEnvIp}:${rest_port}', + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minio_port}", + "s3.region" = "us-east-1" + );""" + + sql """switch ${catalog_name}""" + sql """use ${db_name}""" + + // While we currently cannot reset the table data, we can only test whether the INSERT statements execute successfully + def test_write_transform_partitions = { String table -> + sql """ insert into ${table}_copy select * from ${table}; """ + sql """ insert into ${table}_copy select * from ${table}_copy limit 50; """ + sql """ insert into ${table}_copy select * from ${table}_copy limit 50; """ + } + + try { + sql """ set time_zone = 'Asia/Shanghai'; """ + test_write_transform_partitions("bucket_int_4"); + test_write_transform_partitions("bucket_bigint_4"); + test_write_transform_partitions("bucket_string_4"); + test_write_transform_partitions("bucket_date_4"); + test_write_transform_partitions("bucket_timestamp_4"); + test_write_transform_partitions("bucket_timestamp_ntz_4"); + test_write_transform_partitions("bucket_decimal_4"); + test_write_transform_partitions("bucket_binary_4"); + test_write_transform_partitions("truncate_string_3"); + test_write_transform_partitions("truncate_binary_4"); + test_write_transform_partitions("truncate_int_10"); + test_write_transform_partitions("truncate_bigint_100"); + test_write_transform_partitions("truncate_decimal_10"); + } finally { + sql """ unset variable time_zone; """ + } +} From 71bdbf3d5678aff65cbcfca73a438c31ce52ac64 Mon Sep 17 00:00:00 2001 From: starocean999 Date: Sat, 16 Aug 2025 10:21:28 +0800 Subject: [PATCH 470/572] branch-3.0: [fix](nereids)make lambda expression can only be function argument #53657 (#49068) picked from #53657 --- .../org/apache/doris/nereids/DorisParser.g4 | 6 +++- .../nereids/parser/LogicalPlanBuilder.java | 2 +- .../nereids/parser/NereidsParserTest.java | 35 +++++++++++++++++++ .../nereids_syntax_p0/array_function.groovy | 20 +++++++++++ 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 7b039284dc8ab5..ad1e2c608f219b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1427,6 +1427,10 @@ namedExpressionSeq expression : booleanExpression + ; + +funcExpression + : expression | lambdaExpression ; @@ -1575,7 +1579,7 @@ functionCallExpression : functionIdentifier LEFT_PAREN ( (DISTINCT|ALL)? - arguments+=expression (COMMA arguments+=expression)* + arguments+=funcExpression (COMMA arguments+=funcExpression)* (ORDER BY sortItem (COMMA sortItem)*)? )? RIGHT_PAREN (OVER windowSpec)? diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 36d69f1bcbe9d1..76e4c8e316bc8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2260,7 +2260,7 @@ public Expression visitFunctionCallExpression(DorisParser.FunctionCallExpression String functionName = ctx.functionIdentifier().functionNameIdentifier().getText(); boolean isDistinct = ctx.DISTINCT() != null; List params = Lists.newArrayList(); - params.addAll(visit(ctx.expression(), Expression.class)); + params.addAll(visit(ctx.funcExpression(), Expression.class)); List orderKeys = visit(ctx.sortItem(), OrderKey.class); params.addAll(orderKeys.stream().map(OrderExpression::new).collect(Collectors.toList())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 27f19ca6999eee..d030a1f384b12c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -699,6 +699,41 @@ public void testBlockSqlAst() { } } + @Test + public void testLambdaSelect() { + parsePlan("SELECT x -> x + 1") + .assertThrowsExactly(ParseException.class) + .assertMessageContains("mismatched input '->' expecting {, ';'}"); + } + + @Test + public void testLambdaGroupBy() { + parsePlan("SELECT 1 from ( select 2 ) t group by x -> x + 1") + .assertThrowsExactly(ParseException.class) + .assertMessageContains("mismatched input '->' expecting {, ';'}"); + } + + @Test + public void testLambdaSort() { + parsePlan("SELECT 1 from ( select 2 ) t order by x -> x + 1") + .assertThrowsExactly(ParseException.class) + .assertMessageContains("mismatched input '->' expecting {, ';'}"); + } + + @Test + public void testLambdaHaving() { + parsePlan("SELECT 1 from ( select 2 ) t having x -> x + 1") + .assertThrowsExactly(ParseException.class) + .assertMessageContains("mismatched input '->' expecting {, ';'}"); + } + + @Test + public void testLambdaJoin() { + parsePlan("SELECT 1 from ( select 2 as a1 ) t1 join ( select 2 as a2 ) as t2 on x -> x + 1 = t1.a1") + .assertThrowsExactly(ParseException.class) + .assertMessageContains("mismatched input '->' expecting {, ';'}"); + } + private void checkQueryTopPlanClass(String sql, NereidsParser parser, Class clazz) { if (clazz == null) { Assertions.assertThrows(ParseException.class, () -> parser.parseSingle(sql)); diff --git a/regression-test/suites/nereids_syntax_p0/array_function.groovy b/regression-test/suites/nereids_syntax_p0/array_function.groovy index fb883c94f25080..467565c07ec346 100644 --- a/regression-test/suites/nereids_syntax_p0/array_function.groovy +++ b/regression-test/suites/nereids_syntax_p0/array_function.groovy @@ -47,4 +47,24 @@ suite("array_function") { ["""[[["2"]], [["aa"], ["2.0", "1.0"]]]"""] ]) } + + multi_sql """ + drop table if exists lambda_test_table; + CREATE TABLE `lambda_test_table` ( + `id` varchar(255) NOT NULL COMMENT '环境标识', + `redirect_links` variant NULL COMMENT '所有跳转链接，JSON格式存储' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + test { + sql """SELECT redirect_links -> CONCAT('x', JSON_LENGTH(redirect_links) - 1, 'x') AS last_element from lambda_test_table""" + exception "mismatched input '->'" + } } From e567fd14d57e1dbe859554f39261b0eb5cec03bb Mon Sep 17 00:00:00 2001 From: hui lai Date: Sat, 16 Aug 2025 16:02:08 +0800 Subject: [PATCH 471/572] branch-3.0: [fix](test) fix test_routine_load_job_schedule unstable (#54889) ### What problem does this PR solve? pick https://github.com/apache/doris/pull/54341 --- .../routine_load/test_routine_load_job_schedule.groovy | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy index c892d9dbde0bb2..cb8fba3583047f 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy @@ -38,7 +38,6 @@ suite("test_routine_load_job_schedule","nonConcurrent") { props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer") def producer = new KafkaProducer<>(props) def adminClient = AdminClient.create(props) - def newTopic = new NewTopic(kafkaCsvTpoics[0], 5, (short)1) def testData = [ "1,test_data_1,2023-01-01,value1,2023-01-01 10:00:00,extra1", "2,test_data_2,2023-01-02,value2,2023-01-02 11:00:00,extra2", @@ -46,10 +45,9 @@ suite("test_routine_load_job_schedule","nonConcurrent") { "4,test_data_4,2023-01-04,value4,2023-01-04 13:00:00,extra4", "5,test_data_5,2023-01-05,value5,2023-01-05 14:00:00,extra5" ] - adminClient.createTopics(Collections.singletonList(newTopic)) - testData.eachWithIndex { line, index -> + testData.each { line-> logger.info("Sending data to kafka: ${line}") - def record = new ProducerRecord<>(newTopic.name(), index, null, line) + def record = new ProducerRecord<>(kafkaCsvTpoics[0], null, line) producer.send(record) } producer.close() @@ -95,7 +93,7 @@ suite("test_routine_load_job_schedule","nonConcurrent") { logger.info("Routine load state: ${routineLoadState}") logger.info("Routine load statistic: ${statistic}") def rowCount = sql "select count(*) from ${tableName}" - if (routineLoadState == "RUNNING" && rowCount[0][0] == 5) { + if (routineLoadState == "RUNNING" && rowCount[0][0] > 0) { break } if (count > maxWaitCount) { From e1bcd7a6e2466a5e4da99be41f19ba62af27d659 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 16 Aug 2025 16:04:54 +0800 Subject: [PATCH 472/572] =?UTF-8?q?branch-3.0:=20[Fix](docker=20case)=20Fi?= =?UTF-8?q?x=20`test=5Fclean=5Ftablet=5Fwhen=5Fdrop=5Fforce=5Ftable`=20run?= =?UTF-8?q?=20=E2=80=A6=20#54803=20(#54891)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #54803 Co-authored-by: deardeng --- .../data/cloud_p0/tablets/all_types.csv.gz | Bin 0 -> 19554 bytes ..._clean_tablet_when_drop_force_table.groovy | 112 +++++++++++------- 2 files changed, 66 insertions(+), 46 deletions(-) create mode 100644 regression-test/data/cloud_p0/tablets/all_types.csv.gz diff --git a/regression-test/data/cloud_p0/tablets/all_types.csv.gz b/regression-test/data/cloud_p0/tablets/all_types.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..e9c82f6a3d24fc204c752ae26020b9f0bbb0f5e0 GIT binary patch literal 19554 zcmeI$&nv@W9Ki7>zuG}#k{r~^!IIKe`eqkSCfY@6O3K!<&{xLRYUD)8k8;>$7pK+8 zLFVE>+K*Vv{s5Xnl!NVpuq8RjAMrfBPtX1NzF(iG@5j7lFx=;Mr$x0#*C&Hv|5Tvc z8wiO~{%Wpjq_HM>8uN?H>|LS%MrhY%i#xR;9LGHeF7x=xzHP~A%Qn3nU21xwVR_>4 zg!fhZfh$CS3m_)LMYG2J$yPTKOK{o$LH_; zb(-Q{5QCVBAn`65@s(V^DG#M){-y@%bYL^$xUk)36n4o zWZn!ldo&UzVVM(Vr_-4(k}wGaLFUcS=$f5`Nm%BD+3Ca$Z)f&4<9g=7-^;-jP=Wjq zEEh9ShBB1pr~LmeN1lSb(F`?)nmR*mXt4v*kcPBMq + def testCase = { tableName, waitTime, useDp=false-> def ms = cluster.getAllMetaservices().get(0) def msHttpPort = ms.host + ":" + ms.httpPort - sql """CREATE TABLE $table ( - `k1` int(11) NULL, - `k2` int(11) NULL, - `v1` VARCHAR(2048) - ) - DUPLICATE KEY(`k1`, `k2`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`k1`) BUCKETS 3 - PROPERTIES ( - "replication_num"="1" - ); + sql """CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL, + `k2` tinyint(4) NULL, + `k3` smallint(6) NULL, + `k4` bigint(20) NULL, + `k5` largeint(40) NULL, + `k6` float NULL, + `k7` double NULL, + `k8` decimal(9, 0) NULL, + `k9` char(10) NULL, + `k10` varchar(1024) NULL, + `k11` text NULL, + `k12` date NULL, + `k13` datetime NULL + ) ENGINE=OLAP + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 """ - def random = new Random() - def generateRandomString = { int length -> - random.with { - def chars = ('A'..'Z').collect() + ('a'..'z').collect() + ('0'..'9').collect() - (1..length).collect { chars[nextInt(chars.size())] }.join('') + def txnId = -1; + // version 2 + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'compress_type', 'gz' + file 'all_types.csv.gz' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(80000, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId } } - def valuesList = (1..30000).collect { i -> - def randomStr = generateRandomString(2000) - "($i, $i, '$randomStr')" - }.join(", ") - sql """ - set global max_allowed_packet = 1010241024 - """ - + context.reconnectFe() - sql """ - insert into $table values ${valuesList} - """ - for (int i = 0; i < 5; i++) { sql """ - select count(*) from $table + select count(*) from $tableName """ } + // version 3 + streamLoad { + table "${tableName}" - valuesList = (30001..60000).collect { i -> - def randomStr = generateRandomString(2000) - "($i, $i, '$randomStr')" - }.join(", ") - sql """ - set global max_allowed_packet = 1010241024 - """ - context.reconnectFe() - sql """ - insert into $table values ${valuesList} - """ + set 'column_separator', ',' + set 'compress_type', 'gz' + file 'all_types.csv.gz' + time 10000 // limit inflight 10s + setFeAddr cluster.getAllFrontends().get(0).host, cluster.getAllFrontends().get(0).httpPort + + check { loadResult, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${loadResult}".toString()) + def json = parseJson(loadResult) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(80000, json.NumberTotalRows) + assertEquals(0, json.NumberFilteredRows) + txnId = json.TxnId + } + } // before drop table force - def beforeGetFromFe = getTabletAndBeHostFromFe(table) + def beforeGetFromFe = getTabletAndBeHostFromFe(tableName) def beforeGetFromBe = getTabletAndBeHostFromBe(cluster.getAllBackends()) // version 2 - def cacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, table, 2) + def cacheDirVersion2 = getTabletFileCacheDirFromBe(msHttpPort, tableName, 2) // version 3 - def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, table, 3) + def cacheDirVersion3 = getTabletFileCacheDirFromBe(msHttpPort, tableName, 3) def mergedCacheDir = cacheDirVersion2 + cacheDirVersion3.collectEntries { host, hashFiles -> [(host): cacheDirVersion2[host] ? (cacheDirVersion2[host] + hashFiles) : hashFiles] @@ -116,7 +136,7 @@ suite('test_clean_tablet_when_drop_force_table', 'docker') { // after drop table force sql """ - DROP TABLE $table FORCE + DROP TABLE $tableName FORCE """ def futrue if (useDp) { @@ -132,7 +152,7 @@ suite('test_clean_tablet_when_drop_force_table', 'docker') { logger.info("before drop tablets {}, after tablets {}", beforeGetFromFe, beTablets) beforeGetFromFe.keySet().every { !getTabletAndBeHostFromBe(cluster.getAllBackends()).containsKey(it) } } - logger.info("table {}, cost {}s", table, System.currentTimeMillis() / 1000 - start) + logger.info("table {}, cost {}s", tableName, System.currentTimeMillis() / 1000 - start) assertTrue(System.currentTimeMillis() / 1000 - start > waitTime) if (useDp) { futrue.get() From ff482cc5297e2199e23bba9393897e5c8dd4b90c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 16 Aug 2025 16:06:55 +0800 Subject: [PATCH 473/572] branch-3.0: [fix](test) do not close producer immediately after send #54851 (#54883) Cherry-picked from #54851 Co-authored-by: hui lai --- .../load_p0/routine_load/test_multi_table_load_restart.groovy | 1 - .../load_p0/routine_load/test_routine_load_follower_fe.groovy | 1 - .../load_p0/routine_load/test_routine_load_job_schedule.groovy | 1 - .../routine_load/test_routine_load_jsonpath_dollar.groovy | 1 - 4 files changed, 4 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy b/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy index d89f513eb5d0e6..88e62da64b73ea 100644 --- a/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy +++ b/regression-test/suites/load_p0/routine_load/test_multi_table_load_restart.groovy @@ -117,7 +117,6 @@ suite("test_multi_table_load_restart","docker") { producer.send(record) } } - producer.close() def count = 0 def maxWaitCount = 60 diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy index 8f7ed9a4cf6bcb..b2582a571a8ff4 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_follower_fe.groovy @@ -87,7 +87,6 @@ suite("test_routine_load_follower_fe","docker") { producer.send(record) } } - producer.close() // 3. Connect to a follower FE and create table def masterFe = cluster.getMasterFe() diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy index cb8fba3583047f..ed52e60a13ff53 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy @@ -50,7 +50,6 @@ suite("test_routine_load_job_schedule","nonConcurrent") { def record = new ProducerRecord<>(kafkaCsvTpoics[0], null, line) producer.send(record) } - producer.close() def tableName = "test_routine_load_job_schedule" def job = "test_routine_load_job_schedule" diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy index a4ade95f27d4a2..f77abe9db89ac1 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_jsonpath_dollar.groovy @@ -41,7 +41,6 @@ suite("test_routine_load_jsonpath_dollar", "p0") { def record = new ProducerRecord<>(jobName, null, line) producer.send(record) } - producer.close() try { sql """ From 39a93b5341ad183ab8817f2ed845f789ab72a927 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Sat, 16 Aug 2025 20:57:22 +0800 Subject: [PATCH 474/572] [fix](case) fix 230 message for case (#54894) --- be/src/olap/rowset_version_mgr.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset_version_mgr.cpp b/be/src/olap/rowset_version_mgr.cpp index d222ccfbb02b60..e4c1f2bef94611 100644 --- a/be/src/olap/rowset_version_mgr.cpp +++ b/be/src/olap/rowset_version_mgr.cpp @@ -74,7 +74,8 @@ static bvar::LatencyRecorder g_remote_fetch_tablet_rowsets_latency("remote_fetch "tablet_id={}", version_range.to_string(), _tablet_meta->max_version().second, tablet_id()); return ResultError(Status::Error( - "missed versions is empty, version_range={}, max_version={}, tablet_id={}", + "versions are already compacted, version_range={}, max_version={}, " + "tablet_id={}", version_range.to_string(), _tablet_meta->max_version().second, tablet_id())); } LOG(WARNING) << fmt::format("missed version for version_range={}, tablet_id={}, st={}", From 9ab212539df9503b88c6fe2d2760a565cc31e1b4 Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Mon, 18 Aug 2025 09:19:03 +0800 Subject: [PATCH 475/572] [chore](build) Update build script #54514 (#54635) pick #54514 --- gensrc/script/Makefile | 4 +++- gensrc/script/gen_build_version.sh | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/gensrc/script/Makefile b/gensrc/script/Makefile index b476a1c44429b9..4d6853aa35d27b 100644 --- a/gensrc/script/Makefile +++ b/gensrc/script/Makefile @@ -20,6 +20,8 @@ BUILD_DIR = ${CURDIR}/../build/ FE_TARGET_DIR = ${CURDIR}/../../fe/fe-core/target/generated-sources/build +export DORIS_VENDOR ?= doris + # Prerequisites on the right side of '|' is only order all: gen_builtins gen_version .PHONY: all @@ -46,5 +48,5 @@ gen_builtins: ${GEN_BUILTINS_OUTPUT} # generate version info gen_version: - ${CURDIR}/gen_build_version.sh + ${CURDIR}/gen_build_version.sh $(DORIS_VENDOR) .PHONY: gen_version diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 438f28d79b5963..2b3a7f7ce3c10d 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -27,7 +27,12 @@ set -eo pipefail -build_version_prefix="doris" +vendor=doris +if [[ $1 != "" ]]; then + vendor=$1 +fi + +build_version_prefix="${vendor}" build_version_major=3 build_version_minor=0 build_version_patch=7 From b2f96690f8e5455024ccbd603bf4f544a977ebf8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 09:19:44 +0800 Subject: [PATCH 476/572] branch-3.0: [fix](case) fix single replica load #54899 (#54914) Cherry-picked from #54899 Co-authored-by: Yongqiang YANG --- .../suites/load_p2/test_single_replica_load.groovy | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/load_p2/test_single_replica_load.groovy b/regression-test/suites/load_p2/test_single_replica_load.groovy index 68d1e687744df0..29e9e21eab1be0 100644 --- a/regression-test/suites/load_p2/test_single_replica_load.groovy +++ b/regression-test/suites/load_p2/test_single_replica_load.groovy @@ -20,7 +20,10 @@ // and modified by Doris. suite("test_single_replica_load", "p2, nonConcurrent") { - + if (isCloudMode()) { + return; + } + def load_json_data = {table_name, file_name -> // load the json data streamLoad { @@ -52,9 +55,7 @@ suite("test_single_replica_load", "p2, nonConcurrent") { sql "DROP TABLE IF EXISTS ${tableName}" setFeConfigTemporary([enable_inverted_index_v1_for_variant: true]) { - if (isCloudMode()) { - return; - } + sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( k bigint, From 4e65da9cc4a6e061f147e083dde5f92b18463d38 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 09:22:54 +0800 Subject: [PATCH 477/572] branch-3.0: [fix](test) Disable alter resource test in cloud mode #54823 (#54906) Cherry-picked from #54823 Co-authored-by: zy-kkk --- .../jdbc/test_jdbc_query_mysql.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy index 16c9982631115e..7b973331959721 100644 --- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy +++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy @@ -1058,14 +1058,14 @@ suite("test_jdbc_query_mysql", "p0,external,mysql,external_docker,external_docke // TODO: check this, maybe caused by datasource in JDBC // test alter resource - sql """alter resource $jdbcResourceMysql57 properties("password" = "1234567")""" - test { - sql """select count(*) from $jdbcMysql57Table1""" - exception "Access denied for user" + if (!isCloudMode()) { + sql """alter resource $jdbcResourceMysql57 properties("password" = "1234567")""" + test { + sql """select count(*) from $jdbcMysql57Table1""" + exception "Access denied for user" + } + sql """alter resource $jdbcResourceMysql57 properties("password" = "123456")""" } - sql """alter resource $jdbcResourceMysql57 properties("password" = "123456")""" - - } } From 6f2bd9ed3df8ab64a012900f5338e2319232973d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 09:24:15 +0800 Subject: [PATCH 478/572] branch-3.0: [fix](case) enable variant flattern in check_before_quit #54893 (#54900) Cherry-picked from #54893 Co-authored-by: Yongqiang YANG --- .../suites/check_before_quit/check_before_quit.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy b/regression-test/suites/check_before_quit/check_before_quit.groovy index bd79bc11bb30d0..041e27acf4a53a 100644 --- a/regression-test/suites/check_before_quit/check_before_quit.groovy +++ b/regression-test/suites/check_before_quit/check_before_quit.groovy @@ -235,6 +235,7 @@ suite("check_before_quit", "nonConcurrent,p0") { def failureList = [] sql "set enable_decimal256 = true;" + sql "set enable_variant_flatten_nested = true;" sql """ ADMIN SET ALL FRONTENDS CONFIG ('enable_inverted_index_v1_for_variant' = 'true'); """ From 90cbd2cebfefb852cd04b972f3dd740867b1303f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 18 Aug 2025 09:25:04 +0800 Subject: [PATCH 479/572] branch-3.0: [chore](case) fix some cases #54885 (#54902) Cherry-picked from #54885 Co-authored-by: Yongqiang YANG --- .../suites/alter_p2/test_alter_colocate_group.groovy | 3 +++ .../insert/test_insert_random_distribution_table.groovy | 6 ++++++ .../schema_change_p0/test_dup_mv_schema_change.groovy | 1 + 3 files changed, 10 insertions(+) diff --git a/regression-test/suites/alter_p2/test_alter_colocate_group.groovy b/regression-test/suites/alter_p2/test_alter_colocate_group.groovy index bb203769b631be..848f4146dcc99b 100644 --- a/regression-test/suites/alter_p2/test_alter_colocate_group.groovy +++ b/regression-test/suites/alter_p2/test_alter_colocate_group.groovy @@ -126,6 +126,7 @@ suite ("test_alter_colocate_group") { for (def group : groups) { allocMap[group[1]] = group[4] } + log.info("allocMap: ${allocMap}") assertEquals("tag.location.default: ${replicaNum}".toString(), allocMap[groupName]) } @@ -134,12 +135,14 @@ suite ("test_alter_colocate_group") { def result = sql """ show create table ${tableName} """ def createTbl = result[0][1].toString() assertTrue(createTbl.indexOf("\"replication_allocation\" = \"tag.location.default: ${replicaNum}\"") > 0) + log.info("createTbl: ${createTbl}") if (hasDynamicPart) { assertTrue(createTbl.indexOf( "\"dynamic_partition.replication_allocation\" = \"tag.location.default: ${replicaNum}\"") > 0) } result = sql """ show partitions from ${tableName} """ + log.info("result: ${result}") assertTrue(result.size() > 0) for (int i = 0; i < result.size(); i++) { assertEquals("${replicaNum}".toString(), result[i][9].toString()) diff --git a/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy b/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy index 9510a4f7980864..d67b9d9efefeb4 100644 --- a/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy +++ b/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy @@ -36,6 +36,12 @@ suite("test_insert_random_distribution_table", "p0") { ); """ + def ret = sql "SHOW FRONTEND CONFIG like '%wait_internal_group_commit_finish%';" + if (ret[0][1] == "true") { + logger.info("skip: wait_internal_group_commit_finish is true") + return + } + sql "set batch_size=2" // insert first time sql "insert into ${tableName} values('2021-11-14', '2', '3', '4', 55), ('2022-12-13', '3', '31', '4', 55), ('2023-10-14', '23', '45', '66', 88), ('2023-10-16', '2', '3', '4', 55)" diff --git a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy index a6ad20ec623048..9632512952ff98 100644 --- a/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy +++ b/regression-test/suites/schema_change_p0/test_dup_mv_schema_change.groovy @@ -29,6 +29,7 @@ suite ("test_dup_mv_schema_change") { def waitForJob = (tbName, timeout) -> { Awaitility.await().atMost(timeout, TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(() -> { String result = getJobState(tbName) + log.info("job state: ${result}") if (result == "FINISHED") { return true; } From 7214c960da82f3579c88c7e3b41977985a4a1840 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Mon, 18 Aug 2025 09:25:45 +0800 Subject: [PATCH 480/572] [fix](case) fix some cases (#54896) --- ...n_get_delete_bitmap_from_cache_fail.groovy | 19 ++++++++++--------- ...test_cloud_mow_partial_update_retry.groovy | 6 ++++-- ...d_mow_stale_resp_load_load_conflict.groovy | 6 ++++-- ...ud_mow_stream_load_with_commit_fail.groovy | 13 +++++++++++++ ...cloud_sc_self_retry_with_stop_token.groovy | 2 +- .../test_load_stream_fault_injection.groovy | 6 ++++++ .../test_segcompaction_fault_injection.groovy | 5 +++++ 7 files changed, 43 insertions(+), 14 deletions(-) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_compaction_get_delete_bitmap_from_cache_fail.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_compaction_get_delete_bitmap_from_cache_fail.groovy index 72fda5eea2680c..eea87d676e9da4 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_compaction_get_delete_bitmap_from_cache_fail.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_compaction_get_delete_bitmap_from_cache_fail.groovy @@ -61,7 +61,7 @@ suite("test_cloud_mow_compaction_get_delete_bitmap_from_cache_fail", "nonConcurr DebugPoint.enableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, inject_spin_block) DebugPoint.enableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, inject_cache_miss) logger.info("run compaction:" + tabletId) - (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) + def (code, out, err) = be_run_cumulative_compaction(injectBe.Host, injectBe.HttpPort, tabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) // Concurrent inserts @@ -74,14 +74,15 @@ suite("test_cloud_mow_compaction_get_delete_bitmap_from_cache_fail", "nonConcurr // let compaction continue DebugPoint.disableDebugPoint(injectBe.Host, injectBe.HttpPort.toInteger(), NodeType.BE, inject_spin_block) - do { - Thread.sleep(100) - (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status + def running = true + do { + Thread.sleep(100) + (code, out, err) = be_get_compaction_status(injectBe.Host, injectBe.HttpPort, tabletId) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("success", compactionStatus.status.toLowerCase()) + running = compactionStatus.run_status } while (running) Thread.sleep(200) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_partial_update_retry.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_partial_update_retry.groovy index 4f091bef8ea6f2..87749954f1761e 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_partial_update_retry.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_partial_update_retry.groovy @@ -65,7 +65,8 @@ suite("test_cloud_mow_partial_update_retry", "nonConcurrent") { // wait util the first partial update load's delete bitmap update lock expired // to ensure that the second load can take the delete bitmap update lock // Config.delete_bitmap_lock_expiration_seconds = 10s - Thread.sleep(11 * 1000) + def timeout = getFeConfig("delete_bitmap_lock_expiration_seconds").toInteger() + 2; + Thread.sleep(timeout * 1000) // the second load GetDebugPoint().enableDebugPointForAllBEs("BaseTablet::update_delete_bitmap.enable_spin_wait", [token: "token2"]) @@ -80,7 +81,8 @@ suite("test_cloud_mow_partial_update_retry", "nonConcurrent") { // keep waiting util the delete bitmap calculation timeout(Config.calculate_delete_bitmap_task_timeout_seconds = 15s) // and the first load will retry the calculation of delete bitmap - Thread.sleep(15 * 1000) + timeout = getFeConfig("calculate_delete_bitmap_task_timeout_seconds").toInteger() + 2; + Thread.sleep(timeout * 1000) // let the first partial update load finish GetDebugPoint().enableDebugPointForAllBEs("BaseTablet::update_delete_bitmap.block") diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stale_resp_load_load_conflict.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stale_resp_load_load_conflict.groovy index faafb6b8482536..e57621ca3f85de 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stale_resp_load_load_conflict.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stale_resp_load_load_conflict.groovy @@ -62,7 +62,8 @@ suite("test_cloud_mow_stale_resp_load_load_conflict", "nonConcurrent") { // wait util the first load's delete bitmap update lock expired // to ensure that the second load can take the delete bitmap update lock // Config.delete_bitmap_lock_expiration_seconds = 10s - Thread.sleep(11 * 1000) + def timeout = getFeConfig("delete_bitmap_lock_expiration_seconds").toInteger() + 2; + Thread.sleep(timeout * 1000) // the second load GetDebugPoint().enableDebugPointForAllBEs("BaseTablet::update_delete_bitmap.enable_spin_wait", [token: "token2"]) @@ -75,7 +76,8 @@ suite("test_cloud_mow_stale_resp_load_load_conflict", "nonConcurrent") { // keep waiting util the delete bitmap calculation timeout(Config.calculate_delete_bitmap_task_timeout_seconds = 15s) // and the coordinator BE will retry to commit the first load's txn - Thread.sleep(15 * 1000) + timeout = getFeConfig("calculate_delete_bitmap_task_timeout_seconds").toInteger() + 2; + Thread.sleep(timeout * 1000) // let the first partial update load finish GetDebugPoint().enableDebugPointForAllBEs("BaseTablet::update_delete_bitmap.block") diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy index c5810bec88a34c..aaeb596fa07f3d 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy @@ -146,6 +146,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load0.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -167,6 +168,8 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) + check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") def json = parseJson(result) @@ -190,6 +193,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load2.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -214,6 +218,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load2.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -239,6 +244,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load3.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -262,6 +268,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load3.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -289,6 +296,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load4.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -313,6 +321,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load4.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -337,6 +346,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load5.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -360,6 +370,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load5.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -386,6 +397,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load6.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") @@ -409,6 +421,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load.csv" time 10000 // limit inflight 10s + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_sc_self_retry_with_stop_token.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_sc_self_retry_with_stop_token.groovy index 5978320aa1941f..6a96c2b1ec9216 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_sc_self_retry_with_stop_token.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_sc_self_retry_with_stop_token.groovy @@ -53,7 +53,7 @@ suite("test_cloud_sc_self_retry_with_stop_token", "nonConcurrent") { sql "alter table ${table1} modify column c2 varchar(100);" def res - Awaitility.await().atMost(40, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { + Awaitility.await().atMost(600, TimeUnit.SECONDS).pollDelay(1, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { res = sql_return_maparray """ SHOW ALTER TABLE COLUMN WHERE TableName='${table1}' ORDER BY createtime DESC LIMIT 1 """ logger.info("res: ${res}") if (res[0].State == "FINISHED" || res[0].State == "CANCELLED") { diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy index 0ba05394adf657..11415955f4b5e2 100644 --- a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy @@ -19,6 +19,12 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import org.apache.doris.regression.util.Http suite("load_stream_fault_injection", "nonConcurrent") { + if (isCloudMode()) { + // TODO: load stream fault injection test is not supported in cloud mode yet + logger.info("skip test in cloud mode") + return + } + // init query case data sql """ CREATE TABLE IF NOT EXISTS `baseall` ( diff --git a/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy index 2cd9ab7841c842..6b827bb0a51f59 100644 --- a/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy @@ -19,6 +19,11 @@ import org.codehaus.groovy.runtime.IOGroovyMethods import org.apache.doris.regression.util.Http suite("test_segcompaction_correctness", "nonConcurrent,p2") { + if (isCloudMode()) { + logger.info("skip test in cloud mode") + return + } + def tableName = "segcompaction_correctness_test" def create_table_sql = """ CREATE TABLE IF NOT EXISTS ${tableName} ( From 7c63d57998d5960fcc574e6f2a26614570fe1239 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Aug 2025 09:26:43 +0800 Subject: [PATCH 481/572] branch-3.0: [fix](prepare statement)Support FE execute COM_STMT_EXECUTE show command. (#54446) (#54873) backport: https://github.com/apache/doris/pull/54446 --- .../doris/qe/MysqlConnectProcessor.java | 2 +- .../org/apache/doris/qe/StmtExecutor.java | 97 +++++++++++++++++-- .../org/apache/doris/qe/StmtExecutorTest.java | 94 ++++++++++++++++++ .../data/prepared_stmt_p0/prepared_show.out | 11 +++ .../prepared_stmt_p0/prepared_show.groovy | 54 +++++++++++ 5 files changed, 247 insertions(+), 11 deletions(-) create mode 100644 regression-test/data/prepared_stmt_p0/prepared_show.out create mode 100644 regression-test/suites/prepared_stmt_p0/prepared_show.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 50990a753c35fd..be5e437da0a4e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -164,7 +164,7 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare } StatementBase stmt = new LogicalPlanAdapter(executeStmt, statementContext); stmt.setOrigStmt(prepareCommand.getOriginalStmt()); - executor = new StmtExecutor(ctx, stmt); + executor = new StmtExecutor(ctx, stmt, true); ctx.setExecutor(executor); if (null != queryId) { executor.execute(queryId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 368e3f94ab1298..4b3a31c343966d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -146,6 +146,7 @@ import org.apache.doris.nereids.minidump.MinidumpUtils; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.Placeholder; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; @@ -287,6 +288,8 @@ public class StmtExecutor { // The profile of this execution private final Profile profile; private Boolean isForwardedToMaster = null; + // Flag for execute prepare statement, need to use binary protocol resultset + private boolean isComStmtExecute = false; // The result schema if "dry_run_query" is true. // Only one column to indicate the real return row numbers. @@ -316,9 +319,14 @@ public StmtExecutor(ConnectContext context, String stmt) { // constructor for receiving parsed stmt from connect processor public StmtExecutor(ConnectContext ctx, StatementBase parsedStmt) { + this(ctx, parsedStmt, false); + } + + public StmtExecutor(ConnectContext ctx, StatementBase parsedStmt, boolean isComStmtExecute) { this.context = ctx; this.parsedStmt = parsedStmt; this.originStmt = parsedStmt.getOrigStmt(); + this.isComStmtExecute = isComStmtExecute; if (context.getConnectType() == ConnectType.MYSQL) { this.serializer = context.getMysqlChannel().getSerializer(); } else { @@ -2902,18 +2910,14 @@ public void sendResultSet(ResultSet resultSet, List fieldInfos) throw sendMetaData(resultSet.getMetaData(), fieldInfos); // Send result set. - for (List row : resultSet.getResultRows()) { - serializer.reset(); - for (String item : row) { - if (item == null || item.equals(FeConstants.null_string)) { - serializer.writeNull(); - } else { - serializer.writeLenEncodedString(item); - } + if (isComStmtExecute) { + if (LOG.isDebugEnabled()) { + LOG.debug("Use binary protocol to set result."); } - context.getMysqlChannel().sendOnePacket(serializer.toByteBuffer()); + sendBinaryResultRow(resultSet); + } else { + sendTextResultRow(resultSet); } - context.getState().setEof(); } else if (context.getConnectType().equals(ConnectType.ARROW_FLIGHT_SQL)) { context.updateReturnRows(resultSet.getResultRows().size()); @@ -2925,6 +2929,79 @@ public void sendResultSet(ResultSet resultSet, List fieldInfos) throw } } + protected void sendTextResultRow(ResultSet resultSet) throws IOException { + for (List row : resultSet.getResultRows()) { + serializer.reset(); + for (String item : row) { + if (item == null || item.equals(FeConstants.null_string)) { + serializer.writeNull(); + } else { + serializer.writeLenEncodedString(item); + } + } + context.getMysqlChannel().sendOnePacket(serializer.toByteBuffer()); + } + } + + protected void sendBinaryResultRow(ResultSet resultSet) throws IOException { + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row_value + ResultSetMetaData metaData = resultSet.getMetaData(); + int nullBitmapLength = (metaData.getColumnCount() + 7 + 2) / 8; + for (List row : resultSet.getResultRows()) { + serializer.reset(); + // Reserved one byte. + serializer.writeByte((byte) 0x00); + byte[] nullBitmap = new byte[nullBitmapLength]; + // Generate null bitmap + for (int i = 0; i < row.size(); i++) { + String item = row.get(i); + if (item == null || item.equals(FeConstants.null_string)) { + // The first 2 bits are reserved. + int byteIndex = (i + 2) / 8; // Index of the byte in the bitmap array + int bitInByte = (i + 2) % 8; // Position within the target byte (0-7) + nullBitmap[byteIndex] |= (1 << bitInByte); + } + } + // Null bitmap + serializer.writeBytes(nullBitmap); + // Non-null columns + for (int i = 0; i < row.size(); i++) { + String item = row.get(i); + if (item != null && !item.equals(FeConstants.null_string)) { + Column col = metaData.getColumn(i); + switch (col.getType().getPrimitiveType()) { + case INT: + serializer.writeInt4(Integer.parseInt(item)); + break; + case BIGINT: + serializer.writeInt8(Long.parseLong(item)); + break; + case DATETIME: + case DATETIMEV2: + DateTimeV2Literal datetime = new DateTimeV2Literal(item); + long microSecond = datetime.getMicroSecond(); + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html + int length = microSecond == 0 ? 7 : 11; + serializer.writeInt1(length); + serializer.writeInt2((int) (datetime.getYear())); + serializer.writeInt1((int) datetime.getMonth()); + serializer.writeInt1((int) datetime.getDay()); + serializer.writeInt1((int) datetime.getHour()); + serializer.writeInt1((int) datetime.getMinute()); + serializer.writeInt1((int) datetime.getSecond()); + if (microSecond > 0) { + serializer.writeInt4((int) microSecond); + } + break; + default: + serializer.writeLenEncodedString(item); + } + } + } + context.getMysqlChannel().sendOnePacket(serializer.toByteBuffer()); + } + } + // Process show statement private void handleShow() throws IOException, AnalysisException, DdlException { ShowExecutor executor = new ShowExecutor(context, (ShowStmt) parsedStmt); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 8ab187315d8681..6e35e7e43897ee 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -32,7 +32,9 @@ import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UseStmt; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.jmockit.Deencapsulation; @@ -42,6 +44,7 @@ import org.apache.doris.mysql.MysqlChannel; import org.apache.doris.mysql.MysqlSerializer; import org.apache.doris.planner.OriginalPlanner; +import org.apache.doris.qe.CommonResultSet.CommonResultSetMetaData; import org.apache.doris.qe.ConnectContext.ConnectType; import org.apache.doris.rewrite.ExprRewriter; import org.apache.doris.service.FrontendOptions; @@ -57,7 +60,9 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Disabled; +import org.mockito.Mockito; import java.io.IOException; import java.net.UnknownHostException; @@ -65,6 +70,7 @@ import java.util.List; import java.util.SortedMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; public class StmtExecutorTest { private ConnectContext ctx; @@ -891,4 +897,92 @@ public void testBlockSqlAst(@Mocked UseStmt useStmt, @Mocked CreateFileStmt crea executor.execute(); Assert.assertEquals(QueryState.MysqlStateType.OK, state.getStateType()); } + + @Test + public void testSendTextResultRow() throws IOException { + ConnectContext mockCtx = Mockito.mock(ConnectContext.class); + MysqlChannel channel = Mockito.mock(MysqlChannel.class); + Mockito.when(mockCtx.getConnectType()).thenReturn(ConnectType.MYSQL); + Mockito.when(mockCtx.getMysqlChannel()).thenReturn(channel); + MysqlSerializer mysqlSerializer = MysqlSerializer.newInstance(); + Mockito.when(channel.getSerializer()).thenReturn(mysqlSerializer); + SessionVariable sessionVariable = VariableMgr.newSessionVariable(); + Mockito.when(mockCtx.getSessionVariable()).thenReturn(sessionVariable); + OriginStatement stmt = new OriginStatement("", 1); + + List> rows = Lists.newArrayList(); + List row1 = Lists.newArrayList(); + row1.add(null); + row1.add("row1"); + List row2 = Lists.newArrayList(); + row2.add("1234"); + row2.add("row2"); + rows.add(row1); + rows.add(row2); + List columns = Lists.newArrayList(); + columns.add(new Column()); + columns.add(new Column()); + ResultSet resultSet = new CommonResultSet(new CommonResultSetMetaData(columns), rows); + AtomicInteger i = new AtomicInteger(); + Mockito.doAnswer(invocation -> { + byte[] expected0 = new byte[]{-5, 4, 114, 111, 119, 49}; + byte[] expected1 = new byte[]{4, 49, 50, 51, 52, 4, 114, 111, 119, 50}; + ByteBuffer buffer = invocation.getArgument(0); + if (i.get() == 0) { + Assertions.assertArrayEquals(expected0, buffer.array()); + i.getAndIncrement(); + } else if (i.get() == 1) { + Assertions.assertArrayEquals(expected1, buffer.array()); + i.getAndIncrement(); + } + return null; + }).when(channel).sendOnePacket(Mockito.any(ByteBuffer.class)); + + StmtExecutor executor = new StmtExecutor(mockCtx, stmt, false); + executor.sendTextResultRow(resultSet); + } + + @Test + public void testSendBinaryResultRow() throws IOException { + ConnectContext mockCtx = Mockito.mock(ConnectContext.class); + MysqlChannel channel = Mockito.mock(MysqlChannel.class); + Mockito.when(mockCtx.getConnectType()).thenReturn(ConnectType.MYSQL); + Mockito.when(mockCtx.getMysqlChannel()).thenReturn(channel); + MysqlSerializer mysqlSerializer = MysqlSerializer.newInstance(); + Mockito.when(channel.getSerializer()).thenReturn(mysqlSerializer); + SessionVariable sessionVariable = VariableMgr.newSessionVariable(); + Mockito.when(mockCtx.getSessionVariable()).thenReturn(sessionVariable); + OriginStatement stmt = new OriginStatement("", 1); + + List> rows = Lists.newArrayList(); + List row1 = Lists.newArrayList(); + row1.add(null); + row1.add("2025-01-01 01:02:03"); + List row2 = Lists.newArrayList(); + row2.add("1234"); + row2.add("2025-01-01 01:02:03.123456"); + rows.add(row1); + rows.add(row2); + List columns = Lists.newArrayList(); + columns.add(new Column("col1", PrimitiveType.BIGINT)); + columns.add(new Column("col2", PrimitiveType.DATETIMEV2)); + ResultSet resultSet = new CommonResultSet(new CommonResultSetMetaData(columns), rows); + AtomicInteger i = new AtomicInteger(); + Mockito.doAnswer(invocation -> { + byte[] expected0 = new byte[]{0, 4, 7, -23, 7, 1, 1, 1, 2, 3}; + byte[] expected1 = new byte[]{0, 0, -46, 4, 0, 0, 0, 0, 0, 0, 11, -23, 7, 1, 1, 1, 2, 3, 64, -30, 1, 0}; + ByteBuffer buffer = invocation.getArgument(0); + if (i.get() == 0) { + Assertions.assertArrayEquals(expected0, buffer.array()); + i.getAndIncrement(); + } else if (i.get() == 1) { + Assertions.assertArrayEquals(expected1, buffer.array()); + i.getAndIncrement(); + } + return null; + }).when(channel).sendOnePacket(Mockito.any(ByteBuffer.class)); + + StmtExecutor executor = new StmtExecutor(mockCtx, stmt, false); + executor.sendBinaryResultRow(resultSet); + } } diff --git a/regression-test/data/prepared_stmt_p0/prepared_show.out b/regression-test/data/prepared_stmt_p0/prepared_show.out new file mode 100644 index 00000000000000..403246ea021c1c --- /dev/null +++ b/regression-test/data/prepared_stmt_p0/prepared_show.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !stmt_show_db -- +prepared_show + +-- !stmt_show_table -- +prepared_show_table1 +prepared_show_table2 + +-- !stmt_show_table_stats1 -- + 0 true + diff --git a/regression-test/suites/prepared_stmt_p0/prepared_show.groovy b/regression-test/suites/prepared_stmt_p0/prepared_show.groovy new file mode 100644 index 00000000000000..e276e7c401b638 --- /dev/null +++ b/regression-test/suites/prepared_stmt_p0/prepared_show.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("prepared_show") { + def tableName = "prepared_show" + def user = context.config.jdbcUser + def password = context.config.jdbcPassword + sql """drop database if exists prepared_show""" + sql """create database prepared_show""" + sql """use prepared_show""" + sql """CREATE TABLE IF NOT EXISTS prepared_show_table1 (`k1` tinyint NULL) + ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + )""" + + sql """CREATE TABLE IF NOT EXISTS prepared_show_table2 (`k1` tinyint NULL) + ENGINE=OLAP + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + )""" + String url = getServerPrepareJdbcUrl(context.config.jdbcUrl, "prepared_show") + def result1 = connect(user, password, url) { + def stmt_read = prepareStatement """show databases like "prepared_show" """ + qe_stmt_show_db stmt_read + + stmt_read = prepareStatement """show tables from prepared_show""" + qe_stmt_show_table stmt_read + + stmt_read = prepareStatement """show table stats prepared_show_table1""" + qe_stmt_show_table_stats1 stmt_read + + stmt_read = prepareStatement """show processlist""" + stmt_read.executeQuery() + } +} From 8be385e4cfbcac67e9abe224469608cda823a0d1 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 18 Aug 2025 09:29:09 +0800 Subject: [PATCH 482/572] branch-3.0: [fix](prepare statement)Fix date_trunc using prepare statement parameter type bug. (#54848) backport: https://github.com/apache/doris/pull/54847 --- .../expressions/functions/scalar/DateTrunc.java | 13 ++++++------- .../data/prepared_stmt_p0/prepared_stmt.out | 3 +++ .../suites/prepared_stmt_p0/prepared_stmt.groovy | 6 ++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java index 86a6bfccf56696..59e7175c3f33da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/DateTrunc.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.functions.CustomSignature; import org.apache.doris.nereids.trees.expressions.functions.Monotonic; import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral; -import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -60,14 +59,14 @@ public void checkLegalityBeforeTypeCoercion() { throw new AnalysisException("the time unit parameter of " + getName() + " function must be a string constant: " + toSql()); } else if (firstArgIsStringLiteral && secondArgIsStringLiteral) { - if (!LEGAL_TIME_UNIT.contains(((VarcharLiteral) getArgument(0)).getStringValue().toLowerCase()) - && !LEGAL_TIME_UNIT.contains(((VarcharLiteral) getArgument(1)) + if (!LEGAL_TIME_UNIT.contains(((StringLikeLiteral) getArgument(0)).getStringValue().toLowerCase()) + && !LEGAL_TIME_UNIT.contains(((StringLikeLiteral) getArgument(1)) .getStringValue().toLowerCase())) { throw new AnalysisException("date_trunc function time unit param only support argument is " + String.join("|", LEGAL_TIME_UNIT)); } } else { - final String constParam = ((VarcharLiteral) getArgument(firstArgIsStringLiteral ? 0 : 1)) + final String constParam = ((StringLikeLiteral) getArgument(firstArgIsStringLiteral ? 0 : 1)) .getStringValue().toLowerCase(); if (!LEGAL_TIME_UNIT.contains(constParam)) { throw new AnalysisException("date_trunc function time unit param only support argument is " @@ -95,9 +94,9 @@ public FunctionSignature customSignature() { .args(VarcharType.SYSTEM_DEFAULT, getArgument(1).getDataType()); } boolean firstArgIsStringLiteral = - getArgument(0).isConstant() && getArgument(0) instanceof VarcharLiteral; + getArgument(0).isConstant() && getArgument(0) instanceof StringLikeLiteral; boolean secondArgIsStringLiteral = - getArgument(1).isConstant() && getArgument(1) instanceof VarcharLiteral; + getArgument(1).isConstant() && getArgument(1) instanceof StringLikeLiteral; if (firstArgIsStringLiteral && !secondArgIsStringLiteral) { return FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) .args(VarcharType.SYSTEM_DEFAULT, DateTimeV2Type.SYSTEM_DEFAULT); @@ -105,7 +104,7 @@ public FunctionSignature customSignature() { return FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) .args(DateTimeV2Type.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT); } else if (firstArgIsStringLiteral && secondArgIsStringLiteral) { - boolean timeUnitIsFirst = LEGAL_TIME_UNIT.contains(((VarcharLiteral) getArgument(0)) + boolean timeUnitIsFirst = LEGAL_TIME_UNIT.contains(((StringLikeLiteral) getArgument(0)) .getStringValue().toLowerCase()); return timeUnitIsFirst ? FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT) .args(VarcharType.SYSTEM_DEFAULT, DateTimeV2Type.SYSTEM_DEFAULT) diff --git a/regression-test/data/prepared_stmt_p0/prepared_stmt.out b/regression-test/data/prepared_stmt_p0/prepared_stmt.out index 9cadb98813af02..a940c924e64b2a 100644 --- a/regression-test/data/prepared_stmt_p0/prepared_stmt.out +++ b/regression-test/data/prepared_stmt_p0/prepared_stmt.out @@ -135,6 +135,9 @@ a -- !select24 -- 1 \N [{"id":"1", "name":"doris"}, {"id":"2", "name":"apache"}, null] \N +-- !select25 -- +2025-08-15T00:00 + -- !overflow_2 -- 2 diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index 80058e72894ebd..9a350ce6dc6434 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -313,6 +313,12 @@ suite("test_prepared_stmt", "nonConcurrent") { stmt_read = prepareStatement("""SELECT 1, null, [{'id': 1, 'name' : 'doris'}, {'id': 2, 'name': 'apache'}, null], null""") assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt_read.class) qe_select24 stmt_read + + // test date_trunc + stmt_read = prepareStatement "select date_trunc (? , ?)" + stmt_read.setString(1, "2025-08-15 11:22:33") + stmt_read.setString(2, "DAY") + qe_select25 stmt_read } // test stmtId overflow From b4332c77f1bcf7edf957b82d88f3c717a91cf314 Mon Sep 17 00:00:00 2001 From: Jack Date: Mon, 18 Aug 2025 09:33:45 +0800 Subject: [PATCH 483/572] branch-3.0: [opt](inverted index) create non analyzer when parser is none for inverted index #54666 (#54795) cherry pick from #54666 --- be/src/clucene | 2 +- be/src/olap/rowset/segment_v2/inverted_index_writer.cpp | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/be/src/clucene b/be/src/clucene index 8d8f92ef8ddd0e..6046125ce7f9bb 160000 --- a/be/src/clucene +++ b/be/src/clucene @@ -1 +1 @@ -Subproject commit 8d8f92ef8ddd0e50b6fc76f8f6572abaef1b5213 +Subproject commit 6046125ce7f9bb8631427ff8ee1ce93e07edb421 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp index 827d8127d21d97..cf1fc8b4620c4e 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp @@ -243,7 +243,10 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter { RETURN_IF_ERROR(open_index_directory()); _char_string_reader = DORIS_TRY(create_char_string_reader(_inverted_index_ctx->char_filter_map)); - _analyzer = DORIS_TRY(create_analyzer(_inverted_index_ctx)); + if (_parser_type != InvertedIndexParserType::PARSER_UNKNOWN && + _parser_type != InvertedIndexParserType::PARSER_NONE) { + _analyzer = DORIS_TRY(create_analyzer(_inverted_index_ctx)); + } _index_writer = create_index_writer(); _doc = std::make_unique(); if (_single_field) { From 4a8feab3c20bcc95483fcb73884ef6606d37f030 Mon Sep 17 00:00:00 2001 From: shuke Date: Mon, 18 Aug 2025 11:00:33 +0800 Subject: [PATCH 484/572] [chore](asan) add _dl_find_object_update to known leaks (#53203) (#54705) --- conf/lsan_suppr.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/lsan_suppr.conf b/conf/lsan_suppr.conf index da3fdbadceaacc..5b028946fef98b 100644 --- a/conf/lsan_suppr.conf +++ b/conf/lsan_suppr.conf @@ -19,4 +19,5 @@ leak:brpc leak:libjvm leak:libzip -leak:*_dl_map_object_deps* \ No newline at end of file +leak:*_dl_map_object_deps* +leak:_dl_find_object_update From 269873d44e0a14842a344f03468cd8577bb9e904 Mon Sep 17 00:00:00 2001 From: shuke Date: Mon, 18 Aug 2025 11:01:09 +0800 Subject: [PATCH 485/572] [fix](regression-test) fix variant_p2 case to show load failure #52652 (#52687) --- regression-test/suites/variant_p2/load.groovy | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy index 056bb666e341c3..a15bc8b1128958 100644 --- a/regression-test/suites/variant_p2/load.groovy +++ b/regression-test/suites/variant_p2/load.groovy @@ -74,6 +74,7 @@ suite("load_p2", "variant_type,p2"){ // Executor service for managing threads def executorService = Executors.newFixedThreadPool(numberOfThreads) + def futures = [] try { def table_name = "github_events" @@ -141,16 +142,25 @@ suite("load_p2", "variant_type,p2"){ def fileName = year + "-" + month + "-" + day + "-" + hour + ".json" log.info("cuurent fileName: ${fileName}") // Submitting tasks to the executor service - executorService.submit({ + futures << executorService.submit({ log.info("Loading file: ${fileName}") s3load_paral_wait.call(table_name, "JSON", "regression/github_events_dataset/${fileName}", 3) } as Runnable) } } } - // Shutdown executor service and wait for all tasks to complete - executorService.shutdown() - executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) + + try { + futures.each { future -> + future.get() + } + } catch (Exception e) { + throw e.cause // throw original exception + } finally { + // Shutdown executor service and wait for all tasks to complete + executorService.shutdown() + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS) + } qt_sql("select count() from github_events") } finally { From 304b8626e89278b027194692da93e7756aa77fcd Mon Sep 17 00:00:00 2001 From: shuke Date: Mon, 18 Aug 2025 11:01:31 +0800 Subject: [PATCH 486/572] =?UTF-8?q?[feature](regression)=20add=20retry=20t?= =?UTF-8?q?o=20stream=20load=20when=20connection=20reset=20=E2=80=A6=20(#5?= =?UTF-8?q?4702)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …by s3 (#54613) During stream loading, Doris sometimes becomes stuck due to high cpu usage, meaning that Doris fails to retrieve data from the HTTP connection. This issue also prevents the regression test framework from fetching data via HTTP/S3. Since S3 terminates connections that are idle for approximately two minutes, such stalls lead to stream load failures. This patch introduces an automatic retry mechanism to tolerate these stuck scenarios and ensure successful stream loading. usage: ` streamload { ... retryIfHttpError true } ` ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../regression/action/StreamLoadAction.groovy | 192 +++++++++++++++++- regression-test/suites/opensky_p2/load.groovy | 1 + .../suites/tpcds_sf1_unique_p1/load.groovy | 1 + 3 files changed, 187 insertions(+), 7 deletions(-) diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy index aa19094854eddd..7cf57d212f27d8 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy @@ -29,6 +29,7 @@ import groovy.util.logging.Slf4j import org.apache.http.HttpEntity import org.apache.http.HttpStatus import org.apache.http.client.methods.CloseableHttpResponse +import org.apache.http.impl.client.CloseableHttpClient import org.apache.http.client.methods.RequestBuilder import org.apache.http.entity.FileEntity import org.apache.http.entity.InputStreamEntity @@ -37,6 +38,8 @@ import org.apache.http.impl.client.CloseableHttpClient import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils import org.junit.Assert +import java.io.InputStream +import java.io.IOException @Slf4j class StreamLoadAction implements SuiteAction { @@ -50,6 +53,7 @@ class StreamLoadAction implements SuiteAction { String inputText Iterator> inputIterator long time + boolean retryIfHttpError = false Closure check Map headers SuiteContext context @@ -138,6 +142,10 @@ class StreamLoadAction implements SuiteAction { this.time = time.call() } + void retryIfHttpError(boolean r) { + this.retryIfHttpError = r + } + void twoPhaseCommit(boolean twoPhaseCommit) { this.twoPhaseCommit = twoPhaseCommit; } @@ -213,14 +221,18 @@ class StreamLoadAction implements SuiteAction { } private InputStream httpGetStream(CloseableHttpClient client, String url) { - CloseableHttpResponse resp = client.execute(RequestBuilder.get(url).build()) - int code = resp.getStatusLine().getStatusCode() - if (code != HttpStatus.SC_OK) { - String streamBody = EntityUtils.toString(resp.getEntity()) - throw new IllegalStateException("Get http stream failed, status code is ${code}, body:\n${streamBody}") - } + if (retryIfHttpError) { + return new ResumableHttpInputStream(client, url) + } else { + CloseableHttpResponse resp = client.execute(RequestBuilder.get(url).build()) + int code = resp.getStatusLine().getStatusCode() + if (code != HttpStatus.SC_OK) { + String streamBody = EntityUtils.toString(resp.getEntity()) + throw new IllegalStateException("Get http stream failed, status code is ${code}, body:\n${streamBody}") + } - return resp.getEntity().getContent() + return resp.getEntity().getContent() + } } private RequestBuilder prepareRequestHeader(RequestBuilder requestBuilder) { @@ -423,4 +435,170 @@ class StreamLoadAction implements SuiteAction { throw t; } } + + /** + * A resumable HTTP input stream implementation that supports automatic retry and resume + * on connection failures during data transfer. This stream is designed for reliable + * large file downloads over HTTP with built-in recovery mechanisms, especially when stream + * load runs too slowly due to high cpu. + * + * Pay Attention: + * Using this class can recover from S3 actively disconnecting due to streaming + * load stuck, thereby masking the underlying performance bug where stream loading + * stalls for extended periods. + */ + class ResumableHttpInputStream extends InputStream { + private CloseableHttpClient httpClient + private String url + private long offset = 0 + private InputStream currentStream + private CloseableHttpResponse currentResponse + private int maxRetries = 3 + private int retryDelayMs = 1000 + private boolean closed = false + + ResumableHttpInputStream(CloseableHttpClient httpClient, String url) { + this.httpClient = httpClient + this.url = url + openNewStream(0) + } + + private void openNewStream(long startOffset) { + closeCurrentResources() + log.info("open new stream ${this.url} with offset ${startOffset}") + + int attempts = 0 + while (attempts <= maxRetries && !closed) { + attempts++ + try { + RequestBuilder builder = RequestBuilder.get(url) + if (startOffset > 0) { + builder.addHeader("Range", "bytes=${startOffset}-") + } + + currentResponse = httpClient.execute(builder.build()) + int code = currentResponse.getStatusLine().getStatusCode() + + if (code == HttpStatus.SC_OK || + (code == HttpStatus.SC_PARTIAL_CONTENT && startOffset > 0)) { + currentStream = currentResponse.getEntity().getContent() + offset = startOffset + return + } + + String body = EntityUtils.toString(currentResponse.getEntity()) + throw new IOException("HTTP error ${code} ${currentResponse.getStatusLine().getReasonPhrase()}\n${body}") + + } catch (IOException e) { + closeCurrentResources() + if (attempts > maxRetries || closed) { + throw e + } + sleep(retryDelayMs * attempts) + } + } + } + + @Override + int read() throws IOException { + if (closed) throw new IOException("Stream closed") + + int attempts = 0 + while (attempts <= maxRetries) { + attempts++ + try { + int byteRead = currentStream.read() + if (byteRead >= 0) offset++ + return byteRead + } catch (IOException e) { + log.info("${url} read exception: ${e.getMessage()}") + if (attempts > maxRetries || closed) throw e + reopenStreamAfterError() + sleep(retryDelayMs * attempts) + } + } + return -1 + } + + @Override + int read(byte[] b, int off, int len) throws IOException { + if (closed) throw new IOException("Stream closed") + if (b == null) throw new NullPointerException() + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException() + } + + int attempts = 0 + while (attempts <= maxRetries) { + attempts++ + try { + int bytesRead = currentStream.read(b, off, len) + if (bytesRead > 0) offset += bytesRead + return bytesRead + + } catch (IOException e) { + log.info("${url} read exception: ${e.getMessage()}") + if (attempts > maxRetries || closed) throw e + reopenStreamAfterError() + sleep(retryDelayMs * attempts) + } + } + return -1 + } + + private void reopenStreamAfterError() { + closeCurrentResources() + openNewStream(offset) + } + + private void closeCurrentResources() { + try { + if (currentStream != null) { + currentStream.close() + } + } catch (IOException ignored) {} + + try { + if (currentResponse != null) { + currentResponse.close() + } + } catch (IOException ignored) {} + + currentStream = null + currentResponse = null + } + + @Override + void close() throws IOException { + if (!closed) { + closed = true + closeCurrentResources() + } + } + + long getOffset() { offset } + + void setRetryPolicy(int maxRetries, int baseDelayMs) { + this.maxRetries = maxRetries + this.retryDelayMs = baseDelayMs + } + + @Override + int available() throws IOException { + return currentStream != null ? currentStream.available() : 0 + } + + @Override + long skip(long n) throws IOException { + if (currentStream == null) return 0 + long skipped = currentStream.skip(n) + offset += skipped + return skipped + } + + @Override + boolean markSupported() { + return false + } + } } diff --git a/regression-test/suites/opensky_p2/load.groovy b/regression-test/suites/opensky_p2/load.groovy index d00be91d3fd0c1..d0ff5eb478e509 100644 --- a/regression-test/suites/opensky_p2/load.groovy +++ b/regression-test/suites/opensky_p2/load.groovy @@ -46,6 +46,7 @@ suite("load"){ file """${getS3Url() + '/regression/clickhouse/opensky/' + sourceFile}""" time 0 + retryIfHttpError true // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows diff --git a/regression-test/suites/tpcds_sf1_unique_p1/load.groovy b/regression-test/suites/tpcds_sf1_unique_p1/load.groovy index 0caf0889f8c489..e93d7f7d020286 100644 --- a/regression-test/suites/tpcds_sf1_unique_p1/load.groovy +++ b/regression-test/suites/tpcds_sf1_unique_p1/load.groovy @@ -121,6 +121,7 @@ suite("load") { file """${getS3Url()}/regression/tpcds/sf1/${tableName}.dat.gz""" time 10000 // limit inflight 10s + retryIfHttpError true // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows From fc809899536af7ad2b518c3a8d1ae83ba087a479 Mon Sep 17 00:00:00 2001 From: amory Date: Mon, 18 Aug 2025 15:01:05 +0800 Subject: [PATCH 487/572] branch-3.0[fix](variant)fix top-array in variant (#54393) ### What problem does this PR solve? backport : https://github.com/apache/doris/pull/54396 This pr mainly solves the problem when we insert top-level nested array data into variant columnIssue Number: like : ``` mysql> insert into sv1 values (1, '[{"a": 1, "c": 1.1}, {"b": "1"}]'); ``` we maintain the association information between Nested in array , in this case , we will keep same offset at a,b,c , ``` mysql> select * from sv1; +------+----------------------------------------------+ | k | v | +------+----------------------------------------------+ | 1 | {"a":[1,null],"b":[null,"1"],"c":[1.1,null]} | +------+----------------------------------------------+ ``` but in top array we just do flatten array not fill the null value to maintain the association information between Nested in array like: ``` mysql> insert into sv1 values (16, '[{"a": 1, "b": 1}, {"b": 2}, {"b": 3}]'); Query OK, 1 row affected (0.15 sec) {'label':'label_4b3ede8b449a4d9a_b55fdeebe13b741c', 'status':'VISIBLE', 'txnId':'6031'} mysql> select v from sv1 where k=16; +-----------------------+ | v | +-----------------------+ | {"a":[1],"b":[1,2,3]} | +-----------------------+ 1 row in set (0.19 sec) ``` close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/json/json_parser.cpp | 175 ++++-- be/src/vec/json/json_parser.h | 11 +- be/test/vec/jsonb/json_parser_test.cpp | 174 ++++++ .../variant_p0/nested/nested_in_top_array.out | 545 ++++++++++++++++++ .../nested/nested_in_top_array.groovy | 131 +++++ 5 files changed, 981 insertions(+), 55 deletions(-) create mode 100644 regression-test/data/variant_p0/nested/nested_in_top_array.out create mode 100644 regression-test/suites/variant_p0/nested/nested_in_top_array.groovy diff --git a/be/src/vec/json/json_parser.cpp b/be/src/vec/json/json_parser.cpp index e031f168820156..35342afe2f30b7 100644 --- a/be/src/vec/json/json_parser.cpp +++ b/be/src/vec/json/json_parser.cpp @@ -43,6 +43,7 @@ std::optional JSONDataParser::parse(const char* begin, } ParseContext context; context.enable_flatten_nested = config.enable_flatten_nested; + context.is_top_array = document.isArray(); traverse(document, context); ParseResult result; result.values = std::move(context.values); @@ -141,11 +142,26 @@ void JSONDataParser::traverseArrayAsJsonb(const JSONArray& array, Js writer.writeEndArray(); } +// check isPrefix in PathInData::Parts. like : [{"a": {"c": {"b": 1}}}, {"a": {"c": 2.2}}], "a.c" is prefix of "a.c.b" +// return true if prefix is a prefix of parts +static bool is_prefix(const PathInData::Parts& prefix, const PathInData::Parts& parts) { + if (prefix.size() >= parts.size()) { + return false; + } + for (size_t i = 0; i < prefix.size(); ++i) { + if (prefix[i].key != parts[i].key) { + return false; + } + } + return true; +} + template void JSONDataParser::traverseArray(const JSONArray& array, ParseContext& ctx) { /// Traverse elements of array and collect an array of fields by each path. ParseArrayContext array_ctx; array_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten; + array_ctx.is_top_array = ctx.is_top_array; array_ctx.total_size = array.size(); for (auto it = array.begin(); it != array.end(); ++it) { traverseArrayElement(*it, array_ctx); @@ -167,80 +183,133 @@ void JSONDataParser::traverseArray(const JSONArray& array, ParseCont } } } + template void JSONDataParser::traverseArrayElement(const Element& element, ParseArrayContext& ctx) { ParseContext element_ctx; element_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten; + element_ctx.is_top_array = ctx.is_top_array; traverse(element, element_ctx); - auto& [_, paths, values, flatten_nested, has_nested] = element_ctx; + auto& [_, paths, values, flatten_nested, __, is_top_array] = element_ctx; + + if (element_ctx.has_nested_in_flatten && is_top_array) { + checkAmbiguousStructure(ctx, paths); + } + size_t size = paths.size(); size_t keys_to_update = ctx.arrays_by_path.size(); + for (size_t i = 0; i < size; ++i) { if (values[i].is_null()) { continue; } + UInt128 hash = PathInData::get_parts_hash(paths[i]); auto found = ctx.arrays_by_path.find(hash); + if (found != ctx.arrays_by_path.end()) { - auto& path_array = found->second.second; - assert(path_array.size() == ctx.current_size); - /// If current element of array is part of Nested, - /// collect its size or check it if the size of - /// the Nested has been already collected. - auto nested_key = getNameOfNested(paths[i], values[i]); - if (!nested_key.empty()) { - size_t array_size = get(values[i]).size(); - auto& current_nested_sizes = ctx.nested_sizes_by_key[nested_key]; - if (current_nested_sizes.size() == ctx.current_size) { - current_nested_sizes.push_back(array_size); - } else if (array_size != current_nested_sizes.back()) { - throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, - "Array sizes mismatched ({} and {})", array_size, - current_nested_sizes.back()); - } - } - path_array.push_back(std::move(values[i])); - --keys_to_update; + handleExistingPath(found->second, paths[i], values[i], ctx, keys_to_update); } else { - /// We found a new key. Add and empty array with current size. - Array path_array; - path_array.reserve(ctx.total_size); - path_array.resize(ctx.current_size); - auto nested_key = getNameOfNested(paths[i], values[i]); - if (!nested_key.empty()) { - size_t array_size = get(values[i]).size(); - auto& current_nested_sizes = ctx.nested_sizes_by_key[nested_key]; - if (current_nested_sizes.empty()) { - current_nested_sizes.resize(ctx.current_size); - } else { - /// If newly added element is part of the Nested then - /// resize its elements to keep correct sizes of Nested arrays. - for (size_t j = 0; j < ctx.current_size; ++j) { - path_array[j] = Array(current_nested_sizes[j]); - } - } - if (current_nested_sizes.size() == ctx.current_size) { - current_nested_sizes.push_back(array_size); - } else if (array_size != current_nested_sizes.back()) { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "Array sizes mismatched ({} and {})", array_size, - current_nested_sizes.back()); - } - } - path_array.push_back(std::move(values[i])); - auto& elem = ctx.arrays_by_path[hash]; - elem.first = std::move(paths[i]); - elem.second = std::move(path_array); + handleNewPath(hash, paths[i], values[i], ctx); } } - /// If some of the keys are missed in current element, - /// add default values for them. - if (keys_to_update) { + + if (keys_to_update && !(is_top_array && ctx.has_nested_in_flatten)) { fillMissedValuesInArrays(ctx); } } +// check if the structure of top_array is ambiguous like: +// [{"a": {"b": {"c": 1}}}, {"a": {"b": 1}}] a.b is ambiguous +// which can not recombine a right doc in ColumnVariant::get_json_by_column_tree +template +void JSONDataParser::checkAmbiguousStructure( + const ParseArrayContext& ctx, const std::vector& paths) { + for (auto&& current_path : paths) { + for (auto it = ctx.arrays_by_path.begin(); it != ctx.arrays_by_path.end(); ++it) { + auto&& [p, _] = it->second; + if (is_prefix(p, current_path) || is_prefix(current_path, p)) { + throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT, + "Ambiguous structure of top_array nested subcolumns: {}, {}", + PathInData(p).to_jsonpath(), + PathInData(current_path).to_jsonpath()); + } + } + } +} + +template +void JSONDataParser::handleExistingPath(std::pair& path_data, + const PathInData::Parts& path, Field& value, + ParseArrayContext& ctx, + size_t& keys_to_update) { + auto& path_array = path_data.second; + // For top_array structure we no need to check cur array size equals ctx.current_size + // because we do not need to maintain the association information between Nested in array + if (!(ctx.is_top_array && ctx.has_nested_in_flatten)) { + assert(path_array.size() == ctx.current_size); + } + // If current element of array is part of Nested, + // collect its size or check it if the size of + // the Nested has been already collected. + auto nested_key = getNameOfNested(path, value); + if (!nested_key.empty()) { + size_t array_size = get(value).size(); + auto& current_nested_sizes = ctx.nested_sizes_by_key[nested_key]; + if (current_nested_sizes.size() == ctx.current_size) { + current_nested_sizes.push_back(array_size); + } else if (array_size != current_nested_sizes.back()) { + throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, + "Array sizes mismatched ({} and {})", array_size, + current_nested_sizes.back()); + } + } + + path_array.push_back(std::move(value)); + --keys_to_update; +} + +template +void JSONDataParser::handleNewPath(UInt128 hash, const PathInData::Parts& path, + Field& value, ParseArrayContext& ctx) { + Array path_array; + path_array.reserve(ctx.total_size); + + // For top_array structure we no need to resize array + // because we no need to fill default values for maintaining the association information between Nested in array + if (!(ctx.is_top_array && ctx.has_nested_in_flatten)) { + path_array.resize(ctx.current_size); + } + + auto nested_key = getNameOfNested(path, value); + if (!nested_key.empty()) { + size_t array_size = get(value).size(); + auto& current_nested_sizes = ctx.nested_sizes_by_key[nested_key]; + if (current_nested_sizes.empty()) { + current_nested_sizes.resize(ctx.current_size); + } else { + // If newly added element is part of the Nested then + // resize its elements to keep correct sizes of Nested arrays. + for (size_t j = 0; j < ctx.current_size; ++j) { + path_array[j] = Array(current_nested_sizes[j]); + } + } + if (current_nested_sizes.size() == ctx.current_size) { + current_nested_sizes.push_back(array_size); + } else if (array_size != current_nested_sizes.back()) { + throw doris::Exception(doris::ErrorCode::INTERNAL_ERROR, + "Array sizes mismatched ({} and {})", array_size, + current_nested_sizes.back()); + } + } + + path_array.push_back(std::move(value)); + auto& elem = ctx.arrays_by_path[hash]; + elem.first = std::move(path); + elem.second = std::move(path_array); +} + template void JSONDataParser::fillMissedValuesInArrays(ParseArrayContext& ctx) { for (auto it = ctx.arrays_by_path.begin(); it != ctx.arrays_by_path.end(); ++it) { @@ -306,4 +375,4 @@ StringRef JSONDataParser::getNameOfNested(const PathInData::Parts& p } template class JSONDataParser; -} // namespace doris::vectorized \ No newline at end of file +} // namespace doris::vectorized diff --git a/be/src/vec/json/json_parser.h b/be/src/vec/json/json_parser.h index 401f225d000e2d..4ad2738216c455 100644 --- a/be/src/vec/json/json_parser.h +++ b/be/src/vec/json/json_parser.h @@ -125,7 +125,6 @@ enum class ExtractType { struct ParseConfig { bool enable_flatten_nested = false; }; - /// Result of parsing of a document. /// Contains all paths extracted from document /// and values which are related to them. @@ -133,7 +132,6 @@ struct ParseResult { std::vector paths; std::vector values; }; - template class JSONDataParser { public: @@ -149,6 +147,7 @@ class JSONDataParser { std::vector values; bool enable_flatten_nested = false; bool has_nested_in_flatten = false; + bool is_top_array = false; }; using PathPartsWithArray = std::pair; using PathToArray = phmap::flat_hash_map; @@ -159,11 +158,19 @@ class JSONDataParser { PathToArray arrays_by_path; KeyToSizes nested_sizes_by_key; bool has_nested_in_flatten = false; + bool is_top_array = false; }; void traverse(const Element& element, ParseContext& ctx); void traverseObject(const JSONObject& object, ParseContext& ctx); void traverseArray(const JSONArray& array, ParseContext& ctx); void traverseArrayElement(const Element& element, ParseArrayContext& ctx); + void checkAmbiguousStructure(const ParseArrayContext& ctx, + const std::vector& paths); + void handleExistingPath(std::pair& path_data, + const PathInData::Parts& path, Field& value, ParseArrayContext& ctx, + size_t& keys_to_update); + void handleNewPath(UInt128 hash, const PathInData::Parts& path, Field& value, + ParseArrayContext& ctx); static void fillMissedValuesInArrays(ParseArrayContext& ctx); static bool tryInsertDefaultFromNested(ParseArrayContext& ctx, const PathInData::Parts& path, Array& array); diff --git a/be/test/vec/jsonb/json_parser_test.cpp b/be/test/vec/jsonb/json_parser_test.cpp index 924bd13197db7b..99567dc81d6696 100644 --- a/be/test/vec/jsonb/json_parser_test.cpp +++ b/be/test/vec/jsonb/json_parser_test.cpp @@ -21,6 +21,8 @@ #include +#include "vec/common/string_ref.h" +#include "vec/common/uint128.h" #include "vec/core/field.h" #include "vec/core/types.h" @@ -170,3 +172,175 @@ TEST(JsonParserTest, ParseCornerCases) { result = parser.parse(R"({"a":"\n\t"})", 12, config); ASSERT_TRUE(result.has_value()); } + +// Test cases for the selected code functionality +TEST(JsonParserTest, TestIsPrefixFunction) { + JSONDataParser parser; + ParseConfig config; + + // Test is_prefix functionality through nested path parsing + // This tests the is_prefix function used in checkAmbiguousStructure + + // Test case 1: Simple nested paths that should not be ambiguous + std::string json1 = R"({"a": [{"b": 1}, {"b": 2}]})"; + auto result1 = parser.parse(json1.c_str(), json1.size(), config); + ASSERT_TRUE(result1.has_value()); + + // Test case 2: More complex nested paths + std::string json2 = R"({"a": [{"b": {"c": 1}}, {"b": {"c": 2}}]})"; + auto result2 = parser.parse(json2.c_str(), json2.size(), config); + ASSERT_TRUE(result2.has_value()); + + // Test case 3: Deep nested structure + std::string json3 = R"({"level1": {"level2": [{"level3": {"level4": 1}}]}})"; + auto result3 = parser.parse(json3.c_str(), json3.size(), config); + ASSERT_TRUE(result3.has_value()); +} + +TEST(JsonParserTest, TestAmbiguousStructureDetection) { + JSONDataParser parser; + ParseConfig config; + config.enable_flatten_nested = true; + + // Test case 1: Arrays with different sizes in nested structure + // This should trigger the array size mismatch exception + std::string json1 = R"([{"b": [1, 2]}, {"b": [1, 2, 3]}])"; + EXPECT_ANY_THROW(parser.parse(json1.c_str(), json1.size(), config)); + + // Test case 2: Arrays with same sizes should not throw + std::string json2 = R"([{"b": [1, 2]}, {"b": [3, 4]}])"; + EXPECT_ANY_THROW(parser.parse(json2.c_str(), json2.size(), config)); + + // Test case 3: More complex nested array size mismatch + std::string json3 = R"({"nested": [{"arr": [[1, 2], [3]]}, {"arr": [[1, 2], [3, 4]]}]})"; + EXPECT_ANY_THROW(parser.parse(json3.c_str(), json3.size(), config)); + + // Test case 4: Ambiguous structure with prefix paths + // This should trigger the ambiguous structure exception + std::string json4 = R"([{"a": {"c": 1}}, {"a": 2}])"; + EXPECT_ANY_THROW(parser.parse(json4.c_str(), json4.size(), config)); +} + +TEST(JsonParserTest, TestNestedArrayHandling) { + JSONDataParser parser; + ParseConfig config; + config.enable_flatten_nested = true; + + // Test case 1: Simple nested array handling + std::string json1 = R"([{"b": 1}, {"c": 2}])"; + auto result1 = parser.parse(json1.c_str(), json1.size(), config); + ASSERT_TRUE(result1.has_value()); + EXPECT_GT(result1->values.size(), 0); + + // Test case 2: Multi-level nested array + std::string json2 = R"([{"a": {"b": 1}}, {"a": {"b": 2}}])"; + auto result2 = parser.parse(json2.c_str(), json2.size(), config); + ASSERT_TRUE(result2.has_value()); + EXPECT_GT(result2->values.size(), 0); +} + +TEST(JsonParserTest, TestNestedArrayWithDifferentConfigs) { + JSONDataParser parser; + + // Test with flatten_nested = false + ParseConfig config1; + config1.enable_flatten_nested = false; + + std::string json1 = R"([{"b": [1, 2]}, {"b": [3, 4]}])"; + auto result1 = parser.parse(json1.c_str(), json1.size(), config1); + ASSERT_TRUE(result1.has_value()); + EXPECT_EQ(result1->values.size(), 1); + EXPECT_EQ(result1->values[0].get_type(), doris::vectorized::Field::Types::JSONB); + + // Test with flatten_nested = true + ParseConfig config2; + config2.enable_flatten_nested = true; + + EXPECT_ANY_THROW(parser.parse(json1.c_str(), json1.size(), config2)); +} + +// Test case for directly calling handleNewPath to cover the if (!nested_key.empty()) branch +TEST(JsonParserTest, TestHandleNewPathDirectCall) { + JSONDataParser parser; + + // Create a ParseArrayContext + JSONDataParser::ParseArrayContext ctx; + ctx.current_size = 1; + ctx.total_size = 2; + ctx.has_nested_in_flatten = true; + ctx.is_top_array = true; + + // Create a path with nested parts + doris::vectorized::PathInData::Parts path; + // Create a nested part (is_nested = true) + path.emplace_back("nested_key", true, 0); // is_nested = true + path.emplace_back("inner_key", false, 0); // is_nested = false + + // Create a Field with array type (required for getNameOfNested to return non-empty) + doris::vectorized::Array array_data; + array_data.push_back(doris::vectorized::Int32(1)); + array_data.push_back(doris::vectorized::Int32(2)); + doris::vectorized::Field value = std::move(array_data); + + // Create hash for the path + doris::vectorized::UInt128 hash = doris::vectorized::PathInData::get_parts_hash(path); + + // Call handleNewPath directly + // This should trigger the if (!nested_key.empty()) branch + parser.handleNewPath(hash, path, value, ctx); + + // Verify that the nested_sizes_by_key was populated + EXPECT_EQ(ctx.nested_sizes_by_key.size(), 1); + + // Verify that the arrays_by_path was populated + EXPECT_EQ(ctx.arrays_by_path.size(), 1); + EXPECT_TRUE(ctx.arrays_by_path.find(hash) != ctx.arrays_by_path.end()); +} + +// Test case for testing the else branch in handleNewPath (when nested_sizes is not empty) +TEST(JsonParserTest, TestHandleNewPathElseBranch) { + JSONDataParser parser; + + // Create a ParseArrayContext + JSONDataParser::ParseArrayContext ctx; + ctx.current_size = 2; // Start with size 2 + ctx.total_size = 3; + ctx.has_nested_in_flatten = true; + ctx.is_top_array = true; + + // Create a path with nested parts + doris::vectorized::PathInData::Parts path; + path.emplace_back("nested_key", true, 0); + path.emplace_back("inner_key", false, 0); + + // Create a Field with array type + doris::vectorized::Array array_data; + array_data.push_back(doris::vectorized::Int32(1)); + array_data.push_back(doris::vectorized::Int32(2)); + doris::vectorized::Field value = std::move(array_data); + + // Create hash for the path + doris::vectorized::UInt128 hash = doris::vectorized::PathInData::get_parts_hash(path); + + // First call to populate nested_sizes_by_key + parser.handleNewPath(hash, path, value, ctx); + + // Verify nested_sizes_by_key was populated + EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key")).size(), 3); + EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key"))[0], 0); + + // Create another array with same size + doris::vectorized::Array array_data2; + array_data2.push_back(doris::vectorized::Int32(3)); + array_data2.push_back(doris::vectorized::Int32(4)); + doris::vectorized::Field value2 = std::move(array_data2); + + // Second call should trigger the else branch (nested_sizes is not empty) + ctx.is_top_array = false; + ctx.has_nested_in_flatten = false; + parser.handleNewPath(hash, path, value2, ctx); + + // Verify nested_sizes_by_key was updated + EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key")).size(), 3); + EXPECT_EQ(ctx.nested_sizes_by_key.at(doris::StringRef("nested_key"))[1], 0); +} diff --git a/regression-test/data/variant_p0/nested/nested_in_top_array.out b/regression-test/data/variant_p0/nested/nested_in_top_array.out new file mode 100644 index 00000000000000..0708bce9a2d7e4 --- /dev/null +++ b/regression-test/data/variant_p0/nested/nested_in_top_array.out @@ -0,0 +1,545 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_0 -- +1 {"a":[1],"b":["1"],"c":[1.1]} + +-- !sql_1 -- +[1] + +-- !sql_2 -- +["1"] + +-- !sql_3 -- +[1.1] + +-- !sql_4 -- +{"a":[1],"b":["1"],"c":[1.1]} + +-- !sql_8 -- +[1] 1 + +-- !sql_9 -- +[1] 1 + +-- !sql_10 -- +[1] 1 + +-- !sql_11 -- +["1"] 1 + +-- !sql_12 -- +["1"] 1 + +-- !sql_13 -- +["1.1"] 1 + +-- !sql_14 -- +[1] 1 + +-- !sql_15 -- +[1] 1 + +-- !sql_16 -- +[1.1] 1 + +-- !sql_17 -- +\N \N \N + +-- !sql_18 -- +[1] ['1'] [1.1] + +-- !sql_19 -- +\N \N \N + +-- !sql_0 -- +1 {"a":[1],"b":["1"],"c":[1.1]} +2 {"a":["2.5"],"b":[123.1]} + +-- !sql_1 -- +["1"] +[""2.5""] + +-- !sql_2 -- +[""1""] +["123.1"] + +-- !sql_3 -- +[1.1] +\N + +-- !sql_4 -- +{"a":[1],"b":["1"],"c":[1.1]} +{"a":["2.5"],"b":[123.1]} + +-- !sql_8 -- +[1] 1 +[2] 1 + +-- !sql_9 -- +[1] 1 +[123] 1 + +-- !sql_10 -- +[1] 1 +\N \N + +-- !sql_11 -- +["1"] 1 +["2.5"] 1 + +-- !sql_12 -- +["1"] 1 +["123.1"] 1 + +-- !sql_13 -- +["1.1"] 1 +\N \N + +-- !sql_14 -- +[1] 1 +[2.5] 1 + +-- !sql_15 -- +[1] 1 +[123.1] 1 + +-- !sql_16 -- +[1.1] 1 +\N \N + +-- !sql_17 -- +\N \N \N +\N \N \N + +-- !sql_18 -- +[1] ['1'] [1.1] +['2.5'] [123.1] \N + +-- !sql_19 -- +\N \N \N +\N \N \N + +-- !sql_0 -- +1 {"a":[1],"b":["1"],"c":[1.1]} +2 {"a":["2.5"],"b":[123.1]} +3 {"a":{"b":[2],"c":[1]}} + +-- !sql_1 -- +[1] +["2.5"] +{"b":[2],"c":[1]} + +-- !sql_2 -- +[""1""] +["123.1"] +\N + +-- !sql_3 -- +[1.1] +\N +\N + +-- !sql_4 -- +{"a":[1],"b":["1"],"c":[1.1]} +{"a":["2.5"],"b":[123.1]} +{"a":{"b":[2],"c":[1]}} + +-- !sql_8 -- +[1] 1 +[2] 1 +\N \N + +-- !sql_9 -- +[1] 1 +[123] 1 +\N \N + +-- !sql_10 -- +[1] 1 +\N \N +\N \N + +-- !sql_11 -- +["1"] 1 +["2.5"] 1 +\N \N + +-- !sql_12 -- +["1"] 1 +["123.1"] 1 +\N \N + +-- !sql_13 -- +["1.1"] 1 +\N \N +\N \N + +-- !sql_14 -- +[1] 1 +[2.5] 1 +\N \N + +-- !sql_15 -- +[1] 1 +[123.1] 1 +\N \N + +-- !sql_16 -- +[1.1] 1 +\N \N +\N \N + +-- !sql_17 -- +\N \N \N +\N \N \N +\N \N \N + +-- !sql_18 -- +[1] ['1'] [1.1] +['2.5'] [123.1] \N +{"b":[2],"c":[1]} \N \N + +-- !sql_19 -- +\N \N \N +\N \N \N +\N \N \N + +-- !sql_0 -- +1 {"a":[1],"b":["1"],"c":[1.1]} +2 {"a":["2.5"],"b":[123.1]} +3 {"a":{"b":[2],"c":[1]}} +5 {"a":{"c":[1]},"b":["1"],"c":{"a":[2]}} + +-- !sql_1 -- +[1] +["2.5"] +{"b":[2],"c":[1]} +{"c":[1]} + +-- !sql_2 -- +[""1""] +["123.1"] +\N +[""1""] + +-- !sql_3 -- +[1.1] +\N +\N +{"a":[2]} + +-- !sql_4 -- +{"a":[1],"b":["1"],"c":[1.1]} +{"a":["2.5"],"b":[123.1]} +{"a":{"b":[2],"c":[1]}} +{"a":{"c":[1]},"b":["1"],"c":{"a":[2]}} + +-- !sql_8 -- +[1] 1 +[2] 1 +\N \N +\N \N + +-- !sql_9 -- +[1] 1 +[123] 1 +\N \N +[1] 1 + +-- !sql_10 -- +[1] 1 +\N \N +\N \N +\N \N + +-- !sql_11 -- +["1"] 1 +["2.5"] 1 +\N \N +\N \N + +-- !sql_12 -- +["1"] 1 +["123.1"] 1 +\N \N +["1"] 1 + +-- !sql_13 -- +["1.1"] 1 +\N \N +\N \N +\N \N + +-- !sql_14 -- +[1] 1 +[2.5] 1 +\N \N +\N \N + +-- !sql_15 -- +[1] 1 +[123.1] 1 +\N \N +[1] 1 + +-- !sql_16 -- +[1.1] 1 +\N \N +\N \N +\N \N + +-- !sql_17 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_18 -- +[1] ['1'] [1.1] +['2.5'] [123.1] \N +{"b":[2],"c":[1]} \N \N +{"c":[1]} ['1'] {"a":[2]} + +-- !sql_19 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_0 -- +1 {"a":[1],"b":["1"],"c":[1.1]} +2 {"a":["2.5"],"b":[123.1]} +3 {"a":{"b":[2],"c":[1]}} +5 {"a":{"c":[1]},"b":["1"],"c":{"a":[2]}} +6 {"a":[1],"b":[1,2,3]} + +-- !sql_1 -- +[1] +["2.5"] +{"b":[2],"c":[1]} +{"c":[1]} +[1] + +-- !sql_2 -- +[""1""] +["123.1"] +\N +[""1""] +["1", "2", "3"] + +-- !sql_3 -- +[1.1] +\N +\N +{"a":[2]} +\N + +-- !sql_4 -- +{"a":[1],"b":["1"],"c":[1.1]} +{"a":["2.5"],"b":[123.1]} +{"a":{"b":[2],"c":[1]}} +{"a":{"c":[1]},"b":["1"],"c":{"a":[2]}} +{"a":[1],"b":[1,2,3]} + +-- !sql_8 -- +[1] 1 +[2] 1 +\N \N +\N \N +[1] 1 + +-- !sql_9 -- +[1] 1 +[123] 1 +\N \N +[1] 1 +[1, 2, 3] 3 + +-- !sql_10 -- +[1] 1 +\N \N +\N \N +\N \N +\N \N + +-- !sql_11 -- +["1"] 1 +["2.5"] 1 +\N \N +\N \N +["1"] 1 + +-- !sql_12 -- +["1"] 1 +["123.1"] 1 +\N \N +["1"] 1 +["1", "2", "3"] 3 + +-- !sql_13 -- +["1.1"] 1 +\N \N +\N \N +\N \N +\N \N + +-- !sql_14 -- +[1] 1 +[2.5] 1 +\N \N +\N \N +[1] 1 + +-- !sql_15 -- +[1] 1 +[123.1] 1 +\N \N +[1] 1 +[1, 2, 3] 3 + +-- !sql_16 -- +[1.1] 1 +\N \N +\N \N +\N \N +\N \N + +-- !sql_17 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_18 -- +[1] ['1'] [1.1] +['2.5'] [123.1] \N +{"b":[2],"c":[1]} \N \N +{"c":[1]} ['1'] {"a":[2]} +[1] [1, 2, 3] \N + +-- !sql_19 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_0 -- +1 {"a":[1],"b":["1"],"c":[1.1]} +2 {"a":["2.5"],"b":[123.1]} +3 {"a":{"b":[2],"c":[1]}} +5 {"a":{"c":[1]},"b":["1"],"c":{"a":[2]}} +6 {"a":[1],"b":[1,2,3]} + +-- !sql_1 -- +[1] +["2.5"] +{"b":[2],"c":[1]} +{"c":[1]} +[1] + +-- !sql_2 -- +[""1""] +["123.1"] +\N +[""1""] +["1", "2", "3"] + +-- !sql_3 -- +[1.1] +null +null +{"a":[2]} +null + +-- !sql_4 -- +{"a":[1],"b":["1"],"c":[1.1]} +{"a":["2.5"],"b":[123.1]} +{"a":{"b":[2],"c":[1]}} +{"a":{"c":[1]},"b":["1"],"c":{"a":[2]}} +{"a":[1],"b":[1,2,3]} + +-- !sql_8 -- +[1] 1 +[2] 1 +\N \N +\N \N +[1] 1 + +-- !sql_9 -- +[1] 1 +[123] 1 +\N \N +[1] 1 +[1, 2, 3] 3 + +-- !sql_10 -- +[1] 1 +\N \N +\N \N +\N \N +\N \N + +-- !sql_11 -- +["1"] 1 +["2.5"] 1 +\N \N +\N \N +["1"] 1 + +-- !sql_12 -- +["1"] 1 +["123.1"] 1 +\N \N +["1"] 1 +["1", "2", "3"] 3 + +-- !sql_13 -- +["1.1"] 1 +\N \N +\N \N +\N \N +\N \N + +-- !sql_14 -- +[1] 1 +[2.5] 1 +\N \N +\N \N +[1] 1 + +-- !sql_15 -- +[1] 1 +[123.1] 1 +\N \N +[1] 1 +[1, 2, 3] 3 + +-- !sql_16 -- +[1.1] 1 +\N \N +\N \N +\N \N +\N \N + +-- !sql_17 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + +-- !sql_18 -- +[1] ["1"] [1.1] +["2.5"] [123.1] null +{"b":[2],"c":[1]} \N null +{"c":[1]} ["1"] {"a":[2]} +[1] [1, 2, 3] null + +-- !sql_19 -- +\N \N \N +\N \N \N +\N \N \N +\N \N \N +\N \N \N + diff --git a/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy b/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy new file mode 100644 index 00000000000000..9518868646c247 --- /dev/null +++ b/regression-test/suites/variant_p0/nested/nested_in_top_array.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// this test is used to test the nested in top array +suite("nested_in_top_array", "p0"){ + + try { + + // create a table with nested in top array + def table_name = "var_nested_in_top_array" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """set describe_extend_variant_column = true""" + + + // set disable_variant_flatten_nested = false to enable variant flatten nested + sql """ set enable_variant_flatten_nested = true """ + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 -- 1 bucket make really compaction in conflict case + properties("replication_num" = "1", "disable_auto_compaction" = "false", "variant_enable_flatten_nested" = "true"); + """ + sql """ insert into ${table_name} values (1, '[{"a": 1, "c": 1.1}, {"b": "1"}]'); """ + + def sql_select_batch = { tn -> + qt_sql_0 """select * from ${tn} order by k""" + + qt_sql_1 """select v['a'] from ${tn} order by k""" + qt_sql_2 """select v['b'] from ${tn} order by k""" + qt_sql_3 """select v['c'] from ${tn} order by k""" + + qt_sql_4 """select v from ${tn} order by k""" + } + + def sql_test_cast_to_array = { tn -> + // test cast to array + qt_sql_8 """select cast(v['a'] as array), size(cast(v['a'] as array)) from ${tn} order by k""" + qt_sql_9 """select cast(v['b'] as array), size(cast(v['b'] as array)) from ${tn} order by k""" + qt_sql_10 """select cast(v['c'] as array), size(cast(v['c'] as array)) from ${tn} order by k""" + + // test cast to array + qt_sql_11 """select cast(v['a'] as array), size(cast(v['a'] as array)) from ${tn} order by k""" + qt_sql_12 """select cast(v['b'] as array), size(cast(v['b'] as array)) from ${tn} order by k""" + qt_sql_13 """select cast(v['c'] as array), size(cast(v['c'] as array)) from ${tn} order by k""" + + // test cast to array + qt_sql_14 """select cast(v['a'] as array), size(cast(v['a'] as array)) from ${tn} order by k""" + qt_sql_15 """select cast(v['b'] as array), size(cast(v['b'] as array)) from ${tn} order by k""" + qt_sql_16 """select cast(v['c'] as array), size(cast(v['c'] as array)) from ${tn} order by k""" + + } + + def sql_test_cast_to_scalar = { tn -> + qt_sql_17 """select cast(v['a'] as int), cast(v['b'] as int), cast(v['c'] as int) from ${tn} order by k""" + qt_sql_18 """select cast(v['a'] as string), cast(v['b'] as string), cast(v['c'] as string) from ${tn} order by k""" + qt_sql_19 """select cast(v['a'] as double), cast(v['b'] as double), cast(v['c'] as double) from ${tn} order by k""" + } + + + sql_select_batch(table_name) + sql_test_cast_to_array(table_name) + sql_test_cast_to_scalar(table_name) + + // insert with type conflict for a, b + sql """ + insert into ${table_name} values (2, '[{"a": "2.5", "b": 123.1}]'); + """ + sql_select_batch(table_name) + sql_test_cast_to_array(table_name) + sql_test_cast_to_scalar(table_name) + + // insert with structure conflict for a + sql """ + insert into ${table_name} values (3, '[{"a": {"c": 1}}, {"a": {"b": 2}}]'); + """ + sql_select_batch(table_name) + sql_test_cast_to_array(table_name) + sql_test_cast_to_scalar(table_name) + + // insert ambiguous structure for a which should throw exception + test { + sql """ + insert into ${table_name} values (4, '[{"a": {"c": 1}}, {"a": 2}]'); + """ + exception "Ambiguous structure of top_array nested subcolumns:" + } + + + // insert multi object in array + sql """ + insert into ${table_name} values (5, '[{"a": {"c": 1}}, {"c": {"a": 2}}, {"b": "1"}]'); + """ + sql_select_batch(table_name) + sql_test_cast_to_array(table_name) + sql_test_cast_to_scalar(table_name) + + // insert multi b in array + sql """ + insert into ${table_name} values (6, '[{"a": 1, "b": 1}, {"b": 2}, {"b": 3}]'); + """ + sql_select_batch(table_name) + sql_test_cast_to_array(table_name) + sql_test_cast_to_scalar(table_name) + + // trigger and wait compaction + trigger_and_wait_compaction("${table_name}", "full") + sql_select_batch(table_name) + sql_test_cast_to_array(table_name) + sql_test_cast_to_scalar(table_name) + + } finally { + } + +} From 3ecb67c7a0912e02f0400ed8b69a7f4e7c1ebe43 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Mon, 18 Aug 2025 19:56:32 +0800 Subject: [PATCH 488/572] =?UTF-8?q?Revert=20"branch-3.0:=20[fix](config)?= =?UTF-8?q?=20dynamic=20set=20FE=20masterOnly=20config=20wi=E2=80=A6=20(#5?= =?UTF-8?q?4929)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …ll lost …" This reverts commit b26cba1247c32ed97a7ad73a0afad97094880fb5. ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../apache/doris/analysis/AdminSetConfigStmt.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java index b197c6c2bca113..458116924d7ee5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetConfigStmt.java @@ -49,13 +49,11 @@ public AdminSetConfigStmt(NodeType type, Map configs, boolean ap } this.applyToAll = applyToAll; - if (!this.applyToAll) { - // we have to analyze configs here to determine whether to forward it to master - for (String key : this.configs.keySet()) { - if (ConfigBase.checkIsMasterOnly(key)) { - redirectStatus = RedirectStatus.FORWARD_NO_SYNC; - break; - } + // we have to analyze configs here to determine whether to forward it to master + for (String key : this.configs.keySet()) { + if (ConfigBase.checkIsMasterOnly(key)) { + redirectStatus = RedirectStatus.FORWARD_NO_SYNC; + this.applyToAll = false; } } } From 4486c6b7bcbd2ca9cdf865ecb8a35c421c8bec34 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Tue, 19 Aug 2025 09:43:14 +0800 Subject: [PATCH 489/572] [fix](case) remove test_rowstore_query_ck (#54917) which is not supported in 3.0. Co-authored-by: Yongqiang YANG --- .../point_query_p0/test_rowstore_query_ck.out | 7 --- .../test_rowstore_query_ck.groovy | 44 ------------------- 2 files changed, 51 deletions(-) delete mode 100644 regression-test/data/point_query_p0/test_rowstore_query_ck.out delete mode 100644 regression-test/suites/point_query_p0/test_rowstore_query_ck.groovy diff --git a/regression-test/data/point_query_p0/test_rowstore_query_ck.out b/regression-test/data/point_query_p0/test_rowstore_query_ck.out deleted file mode 100644 index b43e0263960979..00000000000000 --- a/regression-test/data/point_query_p0/test_rowstore_query_ck.out +++ /dev/null @@ -1,7 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !sql -- -1 abc 1111919.123456789190000000 - --- !sql -- -2 def 1111919.123456789190000000 - diff --git a/regression-test/suites/point_query_p0/test_rowstore_query_ck.groovy b/regression-test/suites/point_query_p0/test_rowstore_query_ck.groovy deleted file mode 100644 index 896264f0065bcc..00000000000000 --- a/regression-test/suites/point_query_p0/test_rowstore_query_ck.groovy +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -suite("test_rowstore_ck", "p0") { - def tableName = "rs_query_2_ck" - sql """DROP TABLE IF EXISTS ${tableName}""" - sql "set enable_decimal256 = true" - sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( - `k1` int(11) NULL COMMENT "", - `v1` text NULL COMMENT "", - `v2` DECIMAL(50, 18) NULL COMMENT "" - ) ENGINE=OLAP - UNIQUE KEY(`k1`) - CLUSTER BY(`v2`) - DISTRIBUTED BY HASH(`k1`) BUCKETS 1 - PROPERTIES ( - "replication_allocation" = "tag.location.default: 1", - "store_row_column" = "true", - "enable_unique_key_merge_on_write" = "true", - "light_schema_change" = "true", - "storage_format" = "V2" - ) - """ - - sql """insert into ${tableName} values (1, 'abc', 1111919.12345678919)""" - qt_sql """select * from ${tableName}""" - sql """insert into ${tableName} values (2, 'def', 1111919.12345678919)""" - qt_sql """select * from ${tableName} where k1 = 2""" -} \ No newline at end of file From f9ec237476beb8c7579a9e36bcb684932def15ee Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Tue, 19 Aug 2025 14:11:36 +0800 Subject: [PATCH 490/572] [branch-3.0] [feat](checker) Add meta key consistency checking for checker #54002 (#54919) pick: https://github.com/apache/doris/pull/54002 --- cloud/src/common/config.h | 1 + cloud/src/recycler/meta_checker.cpp | 786 ++++++++++++++++++++-------- cloud/src/recycler/meta_checker.h | 109 ++++ 3 files changed, 686 insertions(+), 210 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index c52f4100f3c2be..6c3dd03c1da2a7 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -118,6 +118,7 @@ CONF_mInt64(recycle_task_threshold_seconds, "10800"); // 3h // **just for TEST** CONF_Bool(force_immediate_recycle, "false"); +CONF_mBool(enable_checker_for_meta_key_check, "false"); CONF_String(test_s3_ak, ""); CONF_String(test_s3_sk, ""); CONF_String(test_s3_endpoint, ""); diff --git a/cloud/src/recycler/meta_checker.cpp b/cloud/src/recycler/meta_checker.cpp index a299c2839df882..78d65fe0eea16a 100644 --- a/cloud/src/recycler/meta_checker.cpp +++ b/cloud/src/recycler/meta_checker.cpp @@ -19,14 +19,21 @@ #include #include +#include #include #include #include +#include +#include #include +#include +#include "common/config.h" #include "common/logging.h" #include "common/util.h" +#include "meta-service/meta_service_schema.h" +#include "meta-store/blob_message.h" #include "meta-store/keys.h" #include "meta-store/txn_kv.h" @@ -34,45 +41,15 @@ namespace doris::cloud { MetaChecker::MetaChecker(std::shared_ptr txn_kv) : txn_kv_(std::move(txn_kv)) {} -struct TabletInfo { - int64_t db_id; - int64_t table_id; - int64_t partition_id; - int64_t index_id; - int64_t tablet_id; - int64_t schema_version; - - std::string debug_string() const { - return "db id: " + std::to_string(db_id) + " table id: " + std::to_string(table_id) + - " partition id: " + std::to_string(partition_id) + - " index id: " + std::to_string(index_id) + - " tablet id: " + std::to_string(tablet_id) + - " schema version: " + std::to_string(schema_version); - } -}; - -struct PartitionInfo { - int64_t db_id; - int64_t table_id; - int64_t partition_id; - int64_t tablet_id; - int64_t visible_version; -}; - -bool MetaChecker::check_fe_meta_by_fdb(MYSQL* conn) { +bool MetaChecker::scan_and_handle_kv( + std::string& start_key, const std::string& end_key, + std::function handle_kv) { std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to init txn"; return false; } - - std::string start_key; - std::string end_key; - meta_tablet_idx_key({instance_id_, 0}, &start_key); - meta_tablet_idx_key({instance_id_, std::numeric_limits::max()}, &end_key); - std::vector tablet_indexes; - std::unique_ptr it; do { err = txn->get(start_key, end_key, &it); @@ -83,43 +60,186 @@ bool MetaChecker::check_fe_meta_by_fdb(MYSQL* conn) { while (it->has_next()) { auto [k, v] = it->next(); - TabletIndexPB tablet_idx; - if (!tablet_idx.ParseFromArray(v.data(), v.size())) [[unlikely]] { - LOG(WARNING) << "malformed tablet index value"; - return false; - } - tablet_indexes.push_back(std::move(tablet_idx)); - if (!it->has_next()) start_key = k; + handle_kv(k, v); + if (!it->has_next()) { + start_key = k; + } } start_key.push_back('\x00'); } while (it->more()); + return true; +} +bool MetaChecker::do_meta_tablet_key_check(MYSQL* conn) { + std::vector tablets_meta; bool check_res = true; + + // scan and collect tablet_meta + std::string start_key; + std::string end_key; + meta_tablet_key({instance_id_, 0, 0, 0, 0}, &start_key); + meta_tablet_key({instance_id_, INT64_MAX, 0, 0, 0}, &end_key); + scan_and_handle_kv(start_key, end_key, + [&tablets_meta](std::string_view key, std::string_view value) -> int { + doris::TabletMetaCloudPB tablet_meta; + if (!tablet_meta.ParseFromArray(value.data(), value.size())) { + LOG(WARNING) << "malformed tablet meta value"; + return -1; + } + tablets_meta.push_back(std::move(tablet_meta)); + return 0; + }); + + for (const auto& tablet_meta : tablets_meta) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to init txn"; + continue; + } + + // get tablet_index to search tablet belongs which db + std::string tablet_index_key; + std::string tablet_index_val; + meta_tablet_idx_key({instance_id_, tablet_meta.tablet_id()}, &tablet_index_key); + err = txn->get(tablet_index_key, &tablet_index_val); + if (err != TxnErrorCode::TXN_OK) { + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + LOG(WARNING) << "tablet_idx not found, tablet id: " << tablet_meta.tablet_id(); + continue; + } else { + LOG(WARNING) << "failed to get tablet_idx, err: " << err + << " tablet id: " << tablet_meta.tablet_id(); + continue; + } + } + + TabletIndexPB tablet_index_meta; + tablet_index_meta.ParseFromString(tablet_index_val); + + if (!db_meta_.contains(tablet_index_meta.db_id())) { + LOG(WARNING) << "tablet_idx.db_id not found in fe meta, db_id = " + << tablet_index_meta.db_id() + << "tablet index meta: " << tablet_index_meta.DebugString(); + continue; + } + std::string db_name = db_meta_.at(tablet_index_meta.db_id()); + if (db_name == "__internal_schema" || db_name == "information_schema" || + db_name == "mysql") { + continue; + } + + if (mysql_select_db(conn, db_name.c_str())) { + LOG(WARNING) << "mysql select db error, db_name: " << db_name + << " error: " << mysql_error(conn); + continue; + } + MYSQL_RES* result; + std::string sql_stmt = "show tablet " + std::to_string(tablet_meta.tablet_id()); + mysql_query(conn, sql_stmt.c_str()); + + result = mysql_store_result(conn); + if (result) { + MYSQL_ROW row = mysql_fetch_row(result); + auto [db_id, table_id, partition_id, index_id] = + std::make_tuple(atoll(row[4]), atoll(row[5]), atoll(row[6]), atoll(row[7])); + if (tablet_meta.table_id() != table_id) { + LOG(WARNING) << "check failed, fdb meta: " << tablet_meta.ShortDebugString() + << " fe table_id: " << atoll(row[5]); + check_res = false; + } + if (tablet_meta.partition_id() != partition_id) { + LOG(WARNING) << "check failed, fdb meta: " << tablet_meta.ShortDebugString() + << " fe partition_id: " << atoll(row[6]); + check_res = false; + } + if (tablet_meta.index_id() != index_id) { + LOG(WARNING) << "check failed, fdb meta: " << tablet_meta.ShortDebugString() + << " fe index_id: " << atoll(row[7]); + check_res = false; + } + mysql_free_result(result); + } else { + LOG(WARNING) << "check failed, fdb meta: " << tablet_meta.ShortDebugString() + << " fe tablet not found"; + check_res = false; + } + stat_info_.check_fe_tablet_num++; + } + + return check_res; +} + +bool MetaChecker::do_meta_tablet_key_index_check(MYSQL* conn) { + std::vector tablet_indexes; + bool check_res = true; + + // scan and collect tablet_idx + std::string start_key; + std::string end_key; + meta_tablet_idx_key({instance_id_, 0}, &start_key); + meta_tablet_idx_key({instance_id_, INT64_MAX}, &end_key); + scan_and_handle_kv(start_key, end_key, + [&tablet_indexes](std::string_view key, std::string_view value) -> int { + TabletIndexPB tablet_idx; + if (!tablet_idx.ParseFromArray(value.data(), value.size())) { + LOG(WARNING) << "malformed tablet index value"; + return -1; + } + tablet_indexes.push_back(std::move(tablet_idx)); + return 0; + }); + for (const TabletIndexPB& tablet_idx : tablet_indexes) { + if (!db_meta_.contains(tablet_idx.db_id())) { + LOG(WARNING) << "tablet_idx.db_id not found in fe meta, db_id = " << tablet_idx.db_id(); + continue; + } std::string sql_stmt = "show tablet " + std::to_string(tablet_idx.tablet_id()); MYSQL_RES* result; + std::string db_name = db_meta_.at(tablet_idx.db_id()); + if (db_name == "__internal_schema" || db_name == "information_schema" || + db_name == "mysql") { + continue; + } + if (mysql_select_db(conn, db_name.c_str())) { + LOG(WARNING) << "mysql select db error, db_name: " << db_name + << " error: " << mysql_error(conn); + continue; + } mysql_query(conn, sql_stmt.c_str()); result = mysql_store_result(conn); if (result) { MYSQL_ROW row = mysql_fetch_row(result); - if (tablet_idx.table_id() != atoll(row[5])) { + auto [db_id, table_id, partition_id, index_id] = + std::make_tuple(atoll(row[4]), atoll(row[5]), atoll(row[6]), atoll(row[7])); + if (tablet_idx.db_id() != db_id) { + LOG(WARNING) << "check failed, fdb meta: " << tablet_idx.ShortDebugString() + << " fe db_id: " << atoll(row[4]); + check_res = false; + } + if (tablet_idx.table_id() != table_id) { LOG(WARNING) << "check failed, fdb meta: " << tablet_idx.ShortDebugString() << " fe table_id: " << atoll(row[5]); check_res = false; } - if (tablet_idx.partition_id() != atoll(row[6])) { + if (tablet_idx.partition_id() != partition_id) { LOG(WARNING) << "check failed, fdb meta: " << tablet_idx.ShortDebugString() << " fe partition_id: " << atoll(row[6]); check_res = false; } - if (tablet_idx.index_id() != atoll(row[7])) { + if (tablet_idx.index_id() != index_id) { LOG(WARNING) << "check failed, fdb meta: " << tablet_idx.ShortDebugString() << " fe index_id: " << atoll(row[7]); check_res = false; } + mysql_free_result(result); + } else { + LOG(WARNING) << "check failed, fdb meta: " << tablet_idx.ShortDebugString() + << " fe tablet not found"; + check_res = false; } - mysql_free_result(result); stat_info_.check_fe_tablet_num++; } LOG(INFO) << "check_fe_tablet_num: " << stat_info_.check_fe_tablet_num; @@ -127,131 +247,167 @@ bool MetaChecker::check_fe_meta_by_fdb(MYSQL* conn) { return check_res; } -bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) { - // get db info from FE - std::string sql_stmt = "show databases"; - MYSQL_RES* result; - mysql_query(conn, sql_stmt.c_str()); - result = mysql_store_result(conn); - std::map*> db_to_tables; - if (result) { - int num_row = mysql_num_rows(result); - for (int i = 0; i < num_row; ++i) { - MYSQL_ROW row = mysql_fetch_row(result); - if (strcmp(row[0], "__internal_schema") == 0 || - strcmp(row[0], "information_schema") == 0) { +bool MetaChecker::do_meta_schema_key_check(MYSQL* conn) { + std::vector tablets_meta; + bool check_res = true; + + // scan and collect tablet_meta + std::string start_key; + std::string end_key; + meta_tablet_key({instance_id_, 0, 0, 0, 0}, &start_key); + meta_tablet_key({instance_id_, INT64_MAX, 0, 0, 0}, &end_key); + scan_and_handle_kv(start_key, end_key, + [&tablets_meta](std::string_view key, std::string_view value) -> int { + doris::TabletMetaCloudPB tablet_meta; + if (!tablet_meta.ParseFromArray(value.data(), value.size())) { + LOG(WARNING) << "malformed tablet meta value"; + return -1; + } + tablets_meta.push_back(std::move(tablet_meta)); + return 0; + }); + + for (const auto& tablet_meta : tablets_meta) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to init txn"; + continue; + } + + // get tablet_index to search tablet belongs which db + std::string tablet_index_key; + std::string tablet_index_val; + meta_tablet_idx_key({instance_id_, tablet_meta.tablet_id()}, &tablet_index_key); + err = txn->get(tablet_index_key, &tablet_index_val); + if (err != TxnErrorCode::TXN_OK) { + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + LOG(WARNING) << "tablet_idx not found, tablet id: " << tablet_meta.tablet_id(); + continue; + } else { + LOG(WARNING) << "failed to get tablet_idx, err: " << err + << " tablet id: " << tablet_meta.tablet_id(); continue; } - db_to_tables.insert({row[0], new std::vector()}); } - } - mysql_free_result(result); - // get tables info from FE - for (const auto& elem : db_to_tables) { - std::string sql_stmt = "show tables from " + elem.first; + TabletIndexPB tablet_index_meta; + tablet_index_meta.ParseFromString(tablet_index_val); + + if (!db_meta_.contains(tablet_index_meta.db_id())) { + LOG(WARNING) << "tablet_idx.db_id not found in fe meta, db_id = " + << tablet_index_meta.db_id() + << "tablet index meta: " << tablet_index_meta.DebugString(); + continue; + } + std::string db_name = db_meta_.at(tablet_index_meta.db_id()); + if (db_name == "__internal_schema" || db_name == "information_schema" || + db_name == "mysql") { + continue; + } + + if (mysql_select_db(conn, db_name.c_str())) { + LOG(WARNING) << "mysql select db error, db_name: " << db_name + << " error: " << mysql_error(conn); + continue; + } + std::string tablet_schema_key; + std::string tablet_schema_val; + meta_schema_key({instance_id_, tablet_index_meta.index_id(), tablet_meta.schema_version()}, + &tablet_schema_key); + ValueBuf val_buf; + err = cloud::blob_get(txn.get(), tablet_schema_key, &val_buf); + if (err != TxnErrorCode::TXN_OK) { + LOG(WARNING) << fmt::format( + "failed to get schema, err={}", + err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "not found" : "internal error"); + continue; + } + doris::TabletSchemaCloudPB tablet_schema; + if (!parse_schema_value(val_buf, &tablet_schema)) { + LOG(WARNING) << fmt::format("malformed schema value, key={}", tablet_schema_key); + continue; + } + + MYSQL_RES* result; + std::string sql_stmt = + fmt::format("SHOW PROC '/dbs/{}/{}/index_schema/{}'", tablet_index_meta.db_id(), + tablet_meta.table_id(), tablet_meta.index_id()); mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); if (result) { - int num_row = mysql_num_rows(result); - for (int i = 0; i < num_row; ++i) { - MYSQL_ROW row = mysql_fetch_row(result); - elem.second->push_back(row[0]); - } - } - mysql_free_result(result); - } - - // get tablet info from FE - std::vector tablets; - for (const auto& elem : db_to_tables) { - for (const std::string& table : *elem.second) { - std::string sql_stmt = "show tablets from " + elem.first + "." + table; - mysql_query(conn, sql_stmt.c_str()); - result = mysql_store_result(conn); - if (result) { - int num_row = mysql_num_rows(result); - for (int i = 0; i < num_row; ++i) { - MYSQL_ROW row = mysql_fetch_row(result); - TabletInfo tablet_info = {0}; - tablet_info.tablet_id = atoll(row[0]); - VLOG_DEBUG << "get tablet info log" - << ", db name" << elem.first << ", table name" << table - << ",tablet id" << tablet_info.tablet_id; - tablet_info.schema_version = atoll(row[4]); - tablets.push_back(std::move(tablet_info)); - } + MYSQL_ROW row = mysql_fetch_row(result); + int64_t schema_version = atoll(row[2]); + if (tablet_schema.schema_version() != schema_version) { + LOG(WARNING) << "check failed, fdb meta: " << tablet_schema.ShortDebugString() + << " fe schema_version: " << schema_version; + check_res = false; } mysql_free_result(result); + } else { + LOG(WARNING) << "check failed, fdb meta: " << tablet_meta.ShortDebugString() + << " fe tablet not found"; + check_res = false; } + stat_info_.check_fe_tablet_num++; } - // get tablet info from FE - // get Partition info from FE - std::map partitions; - for (auto& tablet_info : tablets) { - std::string sql_stmt = "show tablet " + std::to_string(tablet_info.tablet_id); - mysql_query(conn, sql_stmt.c_str()); - result = mysql_store_result(conn); - if (result) { - int num_row = mysql_num_rows(result); - for (int i = 0; i < num_row; ++i) { - MYSQL_ROW row = mysql_fetch_row(result); - tablet_info.db_id = atoll(row[4]); - tablet_info.table_id = atoll(row[5]); - tablet_info.partition_id = atoll(row[6]); - tablet_info.index_id = atoll(row[7]); + return check_res; +} - PartitionInfo partition_info = {0}; - partition_info.db_id = atoll(row[4]); - partition_info.table_id = atoll(row[5]); - partition_info.partition_id = atoll(row[6]); - partition_info.tablet_id = tablet_info.tablet_id; - VLOG_DEBUG << "get partition info log" - << ", db id" << partition_info.db_id << ", table id" - << partition_info.table_id << ", partition id" - << partition_info.partition_id << ", tablet id" - << partition_info.tablet_id; +template <> +bool MetaChecker::handle_check_fe_meta_by_fdb(MYSQL* conn) { + bool check_res = true; + // check MetaTabletIdxKey + if (!do_meta_tablet_key_index_check(conn)) { + check_res = false; + LOG(WARNING) << "do_meta_tablet_key_index_check failed"; + } else { + LOG(INFO) << "do_meta_tablet_key_index_check success"; + } - partitions.insert({partition_info.partition_id, std::move(partition_info)}); - } - } - mysql_free_result(result); + // check MetaTabletKey + if (!do_meta_tablet_key_check(conn)) { + check_res = false; + LOG(WARNING) << "do_meta_tablet_key_check failed"; + } else { + LOG(INFO) << "do_meta_tablet_key_check success"; } - // get partition version from FE - for (const auto& elem : db_to_tables) { - for (const std::string& table : *elem.second) { - std::string sql_stmt = "show partitions from " + elem.first + "." + table; - mysql_query(conn, sql_stmt.c_str()); - result = mysql_store_result(conn); - if (result) { - int num_row = mysql_num_rows(result); - for (int i = 0; i < num_row; ++i) { - MYSQL_ROW row = mysql_fetch_row(result); - int64_t partition_id = atoll(row[0]); - int64_t visible_version = atoll(row[2]); - partitions[partition_id].visible_version = visible_version; - VLOG_DEBUG << "get partition version log" - << ", db name" << elem.first << ", table name" << table - << ", raw partition id" << row[0] << ", first partition id" - << partition_id << ", db id" << partitions[partition_id].db_id - << ", table id" << partitions[partition_id].table_id - << ", second partition id" << partitions[partition_id].partition_id - << ", tablet id" << partitions[partition_id].tablet_id; - } - } - mysql_free_result(result); - } + // check MetaSchemaKey + if (!do_meta_schema_key_check(conn)) { + check_res = false; + LOG(WARNING) << "do_meta_schema_key_check failed"; + } else { + LOG(INFO) << "do_meta_schema_key_check success"; } + return check_res; +} +bool MetaChecker::check_fe_meta_by_fdb(MYSQL* conn) { + bool success = true; + if (config::enable_checker_for_meta_key_check) { + success = handle_check_fe_meta_by_fdb(conn); + } + + // TODO(wyxxxcat) add check for version key + // if (config::enable_checker_for_version_key_check) { + // success = handle_check_fe_meta_by_fdb(conn); + // } + return success; +} + +bool MetaChecker::do_meta_tablet_index_key_inverted_check(MYSQL* conn, + const std::vector& tablets) { + bool check_res = true; // check tablet idx for (const auto& tablet_info : tablets) { std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to init txn"; - return false; + continue; } std::string key, val; @@ -260,62 +416,71 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) { if (err != TxnErrorCode::TXN_OK) { if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { LOG(WARNING) << "tablet not found, tablet id: " << tablet_info.tablet_id; - return false; + check_res = false; + continue; } else { LOG(WARNING) << "failed to get tablet_idx, err: " << err << " tablet id: " << tablet_info.tablet_id; - return false; + check_res = false; + continue; } } TabletIndexPB tablet_idx; if (!tablet_idx.ParseFromString(val)) [[unlikely]] { LOG(WARNING) << "malformed tablet index value"; - return false; + continue; } - /* if (tablet_info.db_id != tablet_idx.db_id()) [[unlikely]] { LOG(WARNING) << "tablet idx check failed, fe db id: " << tablet_info.db_id << " tablet idx db id: " << tablet_idx.db_id(); - return false; + check_res = false; + continue; } - */ if (tablet_info.table_id != tablet_idx.table_id()) [[unlikely]] { LOG(WARNING) << "tablet idx check failed, fe table id: " << tablet_info.table_id << " tablet idx table id: " << tablet_idx.table_id(); - return false; + check_res = false; + continue; } if (tablet_info.partition_id != tablet_idx.partition_id()) [[unlikely]] { LOG(WARNING) << "tablet idx check failed, fe part id: " << tablet_info.partition_id << " tablet idx part id: " << tablet_idx.partition_id(); - return false; + check_res = false; + continue; } if (tablet_info.index_id != tablet_idx.index_id()) [[unlikely]] { LOG(WARNING) << "tablet idx check failed, fe index id: " << tablet_info.index_id << " tablet idx index id: " << tablet_idx.index_id(); - return false; + check_res = false; + continue; } if (tablet_info.tablet_id != tablet_idx.tablet_id()) [[unlikely]] { LOG(WARNING) << "tablet idx check failed, fe tablet id: " << tablet_info.tablet_id << " tablet idx tablet id: " << tablet_idx.tablet_id(); - return false; + check_res = false; + continue; } - stat_info_.check_fdb_tablet_idx_num++; } + return check_res; +} +bool MetaChecker::do_meta_tablet_key_inverted_check(MYSQL* conn, std::vector& tablets, + std::map& partitions) { + bool check_res = true; // check tablet meta for (const auto& tablet_info : tablets) { std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); if (err != TxnErrorCode::TXN_OK) { LOG(WARNING) << "failed to init txn"; - return false; + continue; } MetaTabletKeyInfo key_info1 {instance_id_, tablet_info.table_id, tablet_info.index_id, @@ -325,16 +490,76 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) { err = txn->get(key, &val); if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { LOG(WARNING) << "tablet meta not found: " << tablet_info.tablet_id; - return false; + check_res = false; + continue; } else if (err != TxnErrorCode::TXN_OK) [[unlikely]] { LOG(WARNING) << "failed to get tablet, err: " << err; - return false; + check_res = false; + continue; } stat_info_.check_fdb_tablet_meta_num++; } - // check tablet schema - /* + // TODO(wyxxxcat): + // separate from this function to check partition version function + // for (const auto& elem : partitions) { + // std::unique_ptr txn; + // TxnErrorCode err = txn_kv_->create_txn(&txn); + // if (err != TxnErrorCode::TXN_OK) { + // LOG(WARNING) << "failed to init txn"; + // continue; + // } + // if (elem.second.visible_version == 0 || elem.second.visible_version == 1) { + // continue; + // } + + // int64_t db_id = elem.second.db_id; + // int64_t table_id = elem.second.table_id; + // int64_t partition_id = elem.second.partition_id; + // int64_t tablet_id = elem.second.tablet_id; + // std::string ver_key = partition_version_key({instance_id_, db_id, table_id, partition_id}); + // std::string ver_val; + // err = txn->get(ver_key, &ver_val); + // if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + // LOG_WARNING("version key not found.") + // .tag("db id", db_id) + // .tag("table id", table_id) + // .tag("partition id", partition_id) + // .tag("tablet id", tablet_id); + // check_res = false; + // continue; + // } else if (err != TxnErrorCode::TXN_OK) { + // LOG_WARNING("failed to get version.") + // .tag("db id", db_id) + // .tag("table id", table_id) + // .tag("partition id", partition_id) + // .tag("tablet id", tablet_id); + // check_res = false; + // continue; + // } + + // VersionPB version_pb; + // if (!version_pb.ParseFromString(ver_val)) { + // LOG(WARNING) << "malformed version value"; + // check_res = false; + // continue; + // } + + // if (version_pb.version() != elem.second.visible_version) { + // LOG(WARNING) << "partition version check failed, FE partition version" + // << elem.second.visible_version << " ms version: " << version_pb.version(); + // check_res = false; + // continue; + // } + // stat_info_.check_fdb_partition_version_num++; + // } + return check_res; +} + +bool MetaChecker::do_meta_schema_key_inverted_check(MYSQL* conn, std::vector& tablets, + std::map& partitions) { + bool check_res = true; + for (const auto& tablet_info : tablets) { std::unique_ptr txn; TxnErrorCode err = txn_kv_->create_txn(&txn); @@ -350,68 +575,87 @@ bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) { err = cloud::blob_get(txn.get(), schema_key, &val_buf); if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { LOG(WARNING) << "tablet schema not found: " << tablet_info.debug_string(); - return false; + check_res = false; + continue; } else if (err != TxnErrorCode::TXN_OK) [[unlikely]] { - LOG(WARNING) <<"failed to get tablet schema, err: " << err; - return false; + LOG(WARNING) << "failed to get tablet schema, err: " << err; + check_res = false; + continue; } + stat_info_.check_fdb_tablet_schema_num++; } - */ + return check_res; +} - // check partition - for (const auto& elem : partitions) { - std::unique_ptr txn; - TxnErrorCode err = txn_kv_->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - LOG(WARNING) << "failed to init txn"; - return false; - } - if (elem.second.visible_version == 0 || elem.second.visible_version == 1) { - continue; - } +template <> +bool MetaChecker::handle_check_fdb_by_fe_meta(MYSQL* conn) { + std::vector tablets; + std::map partitions; - int64_t db_id = elem.second.db_id; - int64_t table_id = elem.second.table_id; - int64_t partition_id = elem.second.partition_id; - int64_t tablet_id = elem.second.tablet_id; - std::string ver_key = partition_version_key({instance_id_, db_id, table_id, partition_id}); - std::string ver_val; - err = txn->get(ver_key, &ver_val); - if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { - LOG_WARNING("version key not found.") - .tag("db id", db_id) - .tag("table id", table_id) - .tag("partition id", partition_id) - .tag("tablet id", tablet_id); - return false; - } else if (err != TxnErrorCode::TXN_OK) { - LOG_WARNING("failed to get version.") - .tag("db id", db_id) - .tag("table id", table_id) - .tag("partition id", partition_id) - .tag("tablet id", tablet_id); - return false; - } + init_tablet_info_from_fe_meta(conn, tablets, partitions); - VersionPB version_pb; - if (!version_pb.ParseFromString(ver_val)) { - LOG(WARNING) << "malformed version value"; - return false; - } + bool check_res = true; + // check MetaTabletIdxKey + if (!do_meta_tablet_index_key_inverted_check(conn, tablets)) { + check_res = false; + LOG(WARNING) << "do_meta_tablet_index_key_inverted_check failed"; + } else { + LOG(INFO) << "do_meta_tablet_index_key_inverted_check success"; + } - if (version_pb.version() != elem.second.visible_version) { - LOG(WARNING) << "partition version check failed, FE partition version" - << elem.second.visible_version << " ms version: " << version_pb.version(); - return false; - } - stat_info_.check_fdb_partition_version_num++; + // check MetaTabletKey + if (!do_meta_tablet_key_inverted_check(conn, tablets, partitions)) { + check_res = false; + LOG(WARNING) << "do_meta_tablet_key_inverted_check failed"; + } else { + LOG(INFO) << "do_meta_tablet_key_inverted_check success"; + } + + // check MetaSchemaKey + if (!do_meta_schema_key_inverted_check(conn, tablets, partitions)) { + check_res = false; + LOG(WARNING) << "do_meta_schema_key_inverted_check failed"; + } else { + LOG(INFO) << "do_meta_schema_key_inverted_check success"; } + return check_res; +} + +bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) { + bool success = true; + if (config::enable_checker_for_meta_key_check) { + success = handle_check_fdb_by_fe_meta(conn); + } + + // TODO(wyxxxcat) add check for version key + // if (config::enable_checker_for_version_key_check) { + // success = handle_check_fdb_by_fe_meta(conn); + // } + LOG(INFO) << "check_fdb_table_idx_num: " << stat_info_.check_fdb_tablet_idx_num << " check_fdb_table_meta_num: " << stat_info_.check_fdb_tablet_meta_num + << " check_fdb_tablet_schema_num: " << stat_info_.check_fdb_tablet_schema_num << " check_fdb_partition_version_num: " << stat_info_.check_fdb_partition_version_num; + return success; +} - return true; +void MetaChecker::init_db_meta(MYSQL* conn) { + // init db_meta_ -> map + db_meta_.clear(); + std::string sql_stmt = "SHOW PROC '/dbs/'"; + MYSQL_RES* result; + mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); + if (result) { + int num_row = mysql_num_rows(result); + for (int i = 0; i < num_row; ++i) { + MYSQL_ROW row = mysql_fetch_row(result); + auto [db_id, db_name] = std::make_tuple(atoll(row[0]), row[1]); + db_meta_.insert({db_id, db_name}); + } + mysql_free_result(result); + } } void MetaChecker::do_check(const std::string& host, const std::string& port, @@ -442,6 +686,7 @@ void MetaChecker::do_check(const std::string& host, const std::string& port, LOG(INFO) << "check_fe_meta_by_fdb begin"; bool ret = false; do { + init_db_meta(&conn); ret = check_fe_meta_by_fdb(&conn); if (!ret) { std::this_thread::sleep_for(seconds(10)); @@ -456,6 +701,8 @@ void MetaChecker::do_check(const std::string& host, const std::string& port, now = duration_cast(system_clock::now().time_since_epoch()).count(); LOG(INFO) << "check_fe_meta_by_fdb finish, cost(second): " << now - start; + LOG(INFO) << "check_fdb_by_fe_meta begin"; + init_db_meta(&conn); ret = check_fdb_by_fe_meta(&conn); if (!ret) { LOG(WARNING) << "check_fdb_by_fe_meta failed, there may be data loss"; @@ -470,4 +717,123 @@ void MetaChecker::do_check(const std::string& host, const std::string& port, LOG(INFO) << "meta check finish"; } +void MetaChecker::init_tablet_info_from_fe_meta(MYSQL* conn, std::vector& tablets, + std::map& partitions) { + // init tablet info, partition info + std::map> db_to_tables; + std::string sql_stmt = "show databases"; + MYSQL_RES* result; + + mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); + if (result) { + int num_row = mysql_num_rows(result); + for (int i = 0; i < num_row; ++i) { + MYSQL_ROW row = mysql_fetch_row(result); + if (strcmp(row[0], "__internal_schema") == 0 || + strcmp(row[0], "information_schema") == 0 || strcmp(row[0], "mysql")) { + continue; + } + db_to_tables.insert({row[0], std::vector()}); + } + mysql_free_result(result); + } + + // get tables info from FE + for (auto& elem : db_to_tables) { + std::string sql_stmt = "show tables from " + elem.first; + mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); + if (result) { + int num_row = mysql_num_rows(result); + for (int i = 0; i < num_row; ++i) { + MYSQL_ROW row = mysql_fetch_row(result); + elem.second.emplace_back(row[0]); + } + mysql_free_result(result); + } + } + + // get tablet info from FE + for (const auto& elem : db_to_tables) { + for (const std::string& table : elem.second) { + std::string sql_stmt = "show tablets from " + elem.first + "." + table; + mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); + if (result) { + int num_row = mysql_num_rows(result); + for (int i = 0; i < num_row; ++i) { + MYSQL_ROW row = mysql_fetch_row(result); + TabletInfo tablet_info; + tablet_info.tablet_id = atoll(row[0]); + VLOG_DEBUG << "get tablet info log" + << ", db name" << elem.first << ", table name" << table + << ",tablet id" << tablet_info.tablet_id; + tablet_info.schema_version = atoll(row[4]); + tablets.push_back(tablet_info); + } + mysql_free_result(result); + } + } + } + + // get tablet info from FE + // get Partition info from FE + for (auto& tablet_info : tablets) { + std::string sql_stmt = "show tablet " + std::to_string(tablet_info.tablet_id); + mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); + if (result) { + int num_row = mysql_num_rows(result); + for (int i = 0; i < num_row; ++i) { + MYSQL_ROW row = mysql_fetch_row(result); + tablet_info.db_id = atoll(row[4]); + tablet_info.table_id = atoll(row[5]); + tablet_info.partition_id = atoll(row[6]); + tablet_info.index_id = atoll(row[7]); + + PartitionInfo partition_info; + partition_info.db_id = atoll(row[4]); + partition_info.table_id = atoll(row[5]); + partition_info.partition_id = atoll(row[6]); + partition_info.tablet_id = tablet_info.tablet_id; + VLOG_DEBUG << "get partition info log" + << ", db id" << partition_info.db_id << ", table id" + << partition_info.table_id << ", partition id" + << partition_info.partition_id << ", tablet id" + << partition_info.tablet_id; + + partitions.insert({partition_info.partition_id, partition_info}); + } + mysql_free_result(result); + } + } + + // get partition version from FE + for (const auto& elem : db_to_tables) { + for (const std::string& table : elem.second) { + std::string sql_stmt = "show partitions from " + elem.first + "." + table; + mysql_query(conn, sql_stmt.c_str()); + result = mysql_store_result(conn); + if (result) { + int num_row = mysql_num_rows(result); + for (int i = 0; i < num_row; ++i) { + MYSQL_ROW row = mysql_fetch_row(result); + int64_t partition_id = atoll(row[0]); + int64_t visible_version = atoll(row[2]); + partitions[partition_id].visible_version = visible_version; + VLOG_DEBUG << "get partition version log" + << ", db name" << elem.first << ", table name" << table + << ", raw partition id" << row[0] << ", first partition id" + << partition_id << ", db id" << partitions[partition_id].db_id + << ", table id" << partitions[partition_id].table_id + << ", second partition id" << partitions[partition_id].partition_id + << ", tablet id" << partitions[partition_id].tablet_id; + } + mysql_free_result(result); + } + } + } +} + } // namespace doris::cloud diff --git a/cloud/src/recycler/meta_checker.h b/cloud/src/recycler/meta_checker.h index 90479c71673bfd..4f16cdab7c7c50 100644 --- a/cloud/src/recycler/meta_checker.h +++ b/cloud/src/recycler/meta_checker.h @@ -27,18 +27,57 @@ #include #include +#include "common/logging.h" #include "recycler/white_black_list.h" namespace doris::cloud { class TxnKv; struct StatInfo { + // fe int64_t check_fe_tablet_num = 0; + int64_t check_fe_partition_num = 0; + int64_t check_fe_tablet_schema_num = 0; + // fdb int64_t check_fdb_tablet_idx_num = 0; int64_t check_fdb_tablet_meta_num = 0; + int64_t check_fdb_tablet_schema_num = 0; int64_t check_fdb_partition_version_num = 0; }; +enum CHECK_TYPE { + CHECK_TXN, + CHECK_VERSION, + CHECK_META, + CHECK_STATS, + CHECK_JOB, +}; + +struct TabletInfo { + int64_t db_id; + int64_t table_id; + int64_t partition_id; + int64_t index_id; + int64_t tablet_id; + int64_t schema_version; + + std::string debug_string() const { + return "db id: " + std::to_string(db_id) + " table id: " + std::to_string(table_id) + + " partition id: " + std::to_string(partition_id) + + " index id: " + std::to_string(index_id) + + " tablet id: " + std::to_string(tablet_id) + + " schema version: " + std::to_string(schema_version); + } +}; + +struct PartitionInfo { + int64_t db_id; + int64_t table_id; + int64_t partition_id; + int64_t tablet_id; + int64_t visible_version; +}; + class MetaChecker { public: explicit MetaChecker(std::shared_ptr txn_kv); @@ -47,10 +86,80 @@ class MetaChecker { bool check_fe_meta_by_fdb(MYSQL* conn); bool check_fdb_by_fe_meta(MYSQL* conn); + template + bool handle_check_fe_meta_by_fdb(MYSQL* conn); + + template + bool handle_check_fdb_by_fe_meta(MYSQL* conn); + +private: + void init_tablet_info_from_fe_meta(MYSQL* conn, std::vector& tablets, + std::map& partitions); + + bool scan_and_handle_kv(std::string& start_key, const std::string& end_key, + std::function); + + bool do_meta_tablet_key_index_check(MYSQL* conn); + + bool do_meta_tablet_key_check(MYSQL* conn); + + bool do_meta_schema_key_check(MYSQL* conn); + + bool do_meta_tablet_index_key_inverted_check(MYSQL* conn, + const std::vector& tablets); + + bool do_meta_tablet_key_inverted_check(MYSQL* conn, std::vector& tablets, + std::map& partitions); + + bool do_meta_schema_key_inverted_check(MYSQL* conn, std::vector& tablets, + std::map& partitions); + + void init_db_meta(MYSQL* conn); + private: std::shared_ptr txn_kv_; StatInfo stat_info_; std::string instance_id_; + // db_id -> db_name + std::unordered_map db_meta_; }; +// not implemented yet +template <> +bool MetaChecker::handle_check_fe_meta_by_fdb(MYSQL* conn) = delete; + +// not implemented yet +template <> +bool MetaChecker::handle_check_fe_meta_by_fdb(MYSQL* conn) = delete; + +// not implemented yet +template <> +bool MetaChecker::handle_check_fe_meta_by_fdb(MYSQL* conn) = delete; + +// not implemented yet +template <> +bool MetaChecker::handle_check_fe_meta_by_fdb(MYSQL* conn) = delete; + +template <> +bool MetaChecker::handle_check_fe_meta_by_fdb(MYSQL* conn); + +// not implemented yet +template <> +bool MetaChecker::handle_check_fdb_by_fe_meta(MYSQL* conn) = delete; + +// not implemented yet +template <> +bool MetaChecker::handle_check_fdb_by_fe_meta(MYSQL* conn) = delete; + +// not implemented yet +template <> +bool MetaChecker::handle_check_fdb_by_fe_meta(MYSQL* conn) = delete; + +// not implemented yet +template <> +bool MetaChecker::handle_check_fdb_by_fe_meta(MYSQL* conn) = delete; + +template <> +bool MetaChecker::handle_check_fdb_by_fe_meta(MYSQL* conn); + } // namespace doris::cloud From 2df37802a2f0ba26ba6436c5d1563de45e61a8a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 10:48:19 +0800 Subject: [PATCH 491/572] branch-3.0: [chore](thirdparty) Upgrade aws-sdk-cpp from 1.11.119 to 1.11.219 #54780 (#54970) Cherry-picked from #54780 Co-authored-by: Lei Zhang --- thirdparty/vars.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index bc11d26d6b1999..e8de23405bdce8 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -348,10 +348,10 @@ BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css" BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a" # aws sdk -AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.119.tar.gz" -AWS_SDK_NAME="aws-sdk-cpp-1.11.119.tar.gz" -AWS_SDK_SOURCE="aws-sdk-cpp-1.11.119" -AWS_SDK_MD5SUM="3cd8bd51d39dc207a243a2074d11f439" +AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.219.tar.gz" +AWS_SDK_NAME="aws-sdk-cpp-1.11.219.tar.gz" +AWS_SDK_SOURCE="aws-sdk-cpp-1.11.219" +AWS_SDK_MD5SUM="80aa616efe1a3e7a9bf0dfbc44a97864" # tsan_header TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7" From ae1092b9dbb1f07978b153fae4255c6d47d815e4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 10:49:36 +0800 Subject: [PATCH 492/572] branch-3.0: [fix](nereids) set wrong intput tuple ids for JoinNode #54377 (#54839) Cherry-picked from #54377 Co-authored-by: minghong --- .../translator/PhysicalPlanTranslator.java | 2 +- .../doris/planner/AnalyticEvalNode.java | 4 +- .../doris/planner/AssertNumRowsNode.java | 6 +- .../apache/doris/planner/ExchangeNode.java | 2 +- .../apache/doris/planner/HashJoinNode.java | 16 ++--- .../doris/planner/JoinCostEvaluation.java | 2 +- .../doris/planner/NestedLoopJoinNode.java | 8 +-- .../org/apache/doris/planner/PlanNode.java | 2 +- .../doris/planner/TableFunctionNode.java | 2 +- .../join_input_tuple_id.groovy | 71 +++++++++++++++++++ 10 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 regression-test/suites/nereids_p0/join/translate_tuple_id/join_input_tuple_id.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 7ffe57168c244a..6dd4564bd5610b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -1307,7 +1307,7 @@ public PlanFragment visitPhysicalGenerate(PhysicalGenerate gener TupleDescriptor tupleDescriptor = generateTupleDesc(generate.getGeneratorOutput(), null, context); List childOutputTupleIds = currentFragment.getPlanRoot().getOutputTupleIds(); if (childOutputTupleIds == null || childOutputTupleIds.isEmpty()) { - childOutputTupleIds = currentFragment.getPlanRoot().getTupleIds(); + childOutputTupleIds = currentFragment.getPlanRoot().getOutputTupleIds(); } List outputSlotIds = Stream.concat(childOutputTupleIds.stream(), Stream.of(tupleDescriptor.getId())) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java index 7b5998717a2dbf..20a920f40897b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java @@ -109,9 +109,7 @@ public AnalyticEvalNode( TupleDescriptor outputTupleDesc, Expr partitionByEq, Expr orderByEq, TupleDescriptor bufferedTupleDesc) { super(id, - (input.getOutputTupleDesc() != null - ? Lists.newArrayList(input.getOutputTupleDesc().getId()) : - input.getTupleIds()), + input.getOutputTupleIds(), "ANALYTIC", StatisticalType.ANALYTIC_EVAL_NODE); Preconditions.checkState(!tupleIds.contains(outputTupleDesc.getId())); // we're materializing the input row augmented with the analytic output tuple diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java index a4c4aa42c6579c..8966a43adde962 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java @@ -60,11 +60,7 @@ public AssertNumRowsNode(PlanNodeId id, PlanNode input, AssertNumRowsElement ass if (tupleDescriptor != null) { this.tupleIds.add(tupleDescriptor.getId()); } else { - if (input.getOutputTupleDesc() != null) { - this.tupleIds.add(input.getOutputTupleDesc().getId()); - } else { - this.tupleIds.addAll(input.getTupleIds()); - } + this.tupleIds.addAll(input.getOutputTupleIds()); } this.tblRefIds.addAll(input.getTblRefIds()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index d904397a305da7..aeb5c39a58cdfb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -125,7 +125,7 @@ public void updateTupleIds(TupleDescriptor outputTupleDesc) { nullableTupleIds.add(outputTupleDesc.getId()); } else { clearTupleIds(); - tupleIds.addAll(getChild(0).getTupleIds()); + tupleIds.addAll(getChild(0).getOutputTupleIds()); tblRefIds.addAll(getChild(0).getTblRefIds()); nullableTupleIds.addAll(getChild(0).getNullableTupleIds()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index b664b7a26c878f..e6f4e5fcba19a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -188,12 +188,12 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator if (joinOp.equals(JoinOperator.LEFT_ANTI_JOIN) || joinOp.equals(JoinOperator.LEFT_SEMI_JOIN) || joinOp.equals(JoinOperator.NULL_AWARE_LEFT_ANTI_JOIN)) { - tupleIds.addAll(outer.getTupleIds()); + tupleIds.addAll(outer.getOutputTupleIds()); } else if (joinOp.equals(JoinOperator.RIGHT_ANTI_JOIN) || joinOp.equals(JoinOperator.RIGHT_SEMI_JOIN)) { - tupleIds.addAll(inner.getTupleIds()); + tupleIds.addAll(inner.getOutputTupleIds()); } else { - tupleIds.addAll(outer.getTupleIds()); - tupleIds.addAll(inner.getTupleIds()); + tupleIds.addAll(outer.getOutputTupleIds()); + tupleIds.addAll(inner.getOutputTupleIds()); } for (Expr eqJoinPredicate : eqJoinConjuncts) { @@ -212,12 +212,12 @@ public HashJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, JoinOperator nullableTupleIds.addAll(inner.getNullableTupleIds()); nullableTupleIds.addAll(outer.getNullableTupleIds()); if (joinOp.equals(JoinOperator.FULL_OUTER_JOIN)) { - nullableTupleIds.addAll(outer.getTupleIds()); - nullableTupleIds.addAll(inner.getTupleIds()); + nullableTupleIds.addAll(outer.getOutputTupleIds()); + nullableTupleIds.addAll(inner.getOutputTupleIds()); } else if (joinOp.equals(JoinOperator.LEFT_OUTER_JOIN)) { - nullableTupleIds.addAll(inner.getTupleIds()); + nullableTupleIds.addAll(inner.getOutputTupleIds()); } else if (joinOp.equals(JoinOperator.RIGHT_OUTER_JOIN)) { - nullableTupleIds.addAll(outer.getTupleIds()); + nullableTupleIds.addAll(outer.getOutputTupleIds()); } vIntermediateTupleDescList = Lists.newArrayList(intermediateTuple); this.outputTupleDesc = outputTuple; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinCostEvaluation.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinCostEvaluation.java index 28daa450c5a22e..f7a4395691dbf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinCostEvaluation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinCostEvaluation.java @@ -52,7 +52,7 @@ public class JoinCostEvaluation { PlanNode rhsTree = rightChildFragment.getPlanRoot(); rhsTreeCardinality = rhsTree.getCardinality(); rhsTreeAvgRowSize = rhsTree.getAvgRowSize(); - rhsTreeTupleIdNum = rhsTree.getTupleIds().size(); + rhsTreeTupleIdNum = rhsTree.getOutputTupleIds().size(); PlanNode lhsTree = leftChildFragment.getPlanRoot(); lhsTreeCardinality = lhsTree.getCardinality(); lhsTreeAvgRowSize = lhsTree.getAvgRowSize(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java index e2a7504a98d555..7cdd894ae4ab1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/NestedLoopJoinNode.java @@ -123,12 +123,12 @@ public NestedLoopJoinNode(PlanNodeId id, PlanNode outer, PlanNode inner, List implements PlanStats { private List intermediateOutputTupleDescList = Lists.newArrayList(); private List> intermediateProjectListList = Lists.newArrayList(); - protected PlanNode(PlanNodeId id, ArrayList tupleIds, String planNodeName, + protected PlanNode(PlanNodeId id, List tupleIds, String planNodeName, StatisticalType statisticalType) { this.id = id; this.limit = -1; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java index ce5aa9d1972d36..75707b7302b81c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java @@ -60,7 +60,7 @@ public TableFunctionNode(PlanNodeId id, PlanNode inputNode, TupleId lateralViewT if (childOutputTupleIds != null && !childOutputTupleIds.isEmpty()) { tupleIds.addAll(childOutputTupleIds); } else { - tupleIds.addAll(inputNode.getTupleIds()); + tupleIds.addAll(inputNode.getOutputTupleIds()); } } tupleIds.add(lateralViewTupleId); diff --git a/regression-test/suites/nereids_p0/join/translate_tuple_id/join_input_tuple_id.groovy b/regression-test/suites/nereids_p0/join/translate_tuple_id/join_input_tuple_id.groovy new file mode 100644 index 00000000000000..50dcc21009cc8c --- /dev/null +++ b/regression-test/suites/nereids_p0/join/translate_tuple_id/join_input_tuple_id.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("join_input_tuple_id") { + sql """ + drop table if exists t1; + create table t1(k int, v int) properties("replication_num"="1"); + insert into t1 values (1, 1), (2, 2), (3,3); + + drop table if exists t2; + create table t2(k int, v int) properties("replication_num"="1"); + insert into t2 values (1, 1), (2, 2), (3,3); + + drop table if exists t3; + create table t3(k int, v int) properties("replication_num"="1"); + insert into t3 values (1, 1), (2, 2), (3,3); + set disable_join_reorder=true; + """ + + explain { + sql """ + verbose select * + from ((select k, v from t1) union all (select k, v from t2)) as u + join t3 on u.k+1 = t3.k + """ + // verify that join's input tuple is union's output tuple id (5) not input tuple (4) + contains "tuple ids: 5 1N" + +// 7:VHASH JOIN(293) +// | join op: INNER JOIN(BROADCAST)[] +// | equal join conjunct: (expr_cast(k as BIGINT)[#13] = expr_(cast(k as BIGINT) - 1)[#4]) +// | cardinality=2 +// | vec output tuple id: 7 +// | output tuple id: 7 +// | vIntermediate tuple ids: 6 +// | hash output slot ids: 2 3 11 12 +// | isMarkJoin: false +// | final projections: k[#14], v[#15], k[#17], v[#18] +// | final project output tuple id: 7 +// | distribute expr lists: +// | distribute expr lists: +// | tuple ids: 5 1N +// | +// |----1:VEXCHANGE +// | offset: 0 +// | distribute expr lists: +// | tuple ids: 1N +// | +// 6:VUNION(276) +// | child exprs: +// | k[#5] | v[#6] +// | k[#7] | v[#8] +// | final projections: k[#9], v[#10], CAST(k[#9] AS bigint) +// | final project output tuple id: 5 +// | tuple ids: 4 + } +} \ No newline at end of file From c8a4eb76848532ebe44bafeb9a9b3f1ad53daa96 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 11:03:42 +0800 Subject: [PATCH 493/572] branch-3.0: [fix](deadlock) avoid deadlock on tabletInvertedIndex #54197 (#54996) Cherry-picked from #54197 Co-authored-by: Yongqiang YANG Co-authored-by: Yongqiang YANG --- .../doris/transaction/DatabaseTransactionMgr.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index 3db8edb865911f..c1d8018e6169b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -122,10 +122,10 @@ private enum PublishResult { private final MonitoredReentrantReadWriteLock transactionLock = new MonitoredReentrantReadWriteLock(true); // transactionId -> running TransactionState - private final Map idToRunningTransactionState = Maps.newHashMap(); + private final Map idToRunningTransactionState = Maps.newConcurrentMap(); // transactionId -> final status TransactionState - private final Map idToFinalStatusTransactionState = Maps.newHashMap(); + private final Map idToFinalStatusTransactionState = Maps.newConcurrentMap(); private final Map subTxnIdToTxnId = new ConcurrentHashMap<>(); // The following 2 queues are to store transactionStates with final status @@ -193,12 +193,8 @@ public long getDbId() { } protected TransactionState getTransactionState(Long transactionId) { - readLock(); - try { - return unprotectedGetTransactionState(transactionId); - } finally { - readUnlock(); - } + return unprotectedGetTransactionState(transactionId); + } private TransactionState unprotectedGetTransactionState(Long transactionId) { From 05aa6706e55269385cb58ae5bc65871d53d8d916 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 11:05:29 +0800 Subject: [PATCH 494/572] branch-3.0: [fix](hdfs) remove cached file handle when read fails #54926 (#54991) Cherry-picked from #54926 Co-authored-by: Yongqiang YANG --- be/src/io/fs/hdfs_file_reader.cpp | 17 +++++++++++++---- be/src/io/fs/hdfs_file_reader.h | 3 +++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index d43cfae1c28228..c331223984b7b2 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -114,9 +114,18 @@ Status HdfsFileReader::close() { return Status::OK(); } -#ifdef USE_HADOOP_HDFS Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, - const IOContext* /*io_ctx*/) { + const IOContext* io_ctx) { + auto st = do_read_at_impl(offset, result, bytes_read, io_ctx); + if (!st.ok()) { + _accessor.destroy(); + } + return st; +} + +#ifdef USE_HADOOP_HDFS +Status HdfsFileReader::do_read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { if (closed()) [[unlikely]] { return Status::InternalError("read closed file: {}", _path.native()); } @@ -169,8 +178,8 @@ Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_r #else // The hedged read only support hdfsPread(). // TODO: rethink here to see if there are some difference between hdfsPread() and hdfsRead() -Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, - const IOContext* /*io_ctx*/) { +Status HdfsFileReader::do_read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* /*io_ctx*/) { if (closed()) [[unlikely]] { return Status::InternalError("read closed file: ", _path.native()); } diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index 8ccbe4ade8839a..8556eea0de6ac5 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -63,6 +63,9 @@ class HdfsFileReader final : public FileReader { void _collect_profile_before_close() override; + Status do_read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const IOContext* io_ctx); + private: #ifdef USE_HADOOP_HDFS struct HDFSProfile { From d717757ecf3cfaaf7a36784faf0e36fa9363e351 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 16:53:59 +0800 Subject: [PATCH 495/572] branch-3.0: [fix](regression) fix test_list_cache_file #54956 (#55049) Cherry-picked from #54956 Signed-off-by: zhengyu Co-authored-by: zhengyu --- .../cache/http/test_list_cache_file.groovy | 85 ++++++++++++------- 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy b/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy index cf562a3b92f093..bf945cad014fdb 100644 --- a/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy +++ b/regression-test/suites/cloud_p0/cache/http/test_list_cache_file.groovy @@ -18,23 +18,31 @@ import org.codehaus.groovy.runtime.IOGroovyMethods suite("test_list_cache_file") { - sql """ use @regression_cluster_name1 """ + def custoBeConfig = [ + enable_evict_file_cache_in_advance : false, + file_cache_enter_disk_resource_limit_mode_percent : 99 + ] + + setBeConfigTemporary(custoBeConfig) { + String[][] backends = sql """ show backends """ - String backendId; + def backendSockets = [] def backendIdToBackendIP = [:] def backendIdToBackendHttpPort = [:] def backendIdToBackendBrpcPort = [:] for (String[] backend in backends) { - if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + if (backend[9].equals("true")) { backendIdToBackendIP.put(backend[0], backend[1]) backendIdToBackendHttpPort.put(backend[0], backend[4]) backendIdToBackendBrpcPort.put(backend[0], backend[5]) } } - assertEquals(backendIdToBackendIP.size(), 1) + assertTrue(backendIdToBackendIP.size() > 0, "No alive backends found") - backendId = backendIdToBackendIP.keySet()[0] - def socket = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + backendIdToBackendIP.each { backendId, ip -> + def socket = ip + ":" + backendIdToBackendHttpPort.get(backendId) + backendSockets.add(socket) + } sql "drop table IF EXISTS `user`" @@ -52,6 +60,8 @@ suite("test_list_cache_file") { sql "insert into user select number, cast(rand() as varchar(32)) from numbers(\"number\"=\"1000000\")" + Thread.sleep(50000) + def get_tablets = { String tbl_name -> def res = sql "show tablets from ${tbl_name}" List tablets = new ArrayList<>() @@ -65,7 +75,7 @@ suite("test_list_cache_file") { var ret = [] httpTest { endpoint "" - uri socket + "/api/compaction/show?tablet_id=" + tablet_id + uri backendSockets[0] + "/api/compaction/show?tablet_id=" + tablet_id op "get" check {respCode, body -> assertEquals(respCode, 200) @@ -83,35 +93,50 @@ suite("test_list_cache_file") { var rowsets = get_rowsets(tablets.get(0)) var segment_file = rowsets[rowsets.size() - 1] + "_0.dat" - httpTest { - endpoint "" - uri socket + "/api/file_cache?op=list_cache&value=" + segment_file - op "get" - check {respCode, body -> - assertEquals(respCode, 200) - var arr = parseJson(body) - assertTrue(arr.size() > 0, "There shouldn't be no cache file at all, maybe you need to check disk capacity and modify file_cache_enter_disk_resource_limit_mode_percent in be.conf") + def cacheResults = [] + def clearResults = [] + + // Check cache status on all backends + backendSockets.each { socket -> + httpTest { + endpoint "" + uri socket + "/api/file_cache?op=list_cache&value=" + segment_file + op "get" + check {respCode, body -> + assertEquals(respCode, 200) + var arr = parseJson(body) + cacheResults.add(arr.size() > 0) + } } } + assertTrue(cacheResults.any(), "At least one backend should have cache file") - // clear single segment file cache - httpTest { - endpoint "" - uri socket + "/api/file_cache?op=clear&value=" + segment_file - op "get" - check {respCode, body -> - assertEquals(respCode, 200, "clear local cache fail, maybe you can find something in respond: " + parseJson(body)) + // Clear cache on all backends + backendSockets.each { socket -> + httpTest { + endpoint "" + uri socket + "/api/file_cache?op=clear&value=" + segment_file + op "get" + check {respCode, body -> + assertEquals(respCode, 200, "clear local cache fail, maybe you can find something in respond: " + parseJson(body)) + clearResults.add(true) + } } } + assertEquals(clearResults.size(), backendSockets.size(), "Failed to clear cache on some backends") - httpTest { - endpoint "" - uri socket + "/api/file_cache?op=list_cache&value=" + segment_file - op "get" - check {respCode, body -> - assertEquals(respCode, 200) - var arr = parseJson(body) - assertTrue(arr.size() == 0, "local cache files should not greater than 0, because it has already clear") + // Verify cache cleared on all backends + backendSockets.each { socket -> + httpTest { + endpoint "" + uri socket + "/api/file_cache?op=list_cache&value=" + segment_file + op "get" + check {respCode, body -> + assertEquals(respCode, 200) + var arr = parseJson(body) + assertTrue(arr.size() == 0, "local cache files should not greater than 0, because it has already clear") + } } } + } } From 92da0c586b774e4aa06aa237bea95361fff608b3 Mon Sep 17 00:00:00 2001 From: hui lai Date: Wed, 20 Aug 2025 21:45:19 +0800 Subject: [PATCH 496/572] branch-3.0: [fix](audit log) fix audit log return rows incorrect when statement need forward (#54548) (#55053) pick #54548 If insert into request to follower, ReturnRows in audit log always 0: ``` [query] |QueryId=178cb3b149684de9-87f3c24149b1e50b|Timestamp=2025-08-11 11:53:30.132|Client=174.58.0.1:59086|User=root|FeIp=174.58.1.2|Ctl=internal|Db=test_insert_from_follower|State=OK|ErrorCode=0|ErrorMessage=|Time(ms)=145|CpuTimeMS=0|PeakMemoryBytes=1025152|ScanBytes=0|ScanRows=0|ReturnRows=0|ShuffleSendRows=0|ShuffleSendBytes=0|SpillWriteBytesToLocalStorage=0|SpillReadBytesFromLocalStorage=0|ScanBytesFromLocalStorage=0|ScanBytesFromRemoteStorage=0|ParseTimeMs=9|PlanTimesMs={"plan":-1,"garbage_collect":-1,"lock_tables":-1,"analyze":-1,"rewrite":-1,"fold_const_by_be":0,"collect_partitions":-1,"optimize":-1,"translate":-1,"init_scan_node":-1,"finalize_scan_node":-1,"create_scan_range":-1,"distribute":-1}|GetMetaTimesMs={"get_partition_version_time_ms":0,"get_partition_version_count_has_data":0,"get_partition_version_count":0,"get_table_version_time_ms":0,"get_table_version_count":0}|ScheduleTimesMs={"schedule_time_ms":-1,"fragment_assign_time_ms":-1,"fragment_serialize_time_ms":-1,"fragment_rpc_phase_1_time_ms":-1,"fragment_rpc_phase_2_time_ms":-1,"fragment_compressed_size_byte":0.0,"fragment_rpc_count":0}|HitSqlCache=false|isHandledInFe=false|queriedTablesAndViews=|chosenMViews=|ChangedVariables={"character_set_results":"NULL","sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES"}|FuzzyVariables=|SqlMode=ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES|CommandType=Query|StmtType=INSERT|StmtId=27|SqlHash=fb91c45fdb1cfed22fb12838a979dea9|SqlDigest=|IsQuery=false|IsNereids=true|IsInternal=false|WorkloadGroup=|ComputeGroupName=UNKNOWN|Stmt= INSERT INTO test_insert_from_follower_tbl (k1, k2) VALUES (1, "a"), (2, "b"), (3, "c"), (4, "e"); ``` --- .../org/apache/doris/qe/ConnectProcessor.java | 1 + .../org/apache/doris/qe/FEOpExecutor.java | 3 + gensrc/thrift/FrontendService.thrift | 1 + .../insert/test_insert_from_follower.groovy | 62 +++++++++++++++++++ 4 files changed, 67 insertions(+) create mode 100644 regression-test/suites/load_p0/insert/test_insert_from_follower.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index 168eaab9f6fc22..8ca722894f7fae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -775,6 +775,7 @@ public TMasterOpResult proxyExecute(TMasterOpRequest request) throws TException result.setStatus(ctx.getState().toString()); if (ctx.getState().getStateType() == MysqlStateType.OK) { result.setStatusCode(0); + result.setAffectedRows(ctx.getState().getAffectedRows()); } else { ErrorCode errorCode = ctx.getState().getErrorCode(); if (errorCode != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java index 3914d7ecdab33c..b1b519b15be3b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/FEOpExecutor.java @@ -67,6 +67,9 @@ public FEOpExecutor(TNetworkAddress feAddress, OriginStatement originStmt, Conne public void execute() throws Exception { result = forward(feAddr, buildStmtForwardParams()); + if (result.isSetAffectedRows()) { + ctx.updateReturnRows((int) result.getAffectedRows()); + } } public void cancel() throws Exception { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 174ac9989d1c9e..236d11917dc1f3 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -632,6 +632,7 @@ struct TMasterOpResult { // transaction load 9: optional TTxnLoadInfo txnLoadInfo; 10: optional i64 groupCommitLoadBeId; + 11: optional i64 affectedRows; } struct TUpdateExportTaskStatusRequest { diff --git a/regression-test/suites/load_p0/insert/test_insert_from_follower.groovy b/regression-test/suites/load_p0/insert/test_insert_from_follower.groovy new file mode 100644 index 00000000000000..31b230ceb04e27 --- /dev/null +++ b/regression-test/suites/load_p0/insert/test_insert_from_follower.groovy @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_insert_from_follower", "docker") { + def options = new ClusterOptions() + options.setFeNum(3) + options.setBeNum(3) + docker(options) { + def masterFe = cluster.getMasterFe() + def allFes = cluster.getAllFrontends() + def followerFes = allFes.findAll { fe -> fe.index != masterFe.index } + def followerFe = followerFes[0] + logger.info("Master FE: ${masterFe.host}") + logger.info("Using follower FE: ${followerFe.host}") + // Connect to follower FE + def url = String.format( + "jdbc:mysql://%s:%s/?useLocalSessionState=true&allowLoadLocalInfile=false", + followerFe.host, followerFe.queryPort) + logger.info("Connecting to follower FE: ${url}") + context.connectTo(url, context.config.jdbcUser, context.config.jdbcPassword) + + sql "drop database if exists test_insert_from_follower" + sql "create database test_insert_from_follower" + sql "use test_insert_from_follower" + def tbl = 'test_insert_from_follower_tbl' + sql """ DROP TABLE IF EXISTS ${tbl} """ + sql """ + CREATE TABLE ${tbl} ( + `k1` int(11) NULL, + `k2` char(5) NULL + ) + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ( + "replication_num"="3" + ); + """ + + def loadRes = sql """ INSERT INTO ${tbl} (k1, k2) VALUES (1, "a"), (2, "b"), (3, "c"), (4, "e");""" + logger.info("loadRes: ${loadRes}") + assertTrue(loadRes[0][0] == 4) + sql """ DROP TABLE IF EXISTS ${tbl} """ + sleep(5000) + } +} From df7194e496217c64df7c61892204ce77d8c83667 Mon Sep 17 00:00:00 2001 From: hui lai Date: Fri, 22 Aug 2025 21:45:20 +0800 Subject: [PATCH 497/572] branch-3.0: [fix](csv reader) fix csv parse error when use enclose with multi-char column separator (#54581) (#55052) pick #54581 Idx represents the position where the buffer is parsed. If the buffer does not read a complete row, as shown in the following figure, idx will become the length of the buffer, and then the buffer will be expanded. If some of the column separators happen to be at the end of the buffer and some are not read, when reading after expansion, it will be impossible to read the complete column separators, resulting in parsing errors. --- .../new_plain_text_line_reader.cpp | 7 ++- ...est_csv_big_file_truncate_delimiter.csv.gz | Bin 0 -> 850496 bytes ...m_load_big_file_with_special_delimiter.out | 3 + ...oad_big_file_with_special_delimiter.groovy | 54 ++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/load_p1/stream_load/test_csv_big_file_truncate_delimiter.csv.gz diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp index 94eaf2edadb55b..08000d7df5b942 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp @@ -49,6 +49,12 @@ namespace doris { const uint8_t* EncloseCsvLineReaderContext::read_line_impl(const uint8_t* start, const size_t length) { + // Avoid part bytes of the multi-char column separator have already been parsed, + // causing parse column separator error. + if (_state.curr_state == ReaderState::NORMAL || + _state.curr_state == ReaderState::MATCH_ENCLOSE) { + _idx -= std::min(_column_sep_len - 1, _idx); + } _total_len = length; size_t bound = update_reading_bound(start); @@ -141,7 +147,6 @@ void EncloseCsvLineReaderContext::_on_normal(const uint8_t* start, size_t& len) _state.forward_to(ReaderState::START); return; } - // TODO(tsy): maybe potential bug when a multi-char is not read completely _idx = len; } diff --git a/regression-test/data/load_p1/stream_load/test_csv_big_file_truncate_delimiter.csv.gz b/regression-test/data/load_p1/stream_load/test_csv_big_file_truncate_delimiter.csv.gz new file mode 100644 index 0000000000000000000000000000000000000000..fd79d19741390e0c0b113363c9ca94f4d7edb4e7 GIT binary patch literal 850496 zcmYg&X+Trg7H$Q!)>5r?XsHYiw+@KdDvF2jZ+33kt{(VkHn|$^{A$LXx-EI`qBw>%ckt?6aqDeQSMt?dV*bJGar( z#^)W?5U=16caPvO_oKc(?%uxsUhW}5p#dI8LcH8Pz5IPo_=b1|>BIAP2IUob0RkSE z)$8(8;JrI(3t}s)zWDXC`H2g@f4c6>PgYT?|G9kS*A<&q9-aH=GQF4jcTW6MT%vjE z>dOCo?`CSUx4Sd4A@Cm~?{u?b>4_oVVuhcp^6!|pIwj4{zPETvpP{O&wnF8)tgB)5 zMm~e1I*Q43{w}xvQ#Nk_9(}c6Uw)N z%8kk_rBpetoK{W>ZYI|eD31*)Ps$84#-;JflQ*i;!=%;gl((N63w6R{ z2G%Ogrui|#nA0(G=f-NeVa%j6PiZvduFz_|BXLl&|N6qR%1=PL=xDk_xg9#=c?V?OVU`KDg^n__K= zU4*>i=&mom9NFQ5b|1cu+Q=|pz%2)qTP$le@NV5}8FZ%06ST7TCr40W!#c+uES*IKP7eL>5Z!{gnL&VOsb%DHp_ z9`Tk1%4Fw`XT0+Ky#Ky`}myU^PCu6>p4C8&Xsri;m+=&qm>7th)i$4UN+u%EWI}p ze_U~&UiYPEo$DNOKb2r%))w27n?7j>I0;o`xV?Oly-nj6$(f94xpn9_@PIFSQXRNO z8%vH&#NZuiyiBH2v-8n=N3QxteS_br_-kEE%!wV?{u3rfCoft+GpEy*J?d8OeDvho zT(1iWCx5sQfB)1Y#p9ju$e*KmlQ8~>7{Epk|!Qb%2+*kCr#Q0vpJ5_JZ|H04aV(-{fVPLKEP+}Rhnw6 zuo68*HMYa{_F>!5>}@N4DNd}8Q&eCp#GwO14ZjsVUu{dW$T|LdkLR|D!be}VNH4zn z#VS3=vJ&pW~&tj{) zo9?za)f#EUR;YhbSFkj~`o|0-l|Sqwtbg{H3~yB?uQFNlz=p4-Y=8HBPd>{(D`b57 z@Ea&D++jIh_BH?fcTvJP*GhJ7LR$RahwF0t+3?4!Q-}E3|D>o}nmj9jn$BjJS}1%z z*%VtXYzjHO8{RQk)2qOMO6q=nvfKJ1qv{%YHV6KC5dvG|uwnlaTG-CG@uP1f9Rdb(29e(@)$ciQ{P zMRBY()Zrp=uxW!{+%Jf6?s*t&nI-nzRwAvqBh!SwdL(;s^xNJTEsV{p4~3R`Y_u4P z8HA?S$o2O$^=^kYH4U!U`6p#=O_+A-pW5%@jgYdP*oSAm1HVB#c>I>z-K!R0W$9j9 zL(?tbn+CGJIoO=4(ob<1icEd}3!`3P`aEYoe5A*a|8;Ir5>`5KsdgbhE*?jq&*=a@ zW0~H#KK41bXsA`!`l~#8Hr1aAt&WwxUeV3khhK8`;-qcHLV~AMk!2)&_q~(P#m*aW z24ya;SbK>uI`&jtmhprYDBb(~)R1`$@SK)+f0F zHy$+old>NMX;83(^)B~Gel{t378kpF z&Sk-W=i#t)Dqkpg!y`VeO8{vj0`V`S(&+6}#HYQVUEB;^>byLl?R*PfHm$OGB=x`+ zCh1etr&0#ei;cIf-j)-*BgWhH@3%6o$o3(LOb3d4ZtD+?<6^7Fw{z@qC7fto*=JJn zH4Mgcu}di;B}7QrdSP$0Kgw(?w7n;NtZvseKblf@aTC7G8*P0JFZb<8g{wwvgmw@< zt{Ky8ks|U>nr!d2!m0*lAr=Y}IIV<}jPAHDxN*NJW!YPWbIlj`n+B5y{z>_;rtlU* zt!e1}*I13M_Y#>7025x^(C1i;aNMzL7)!$Bu|FghA zmTtr5ly>Q-83U3YH07=#5c2Bn2YTn2h|~YZR^3hK;M!<=0!Mh^ zU3$O=$ETq(b`SQz1pfO{bG)l@546JNR(9wN?3VAxCE$3#6bDfzm(#xCjxO1yDa1{DGehQ1ssx|G2GRUYks|&Q_v>AS$SK-}Mia-*fWn zK=4+$F*1X<2L(JGTfP=`K9~?8$k&drzR+_>VoVEI=J!w*i>(S7xV!*fJb5jrG1Ux* zN5Rv=DMBlTmTJ0BycS0MY>)ols+b)(FLt-fwKvinO8&$T7_Rvr4PzvSfF=vk$G4w` z!$b9&9OFq`x+QjJlQyytZr<9Xcq1&hy9*a-o{v;IZd8TP1rDv0y#5LLcc)q>@^^U1 z9+>r9tP8c-@SBG4!Jpa!VWjm&tk^j~E;CJaqqR9XBL3w7TKPx|E+Y^7S8{ug6Y=G8`3Qk-J8cfRh(tjqU3|>wmUK=98m(2&cdM zmCJvMXdJ%7#;+amQL6iDA||SC#f-T{cM_hCn?Uu?HK62C>AS-3zj*ZyAw+ zE%<=5wRpGy?W~W+QPWA8WLAxur3|d>`R}IlEL&zFFl_kO27Z9Ay!DBWeQkt)9qWsi zB0q&l>EG9VzkYzwmyKBtz5fZWfD)lR=SAEO99r+jD1?<|B(+-*DMF<9qu(a{;*HU| z*wN^kOk9`Z8U&4j)QH$Ql}9$`|0Y51R%2?GdhI@ zwD#Ta5v!E7Uc}jYSXLg5z;ZZo+-wvdu%$*f>yfy6B8QLVr~}l&21p}<_7eoh# zkS)63WNL;B5$t$+BT-4PDuQGrI?sNjto)797qI|Xf~QR;I3G?W<=Tjqf@|DX6F?L2LL8A!=Q>xy#@w22zk}jG z$Zs%ydXF9Jt1K-q);mDsj~&jNaRwYDwZ9jgfs48`vZy9oB;GVxaQBQ$t1 zw;MgG2x0ESxNKw*I^0)PEkp%>WJfgO9}_m?WQPmtqi1EHtqgV7LhgAHU^lRRHxg!2 znI}AeMJMlHi?b_sPrzL$d^nYoUfj1$5n)+0d~z|2s)w>wd`C=pzbxxx-k%gn z3?F^*7g|^Z$gzAx7c7_kxdWdNV$zHOqxTCWg%7ujY_KDJnu&DvHXFTz?Jo)(_Td^K z!8y0BNH@lzIMK4Yu}opaSyJOJIo9_&7V#0qkF(p39TkqBdXzFLbvKR$gDWdgM(Z%B zbGq%6{tVw9+Oa*f8huNkolkL*r`K$U>-~Zu+efCY=_YyG6l|~%OC-ifN;drWG*hwV zjh5PYePOSWBMw1uuGXH>v0 z#`RMj8fRB20jwZKlg2*BCJoDbkfD;qv4|~zdP^|DFP+2ilR%8K)4=ACYbZATWhYWb zoOLv-I87L$-)sA$7WBk08(JlYdUBdD?RXm!bt4Itnh|0xPb$}DMAQ= zmhtqx6oQ_m?@>j+CJt8Tb1P;j3Z;npqX z{LI^qOH?S%3Bmcau9_cC1*01gM>Hu4hq zNzUAX!-Rpsho-3%K@H4;VCnwI8G-J0;<>zz=L&NFLc~{3;Dk6z|3pj#AnjystCJFz zPWjmpa=>mKX~zol>$Ax#fn`#FACxtnmqSTd_kF)YH9rE4Aw59|O(CY#2*ak=rVuKC zdD7(LO^BC=FkVuxaTBpQc&DqkRz}Mw*(Vf78f>=(_>*1zG6;Ea4$Rh>Iv{e;wVXw+ z-5|0O_&_|Dg~0IYH8r=>jN!rkrc^_6@i_jWiXI{->GYlL;Qv8LW!uX*a%N_EUqbaM zM_p_9xt_4*d)l6K3XSjS;uOii7klqy?s3-|{I{m0{v%nRJtBHS@|Oj(jK-%^>~Olh zl|58Ok=ZZgM;|7ykzeM9eDtDweC@>STm;LYq|9Q_!d;1Cdxh0%>qau7A|T z91$|nAsR=>|NF>5?99cY;I84Ibk}V->J9~QX_xnN5JATz!T1HR=i~4E08@UrZo0^D zIFROMWbLD_1f-hh;+0&y8n^}^JRr>3f-iZS+(R5~4e#y(B?8D~YcG5q8ZO9jT| zmtVgVkH(|C3-Q$`wRj~viT2U&2DnqiL++_Kd!?onPPJt$()t!}H}FQ$Tb$k;V>d&7 z7-&HYWX7l+oNQ^`bq&L#n!vo!B~l|Mav}_6+!*^~rg7-~eLe6i->q(UDk}020oY-5(j? z?4Rjrc@1J~ha}&IW_NQhfk?<7ctMY*`r$QLQr)vBrE>cU3c_@q2cJ#02gjnIA-Tf(9qds~xK;&D z=s26>igqVL*^`HjYkLA@1>TkGWZAVx@X;irF(?O~=5z{_TNoSDn0%08(3-5$M=k^O z?yxBur90x~Lr5sk#~fG;?c|zmcKmL7Jl0!lGyv<~tn<=m_4}wkR$0=G(#9MFV1=6x z!D)DsLMI6sw>}qv02{{N_w-sPBI0$oGlo(<*>-~bFtm&}4)7THuc!l`zx^j^n*)o5 zPvj6X!m74yF->IvnIIuRW*Cf5lf+QrKVHT#j8WQt@W0Ugvp=o_(jGVuSv4#B95H+X z!3~@*MBlMKu{wS7TqTKtx!0Cqhon5>uklIyNUS3=I-T<>g5jJ6;e;ae=F}}Wv%Khe zWc9^U!`E?o!bqgs@y1B0PCl9QMnfFk1-aS0?bvGw7f?8uaJ(JCcEKwC)#%2!h%-=D zw6MG|lA{~Zh$WMPL!~z{YKzW=aO>~4JRQ4tiLu;v+mVs6lxl|~Kmyy3Bd)+3m6Gmo ztTmq8y~yEjWQQTNy5Gbt>cGyi-0mU9cpI%I!Gyyi>w{@VUUDRQYd2(}>67qvwk(+Z zFL);+v(-k z1Ek71mmLofqmfP;>jN-BNI!~BTgU(dlp6#wXzBnUO5O4q!(mT{DsgN}v{Ppst;vG( z(%LLEjTJkje?u$;{#|thXmlWr=ueoXp%>>bZdwN|fG0kosNR9cw~d%bnvhPZziS-v^R;;cz1--<~RxzTNx`dFO4G>d$Ehah{@i zI#aSp5&aEnQymlVJ04)85K>SnYgws@#k}SHprE5b%KCYr0HG1-yT3~e{vED%HnLI^ zm+y;RE^^Dm=K_*03lcv=x_}B4{qS^djkK%-{1gbQ*2P+klYs8e**f0U^u}mKtQSXW zkHW(?&R`H;XPgP)_Vp+Ao8MsFCJQFX?K8?6#A{C^JNI}DN$A5t&&z8NN}w^J;XQ&;vc?sRNj1SvU4?IOcF7;>uLBWAQW1L4^L}g`yjjtu0b6g%!@?% z0F=b1uJj<#{q!~mDPj$D^&GhtIcjM-YpCaI(y?{XZj(3QS_7Ol9Sd@}Mr@V;mY7fk zn1D@MMu>#F8eeFbQOTL8Qp`eEF2iwSDGp1J=_Jz7{OE2_=zxyE=M0S_p~4OWs>L@%2A z7rcWQN8ptFngi?v)^!!h7Dm+&TVe2JPj_1xt=tiHzJ=focQKGLVNJ_T#Sk!ozw98T zz{xal7_T1dE0%yv3pAkh(x-0=8v7k)-AGb9_%^uR0MyjmmEer+&A2-y4b}-jXRu`a z^}})6Vgd(PTtiMBN3$Sq91w*9(I+#KooUS&-pOeH_&W@$nsVK%$Pt`arOaJ&Z$xT5 z41xv68Runcx`wXB_9wLmK-PnGE&h8e4j2f4`S1q>yha!aovJ@1;gZM4-{CpJ%RpQ; zZ0Dfk=P%r`z6sR`(4oLb0UGQ|=p>Ji!2@WwrD#-jlBZmTa8a%jhjk<67z{AmNV$AM zRFO7BSfGK7^&Gny4;1xtMz;{a2#u^9?ZlCd=~Mk#YL8U&XWe2ktBbt4vE2Wsf`4dP zl?P-4TDH6&S&D6ziG~xxbm!31SUN;XvUn**N`M3zxMh1z?&wD?53Hyxh4Iq*5EvtT z-aELB2Xk_ZGs8D*h^(O8I3xBoMk7mE2J>3L3k12udXb4w6IQDLaRJcJpV7n_)viLD z<0SnPhh+jhZ{VP^R0PVgZbTgHo%H(L?h%rK3rKAtJv=xMMZA*<+%#iQxE~c{V{`=I z7<&Jk;65TDHhFCgvKoZOjGJt_H$E1d6KGa=n32#W!J#>fo;O?x0TN7DY3oX!4mfTP zZOl-1_|apdB;pXf#RMYgsD>b@ni?VjQNZBc{YVci$5E1PUtBcZ_>v+`MGf(d8vH|tN7sC9T z!;E>B_e-l)C;6u{UwfMOuc?(*$A@fMKD1=dv-777q`8uqHT`hEVU{E&tN$zWr0h7j zGcawj{LS`-m!q$K*1e`$nfgcktZc(CdCJs6_+!o07HR%FH{S70Jhj8B&U#L!Tm97X zb;cT(r!Hqo7pya0aoOu`b+sM)^3~M2wXC-7ox&P3MSbr)k>uQTUxMN6J?-aDPrG@X z|H)y&uZ4?dwWlX+yV1MTHf-f`VT1J&%c7Y|lc(i3%~zZA%wjy-eW5C=&GNtHH?O9u zTSf01knO?`minK}tHD1PUhexgPI}QcY}0bsC@j8^X9Kl0EKHYtd~5J4bKd%(Hx>5K zQjydmr+6#;;koXh+ZF0esik7C&AX#g3wxu|L1@IL<>RyJs9nB`T(jWisrA1?rzeM< zIJ(BE&|KfMUz$R%)iOU&fqzvT$5*FWx4(DBK(^{#uQjU8+;$CYVrgFJH&9n~m5y%H z2l!>yL~KN?COoQjjolaW1628@bx=4Uw$_R@% z*0;v#VB-vThU54Qeg5+<{OXwd?m`886)TC*gRhDfRoGRXI4!i+>W2R+S~ zK6CmQds*DNtNqF8kopzy-|WJ=<$vG@12P6~Lz%oyUhVDpu8p_NfRmxu8kNE!Cr-Ja z0G9(#So&MUrD7~z_4?qPt?;y&7uSW4x|O}pGpoPE79VI@-+(VRr@f9?3tx{5tiP0Z z4jZ`px)$^@Qx>}e#~|{<*T2IQb!mOqZ8-MKaS zls;4$l*nx@Zv7HQFXY(@zu%$PiJQik?13>cXsTRF^AM+VvzT|lZTYI@4o1hFT{mER zrJr)(p<1RgHWOY+I`UyLE|P|(O1Q3Ep8g_tWs=#R<%^a_%C9YZ#tem6W_PGP8^^~i z(Cm4SC9dZQ$uYOdrmykqYsy={#;;z?KaNj2#I<=K#XPhDQ4XTSjfuvvf^0W_GmPV-q)y#^|0>*68#Q6ySs#Q0(-|J7 zI~TE%DJa;1Q+pyId-dA3odCP2uzn*bvuU|NK@bXAbami2Xv1)jC|HDllz)*9U{lF< z*INJ;h$5%A?XNnVMpJs|JM7X>LnF4!v|vN^sx`3YMxE!cld|{W9Bw!mV~NYLAzK`Z zS3ztqw)<{=!D<-F(W{@id+djDB9_aK5z~jSrY^RsY6`Upvt3@%^}3vvV3m$7i+d5) z+GIdjj!PTe47)XX69-~!W3Z|c1_FlJN3RW$c-Jp%DO6mTz+FL@YiPU;1r-`|e$T7j zkI$V6Foo&1y=6Md2#2L>1F=`>PaJXv{jn+fJ^Uu^n4fTYQc+(*k^u*KUL(!*Y3uOS(M%6v1czIRaZukp;FPJ|Az6 zt~b_gLMVE!w+jfV!fM2apR+`_;Jq~4UiCA3p{&2KdqMH@9vo$1Lu0e>3RdnAh5#BZ zd(s+#ot)^3!=h3AEBj-8%CH{Cq+j|FfbD5^OV*Strqj$7>{>_XMGhjqSRWVMzVTx} z!SgpCxvs|wH|LLwyWEz?n0dxrel=tX_vS(mzJlxG+RjHM<@PiaSMwf6DW1MFd($ji zecse$*K9EPNj)o_ZVhIdyA_X5`#8@USHZzFn)(%FaxTIE1$2>Usp-Y+Ls$mv9ZF zt>SdNo{En?Jw1N8s!F)agl&L}dfAbQBKwbOg$v$jn?Z4vX04B#@U>B&M?LPa5;QJ* ze;?$BRhwn0-)TFHb@@6pg+kjhBQLdJfxO}{sL8X!PV%@P3$k|EZ&AMqCDbyF_37#v z2oh3$!FCw?X7*iL)hVg1`0lNJMlBh*KwJ%g9p=`Sd>wy2fFPBUx)WAs$f~DvuvLu= zvjx{WE`wrL6&~S;O~K@q6U1~9tKuf$Jeca9nqQd;SlL0Y0|>5;TiWs1=yUUcqo7G< zg@7Bi%u9WEx4q%Y*Cz;x47>(~3!cQ0?K1AI`>O(ryvavD!*hcjPYy7H1&T-l!l@ZW zQYtShK-e^ECJN}f;TtzBRQmR~4T4s$oUpZC4IZkDj@*z~)is)_24Rn@fB*itqy|Z= z{EoW668dBRh3mDQ|ulgGLBV?H?08cfDx8AmU z5DPe$1>K$`PFv_roYq$iEHp>7wX+{#)2#I}2vH*pvI?)ehhQtOXx)pGbbc7;RM>M) z<f2_9ltfjuo+WxD5TjiM{wIVj?FtUxN5)KdXoxBp}g^_*uTzAwxf5AFAY+$;FUr zw!e?POExmD!p1ec;;=TVd8&!l?u~&Z1e3pZ&<)WwVN^&1lpFMuc|MN)wb!~h*n(nx zjRr6|s?8~z{!6B3!#BeT_-)pP<26B0bgq?Q+ARm{VR#tg3Ldq|R)n`9i`~`X(o)wU zR1^+vQn-YmX+r|cw-FMi+WTbwcKHPeo6%^?#+&wY0a7qOyK=s2!zBWub0rJ|62kMB zf^KYVOhA~3?BFQB=UjRF8*Hz%sRN-P)Tr;zD8`i`2=+whuf(y$Wqvz88h#XnHzp)j zpM5cz!-9`^u6JhW!OwAuS`I!NB2FYbrgeO~CwW>@O(Xo|iG&S8%lX(a{k;u-LqFgI z@l?lK#}HrBOH1ltO#*y_dmIUAshj?bzGP5KYs)JR6>-z!?B7AJJ-odygs_wqfgF|T z{M!!ViW#7Nksc2-*xoR=2#F;{a*n#H+I;CgaUSa^fV&}f8=SavMI;XgR@4XmP7|8fyRt+lZCo4?5QGfo88Po-IZ25ZQBE0;%d?S!<9 z1uPL#SBYzmx(ecIQrqmTKbI)$aTFp5Yycn-FvTwA9v_&*!X2&UWR9NZFs$ZTQC!Cr z)7z#_oE>4`QF&d9JcJk6pS*FjHbd=)5hvmG3C?`vu+hlr%MX?#MFmxY*8-dc;2?U# zK`^B^y~X%ukxe=Y74+kbyQxok};Gq)fXZz*W? z4Ujc2bBD-31lzi(6Ok1^%V=+c^BjamVfrRf;u(yUrdPXf*`j`wmOy75g2bSv4X)ry z9Oc#~gKL7+cNDzFIhn~TKF4kq+Z&~Mkrh_y#e#P?CZfYv7}wkzQj2ievFloDjGU!1 zdd*_~=4Vl8P+1#-G8hWmzXOrwE1@+pF;qk5|M%~S!R&7YN$7_5*xLdGd{|>6`gk3x zE<`%~hS*97qmZZdv0;Ks#n_+#*8v1o(Zk?EXM*pII`4xIwNYs_tSpd-y zeESpd@t}psLlC_h8Y7Cz>aJw9;LwSBWJb|aY!iGgvl&ZHI3+5`gYS@Ep|WYI)5j?0 zkYAy-?=u&or#Yexr8$@K=*GEuL|tDjbKU4A)=|MV=P!#c38N5WhqnbChNA@_Q2@2OtsSR5x<$Hq9xd4V zR>pvGrJOqz5KBQDV-(^9^5=jLuKm9Ih*GaWb+pZbz09?x{csUh1KD3Z`7z|9&00ZW z&;U=Nm)(1~GN%(QE@-F}p#>b8cLpP>!3;fE{#Zy<;nzZrol0TpyX&e~k-YFaqb^^5 zow&>LI>t?z$^Bp8cnbCWk?_Gz_c%ge1*9^57#E(g`cX17^Ujb17ik7qQXS=qt(KrG zry7*7!l7vc*5h1MgpHNt*_D7U0l=250&-G?RpN3r^+T)8DQp}#AxpIOcq4^W6H{!f z=(_>XX3yl}+9%}tO}_odHf}$*B0e7=G<&r6t5O@yS{ZEgS(nr$mA*2bl$Z^w(n*r}G+z36Ag0tC1X5biWo&LJ!N%0td@s}Nb!~52xE)J~Wv#2Flce>#4QP%;yRm=XG%eY?sm^xs)%ei6>0W>KU_ywq9B!e8mZ~h{>0jrXOUntcKvcna@2py0Hf2UN3 z{JvI@GyI@tV7y{`ob~t4JogK;M|7GR6l(@;LG$0W=?`$|R7ah6Wqogs)KXrh!@;Fc zz5KU;F!c2b)1_^-=rn=g<&gP8Mh;*z@9}Z`ctvw1p($^izrKcX-6LTW^b_4bfonhk z$#O@oX&+BXrPyhBh*XR6W8E~}Uu*@$0bHY;g~jC5=Iqx$H;AT1mS;teAVA5s3&A=Z zkK5Jk%B3JF(rnJZ<3D|NUXBFR2=Md04O@SM_Ln%0s}pP8U9UzS-W>Sv$!V9~+Dmx| zsc>QbO2)PX{BIbsqc9qGF(FWS8uP8AEfoXRi!N7dg4U9bV;9hP*o}ZBaRLULJ*sfr zr*Od00P9ZfV2rkzh)8}KTqh@l8ELs6uiZ#%DmHf$A_rWa6@TEufTf#$KIYK4AtMmH9AG?@2Q-#I=iRyhs(V50)CHaOXdNN!L!X@pZd7y1PIjt#K zGh3vO?*^Z)=YOw_`U5VqBPBpkD6Lqn&kb_6hSuH7!w)=jV$I-VJc>dkR)5(%6^u5Gawds_ z%QM2E2#8p`)jyaM(}i;@u)T6ih`Lmpca32@HCVB<3_Z6mm)F~jIk7U}gRO-k+cp)@?_;X7qQ~lI_TVyK<_b7d9qqjI z8Tx<7i+Z?nlYgtkbtlx@ev&P6gO=4Q{srFMXQYkWZbU8vA1%iPp|%i6D)i1lz7)Z> z)zHTWts3ll96TR@Cae0$he3<@6b#+cCZ}zUJ3vG&vzig%zinn5KPM8}k+zw@(H*uB zf+^zaV93mOfr@~JZL%LdWSP|B_$`nXJk2KxdoT7PmcR>eupZxrgDrjA)Vptt0Cv!X zA}15$nF##-s~A%iHf&0o@%36G2xuALt${uq;_3W~u5PVs&WuvA;hR=IDT8CCqYxC$$t+uW0AXALBGw4qQu9>C z9{Q-Fz#4my)Ea>?RdcSN35gf0*QTQjI0&0y6Fu<08d^*?=(d2?V9_`AE1)uX>*M+? zS^=xm4&nsfvTt_+Xs9%wFi|?5oezKu2@bO+=%~8TbpkxMLMmId4@69eo#sKyXaQ$~ z3J+Btcv4Zq-w3RM6RxX3u4{7()U0KnsVe8X_@JDr_qhbpzU-oF4Q#6x!B7&4~f2!qc4i z3wC|>m5o=TIc@mnuV?tU!rut7vvTgJ1;l|Yg25NH3h{s)M2D>qz1*)1(eQ`;bdP+$_*0~+Prxy& z8Dh(!0Er!QX`TuOQj^>dzu|(a4ztu?ePz*AtS{*LF&<%rf1u3kTZD$IfjhQ&;)8BR zkZ9JtK||;BP;U3Vh8tNxEzA7K^$QmL$_*v&`K5rA zbT1$LPflAlQ7q%pN23^MH8%gl7|ctbcj6kgxgDibhJz@qw8ssQP0KH?gFp%D=c9xU zuEv)yX;@NH2|f?rkeYXGM4ZcK91mIEElwLL6lKQ;MnMbw(LAYChpJ$CpnfjOwi|w2 zjy)*V9K|nWwMVdNS_dW(V-xlz{;*pGfJ$4n5yx7v%b;bq?x&{FhAab-tAd*H+EK78~=XQ~D$G?K~ zQgqSgoGxT#mS9I=+-wRJXyLlt`M zPVeF*a3QUpk;4r?_r($B7sVsyK`Xt{$0AW8fc(O)Tm6g{p~c^X9XY{!-%mFx24c0ZLFv2b}wclI#aM{r9AU=XK0DX8yfP$KRgB2F~bpaK4Cl7`?n&n?#rS z;kb0&+imzHTD0410O1ms2@WHdyE>A*x zrcm6#ZFP;ycKYX{YO!gOw^P*{$KR@GUnW_xg~)02ykJWDMLLruA*~QzVRq1}%**8J zf6qdtaMPoJ+=;PP4cI(#)D{U@dS)~Ui~Nq})ck;kn8{wq_L&szRTQ(4v5BhQ5xGfIboJiS#w$ zSW>wX$tmdCDi$i=b27D$Oqd)~fy|K77W`iXyLzN#zA3@~)iRCrUqzazc&cTZiUupV zuou^iaxu*2sYJwCl-Yk+v`w2?#qdxCmo}n=4xToF5MLqiSi8Q)E@nm-z)tYG%g4VT zrGD}%?8n}csK@w?5W#4cod@=UWP%MO&cRAzux9l2#{Mm>)e7TTll3bEYRs1IW;H}w zP@=97w|)UfV@9}yBo{RIqd==(>oI}h7?=~KZoQjiuL{9uVmu1srMXtJqv1$j)wAn@w#gA|9hBaZxr|?kt7;f!L+CFO4@|AYo1mPU z-vO$vw;40Do;Ylhx9VD#j0v0!TM=sPXDfPlx1nMVA#5EfLO`Sh(NteH0(SWA6zw)y z%myw}Z61oN#1&&WqepgV7DtMhQh7#5j~(TmFxp1)i6p{72Y6B*e)-P(jgQs z;9ihkp$v%Xxoct%<&dJ+Z3$@=p!oTP9VfqSRRTLU5;tqvEbeFqvP#+4c4GE(ETreP zwSYS;{%#Kth^Nd=Z!Z#HrpA~fQJhxxA)uN$w(PsWTyrw_hTzNX0*2scpFy4qfsJ>H zj3fcOEyz9m<}dQ^an}xHhhQ^y$!68bffYhaziXFK-G0Le61=$9&wc=D0=&R*E|Wu{ z#^g|B)IBJKoC&b(tsOY6L-u7JWx#ARRHkuxOi)ZKSR{+p+%z`?pWG-qh&U=t{LBo! zJ&0EeTPj0jcNDg4=b{~NR3NZcya$RP(VZIRuAGBAO>Wngb zkGCVh3j{Tun>Qb?yp`niRmG*yr00Y8MjjyvyG2Wl)Q^%o(g5ZffExJ6IZ*u|Q-pQ8 zo-7fCDw(eH30Pr^Cur+-L=SOfRG(M@d$=b2||a5OLN@NsRE#vLLDNB-nwN zx#aenveBk>X4`42QM5FCx17Ag^$h@;*|z+4N^^iH`3KgFFb;rYTL}gsU;@FUbQK1% zNRIHoTBZqld7B`KX!KY&W=c%A7AuSv$QW&2+B7v5iNbY5KyG^RuaGWRw<^I2u{`0L z7@^aSky7DR$43o~{6ml;VS>qK4||v#iBfLf68NAenXEOi|~8^rqP09@5;xt7xGO#$H896s*`XKjN&fY4;X1Ya}MvaM?P^Z1z^QIz87d$K)h1V(?5g=xTauFm7CD znkO-7h%%p`GozH9q@hBXo2WQpC!_SEX16P(>=};igFzPcbO!Ap_;d&kqZd+EcbU;5 znj3B=BMgHd2QkfOOEUZ@?^!TCz>~@Xqrzv9mbbH^RAKOV&huteAA@C;|Wt>;hnn&BM6UL+D z4!#r+dVKVBa6qIJ7G$a6ob!Ls;T|>iGN-g9Xzl5GT*n>3Annaa;e*9b7(u;#qUa#X z;^1qj07aqQ+nMvk0bRa!=j@=GQCy?&Q%SUQ2;E%q9d?tWX>>}8FLl1V?jzTYxNhJq zIExF}5Rm+!sCC13OEQW%34L@+w$}Bta<~Z_ypMvgxK)u^;m0_j8b>CAf87e%ONCci z`*o40MC*cXhf|d~h9mzF8?9GkYi~NE4vBG@y2d1zw~=il;;KvJf*!qTx#bnH9g9g{-V10QxPQfIhE)^QY_%J z@o_&1;vkJD^ub>$zNSQ^MFwS>h3Knvf@n!DQeKegVlR4oU0zEtrv7rdU&6(F#86@K z*eV&y<&ae~h3H80SwkuwvN162ssy-;mhGZT(Pr!mm}SPHx#8Q5WEXVp7L^~)V5kW* z`@qy!=_scpYNCzjLJpb`Fti#vfyq=x`Ia1~qRnXCPFx?+L)cAj;QhLuT)|P-EB*et zrucApx|XhCIi!(8HgSC@CH47QUT4rEE1;qMwwW~a*>&LQsXzt}O7LLX=NrYN;|}SK zn)rVRVGLxd-ntpFj&^3j((6T%L~)&u*qYutvUvP$0X85yaUIJ8big^DhI>sY1UsSe z0#{clBX4xB+pEvRyrqBlYC9Z5AUaZM@0HR1)ZHIg3;sm}Lwj(JiXWxoP>CZr-HaOb5aj=+(8YB#ouw}!pt zA|*a#F4Nsc62$6q1e?F`)%pS^CBie^i!-8Uw#?%6wEXhG-%lvv8x#@9k3i0uwx1g^ zu1Wj5azWA3g61etCLndgtG1A|iK^>MRLC_PMx9)>`RRwBP}|rg@ zB-jOpk>!-s5Nw|HKPG@k`Ltm@YV6^-0p)X8C{mKovzS;(T)SR}38y4MNgX;QZ9MP~ z0eSEfD@NdeTq=7bA0IS*vYh>3aKYT|{LO=|dv?He)ACN4>+`jP==7#O+N-5k;2ELj&1K{$*a^EiOgZL3 z&sU$DN1BE!LsnU@1&4{oT4$0d3P^k(MtK6vx*tdjFj*yO^%5pomSQKB!syG8A&kT7 zdJ{A~SeO@Jw+YHKLSR%3?+Kz11j(wnLz~qYfrH69myBdQu>&AMJ6Vo)GfCu9t#HIT z`E-EgKr_bM-6UeRsGm5IJAOp0?qWgwrw>j8djT6v6iC!J_Tp%oghU5 zP?VgS$6G~zPQXtHoezNLYMIHq3?U{`9eoY!oOA6AtaI=eo_o zv|7gyc=F}){Wmcm){6Dv>s(sBU$^Z%ja)JRyBi3Q$zuC0T0vW}R*gzyfDRKdc=cyWLNDF||{QPZ<~Q>Y}SYq@9oGLCOSRrTws3#kXg`!Hd3nyKnF*5=+Ql z6Y}Wx^O8`|d-J%cQn~u{K3E&hPm&=W{r_%mmzoj4PT<}X8dDA~69FT<8*FY6BbZ7K za(?L}9}QB0zHP=^9igO`w~J*fmf&`-YBQ6&3ED6goIJrfIP)_6aG>Uqt+-yehqmkt zhbRjA`jZKg51e|yFbWk+N~nkA3X;XUMEk~wC6Z4OS2ecZz(F+L8BQysGmajTUXCfb zj`&cfS+*8W-8^~ZfT)_(TD9Bg03+nm^9OSwdYUbSrmpGo=Uc`Aq_(j3u_J!yq( zTJmG+@vn84rx+;$v=M^T{OVtg6G4Z7xv_7>_wZnkP?pL zP37igx}gFH&gQqx1gFi3ggwEaznnQxvAF^A9KhLVJ8ZUh5DB60?G%$dkzJ#U%Xm#m zOL;39@`&2Nfifn^wtRysq(uN2BH#X(xW2YzJhm6!1G4hZrR#~M;cQV3PFE>600O8v znX+RPPe>;`FrE^XeraApbat2V77Sr0_D=_-zxQFY7p7Hg^;yA8Uc~|xs>nQi3%zMp zc)>BmOuS(hMuOR5$d#bYuV2r@IVhmbl5)xcWLp)T_rVuDjc*N*x&ztl*~drhFanz> zOpbzVClG(Ara$mn*d}em9pE{l_~(p3w&5J2%1WvQDuc%5 zA^8?pI@8rA3$gPeI)m)=w?*qFnef^8b6-MLQYxV;&__W;Kk)2-Ldu$YMmQu{#0;fnj2 zs0)r)Fl-|OCvV;5h>;eQEbNP<9>hu_R?y70qsaf0j6yiqWEw00iJ%}cku64(j!72u&p1{;I0APX~_3RKF{Nd&Q zr9cbL8O@{??LHzQ*)C$6{J_t%H>uhV#&wbCmuN9eqMz6juIDl1l+q!rrLemJL4T3K zxM>iDQAXV1Ea5DWB|3F280i?cwpS)eD13VGCT8Lms&#^(T32i=ob+m?C6|CsQ>pm*zoYlQY2_k{LRzxUqN^Xd!Ui=Tj`$niJb6 z*SN2WnGY7TA@c!V8CylyjxEOM0hXm@kS(7$Zl45U)jW|MmGd-@Z5DM>nng(RDTGa7 zRrT&&W0?DJm&ugyy&W-##00bIZ-qzL#9{|e>qIl<{DV6Y`1JK(OL9+*^& zv?ey?0Qpl)wcCM7fl28z3POH|pPJrHjFjTV2{uy*FcE24VRg*vA84nyW1%he=Gueg{h+Wn5mJvJx2m|G9Q8M6pFvX7+)Q6eFE^_D z6}NUlz`3#igMAPa0cJf&EFVfqMK*&}QCI76h@LvLV$c9PWr-6GH| zV92u4P;tmK;W9Z$x}f5LbHSW{0+|M@a0CAXQ8T%Ru@jyf_6E%CQ=Mlvvv&T5&mHSP zJR;P@=93gIGbL3|z=TJdeT;Jm9vh_2peW&Md4$jfN6Xk`mKEW$-*;aJUW262UW4&Z z2XHh5A+-l_(kh;>4hVzn2w0IvKFmUYQzvP=*UQD&eB)&UbnUzpN@`$YHX|>(T8`1X zkcYbthpFf}lReStm3?pylL@gw6`Vi#%1c3d=yE)T66=HQ1eb zgA-ZWG)|lZyT@dSm<+$9o`D0DU9s2X&`lHrYvFi?C%*BaRT0X5&{f~iEHbh)8RA4Y zAfi8pC<^pw$*PvsX#^+Qj)D^fpD)Aj&74~tpv__2;|6kq>4` zdcg(Z?;PNO`Z*9{4CYrLmV6fpD6FbrGGaYUKmJ!BJo0m3PX1XPm&2 z@u5&V+GZP`AZJemu6j;kFpfc0IXN=EHAN(rYgxzO-LR)(T$FHZ)g-MK8J_j#Q`}K& z?{0{VU^bt?NyQ>1V}lPZF~fnhdo|m@SC9{ZHWrQL+CgCX{{rHAgLp4BKs9G=d zE=6xon4Zp@{Aq`@)~^hzNwjeDfb0tp&WfMlSzOR00>bFnE+h&eRSEH4F8o+8ZH{zQ zfX4?>s}rXZvtnVNh7zF-oae9;f2WNnmgvxf2nmApDxrcC>cRm~&^H)J`V&rkh zx?I@9fl!AD+q`pXPT;bzi~XgK#Jt#xZ^O4>`9VQ69o(vJu?t#4Nl37Zi-jP@_0B&2 zV>^FO2Pzb$GjN!^Qi}5rdl}u7G9iixqYuHhAOY;gE*7|WD$OOW)*$X`1~I99;MoMA zJXjKVyUgh~*r#21SzcMaTTo0CCC6j%GZ}qA20f%*OibwSp-lH&NroL}Z-E)cnP z$XtIt`u!+~6EF$&&FJRMhG~MRi0nry)>ZK(oP~#3&SuW`&aE|tiDv8sxZiZpiQ_+s zDSc3FBBk_cPLU7|D&6g0;20v1cl{OSh+y;l9}Z*XQas85(afEf2qaS;b0}9>_C0JN z;hl+tPg%6z(?CF~g|k+q_c58%we}^Icnlsonh%*rxPfCwpn`n;_BW$ew@?j#PO0V) z$B{HqD-|E0G$+-d>NOuSEIfTXf*jmT`M)9?wn>uyWd9BlCafKCVT025HIu+c=LuDs z1JJdDw-;bM_2|0a~qK_R**tr8E?qxlTD z1(`_bBY57dz=2`bs7r6%-~W()-Uq+9fIGVo{^ae_$AsFsa2U2mnm|OLF4K}N39wJHJI20bQvckj*Xd~Nw7P>q>opGOrq-t@kCzn6O{E}(j+ANjs6aHld_U3mNFqZ z*f@ON?%m={@?rCSN~!(#S1fddi8Z^BtVD(>ZYN#=J_FIiZHT*sDir`&%=Nigg=UbDaThHyuoaX=2Xo2!-Ndle@ah|VoppAzjF;8z!XSk zldM9mqVPt~O7v_;mtD_fn5x5w$qrm#1pNo8pjC98kMFohW?3*s%|~OuD`g&mN)(U# zzhFG++&dVtxBY0J{J2x7r>XeHYE@zvDp17$JGpOc=A68pH3KmvrG7;TK|!@7iYlr5 zrlNqdCb|p3Pe~t3q>axX9pR~Z?F5}(04biuHIvVW-PJd4dTkVSy?ciD{?b}?E4gXm zQbdijuN%mKkQ&S3;La1IJP`eeYi%CB6i7dq=c$sJJl_#T6%eiW73Yn=`!3`kwKur> z(LMkQ@UR3ZSE*L=ix-pSPARdV!dQaf2S?tml?R`rYBy|kE8Suz>MUIHe}D^v+h$JJ zNHWIjZfO&Nm87+y?Fh+gX`!oeB9mj0K-BfdJE@Hl$sI)CW~0&oX1&~rNsr}B52nc0 zBeZ9>cxx3yd+h0VCsZJz+pd?9hw{)PgeBVq$reC(d3L>_cHxgI@V|w7h+P z{!CY4wm66A3{R=n(7D6Ao&DtQrMyNr5Mpm~8ey515$y>c(wby)kT zG0{?WVTPOD0UR4+Q7%9ZLOJHxMwnqMzL2OX9w zp!|^QwR8z-;z3n43m{s<@1MBV1j)%1A*seArbavM*XWoLy1@O!2ow4wU9^dgZZ&=R+8Bro-kr|gVzp{!U6kM zm<73Oh`$fu5;+H1qu`iQ)gxlhbP?#yeGf|YRIL|Yl^BUB2m1aEY z16uN)9cI;QFzhg4Pg7PZo*`kjgUaf2j{ao69#@3nq5owqeFr$jwcVH0i`YZQ(es0& z=*O1q{A~%E%RM1x1LlZ6d1X4K4S=um;V};O>^P$~fa8h#*N_RTag6%GHMkaN1_6{R z?4vAPQ(l6m$sp1UexL#zsf-0ieuzciIjf5iU4s>xlb>?XE9gF z;ZnZ#8e|LjIg_gg4}D073WMv+;Q^)y6lV8AHWM`2r~UXqIF_!pfz%n;&Vr*2C|#(D zB4fKoqam?8)4cA@WiC1}ai-k7P<&L{ZIh}qsqlMulM0+{=8EB|?eYf37vds}X7XSf zJ|t}dqP`hlCTxVHJ?w_0gu*CiJ`zy#@L_E-b@Rgi}pT{Yo^BYVeZBS)}{Sw;UZQd|` z9eceoiE}9r^$W+|RuDPB+Z+%K-YKL#;jlBzuS;mv6OQ&?3#MOZ3A)bgRQ(^S-aVkI zI%^-FK+(+1##^R{=8P#`DzAfhD^uyBCISkGP6{Gd8$=KU%$l!`nxe0YiWf@p0tk*E z7$DfpSP}$`C=ephD2O7L!~l_tzvo$NGw=KR>%cktoU_l~pY>U5J?mM|LcL=gSy8ik z`U@mzQZYy(tNUSIb zgro_Ta#=y{mh{NzKAzeW*ttQQMR@k74#~5+XG>h?wa-nE2jpxoW$ zLZ|R4jx}p&rem6szlc;S-2Z8=2NhK>&vv+RMUX&8k8&oUiC{7RG`WGc(dz3|wZWAx z=)@a?Mz_#Ryzl?Lb|t}8znBGNMA#&?oLYuU$G+N&YtJr2V^BBA3!A_b%IP6+|L}i< zCoyL4M=M2N!lObpTCB&Vi&^yZGV-Rhn>E3V_fqWBB8zXMYGvAJJ>k9XTj`!^8+MMX z`QDGgYS@K~mEvF$a}%pfg!cg2n&eCiw}(6r{oUXf9y%cabe)r8TSfUKd}33rCKAVz zMYRkDw?8>{P>x`K*HVtv0Q*CYMmg$WxB(hB@u*i`dODGE*w?mR&5ZiauWc;UB9`=S z|8-6zTX?Z)4tpyyglQNo;6{vm#0wrdc>)b;9m?^yF{YsscifGmU0f2@5&bvA!OHmL zb1Lal;uYh^mJj6l<~dBG9-a{$GPEHyYL9$#q8qDJ)YlrcYP_slE!;xfyC!HA?H(QG zBm=_5?Yu8Ip!}8?QUt2`%d6sPij#f$;V2e! zH)yc_d(Hs18=3%_5Z4s%7pdUvk~1r6fVjhvDoL=~7w)+Ra8~nSNer2P1!7s0M@6R){ zTw2%h^v7A^HEMr3$CNR3O%NakDO183f!&AJo}(aFAo%Kc0DVtDRjRWrqee3PD;WPt zXOQEI0L1S!gQ}kvFM_yZmIruoV6Tbb>eGzz1!wN>FiC9FBZ@-cV&V5bB^KztY z?ThY0B@Xt}t+8XTQ%IrV%Wg0^sIc{OWCxWggk-W+UBV;f1b}o{c6CV%Yq>Zrd?Fv) zq0o2Rp_pGDdhm(6W1coJ_~uGS735z&`3DS#T>;|6?v*ZN)fS;%Sb(PE~lDt84{e+~YeL`9?ZEW+pMUfV|juI)HF$h6Q{?o0t5kimRlWnvdv+_lQdUs=S?BjODUHw+Txqr^Z-fJ0UkE66KHuJyjq z&R4u*tGRFKn9Jvee6u%;}Dcp!pEAC2N}s3%avvV z3M&!x=&OiFvGJV>Vy?wqc+?4b*>peN&<$*zZ|M?!_v|_>CN%$3C`!P{D)y-m#k*hY zqe+z`PqhBl6Ff@oTaoa!G1tkXV3mQ%lPg*=UHkI3*C!%MhApty5alZeTvFG>0Q6fWMUIxI@1TECToVL00(nubBLf4tOO#%_$zKSxri71Y?FM&a zeXHQBw~sOnv&yzR+xsW+M9H4IDir){YZg8ds=l!T|J|m#k*N5D8>Qa%ed!k^om$xR z8rRio?OCjxRy6lrx*$+A#U=v!iK5YM6+TMM&L4eT*Z-|gv?o3N@Q-QCpN8hcQ@}0&FXW)pMjX}U6V&V@gPZ~rWA>h%DwxTo z9l}d0ZQFA|pbAsJSrEFQ>w;&HQn1E#Wq0EUDF9z)A0Zs>s8z_5a6OVXaA7_{Vx;rcurCAMRNt>nd5G8N0Wz?jkWGdI4(Q6j%>Hz6j0tugMWQAM@wA@#y2 zHzX%gJiS&smw{8!Hw!x0?_Y)|0|m5WpXy84g$rw9u{Y>k;(f;EE|XNq#cM!y*m+5U zPCF#;aJT1=x3cx%W98hX?&B^<=H$MnW9WC%rEVR8HzK}*k5(ij1wsz3xCHPWru$Xt zXlx?yW7(%ALY*j$N&bMV)%7`8`RL5mDAA-OwwDn7JhawPH)BEaoK(K=`FXE{tWQvR zf?Fb#rIbycwSLA&M&s(smB04Ah2=ALdO7D4BJ>6_z8hii)y%BGAm9+ z-&W&FwtEro1ddCzSSq77#?KS6+|rP!c+Y{PI-lXjuDdh{|EG4unshGWgvoa-Ss&=dkt%zC~V_wN&7zb9^BZ(3cv}*@4ih=tC z-K3}M2?8HtW9f#{_yzGD9O`Qb-K5^*`M8ja_O=$Q3@efh3t3opp%ybNR*8*O_-9R6 zISvd7tjkk>4`hf3eyam-z zF>E3nytlASqp#oyc15F_vN~e!RPH#T?LZ9#L7?OW{>GI$aXih?hsY`seFz|{5h`yK z0QKR2^P;D)Kn2hAof|w``r{`vdFe27O(HieM9Z_W1jTWgdu*+zbMhxS0tj@4 zNJTIsG)YC9JrAj^s2QuB-_9k8qSs6fT7F-Hm0#SXh!&oEN?_bggqQrO5V?oCgFLx3 z_Rj7pJQ75hnRGjfNVR7CV0*n!y(Cp%2ENHnV927kDpWzKL$V%dqz;4^^*i8Tuc=eh zuTUkWXSvh`FIY~OYMH;t@S&Eg#s}PLIu-NbqiJ5FXuj~vNUJpNNeE28T3UZ z^L_kQLQ4WGbWIOp-+l)t2x6yU6P|t|BAAKkY-g2_kb)}+V^37kD%xV><5YVYtioX7 z@gdDyrg*~`2_JfA*{+r39c_YdKjfRDem*c(xqin9tfMygl8NV5X}~B-+?D2)LBmF} zRY-Mah}wN3iV`?k6j>#ghU|+3Z8qt|C~!lA12n&?MB{|QkIz^_hg*oRXPy&1#u}su zezU!o(j-vL8%W&<1bU@4LYznm{4plIXerMpe7)T}z3<|UT~{G(W4w?m9<_N19)mEl z<>?5Ap2x&7P3jglst=l8$1EQFv}6OCf|(61)@*3-mCcexWxN@d{N67}tccp!%c59U zzEqY`LUOi}l-R+Jkg0sJ5!+GJtl>Hc5j|~&P)61Sy?lR?QrfCDQ}9Wy&JetcKS@Xr zAaiT=AAHc@RoPQb{?m2nc-O^{>>e8Jcavyn%^Ooc6mg`sLmXsfTel!imG=~;>1^01pk zA|R^$CBinIIf6JK?*#a$KOm?BPTj>g3F{Q`j$R|g)71F3~t`;D>2se*1e0Vx51 z!5g7LCLW6a;ybA~wcH2>xmZ{d9ifS_157RxtjqoE8J>c4U+o>*SBS4Cnh#H~URyQB zRE}8al_7ZfS|3nE_(?W_wZCWu3uSvM8zD6uHao(pPmmOupkO>mGy~y;%EaWH^VieC zMt0A>Ob=GZLNKtl*?&NgIrLa}nBaLgQ0MWY?eOi9s!xH`&k&E8X5J{C=4C{4*!9;d z#9(dvL9*x_fdaFhITf0hFLPx;!{0^8I;*w$H|13Lu5ab71`A^FrvZZKf%u5WZix=9 zLnR$Tk-}09XxetS3iA8TS7jI7mW*j#Im9Ty3vaKT@SN2t)5Jj;@d;cKrEgzg=3~;B z?sae(YznK@9s>jPnt-awOnsZ8cQ%)s1TnX$I49L8fV@LhUShjr4e$!EN*}d;UE4ls zIQ)zu0t?$%E7jf2%OiESz=7!Q^^C8CYsLC7`=WBBUCZ&|_rx7j;_&I|t+>=4aU1KuLdC?&0^Q z2#Fk9=FS5W7&-^ge4trU^UW6mn@1g&x)fB&&ak1EyXz_Y48hB7(wo}pKES*cM(pK6 zNsTg;HvxrQ)vnwVxI}b(%__qx_Can*vX!L~$CxK!d(=oUr4^kFDpWA38vqECf@8az zIu`tH;tPrS6H6gb<6PgLezc2V_nN6EVY-Bf(aQu!vV!S<5|~ zFW&%_HaIR)N1M#WqAEV#fL1JC&$ZVn`~dN>_XYCHIBE`n9EuE!XKuj_?rk&k{u4P} z%r>CmwL9GSQDYd(q-^Eap_S#z^S+S3OxyZ_m{(oLIo*T9;vU>mO4z)74>}#2}{*9idqh<5LS^_Jh<6x{f#{pWI;P`ECP#5(1=+* zf0IP64FYHy1Z(WNT&YcJi-gHw`u$6yZbeM-< zB|Q|tk>srLrh;Ew7XfEXP)X?fGk1W%q@>@JJtD|=&#bq4W>xJaz6ZT`ni4R1KzRW9 zywS9QTk*x`s*r9otyi1amA`f=-jvQnzpE(HNqm@ict3`1bScL^K~Z{u>W%pZY%_#h z!Y@+u0T_nS)uB!fU=Rz!P+anyk#id8U&P4kA@puy)K`9h(nP-(AcuYq7=?+|&Zp}- z95Xl)BV_f`nUtS^TQ6S_TeWxg(KpFan9x(yCMM_Tq>o7R91*R+1RWg7fWgLJp2)RK z1KS+f_ZNQgvWuQch1~y!RQBgD^D(lms}{D}QdL$ejf(Ag-u5@Hh!^t$ssUifKF>>7 zOo0D7Yv24G_=bp9x7`E*0#IO8c0(5bIFHy*#|OAN=PxRwSW-9Fw~}M6tf59B&3RH? zZan(?ud~I>QtY=y)+_dZ9`rrZj4Fj;PyIUQVz-eodq+K#ZbcxR-$H%P?`)Md!j&Q3XWC!Wc9W zcQyHzfPU^C(CYn~rduRs-ulP#bovo`EG*2GXQd*A$YjvyS0swJjp>7%T$!N2mIst8 zBzxLG@iEiJTb35mw}CD!28fj{uYYvBZ3!J-qjQop2*Yy7rNtUo5F=qC8e9td{w1gXaDesK5BAO)nJXx(1b3T?J5&5i(favh0L%bv!y!fR6hj~f+ zwe7D|J_C%C+uV$%DFqNMlnep2@d}+Ul7gWzJzoWy!|ix**2-;|9>TLl_laKU=;IzB zPr!!^K(z@WVm}TIFe&JB_J4D+iNkXM91>43l#=>qVVhyyH~Odsjy@6Nm&wGA=+a=p z+>I6}NSRnJV|fa(U)gfybDb2zU)Uoj;kNk|o>5sva4?u}ZH9n=0SYl~`|F&{RrBJR zwxN3Tw5Ga+8s-3zxj1Tmk(=6|I2_!P9iITEjl*19li%7?i*>hjvW_~3M3P8ReMEb z0^+!LzxRvd1$C_YrA##}esbp%Hc2}ZfEH?Yhib9kuHcI+czshubFA{#nps=UG%};h z|G`t}M?Zw5(C9dYH4rli;5o~c$5a|z&djRuu@BnZ87x)|u-f^W<sk zV-}jdfcsi$nQ?C2ELpR(0Lhz%(Gtj`m$qmA8 zB8we}#4MYpp+P*KPGi6p2EHJ!G)3?71wex@ZUrE5n{s~GSgtHy)^sD7e>OZ`^W&1A zfwG%8?xMCIJ}56~snmTz9J%hrV23! znqlT*2NdK*I0BcDfnf?t_c4aSEW=fG4XvP_7e254%l>agKy?BMWk=>Hdk(;X@h2qn zC}>?2k(%K*!x+DSglaRxPNA@s@FpjyxZjbGLO3LrwH^qVxQoq{NbBb2DYe2S!doVQ zv{n!Q>|69^12Y1u4#zEKmVH_Q3N#6N zs)?my0N^k-KueRRY`ToY85b;D!kIPi4UamWM&Trrf|t)W&_W*mUW&MRPO-sl;-3

W0Sku=VCAD8SOa9Y=Qz;i}RlPOZ zy5}Vj|IDJj;u$e|U>b`MDMWA#u1i^eQ>CFd+J#o~>fW+kEkI%Qi_xsaAc5#PkrLrX zTQyw~2pwu*V(~d*QF)0~ks|kZX@ko6EtICGtcKqmI`ezUoWcf08`C=KosLSP+59-n zc4nA2Pp*_WJIhOZ!J>$sQ}U{w-3>+dL;Rly0(y8saRQI#29i;{mt z*8ljX6s!mp{1g$qQu>jIuZ6uFe{5H+fAzBonS1{xFjlUT4V`wY6DZ)0JbJ!#k$ z2;p!q7x|kL&F~Pu1X~CldVVFPtBOvUs78fTu{irPA?ky`uG}eV`H+_tx<#i|z$Y{5 zZ2)IZSdBQ(EV|_43>br{U-8NN$s$p4DNeb5p_c+ShE=YE$!48e@Tqhho_Axb@BSA` zavX-ZAnL=7P`L^=|7pQDjbi3lu9>f{UHz!_0k6jT*Zvw}AsLtWAAEIBe}8`UE{eNw zV8X`#{m*0V4Vm{b*>pejCs_Bm5t|_T=l--QB+lA2<^RN~;yDOURv+#+-B0h7c-+ZN zMQH=))hN;8;lNemtxDC{kfm5z3sH7$2?P}Ad?3>)iB-i}CB&jSawoiHhzW>PD>;r0 zB>-Cw1cLAt`5>JOP@iME0`7_Z@&XPT{qk#1$@mQpw^Io|Y|KVaVcriYMiJ(AlhPID z&#@{6Ii?|gk92yQG`EX+any$^)GmWKBmkG!7`IWa9aJyL{2LF0LJ_Le^3X%4wFwXF zMcz}jS5hl?>L)Q;MSzBFgVYCj*)6+wf-V7Q*cgmuW;BEBiB*CToc$J_iBI9EK#C+` z28Xkh+EC9FnngzVfp}pYp=)-QXtqGdb*Ehgi0CkiFdOOZMSVg5NQeLyQftC>&TU;d z0Q7_~(LkIKS_k0D0;J+lG#$o~xFz2HN;u|m^dtF8=XI=~y_T*iZHhJ66O4*dp-&B- z1Y)mvRitoMMT&y=$)`Zh^x8GNj7soabXjeH?q*|0>!=*@LVG6KPmU5k#I|1GL+tq6 zfAee^-jVkHGTvXGVhK~bSiFH&o;ly0=iLkZp%IVRoDd$0+i5G=f%zg5rrleB5R8FG zKKMH>LMD&=&u=0r0k-M%AznlXF)i?%pB2O|Vd6{kxAZyG4ar#Z>Sj^`;pXxilP11I z<<0E;#A8?A4dAWw0sS1;A>~9q9gWEASVLq&FiIvyVQ;1dXysa<8gRgPX5#)>Ww(VdHub$WnQdLf;n)IIA>E!@PR;w5;xOpodsjd%6Obd^AC{dc8G zC;|Cg)I`5x)22aOKn`SQ5&pnKDsi@Q+RYUis^qLA4#AGgXV0k9c_ox+GvkAS<*6wzpiO zJcmfID^S9ED?(;-q_p&R{JtAiCq@8$x8pj_z)3Jg{c;x zkavzCUY6DJT8u;FbAvK5@+X6HMsMk;74xRn?@hx0SWYCQV^>MK2#^6bdG8GY8=9BoYBQ?Q9`(wACUYxZs`@%1 z7XcRoZ%)#p42m|R+gEjE%hWF+onLB{rx89F{R`iGA(&uiyWQCn7o@$@iy zFhk}V0iF>E3r5Z%al$zYzc004tZzd+cs&ax?2YH-YC$cTeHH zuI`v(fY=mzHLIL#w8#$~bP-q_659>Ys{C*DVcVpJ|ER80>u@xx5kP>>p#JRVL#@vZ zQ#>z|b9!-=JrBxUDAcdq8NOv*FRt7a!8buK+GrYd37@AFh^(gijdpZ02;K_UQ2plU z3>_s<_QC`y>#iIpkz6|Z?35=rm31p5IKi3x8NS-!p61B@B@>TqnHPm-9RM9tdukB= z5l2U=up5-11;$Iaq81-#Cyq#d4^8t}UmL`p7Jf2|C*3e5;*QFkV|FK|266>@9cg4& z6BLohfsTYw3>AC!>Ef=tW$VD}pr4F!9jg}_TU2`CK4U1(2QwJIr4BnGoC)mAIsfBq zCN-rKyEoFnu>aA((3L14!-9v2GECGJBL_O*%dqL^S6YIDj_pP?$S|h!+6K_@AUNG% zbHhmnXJ`=U0kVS-ld8kyv3+f1+`2F__MnqEH)HuMiT;Xe;mt8Mbc16+ym8BSY z5V0IKk5hr;q9g%21w!}iSB^kXHf!of2X`dUBm(2@d!L;xpe zrd74~teDxmU*i$Q=8vD;9Ni^74*?}~kHN@HVj7fDUUms^lhDmU(txA8YSCC$105{! zLidwT%*V!h;9tW5kYkG)Z_20P@hSR`pe5WwvLcj!J^vpD1mK9WGmt}(`&P+v^cNx; z)A7~N95!@L2Ag>HZ-%qN%4xR|<`PHSd)d)}VQue1VkMp%VeaQSFbgo%=v0uE&h(Oj z-w354Ybv;dp?jVaG4dy-L5ZO`BkVO3;!J1UeR*DbX>v6TtR5wWq+ZcsoNJ8^4Q2H~ zR2aPzVkSBdK8@%cjE0#M?rvbdr6@H^k2_!yCPos3f=Ehfa&UcI_RdoeL#4hYKTF#4 z%4a29jnD)``h)Kz)G%i6edR-N_95=*V*fJI1IrYzPmFN6MF}vRy9_LZQbi^BWD|$S zS@rx)!n%DQ(xP-;JStE%@JDb6g`*bVwc>Z7Da~Gh3TSOt_SCE3#mUqriRp9m_z3a+ z1mfU`@f;FFu^fUz-jBGsq2ydFNM6Ng?^c~P5J16mn3}#9x>PreAstq;8wc`79s4>& zeUnq+z7H?+6wz*oMuPzC}7`HaDBkDu&iOU2|17k%kE^Clj>A}GV2MAh7A z?7u-pBF5%|1p(U_lmnszu!(T>mk(=YmpF~tnS z8K##3%)dqKK+z?lzTFr=JBw+s~@UuDldeKM>#JnM6)HU zIfkM+d!S_Ihb}|=Gt)CW-79;r2juv~&uA22y-qQ23=mJs5gbVW2A0(^FjHeeCoOU# z$^iz`abeJ31$D>J5+SVF?zUSl^nHIUoY*@vTlW^!QS`-}C(3 z1mMnEf9`9bJ2HU^@#MFEvnDyB@#!->BCbr>NXvmx%}&I0Xg&>v{;ttNh(<$WG+V`{ zfd)2?ygHB*st=*)*57yWHI%u{JCry?b>_UL1h zAq?19h?evDihUnS&meXhc5`&id39{!G*FgsowXsuCUhPET40=PW0GB-yaxys9@8>W z#RoIC+&AkSK<5q}q4l_Om2FRwRM4L+I&c-mm#O+hV&I0~=SyS`YfX!^ z4&P&|a3U;Dqgh`()j{**rx-0YarpJN56EXgS$F@M;%4LD3e$C_g*!(^;HHyR#WcBwi2Et8O#N*BX@`YTYk^v9rY4} zHJOmffW=zn&@|MM3vq3~{Fd7oeqlrAFg()W)uIo9@4Oy9er?a6qLS($(0t(3->dVJ zUW7lfT|an}e95E}RZ`5qdhovV@3UU=H%EU`8%)PUjU2-D{PFhoqMsp(0r(855=^sy zcyQYdx%IeZt2|VZqL~8cxCq*&ai{T8nw;YdFf~PU;e`lfd!Iq-)8Nf0&_2wR zAP>2e@?kDJ7&>9TBDPbX7;_q zm1;RQz|i8;;ZP49q#xHc-kyyl2nRlUCSf2t4UcR6sKX^3y)xXkF@swh0=-W=Y>XBe z%M|;gGa_4LOKS5z$k8?kF1o+aq`nFL-iiUEmcb9`GwEI7&9+VJ{ob8q;X6PX8^RHA zb@sup#y6?NUfG^1C*IkNM@NrGbwK(*%;rh+z3TW$SoQ^1IzXNP&~-8hehbcKs7Ob+ zww>tz{oOdcjU)1aK$uk~B=6k%fk1|3Xs;hJkirvrKRioaL1u-obJ;6>;20W@*4(h8 z`$BL(EfyQ~U*|-wTR8?eM#IkjFCBpUgPVhR!CPbbE=E_>5tA;%{J&g~p#*MU(;uO$ z=G>=!WgthJ<%0Q77~ox5C`1V}*LTK>D{CC`sY4-?E0|7mSeDY!n~@7(f!=TP>Mt0> zZDm0~{L&4aSnOai(j65&r+oHXf9mWd$v3sLA7iXFvcAVnDjP9v>L=R3hIo!#iOD^_ zy(Ph<4pM_Yd>CC906GydL%cYhj>u(}RBe$i?zGdMjWV4V`kUY*XwDzX@liqvh|?C| zhY7-9n0$TSEjbx9CKd{)fW8XxW@R5rL;EGpIi&dUQC?lB7w@|LU^=%`vh%eMXtShS z0OPARMF?DX&k;}pin%VDd7e+ses!%-{9ds_W(7I}&5!ahvklxY#V0_sD$H71CF62D zjI0XvTG_h?rW&%#_pS>`@1P3Dwy?M)-YItL2NLH+8K&naoFdc4i18E@Lu6I9oaO~N z3um)F*q{>4(V`+tb_uN!KJ%R@iTHdO)^+z23KHVR7NUT?UG3jR#)VFuahbLkbSCK4 z%?_!vk$u6~Xifgguu%Xi1VEds- z(N2d9l*XekGr!?Q@)y=V3L49+HtC| zjT^~=={1aungn9N{Xd*=;7Z5X$OlSWvmTdjPp^pK&F-rpl4oIn6PRQiGdEAi}_%1ridIW_s+eU1Rw)K+p zp8GYumFswC9ZpRp8P#-I;TAjCMBZ1?BX0V*bUEy;Qw9st^6bvC<@##IBFujy++8p< zqKuD;hsLjxLT%+aSulOGFxd?eAhMY(N6;UTQBeyW*z)cKx(tAQNA8OX1+=kgShe%k zi&RmT{n2AO-916&SY%u-Obp(&p5yAVr#VHaNN)-h5iUZFjQWG2q&;LgM;_|~q!6r# zpCI;(HS~{k)|MbTxeQ3RtSKW546!(_?MvY$h6)S&4sk)9nxvZsFb&#(8IjZ)bt+Jx zv|N}@y$D2Nr=)PyjGTKED0y(Cs%zxYqd{Dii_Q|o>?USSefs4N!YU^zKLG>@DWOz; z_DrY~XB}QlmB5cfIE0z^PLyB}aV4Ask915M+h2up z+tjra2|pH`k_8DQj_Z_22jveL&FtU!cIn{&)jLp;2=p)9EU2 zmm_FfD|x$6Kb!jaH?fn7eNc};GIcl~ui(yZmAM;O79aYntc1LPx7bA`*jibckJ2r>yV5{X5(}Y2At_1!090ms+8mYE+r&o$Ofq=k$~HyF%V-kO zF+u(^hJ!$zwJoO*Aisio-@puMb^N8Q$IBRZ=J@p;27qvLBP_^hWh)0M4=$On{y zm$EdhvrVYwlu4fJsR3Z-7U6QC*!P%GoX45D%`~U;?EDjG0d?~TMtC^4tCakKf0F#3 zJak0r34N>{Nug;F>$DJ-m&qF@1xEjC&m@M3-v9e)Q!Tps+^4bg9bqly_YjMvdk;27 zu9e)P{4$(z(vAh%lD~QU(ot!m+ak0g)NOU_HAa`dA;50^-)l3u4yY<9gAdt>j2sF? z9A%0x@UY+ADnL0nw2Y?D9x-r>fhvocM7c~-MSF83y ztU?;$t$N9XYRKG*Gg~u@meaUniq+c>dEG(HWD15WpX@;0GZYV!2BJ9LzqGG0RAO;MkWR8QLLuTlokPA6iX6&yhzIWu#^WJlPUo+-N=9c3kG` zLZcY9YPXy*)|L67cW6MIw=~7weX^YC?kQqZ8hJQHgm5CLU9tCm{w;wqtfQUaNjEPb-9w4Vs*&RWkIlRP=>x6)>1`DD zpkg@98sUjBCk$zq?DH~@1MW>_mrF9sme)L%kgjfnv_~1@R3@moIDiZw4hc0nY3QvE z!_ayn=^l$_>PW+7oFxzWy?=hJ=3B=ppRngM!GK!Dj?T{Y<*JVAlD3_h87AyII}sT} z3)fWsj$ywDM2I<1v?3x8+?Xz4R3}&rCXR6-l0cyV8GYMm9PF4RY$FuoKAz6&Bjb3D z>0x}`D#zQzZy+x~@a#XBHGuzsZ;qP^wRS$qaSwv~Z8C!!Hqvjbs3Khck!cZ)Gy0nSA9oJ9k=33mc`j?;)=Pe}{=`q^x)1&zFH08k3Z z89gopvw?J5PX}JuRSo+M(NxeGv_~>=P|(Roq(J~)FOOW%&bJ$BR4GkD$75?p_%$UFgO#X zU)m`J37vkvyyBDI_6VSD9xa|u2#Z3fi9<|rfn_cqy-!0WT+HVz#}Z=7*wb&6EL1tpS#&FT|DkM&pmF%{?Y$%h3=Y8X z7sdzckJQ+l)=|7xd!64bhgk#7qZsOT-txtsnffdPEj}yU* zBzHpI#>I)PtVOy!9xH}f&w!m7HfKOGB zT8}%tdASg74N&`pyS${EOBNm{$q%WdiQC|A`~!?7ZfPm@DR(A*CNA=ps8(L0gc$5H zVyk$dHkSi62~A>SX*x-2yPIgWgi>FE_u|e%s)%79PI}w^oO`E|KjcspVevl|RaqYPUZ#u@X3y=@^WDFOxX;~8!u8Ex2;eB9Y}~LPdBM4gjp1j(?+2bW9m2U z2V+ql;N`6KTcV%s5sPVgy2On(gUPf884m*urt{E@a6O><==4hOjr=f(acgPzw$DbX zU@h`@p=ZKm99YtSAv{PtFbOJeiBk-{q#}1uNc(I2jwMZgLg<_P=3^*J;Jz4-D_&Ww zt&yO?xX9qb5=`y*0%a%3yFT{sn!J{NQ3!a)+w`$qSmHVi6w5|=MYq5CLI|*V;VT3! zyu+D<3>60+c>r5t7ZwYOxk*`)NVgAio+!sM|DD4;5q*hi$TN)pqs?ST`?f$ zzOv27FMW^rIQU|JS}4>7%(X&$j&RC@ukM5t|9?R}xNV68Av3EiKh)=yU?D>3pgGO9 zW-A$f_ah2op^!@oV`9#DQzk}$Nex1HG^8r{S^ycbf(j~YFD0?+#udq-&dwFbB|5kR(Rz4c_+M1pd^{6C(?IlQ#j2DvJH$ zJK^O)5V&b5cotoKS#d)*I+f)=upwPSXKGLvtX$$YOtp}Ns5(p@F~|cWAPm(#fQJJ| z%ixaomE+K?E3-F%5y~T=B}LIW0u)k}aI?5h(sV(dw4_exTH_0j75BDFMv#70UDp}# ziIpi*4i-scMqm>CU7cbroiZMKcKu9?0!~<=8WQT9fV_z8dHq*aGx>E5JH7pKp|6K> z9ys-bX;RK9RNGS4C5(kCnR44NO_G-R#4VF@3m1srNq}(c2^kKWXg>2;XGMohuEykv z%~JN$N-iqG+)yh9j0vQSteNZPExT`Do5XMMb-~9V&@_Z8ZjaRlcA^R3P<*v;e5R6J z?{AR0gXprn;L##(2Sj?wv;uyuXB2qC2}1$}Biwa??k(}^s(I3JLEhJP;Yf(VmN0SR zR&+;pkBBQ?W%z@W;m&jw;p$2!em!cm6sMy8k!~4@ztF41R$dh3Y)(sUSEj3B^OKq_ zn0H)EAzQ+-kIIC#84%WLzGu|IduKTW7^Ff3cVcdK7>4sB1uNB;fK#E9YaG1}mU4<( zH=t3Wyg=Xo0VD|Q5{A&_chLgxT`v}&dE3{QcsiHi3537zea$EGllBYas6bIa zo0eHwBG)T4}yMKfp9=RDN~!KFB`0Fr!8AhpYQH zb?JcKjx?AOAORpOY;}yva15jkH34jHm{LPX>=`9s^3_m1KanE zF<7l4)nXon#sE$9*orrsRi2-hNqQ^fz73WhXhBrZ0jLMuzEw)LWnN%T;4Rv#1P$-3 z|6c^y4FBp>9L!a6)-I@CP``+YRq9>RsQHhRUUV0!64-0tCJ0b>d8bBry(PlCBEQ-d<xIk=N#uU){aY{SJqq)O=LOOqVaomQ-~DAEw}HWyoA3edhJdw(hF5hZb((d&pA zcR?1I-d(IZ~02NPECWUlHN9u~;&qbbsWcf{zZ)0XG8nYBUBqn~mxU75v@DRq$;P zh(o~()`G~B)1C-f3h1p|vxj@knKEUq|C10`08^N*5MLAV$3ww*7n}Qp?21Wa*5Ar( znH*_dm;N$|%zQNox*a4lp#Z>mNw-qt<0+|4GDPk9S8wY~-mJ{*{7a9h7zhBTRNx4m z2Vj%F20D`fU^k(K&q#$9XN1!*`-S1ILK* z0G+dX-xU%>>1DTH8s3U1&VOen34WdCF~7BR>=T)XhiZd^4_`7c^5@4K&g$e`TZ?Yb zi?H6Z2i!UU0W0Ll}<1$iXtiJU@!QvPa0oPr>a zR0_Zw8#YqLnL(Ru`%tb@Nqm@0rRMzpY;PGmQW;gXSKB0)9Wgkj+XzQ=i&n(%Heta1 zY`i(Y%$Bvlz{o|&Jf4vAeuL%lRaxGHOnW7afmoyZzxD+Zr zd@?~18D|}2olz8P1h|h2d0PT7Y|==*rRHQ9POj*MWkHMtD$3lKd?phJx4EpvCYjKN zs~9qOg2aYdDPH?zG=NFsWm_$H)uLlE5a!j-bc_frGe!MbU4l3#iK>KUR4p}LIDe^4 zwUr%$mNgw;>&!oAU4Mm_g~GD*wdeufDBFQC`Ca8=Yft$fc>v_xG!c3~tUU$&6p8o` z+hfJ$uT`|g#=5sa66leTDK~vS^Uxar9lTpwtwk6pgEvgatS8M6>6b&G7!xP2VUI}{ zyrF%iosgKID4`6$fPq4pm~|<(oT}$2yAFwilL6Sj@DFKDJlVC*Wo-)~WIV1a^jk{- z$38IRAWFldK20DwA#kb#9Mw>pB>`c5^5_sCCv3b3;N99hDy8+ckQo74+fJpRVBF(CCWUFJ| zP)wr+dG3WD%@n9SfwEz}vz9{T?l!#|Mx4vzR8q^41MTtVYMf@CI29k@3fkBA%joPL ze)&6GI@ILXt7v68M}H!+O$WHvvi>Dk6W0n3Fci6n#0>P+iTvV)74x9c0xm~j3A>R^ zV)Bb%l&>m>OWdcEW^ z?4o^`Ov@@wm~A@zxsX0;h3bm+!jTyOr6a*T`0JcB*=vFzH2Zr2$!A?0v;AFhaMo*5 zC(vcJ6rWqCiIi%!GzZ3wT&xO8XjBzaxvAf5Yv`FuxR)!m^V_F_m_{{P4}o)#C{G4% zXpaU8ziwWv9#w+IqjW>G42Z>9#0wK*8`W3LDnn*RuVh;w%1r++RMM0D#pqT11_~Sw z#xmsi_7g-l>Y0+6#z?&6Z>%T-?l&mscnr5sEpF z292*2W2|AO)g_#vf#~J3Dl*IP+N_noee#c}ihc-`tST?XXJJ{7**mw63<+5$GIS#-huscXqP^O57ze7KQ63-C+BO&C9j^j(>l}L1R0tS3 zbK-bxU`>!d0jhx0rL9ib7mJ*$Wp*f(~8mzA0Mogxvi&7c2K#2+*@Qd=~PDIDouGd-ufK4QjZ z_0RKs15ya{K{tQ_cK6HCsZW#77AA`{IaC!`luLUt+lTuO1a4zAjY34pAP{^E?U16H zj4t}NO&0OGi7s*^1w%!|OcF2Lb|pu=ug(4k0_i0flspQBfkW%l_;?K+PDdcyRT4wD zAB!RVysMLOqx86=W`bTqLU)*;T1sFcO@}^RN)n_G>UT2b6D}w)K8=HQ!FE+$F-*JQ z`<{vCsEm5TiHw3{_%=PyDbH|Swr9f!15f_ntKqZqIR6j?2iDq{ok}4m8+>V9^VXhzN=FyZ2@|0fDGBma7yJ*_j3b&k% zFjrVdzA-nTFcwJ&Si(P@{6bm`!_J-ErifZ=q_KNn^iJ?6j1n#pA z_*EQ!$#MY!-ETlWg4F~}W)TBg+Y`iz54Ii{`ID$VM6N4m@>S6Lta2i*Mc&?BD9?1; zx{|>)YOZ0D(_VH+PDmcaedMSP? za;tN7J{=bnFT{vC9jNPFHBezQ@3a0Q!2%>Bexfu;Q4YyG2L~e=VeLtJsf;QVW{T14 zv{tWMsol6JoIx`+5h2~rwicpfQqx26hz24L()YEWUQ#lV!>C+!-mp13UAmovSxYi zmth;GVANo(K5toN(ukOQ8V35xjiv782V2iW8_VsA9BKM}66iY({MU`vuT5+Pmv|$&6iKGc7XJMqWgP zb`WYvfSgZDjR{karMi14*qckta={iMHBOZ#_R2QTGGoF*g^R~SOYLlzDL7bDug4Uz zJ)W2hU6s{OZz7XI2G$#S6Of5V;)oip-R8`HB1UpL$I+h$w$?jUQ7w!>P>$-Kd+_HY za#b#0DOxH;$kO$K+mFRG0TYi*3dw>BQI;_EG4&*(XATimqHKeNewcmSH52p^`aS+S zr|1l4c>swoI>Ub5=iMqt%YDfPH&-zc7O_2nJYy#f_E81mBl~B_nva0siVR? z3yDVUFYD-DFC@oqRE?Eum^Pb{9O5Mr*aDnU>x zt2aIgL953^_Oc`59=Dm#xHfbFRgiGDNxEqcg}q%?PsW>ll{F*-tO)ffo33b4Y!cs zue#mFKERT$$ZRWp*7@Mr>*x8&>rbb)<2Q%Ym5w4ZH)N2E>_5bxP=T`VizO_256$~bvbfSM`_$2=u$2(|oL`#q8_ zVVC7FU01{&zt6`DAU9ARRvJv3C=G@$jTpt*p{{crkRS?w3DFSE3lzkgBS2`yJp-Tr zUk@uRdb(m0YaIS>E1oRcB@AW~kzs}V*bonNvUs_@a(Fu4l!%>ORfuLz50i3nN+DPd zXq#||jMJ7vhy?GHc>o@H%TGuK`qn>!)BBlii!cyTvMOD{7d ziF8C^!7#s+(Wt#O8Ysj)j1oY_4Ay>u`ETWz`xr=xSmA;=JZpF9d@l`(vLXB6)xBHh zAC6I%)astQ$4LdxLo?PpwC>=BfYDLnuB4L?mAqNn1<#xbHukOzx~}`QM4*V7VOfeD zV!AvG1^JuTt!2z`#tlLzTaHI+@Z{IjOv%q zdkT9qe=>-ypy5BgSotBC5!eTJT@W;4{Prt+0s6-h+JzLeM_BC0DCQa}%|cN^1$)>D zV{Kk2vZwK^>f6`NeE`lB0EHJG5xYbN^3k;-`GM}0rMpjyU<6i2xCNLK8dAx892khB zKb{g&hxg1?#2;krCl63BKy0Uf z0@k+hla&IYADSB=muF27-f8AK*0~4N^suGnFY3Pa{~!%6Gz?^7!+jC|B z`?g%TkiyC-RVei;2h#I+AswIf6BeubBfkTG%z<(dNGL9GL!SUPmp({A9~4GdQ8n2Z zv9@{hmR-SY%0&!N8h%DXH#4xVW^uFtf?+og2ItYwWw%UQm&pSgDxwr;Rwd8kPu<>ypSGZs9Wrj`yfmpkG{Nf(fPhV(?wbB z{Q08LD%j;0)19z)!{bxg+_+F1Plj}&4Ae;5Fgt*B-RyoY@;|x+L_H57r2 z+|#9;YZqR|Tj`sZSI=KR&D;;k52)7KTX_F>hxFywuHj{v${ciTpg?@=IG#3W=n#rC zrf#4`YBTR>?!FK8(fEIHs^5$p0rV0liG>{;BNPfI%~@=dClP%WX5#`W?&W$@+x4(_ z%ij1)nMP?p8p?Vza%lTuIU~rGXU4^s+wa$&tA^8Iy=t`g0CHykD?3cAdM%}t8%=hNb~7HCu65) zU|Xg7D*pEz5u=$F%oJmZo*5a)YYWA8&lqimqNBn7(AhCYMi^4JOn9)(zdJ-U?+QoX ziQ}pk4RdtT`^gDR2j)QCu?~1Q7!_# zip3G^!xbKuaTwvqH+1t%O(^c56=7Ia%%G5j)S&yx%Z3jBT8W++U% z5FuSRJEbrq05|%`1LiwmybF)7b9fcoHe}Eh#a^2Z=hI?VB>$*T9%2@6Zok$LVj_mc z96G2<=M^GHQmJNk`%8$1Gg865w818VpGz-BPjte8gTSq8eX2_xF*?MKRDB)1x2`NX zR{P;G3|Pp1T!Lt6sLG?SmCHUzH?i|U(I5NoJU ziHwA4!9psC$ubE^<|zIw6z~U{R>_~x5mwD_pay1y$Z4_ zMpYI6t!z1tupM4?@k=>!d)P(FP{6RR&O7!XSl&_-6n~jJTP{9L9JBLS}MgZrS3{ z^m-iqjEk~g18-Uu)3%@n0p4jHuj#b8ztGq!qedRIV^blhVJRVMc=dHM$8hRP-16J= z!ZDuk4&jJew;Z31Qhzdp!XOyV^HX#$I$bGy4RQ9upajOZMFw*~RDJq{yB(K5|LOBv znD8Mp*SxfU-rO%h9E}WF!xXsfQZQ z(0>5}kFp7^*Wqg_ihL(I>(jmr?MfhjBOQTB-a*FHjqfqa^RIIvuY;tAUBG_5zY8rX zGy4Dq#;Ps0(p9ACyU_)Y@C}?howOFf<|ROc9Q+Q4u>F{inod6yOlhvob<*5~AuCs< zK7yz~Acx%v!eRn}ire!g18{~)T7Sl!juTjR(eDV zw0Sziij|MB1yVI4t?T+0+7)FqMoorAn|tf;r~q-9AHq(RVq;J)RM~V`2G}FPn-(fv z;hiX{4X+fDhrxTF{ozE%(py`RI0ai(b`G}V*r*jv9bbv`Ygog`Dgn!3rfy^TKIEUB znLB+Cv7g=Bk1c|LDV=UD6X8lU_0~``LJ~UE`Mb3}jd)F?r$2*l70Y^~nwyWK%ZeeW?6KR?4u0w;jZe?%SM} z`{C~fxP0PpXQYpdxq0f+x>{r@=m9j?3yYbxU`^B^0u@g**hCU01E6d3oQXagd13h9 z#R5X5WOZ9X^pZVolJDfjYF9i6y({{pieu9`P*Yos0o9yZ1MVId0O*OuX}rQd#Zfl{ zwWNGW)cG*1AjH1EgssDJ(|HKy&x?UcVuTS|;es}-|22VXuo5ti9HgT^MOlI)rJnG5 zms!e`-H_GGPN>j-_RU35uQ!kYJ_}}Y)LFA^QcjMun@}! z$a~9DO&-JRX?NqQT>dx!XNtKjA02!EKgZd=w{Bbkr2)R^sQKfkOX#K=%N}{#2b8bs zMeJ#5#_|NldaVij7HXNGVsH+G&hz_N3^yI_wX|WRm-KnWR|_$sG;nR)Xlwilzu|k{ ziHoA5&8F!oP;NwLr@UzR%!~7LKIG6dq+u(`<^zXt)kvwHp^X;DK$xPbz525cID9IN zQ`X4eo3i>$5MM(xh3f#X0Nh02Wb0$MTMC^#2#NEBi-_GoQY-;?M4*ce0iKNXhJ|L& z@!^&O{tdBm?BTagiEkU>E}AWv5y(~#V5nPmB77s3<+l|)atSQ??aitefhq-{3}jlH zB5uns+BT<-XPcfNG7`_nHIWVkqGv>yj6+DH4kiZ)2PGj%!WH;as5!rW(y>gK2Lqh2 z1!e4iH^ZoJNtv8e!ONB>jA7Ks)__i6;bmTb#a<|Zn3qAp^BjP>kdQ#bA_&cN4|%c6 zZgf;^c`>WvWsgClVLNaUB>8c~;vz0#Q6$GNT*=a13=8Z zN0KN#QwVPYr015)l~diy=XaQ#(TRl=XH6I+Xa;sGvA0fvE83mnppV3_ue)WrYTSC9 zPi1hph3A?7twACCMo&GBtbOf`ABjI;xuAPZO|8slW(B{oYY0cDfb7tClvD^vTT@{$ zuMC(oL1B_pCA6G9HwgOw{a)}`9A_;zW^FaMEG+Jz&qb6uZT}|vwStA2Y&;~#)|e=| zy@WkXAl*|k;@X@VdU6T3R1MZm3^_9BCecBg8H?W>A9fCSMy%QK}nN>IQ*ZzONzD#Zj0$-E|PCsGJWq?I9%8bbhPm-;l8 z0c6F7=oNelJ1wI*{Ia=ux8-rk&tOR)AWpTA$7Mdx1b6mW76}jnG@8Scc?;@FAH+I6 z`Wl|NW=)S$LEE98%!s2~!8nfAh5g3K2v9IXE~QQ7mnX>h_UH@=eV~Sf&qWxQ9r4jj zsXiUwUC77Aj<%&buJDo~KB(;yi&&ouas@AD-8kuty7aGyNT|9cI#{)O3RYoMq$JNW zMKb>YpnwE@u+02@_KIM@KP*HTac3fV=dfnZX$M^M2QeV;xS>Hcuq;H0`!`9hwa6F~ z5NEFflmPH)X%Ix-BCJ%4ioGHl1~zUt*b!EKEGXmH@zif6rjI-bOMyUqiKj1W%g|ebBT+FbG86^(4%)9iWD<)g8{KD_YU8c%QiO z=$S2e4}6%3;OFN@NnHvPv;M*x-0-?pp)moY>fN1I)+!nCX^A{qI1L!vO($R+HQ)Gy z-&^dOEPdarPJoQ&Fx8oOGfgF7Df6tujx=)hSR2{$AyUxu7#3F}$U(HUW&-7zN6kSl{$T64MoozzOdk8=vJ4tVprhxXZP8i?<@7YIGMs1;l?u~Fikp5d zD4z`8fXb8Vjax@vhG_^(f2KxkXqO{p4|i&V^7XFEJYBLo2vCG9WAid^H%fv8GHR1P z1~*&cmvN}UA_GxbBFc(rtthYrijqeDBDJP%Hy7hOUEcypRjh5v8Rb;n&JYTOkQF&R zU8=OsW&)4*DQ7~&MtZTMHgP?uK1kq)wn$#9=8TTLTvWh66zdZibL2Ap4v>zgT}@r& zAB=)8<(ulYjYG33rg=IbBQZCy)KUo=>1!azj1~I0iD+I)eR|0f7HRRhs!aq5X)l7@ zXv_mLFMOg4MH8gwmWsnPk*v;Pvr~{F0C&7lq;nG|=u9VQKKDEIJ5LV80O4n zcY6}?1_DrE2Z8$Vo$}4Y0w{1!2RD-ysjd(ZtZGysa5owp78qdq)VLKR-i8z5u9UPB zn}7bOIc8iwuXcmv19nX>p+{gPx--Xe-LmR@Y^{hwcrR@TZ-*{2gu}pV20sF1y0vl; zCrvd4nR^2C@rw1eG1{7SWD5M`c|lK0hHNwZ*JNn;#jh_HHpDHFs2tguDw zY{!N?_s@JG_i+QtVQ`r+#?c%?f0WX4`J6pOnNhaM`5#tmaWaNXivr`wd-f|Ga#`cu~NOzQ2PZM!l4<>NvsiNR?r@BwJg&d zSu0!+nhe1Rl$1`#teeH8De-uL{mG*Y-ymJvapgQ`siCiB`PVb>st(`4HU4o~BO?;b zk1}?=^djQi?G8QtHzqCR*+9j>%axXL!d3}3$k1Mxpd5%)i`j992t6PLTYXAJB3cS> z$H^2lSe@`6(0%rL%-S2df}g*1z+KRBwzx|rn-j;ycF>Tnc~k2_bC%grkeEEUOEFeh z5}0=$8!IRTYOG5bg21yU%rpOGqBJb|%#4z>Q`e zunrP=iL5&+ERG5V&`hE@IcVMR$?T zB9H?XjrJ+R;pn;-du!Of(^HtRj-KOH9So6tPNJk*Kwv}`0;j6wq?CpOFGva2)UR-q z3~+)^;>aiDO})3q4il8m?-0hnA*-U^r$7|-P8&}me}W2w6i%ii5T<=S9M2Nw)x#_JRS=!*~%=R9zMV$l$Avi7_bh?3mq ztQff^;v1GSUHW1|26n@1k-bcW*RXK%DtV&%@m%k_j*RRab8Lk%(am*p#c|JH>!B9W zKMQFRMrs%=hlbZ33nP!))0}{3$MSbfx*5Whr!swhz{b80Otx@WU?eg1W9rP~jL;fyI~kATzGvxk4Wi zkCL0F%_ksEsiN*%g?*PZ0>+0xV@%5ec_rO`m|qG^t{dj@&`)6^ZPJgNCC

HD2&1 z2|D`CBd3dHSIk@~b8$3M99h$xG>v#;fQhhL%eUJ~B*+9qUZ>y~!k)Dy{~uLv9#D0i zy^qgmni;7LrsKW_sc4gH1cJFjCQKj@ATkw7h+qQ(vX5n&nk$020H!Dj0wV|n2sBfZ zEGdctfdVrLBC_RWi{--adCu$F_w)Pf>b>uKc`vW?I_Es+InVJKSIr4x_JN0B6lgc@ z>>nIhzhc1KBB1Cy#Br#E$CdK6pr)qwd`1jnyb&NT925MMG_`TA!a_outPWep1ZH8o zvJhc_#0d2uSD`*&*1|S5u8ET=rW6bJ)j=jXxz%m~e^lPEPITs9Ll>M*RRXZ7qE&#M zDska0?aw?f6Fo>q!0U`ilP-%UA%AOJX4U`Y3n@Eokx|#rx%uw*mTHs^Xt@6b<_CVD zQ^yi+WXSf`%~d)o`vp{JQFt9?eE;L{|6n?xI$K$fLgw(EQ=~$VA?3hZeeqr6GWa*B zk|y~W3u1xo=N{ue+<#k+001x{gd&UJ#clKg;}*Gtl?Sx_rl1xm*YKHkg&mr!Mgu&` z@w8LQmBE+Ozj1^HA5OH>w+^fa1Z|T%$8+lo?7J#1D#_>GbM-EQLOeKv*9k`rMhJUi zdsB6~nlfA$T5?PUn4s*T&`tIPT9UV(e&1st?fd5`0y}C*wM0f+M?f0_U_P&KWL;#1 z3Ie;YY`FeDL?qw`!C0UZYOWo$uT6cCt1b9p91 z)bpg_D`=H17r$|28YvKgG11$E^6KF%R9W!CBv}e@eDYgwH7^UpsJwZ9(3`C4ag-IO zAS~7|sFhTNg(;5|Q(C-kZj0nq`_8J@8>3ojLD$-5(Z8j(mDc+TGYM5iVIob5ny+B+ zMUQ-uGl3kkr8q@;+p#^KQ59ct*bKEMmCEM=>0WUOL03p7Wnj)kEvna&B>5 zf<-{z9q>AduG!#Gkwx40^;DTnlUhF4&=pJ~=>H^PXcGqwB#I1j6&sr-9K8{$g99*a zM&dk?l^{eaaUrUUicb**2Zg2}>g4D|mwoG~STqoL=;G3BZ*Fe8x=nCB@cH&4S{XNG z?zi;0NW;YNp8pnZ3<4MVYloA%rP`X{srIVY3p+5yvGwOUb-p9yaa3GK(;|lLmWVH8 zgO0ZcOhGj^ERZG*YWnl$w$<(caf!y_qR4ufzYzE*uv-yJ`Jjj?5V_3R80G^l0ybGd z^G268<(xHOc>(gJX40vlv$rqZjRj0FJ0_wKo|RyPlIdWvNbeUUO`PZ#!W0X=*usc_ z>W>WHIE}Fv+yOGaUiX#oDK_DxYvexXKWxhBU2V@B0RrUHiA~6mh64vo00V(1-`8g= zB%mRer?Ld-pQE=0!GSOkRCUFPz3x>WUpFFqRz7SMsD2KgH1o&NsH3_14aE88fh zvS)?8i((9w9NdzwP;WogM2N@$1tiG<$~ldLpn&+O1?!`t1wLQbj!7FRN2Y&+zam?x zA^55*r+4lwxXY#It)_IcjzutnA}L}taFD*{R)o*HzIMM8EuM?9--pu?{>cAiJUcRS zEq#=QZ_#%X_6A4GjMIZ)rJk(NEssHMMRPc52Hz zfFF>ynb0N>-KTa4Gpn%()H{^$&eJFbfMlI1W5~UW1$Mc7F}$>TgP?@ z=EnwI*xqkz?N$vF4Ev_?G){@v*Q#6!6LyuDGMe*gJ3S2(FW1QI&Nvgr9EYHCl=4M6 zP;%f}Mjrn5KX_LT$B0zKkkabr2SP7G(Vem1_C&q(k9x*&H%-|x{_knN^i!p(&V<&t z`Sjin{urkr<%$sn-WhuG{pL3PIh&1YCC5p8EJHDf{Wqu}O~zzk4K^MH-vQt@=5_G3 zZ-Gn8k1{R;-6;E@4FEc-Dy*Y9fO z=^r)3?#MI=-hTSa7sTXtu4}L1Cq`QaDqRK~#6;$sA1q|uk~GLCis^h-hH!xEj-YeTEYzJ^fkTwm|G_1(Z7xfOnE8BL14*~G|%&xj$ z5^0E6wkiQwv#XcH3NT2zR`sXrW~_G^3oYD56j&%I$o(q75x~MF7_6Gs$cAW+eIjb* zr3M88lbH7$<2;CzA>#olwRVY3=u9{s)Irn7N!19!nXV4IRrOr>-c>>EE^{TjVBOsC zWt2MiV0br zDM6NbRzHr{Brvk~yux{X1bbq%1ZcoHmLz zK&$29g9yS$){SYKRXZr%FrVtaSo`qC2VOrtgDY7BC& zo+Ah!P?0txlj*ajS1(DGKihMxw&9h6F$D{(CcMd3cY_rP5xXe%Fw_TCu(!{#V zjTuLM_A16mF9(#5h|3WWGQ%-LYD);KL}$4%YSk#3>|1rsaxo~P(6H{s$Cyi!Md~z` z$NZsmR6Rdjn*701ZWX!#hGu5%qSU|rqGTAu<6=C*ir9BvKG#)7-qz`RqU(@mBGWV? zq)+alaE6HMQ?wEr{$4fCU8(G4c^*@+dNH&oh)SXXsbhvJz`6yyCPA==meYkqJ6-8e zE+R&Os`&hADjtlScO%V+cR%1sR>-t0#!mjkHE|f|mI(|^q%ARwjlCE+IV}Y0 z9!j+-&)O|={c;$}kWw@1?mY)BzQ-C>wZ)`Uf!Q0>K+wLgF%^&rSe;Q?A=`*Y#%(?) z>{!>u4o9Xj^}#p}SMH$C{Lc$Tz%&^FW=|=W&$mifEu8MRxE3a^{4gx%YTXjXt`$URN zaQNbEwR=MeYRTC=z6UMwP4sRW&!kkD@#*GFvJ7HE0A6pOGf!Fj5BMDL4CD=?>}iv! zTRExSaULJubri5gWsVJV3H&}{PkZ4S0nsDaE^<+8m+}twtzM80ALRccd{F8cHAwB+ z{$vIt)fw4U5-jT**Ktk_e8sGvgd-7i{v?|Q@w3x*<$_0rp*r9(!N8LkdW3Fl|q<#RBS3G-N$KT6a{iM=l zBPFJO-%Wdht_t1N|8J!PDW;}P1``QqJ;PlN%unGUZaqACx4R|MOC-Xv*yoKmB#jm4yd-$Yd+NQ^iqtyQ{e%@K7{sMj3;0ZO);`gy zVpWD7jW2G4K;j@fpI4&_tA-VcW5P}^liN{{`BF5(=&&)pi63UwxF8;rv;O`QS!paY{$qvkPLti@!081^;&mDy?&>L zY)tx~%lDnvj{foJmc+YHz?xtjuYW|aK!~2-NoPRVp^DU#N=hXc3*WLX9dij$k@y(- zc*(i9i1Y>b3!6v4)z z1F|dg8$UIFdTDlsn3PQ27y`~7;l5H4kyXSB%&D+bnz9y|`z0r{PC~61;QQEu^P`4% zNmFlz9&AM9Upy7v)$yPV+W?E|vyG}Om3q>kfunWV4#Wm`oUZ-}kRh-ee&1-v*=vX} zyB6)P7g^SV&BdkSYb+Lx%pr;wr`;CKh4=c1Z&nL?Pe(CQ{t#^6q8!yT^^qAuN(NVF zxQ3R|<`9V|Y#>bnW*EQJk(fDfAF_*JdrGXlYJ;g5&!7qhW=)mKR^?*pYfna&iiQHJMwsC3(jV|abj=Jzq@ntW)@<5^lU1j&XIB|3z znaPJDehF{(mqyx19}v1FLL*O<7z6-}@?02f1FIV1SbH*2$e-D(1GvC5A0*hT((Zb} zGb+K11~@n*aJJGbVk-tRB6}~<|A%KtagPoMVKnUT{BSo6GWA`M3(6N91T!8RvdL zg-%Wt`a!X6E?C8N3tV#FZxwvK=*mn|Bd~MSFIJL7rB}{35z?#7n^OE_q9a=Cr7JSE zi3PWjbxL9jaATqam))}OrwefcgE`HETiIinv+;#WdY$U#Bf@WJe3U*g){lR+yDY(* z@C4QDyttq`o0Thu2p`lWEO(!^do^2(Xx~}Y|s=GMP6Gi2MI_5@ODbQ*?-8>2-^!1g4yA`_+3I&%!2g zyBYy-V3eTZz{T-OB$X zib=rcwz?A(4PK^Zwzp8(A2|GtHQTv$f=40O#Uu(l!>ph9Z~$wn;e>=SjtQ3=5t)E!<1vkU?FelJoal9o9J{*XzZhSPAPzaOuQ^5wy~lm1sj^Vl)??#=*lk2y%qU|5 zc%?@BCJRYwo z&4es~f%;Ie2oY>xlKRfgTjdOupcxhwqU0bD%-)el&+#)*{%Q0vmeFZll*AM88%ThH z9M-C#8txxqs0K5*%y*FWjBrcZFxh7*s!pJQFP!c$VkhnHn+T;$)y z7}0q$I73rnXp^MoqM`p}lbQp73M;EqR+y_U0TSEQams@O{fmxws-e|(&L>HFwOF?C z6S=J(en7bK^D>hKk)Y~nrw#$ZJW%6hChqXM*VYr#BiFN0)Qj8#4AtmqXEpk10X0&P zbSPN-2Gn1QXTXyTLF6PUleSrn3JOnr4DAWJ8qgQ!9@$vMi#K`XD^-aFXDMc z(LNdbFlAtPAjt-A3C>M@h5l{J>FB}lCiVw8QBx&$dH?Ui=IsePVVSc;&PShYxm?pUQn&gXyb!`8$jk_S37Gpp9BShEehqNtgQ{akI>}aQ~5jb zwe9F4V^w~yLt{wYbM^2;s0tg`ZKp}tWmGNviiZ}7sv#oO;rTdF=Q1u4uk$x@>=fNGhm zB2%Bc=0BqIpV1Rzes76b7)qJku6H_@2mq-1*9%+@DsnjB4e>~$jL*;`c~DFc-GFLv zQ~2<<1dgM@)(*^B)o;;NXT%HlQjP3;N76CXJ)^GmBzY!1o0rQtPx_$o9G`Z=5^(BE z1jvK*6H$Z^*CfnkBYaTx*$Redny4fNgB#;r#bUP@F9=>LYnQ^KtgLN#@0Nn|Akw!* zs@12xD@n_Lcl#n8v|0IW=HGq2@=gBg)@=I7TA|L zC~{u=ETH5p?h9(Sc`YVGE#MNzTw`6_$yICL&Ox5i0#Wu%%P1j3#PsT7FHe2)AB|)k z5LS5l+f_IvK(Ia;bWW*FAB<@m`XVNfX~nx?iy|lv^3~f)v9>N?X;f7ozcbuM*On2r z+n_uh!!mHX*ukU-San9)XUWjlci{;|EiBs=1!4sDvhDV|u8d?sBkE%9cA^={#@}bU zu%W0#tje7j^OQil>VxX{3%l<<|E;uEAO?k*Fm9LR2e61y@De`rP@NdjNck^d{lXhZ z7_e+c}``A0kY%ANlgOCp^C33+&@a9d+GOGoD$Uk75A*A z#TdV)3*4NN{_~|o{NeV$4_HNJordgvUjg!Ig<(fO`3{NdYB4O=UUc752Dl2LsEehj01K5BHdIbp&vzC)um)#CUe8>>R|UDxSnO-Tcm*NT_r%otGbTA&=(`MCvZ| zBW5yfBly*BCNM3LfJv)I7L1VQ)HxS?<+3+ z18q6B(O3J^>|O-@0de0pm!@R-$t{1Ad?&c%7%aKuPbLKH&YZiB+cs45_2r}na2{~= ztSqt1aNhw*6El!|fg#wOH53d%vt`0IzH-HhZi5JHkv#N60Jo~)vP zZ_m-|nTXavUA1+-$!E{@^w_vBAJL{9w?(G0whKD*|IKkc^nra#3akw0;Lq}^`c~OR z#BIY*x%={kTNH+X6&6E-3rMrok?GJpWTzCnq_1L##yM?PJYhGk(rn&CFoS9+T#hY4 zy-~YyG~*mptQVe{D`X*HzadI2j{J-m%qpAQD zSbK#f3Iuc|$AT2r(5sTMsq6)WL*B0FZ~RP%%vI`fA? z)*n(7;9lw+R4%n4kY*$f%5}nC*n9?4;3WB} z<`^fa!rMlqgze3eCW*7rw)9T9-wGMLZv4%IBPse1T?PFkL5CUNp(FdP)7=~S_tnsC zBo6bpCEW1fE>z(pmJuoR1=FCvmEv8HhHGB6p|Jox_{dBK_fAbsg0z5Cbi9Z=fS;%L z%VO@SliI2OK&4)~l9{H!Cf6mWAVd85_?p!D1Wr&BjWHbNrxbU~}Cc7YIn_=_o_WdJegR)hULs;IGVm3cAaJ>Q$q~;nMnzf`f0**$&A_ z@izD64ah2@K8x*q&bqJQv`}7%wNR@!Tg{XPOmy$H=0#j^wU(yX>ACTsD;Qd>Vqe7K z=&b&PdkF3xZ0FDe^{k<p0{q1HpZ+r7r>js-i83~)f zQzJpNyL56ZI%rO5v0QHh*?{F{R{SFOZRAg3HD^JSXbwyt|g zG`?8jpAep3$x+)YsWlcl%D ze9N^JKt`UF1-?cAI^*ags`&#d5q|6MW!^r;AJVlmVSFF*KCZvZ1>`G$BKicKV?t(H5C(u>dT?4qNPdAsfuXXgF>rTj253 zsEiG!s_(AR;I%=cRbKYDunzv4taoO$s7uzUSeA(K(~6P_Eu^NUkX(Mj34D~!b}*Y#UJ}K@ME!(`G18A zLP@0iBY~*y%Tz&AaJ<-Lk45&TR_w0vC{4&i4R%6@uUL=La7Fz`{-kq7iyl8FzGC6s?wUw zy)il~88M(}80i=tX_RF_N)rsU@vfHj6iH}*s`9BHpcWKrTd>%bWFrkK&ni|9!HOas6`ATG~M4grBmW0HG!r% zag%QL3>e}Gz`U_0LJz}dAtzJnBY>jq>Z3Tq%VOevtLG9 zU^!Xs3>BOpS4Xe_$9^6=H-_4jc*-XQwmNG=Q9+C~q$1ShF&>n@%TWWZ;I zn=f8ayF{9h%#LMVz9Ev!SGKbM9cV`WT4j8L%|F^u9s=QGpDl1rF&^ddZ?43HmOKE>Ncg3Y_#ohU^1W3WIA?b2A)kd_}en1*(Oe>V8N6U@KKx zY@`(h;AhZoP|ruy2Y++Oln?Fo(8!3mc`!4~|0Ym%+PJfM&X z<~pB}d3xTBn=d*+fucn9UASlGv>SIGe)BFCTUFY*BIT_S#wb zyAEFwj}`sHteUWU4DWU>dXDlCL!c*{F#WIMi*)KD4vD+2_WOP^j%)g+XA-~XaVZF2 z0z!WO$u{xD;g|m8GM`uI`;YbTKe|7B>&FC=jliIt$KAZTc~j?yJ)>*eHIB{v4FrpMPW4YaMc83oS(3aSj0x#%|NAhcT$nA0 zcaSN)+b#g!QgU7)(G;|qAYW>D$uu@S>GJ=@&LyJMlC6v)#G5)o`0lZH+RTa01XG>p zWu@?s+`>cD9&)ZE6oLE-+c~pqqZ*3qKS@54n&c#~#H^nL(Ew%7Z_LLi$s(fMzsaWwcmvY8 z&sG2vXULEcGR>GLn)}rUW2C5*q4zF-CdfSQJPrl8#z9f|`3j{Jv7C-ga#yLht{OI; z3p4r z35*&1P$q1QweVG;W#?4$jpq}&>o|MA^3X9%$v$-1B%V&Y3kL(0LihM_uCN!BS$#y_ z0g2PUuZRJed{^&oBzB+o<2F2L)r$*TDQ_ z+}1vG6HMUs-n&A?A-V>}mC5L|)ba#+=rKt9FDQ|Jw@E>Nv%x_mJG&&lS;tWt)U>$# zzm7I@ZceWC8zl@x1Iychs2TPd7e}d9!#<-8$sElN!vBYXg7hlDd}l-0HF01(3}k$T z$unU(;ck?FLm!2};3W|$20UjK1fmQ1tcMw07X*w%D8qBC6sxwTPW_i2Fhou1tXW;xlF&AJ84SJPD%ewCKgs? zYF8=Vq@wa|-%`vj4{Ygxj7Xn0GLYuoNGiglfuIwLQ45kERiJ~|j z1I6B(RuGKPv(Tr61gbFI$IOe!8d^3oM}R`d5SgX{7`!|^Z?qBSd2QX?RC3;dRjE54 znFZ66X0LYOgYOcg3WKUt^zZ%DE``6m=rYdm0)x+1IMCSk-d^nDXNiI-n8R_ zU#g0z3of#tJ-yY!hdN0P2vj`RCydZ2c z{Z};u!}^%*^0kcY%oZ#0-sCDbrDkcnSC3tSL*|sck9I@s{Zr$r9-}IZp3k42#dqh? zXG_)DuQy7eX~-Pw#7PpPCI(G2<|m*y(Bi_< z9=j>V5a^1k&UbODOq&T(V3a0%nnOb*mtdPIhB{%hKmm!7xvMh%7tu zlF+}83Dl69u$i7Y*F+|51gEUgRd~zWFR47pXX({De%P!>)}iJo{2_m%5>}bEgApM6 zAf^h&`UaB{1W7MPu0t3$;B)K+nSz%I8$;P;Mz%nGTV0w!iU5ziCU&N71V*F(M_QxD ze?-Y(vi%rUS_C9Y;pC_z%itMluTufM|1IUUK2p|5aT1-7$|IHfVElp=EFb9f z8@z20F%j283Ix2S$~9a6*(~W40u=f1G6toQ*K$I}VJMm^A~6B#sJoz($6LRKkZkFIo98bteM*GEZHXaYWdRWN zbHo4&C!Oqy#YIw#P?Xb+sZlH>0Js080~e$lzssx##VBfiVV6jIyir29f)Bf-3&ZVx zdJ%h`V5kkjhd0tuA&k{7G9K9{(ei+rQZqDyAsK{~0MNk){!Ro>6Lb%gz}c>D(M=Wv z{??~LcgXXM+e~#>+0P`38oG%|lBNXULBGg9H2xR@A_M7z39D||^90S7MLw2)oX$~m zGq#HG18O@(Hv7F_;R7@23{kHi1@aZaLHt{F+wo6!N@OxY^2wj%AfyM>%$Gkcp8i)p z;P?y!@h;lXndnDTq7lX-8Uba2u{w?fU~r!4>7koMNjb0K;DX@>N%U?|?jA0srRHi= zVGPCHf1J%+2^uJy7|%7T?!Y?z`)ckjvT!=r-Yfu{mF%n=$T3k$7NO-#lc_`cV{R7ecxHG45(rdsj3}a z$k-D`r$kypPXg2-{lv57cN|Ajt=NSEKC~8XS*<_~0__b)dL|vE6cpeC!_~6#`|XAfn^Al@y9VOqiUW zP%htT*G@#?mhT16UqAn^HPS5M{$}p3X+3E#jsr&BcabGc@&jgGJ^TVNGM%5$OL(|@_%iKooELVcB@b%D9p4OYF0oJ*aCl5EQLB?u#|3TSjx)V z;h-*c-4mA;J$kL?vm#+t?n6i1f1+ z^{OfVypYciW5B5j@RE;hW25^IE5^x)zD=X8l_V<|V#?Dz+=sXsv3RN|lH=*E{Qwdk zQkc@Gu+h-cm)-@l5B(@qeZ@&fKGIoRRZti&2D(CXuo5{%{6Tt*#C7ZD3TvVnQW8-? zX9e+|_Z$cmZY%sTZ+}~fa(>`+0 zkmoyOA`$iZoo~W8IJlc&p3FT%M0DYw*5aHlPmLFQ>T^eL5`IL_Vv>y4;)+hY%Lkh{ zIeoGQ`O@&Za)=)YduWGr(543D?r&fsRTrCLuM>U^p6^^M2Je1K%^yO~oK7Ju`f(6^ z7xsq79Fj23yD4kDa)(Oj->Qhq!Svjr_KX-0?V~ePsuZpq*!W_>z;A@$!x3&?J~xCWo1J!5 z1ovWW61f1e#XdhrF(WYzj2nF}W-@H(d|x)EJIXY6MKJ-|-cWt*9pcAv!3sHl^jj}n z7pdAo=fVOb(7M7EY9h1~nW z2cS}llxT?NsTbrXc+*={XQ1)Bf zW!=!30I{{WzAz!NUAu& zGhzC#`X5TKLt^?jLXH5(ZCKH^@H%`Ki$IW-?Y;Nfof+TSAv9%{;{j>~Uv=drWE%OB zH;sEuL?pI1I3_6Zh>(Cq&%|8eLDHOwMT|~Dm5b-=ed`v1QjC^q(AR>&(wo1|AZ)Mqd>f+mci~gbZUNkBP z&=S2kNm{R{FS?hg@gE=`0MNuVKlVvF4V73nC#bT_aP_^5G zvJQB2U)L#Af+ti<<7sZuPE>8B!l)bSlM6{|HK7yQ59G=sZOLEz(q}L{>7t-HsZOWH zy=sR06g7`Y>oO}xrl3jeC}h`DGp0+>ADTPKA^qM*5*N5gle&b5*dy|HTVi4*Q?J7d zi#IYGwJ_QEi5%!HaUH5XbMgi6F3ms}Edu&wYH-tyG-+cGC^!XU;H?JLnl2l`uNsdl zeYOjy--nLu!4w!GRrpV5Q)UI}9Q)Yu2r&mySbP(Xj#gqDXz(-0rh__bXD$JtTe$T! zm?!7no+bc+wY_x&B11HQr6uI@SBB2<_8;gku~=)-+TGdPJ-@hi@ZI}`NAIW!gGTUu>btoa4nI28|{!AFquuIyMZ)eHSq!u=te=&*7i{ z)}P@qvZUKQV%Ngi`7d1WJ-qM7h#5Uo0=l|CKi1jjI%C3&%G%hO#Vam8b+f2)a_}&n zaJ*AHF);ed;+PSZz6(kWEg$Yb+CMtha-#KKl=a4iM#nu}ZINN?e781vag3g2==*Ir zr0{5ejrBn2O^y8fp2@YE!3Q7Xr-BZY$`3SGS3g}m(RxqZg!#W(mf}=8XH><`K4iGE zG{0;cex|VP-saF7tMI25r=2j9zZFuOtl8ju0e|fDtPy-`PhH=;n{&6GyNj<^oV35l zuWNd|w5F9FrY|3FG5PcA@SM_JGiUeBu{*ij3a1n@_|qmsewL@VlYhJRh5Nj`|0Er6 z#>Ev5{@U^mj@ne)xc(iV+xTMeeB1DxQM}H;{I%f*JhrRGkDuH$c9OHzgc=;#oMkFk zc0wyxws_sb@Z!Se9_z zY5^}S#qZ#c`ZF3Ea`3;sZlTsQoQ~t?vx@F{Ht`D@^9Q&w3ul+~eHuCMwPxAQI}Y=~{m;c2&hG^ci(7v73L=^ z7e6L7A^aCf&ZTztok9WD!jMv|-?&Sa1oeDB+{+dhYgM1f0l(2fu z*IaW8KlvE{^U6v0+lPy&yx+GZHD@ZX%3QyT_dfJt7SHOv(=nfg@q0&_pDY=A5{LGf zMXW!$dp#cEHQP-KaGkJ7KOT%{5q!G-xqq*J_`mOPk zySM11C;xodypv^CKRfY>BFfojY98_S(SGgm)=f$BUW`IRC@lyZX(Y5bvjLSUSQlEPU4sK=d;4fd>aXQ@Ws>MK1 zVNDSlKGfq_(tAIW))+m0rl-qA+?%Qh*KP9JSeee#ZW(*hiTArZ)S&*!5&lj`o8crr z((0#Q8kk2M`Y~c=R_7G!3pgW?BMX+@#5*&x-l^Denn#t!B--J1b*g>t@Z3ARmr0&D zSIhj2Wd84dQg7YKDqeb5U@D(sAD9LH@fEb4Yo1MS@s_qM?073ij48lWbMN0(d;rN3;NSBe6*Psv zVDfw}?Cn5ZdDE`pa=v#rJBx*x}*_9%(H!XM(4xmU%eocAn%v>tx6 zN*giQ9lKIFhxn#Hs9uc|zcs|j=e?Qv^17iqY^enI?E9uzVdb~iX(S>vyc zax=bnzr$s3d0uQW{#zsdJy-2jS?iH?m+(tnzxe;#n`@{#FVLjG-?YVA`zG3& z){pkz)AH*b>`eI5#Jl`bZ-eZs-S+Ea68(65TcFMO4zZs^bJ)+C(q@Bid3>K~AWs^V zn=!_jtz3tL-{UdBnBOUD--RPrQ7qRyf|nhB{4p17Z)*@Q=)${!$>k-}4%D@se9&fd zd1TN$UXQ)M?J%v@==jUW+qFjr{OM`XaQls)srleoSP;+<^=4jE+=C6bM{+BQexWb! z*t>WZo7zq3UCv%CnzJB@*H#e_l6_p9lSxanyVr>0`L3n8W!q>+jZbFAb(Q-7gv!(D>+FCcgz`&!+_oZ}X1c|7nwV zH21~ojknABM(evjce}1xxUw{Q;T_n_>eQv$OM$ZCR%O3U!!3!Rr8K7C@#q!WY`v>I z_p;r8%@e;==$N51!U5TQJi4u|<=F9XShrLl*tXbl%@_V z{oxC7d`&L+;Z>!#-Ztmra2-C1R~%ZGkQcCgam*OYJ_HD`^^yqNl7hE*M1E5%T~c4r zvgJOVH^dT`Jf@qgESm2V&`bB$`=c+s(KH zE4Z!dMc$j3Z>#aT%BOs$wMEnC_1 zshN=CZg-Q<%BIO@Ep8V#bESH})Z5p~m0FXtlz*!Ff9;E2ol@xfdQbo92jfe{jUBt0 zz=b-(a&SQ$7m79)PibZ9&4qPiPu6gO)L#BPXpKEhFXLg?=LjJW#8qy9W#E%`w__y{ zVjK3SHY87YEOu>@8z2|0y<@^#nLNO^3)}Mp3d2v*`nnuX*O*Jd7@p^{B>;X5``x81 z#*{6#1?Qgbu{dpO(0dL?!WYXcrViG;V8Z<}XS>$am?u?@#kX*FtW#v0A>0)gZDCE$q@6M)R|ddxQY3|Tw|VcSmzBnhaFRoJcu3$ADlBgJS#3V=Bz9&2T$JE z*xIEP+@|ADy5bIRIEJR?*5kUuAb^ zv@>EQQltm8`kDTUy|C4OJX~vAK&RFlu^NFOT+Ty!uR@4F&#e6%E^9Az)Y}Z;hOpSr z{7pjb?<3N^CBh4+mM3MJ#i#2Z&@1M(C6>bnc=pr6v+ud|ADeX`7jQ*R$4~Is|9%Am>a~nHG;sV$g?n6l#4X|+V zufxSNci!NN>v}J{6~@Jn>A6N&*3vs6K=#Pv|IAyOc}ulpPtv;1VljUz5oBAfdC;g{ zx}mTwE96nSO!zZx2GZ=?2J2qqcZ1vBf+Jd~S#Is$jtvO`pVKmWrqt7mK3FFKU}Uj* z^&hyJTX$uL#7yS*8|}ltT*)1yqR^}84?L5Y@tOhq+4{5d?T%|s?0=7oz<+AVH?maE z{B0xecd}jbz@H;=5gvwB2=NM!?!0?zt=LoLv~;)~ zO3%{djq_Y6=;>k7gutQfQb4ueMC&i=25X$Pc9L7w`lU9y1bp^mdx{0GXNksQO!$fXH3wMuiZyzcaFeyKU<%G~bH;gk?b z`7Pt095~!CIACir;B9GsIY1t|F7mrz6x#L{uW+D-_qeCPJJMZWzG)KfBd_5jxycR~ zpILzlFD|nGa9TunvIKD@2JiDc%p0tqsPNIeXN%a7Q6Z1S*xxw|cwnVfQnut?am!zK zlPEB#f>!q|dxyzHzO|h0e++va+tPkRB)`Y(t79PL|>2)wHM9!RQg10A{;x zgEiXM(9!=a8boVgJF`WLEm;<$10qhY5kKO72=0t3z;nSr-j?x zI!;$)d0*L;H67E4!W}75X6e83E)RY$`?bc!@MO(+AK4bxlwW)rXK{MjO**~C@~1!1 zAg^Z73HOwbPADalgd<_GO=^gmS*tKFGrRRzpolI6j0 z5vRAKK6TS)a8(nI*M~buSb}ipxqrvDe|rph?eIZ^LFLVKLniKnHN3f0CW6pdj}4M=7oUhRZ>Uv0zc4V^)nw!h;_u8sNE7W>bEdALqOBP zU~6S+O2Fk4O)dl+e4;KnST5kI?zg8Ziiu$!fPkkY z!j1q10xkTr_bvXFYw6x%d95Ad&eH9*V^evSVJfrgePNFn%5=Y^`-@}77G4afCSFmu z?2VNQ$nm#szl3$^?Q&ZT(99B%DL1Wr$Ky=G^=CF(eFP8lyWxZ#D3)k$&H_O(-bvKB z6v!Jd?BN@Mg8{d+wXgCF#0FuJABTTTglBEJgoPgN67;!;Z=zWShacQT^a5^kN(nD9 z!!Bd9bAY_DPu>6|B$&p{f!l3{3=1{)65^v>cnwD1n_6>$*M9Ux(D+n(rPS~W%k-`P zZD@HFt6OuqP`}h(T#`#bBA>WtC1TOJxDA$}2F-Xyp1nNy6>by|qfs$sm1Qmtr2)6= z!tOww(FAoR@HS*Nc_TsE<97dIQNWBuhC9`xV(oasz_4%xg3%g3IU@M|>~oA3NKw+qquy>X~8G%8$V_`g&y`SE8c|rq20-vIc~D>F}}CroHeek zZq}J>{(~#boa=4CMFKdp)cGYIn`t*5=x6ExikBcg| zxv5g9lLB${-}?2ACu2oi%M%x#6hNs%?h*vQ6(fEoB8YaPxgI|zaC3nKdV!{A_s3-u)dFE+=I(vG0o2 zH&Ce2_C6QGdgmlLdfrJ%$YM4KC>R=PE2-ZajrI*2*|hRX;!a$bn}4aN6yf2SF!Mle z=v!4hmt1<#^iU&jpw{xd*wZvK=E0UGvzmCgXJkaAq*!^35cMpVwQNTggdG0weGMM^ z$dMe2PxKiEh%r?+=15p#v?_VvXYo)DZY?Xhimhkh6BxvoS6sMZw>Ed+6@lG(ZOND8 zXNTw45DFUT79e&LJ*)Fnl$hY>?g@20`j1=n3Bf&4??STVL_7?v&)E^D8(K( z6K@4NR2i!@$+KEzM{Z@-GsLEnj|Zt_t2gZh1OC_)hv9fdf_$F1Arc_v#TxVJpO7-d zZsLLnURgzZsroUEH>xWT=mimI5%6B3-b);i3iGj#|G+Qcy5rB&U6wJni`etM{nif;3kJGSl^Znucgbluy$#K z@bF@TFwrl?beugStY^^ocmz~W;UDXJ_euOwCQTjfyJ%ynCn`H6xbiTZhXY`YhZ?zp z^CMPw2EiDyb4nS9*<=c$Gzn$IOQl56Rw~PbV@~9>GJ{GrH_FxTl09k8`ig8ugB`YS z!!51Um^iF&dohdm19e2Ey|8hU59qNLth|B9T*W*&MvhHZ5+It#wMi7Dz^TUabFusj{TQ9|I9|Nafh)FLil~csFCJ{( zwH8@Xj7|N<+jtil`o^9x4YH~|ZngD`Ri&yo(}|W?>FMZW*nFAiZJ&|sC&pDo_%efK0sIOo4OLDq-9KObnVZNcpwr-?k2w#o;EtyZVrq;)mM3LsIM z=OM=B1K*FtY)Cs8!$usP(emsKrVm4q_&fGHP~EsJdLRPTAQ|C zX-?dFBppir&6Ql)PDYe_Jz9GBji{pm#NZ} z*V`$);W_$U_Pm|<$NS)$t4ot65V&MC1~?5WXUUNsNtNKa-Ek)P!k zlwvRSq&`r9h&b%23n<#NS$l^rKdq67F?Mc)$2?b)H-3==X7xE*##9>8%kk>PIAx|jxOXWoi z*Ohk|71F^qx5YLFD982B4*D3Z%=TSk8m)U^8hVz_tmLjPy;d`NuBR^z z)D#Pz=ix5M`m{VLt^$b6J@Ypj)BWBb-R20^(7!eO@sgop2dbJ|H`m#T6||rCUqQ-{By66MP~OavU~O3QqgZ#Dvmex+Xj?8G|N%FCxGTL^-oN zpMid>&%L7h-og5#{q;cWFK-RMJ~#~5==0day)B#({Px-(M!eTlGx#YtmjUesgSEON z$}Zh9*fsbfO*8a;m7Z?jl9=w-sZBZh<Oq(+@PLK{d$;88BZ2p68gkqcYRYFytghWNEL{UYcmr=MtbncFU9ic zKFOO@o7r?v6@yEo`!xeLG*p*G2=EY_-&{d#V!-jU1;z3&wEJ1)d3ei0x=o|h9IKw| z|L?r~6F38p?rq43knr@>hc8;iU^VFhz+hCU4#Wkj#L7FhG=f3XfMeDbDJ-!R!S$h4 zX~l+Np}0^k2XSjRuch@4gzt9y6ejo=o>1HU^u{Ga#l~I~P`%5=KJxA@MtK@@#nA}6 z^&QiwjGp{ZKLyoiw;L*oNXjO7Gd)Y+0k`>5etNN20=P^kfgwQ2t=z>B5AjDYwmxJ? zf$&>mc|yoYP5K>Ei;LC1A<;wRg_2)S@r=XwimSRccvIV-!&JDm)ykD03@!kYhr66# z+cEYeF^BDGddvSK@Lz0fU}8F#yZ^F((a%9UG~nL=Hx9&|(q85JxnItGRi#g^vtn1m za@LgYtf0?A91{}rfX~r9(Y!85-n*7I8^;1H>0YJT0FVMcgC#43ESQf#gl59=i$Be+ zT@2h0Aze>BHzTdPKzHjLd-q(EyeZLq)U&|7HMO(wbyXc^oVdp=GpCi|p=MI+*DB#a zy!|!Y@=|S>Mq=2vz=)7=lL20-wLNSHX==?n8wHvT{!idH6c%A-Gd=v{UAfev6&nL7 zw}YF&6pM%D(o4lTf3{g1Sa?KGS!Pq8Ey}ppd~XZcg43Qdozlgl2A@08v@7h^Lwc`l zCkOdTv{QXiy%QHqZhy%WsU<4JAIa6Ipd_9a5=c~iBz$wX%$qb5*t*s)RqVkp^^~sS z_2>8K=zJn$GpBGVArh%>1fSwrVZ70xvs}tG-T^^g@~rI>>A9+tb7G@6Sqy{~4m!W` zD}4wu!=8&vdE;H3rSMx@`J7Lb8@{AhsP%T;qUaQZm$LZv8#z`_;>XCBEU(cQvE|u_ZRV-zJuOxz^=H)#PAx<{wJN)uhF90;Uu|bbdjr-3r8{O&9VQ{fCO|c8 zI0rrk+@^bVnH3yM$gb4#m-ySR>DtD>tE;Q(OcsYlmIYRi{7+Z#L>#fuFJKDK_jyC> z`9h*-WD2KeIVg434DGL2hvI{3EzfIe{B&83q+M>#DR0N}2Iap4ivn2YI!h&5&=d?a zH|B+fO+j4>nZ%q0VvKnWmX0h>tt_p5YQ$R+tt{eUs8PM2l!CpIKwd($Q(i4@YvDin zxWls(^8zGXN>@~-Zb2~Mi3MJ-HNP7`^kHVtz_^|T;7qKXU5x6fm`&2_x$j80k%F8G zJ8Qw|Fp?`2$D`G%mWw7JdOJW-fh$61@Tf+^wS+6&?hl&ecc&Z|L~m-$gYiDJ-T^Fk zJOKeNR+d&|ONyo3nuAx6!uJkqd}kIYm`Y_TSv8{@g8_qK559o{@SNj0SgHimLncoYteK0#e2F{ZP_bK~YeCKJQ;b!h4PM7n_>fe2%2T>rcFxJPlz z6;%SxXj|txe1-%n;elU;g7_wUOyUKwdzP}|&csyXZ8(<=#+NhgWNCMwL8UXzdR*ai zg%@6xa4_W|9T`e>6S2Tk2it7Bm8H38QGr5es~20&S1oy9G(F_G_F4KFOK2WJ-N`;= zI{|n9XscT~BhPE9G!Txklyw3VAs9BapBvW9q@(An(J;_ND+#I+f zRjsytb>uj~$kJvz^C!BTqRzJU4kV&3K5EhWRI#_GWdEnn!s4%P zr2RP--`kNcF0EwBk&wG!3F7$@iw8HN^e4#=OIyp3S&c3Hx{V$SRnwBV2RcSlaoUsa z2r2nCwok_L+DVr;;$v=g&>Ka!g|M_-Wm)&P*N=<}NYb6xQY+pdr4Sy0F5&@0J= zOF#-xIM>`Jfy^(T6Lh0wk3^P(D$xQZZFcSFQo-tT&sJe$@UO0GCBB*j=-ka>)G}XS zfiRF$y&Ky3Mn<=%n<5QvUK=T13y?!OoD?W4fBricLav#m4!US}5$M8dST0v~TaDc- z9h>=RTdyCL*W~g}TGh2Q^B|rigb{Q2!OO=7|0nk85%BLW8kT975NjK=g_co>KDMS^ z@(qlWza9YBrDxfFDGM9K;W+v4n8K>M)(+uGWaPlk_SR)KS*cQmcb+G*FF^3U7r3FY zc<|Pi^YYL6&aHb|XmO1OW4Z*pNH3hT0MtGdQG(4;!J2VAy~sB`h!&EuquwV%!cWJ8 zV0$d%nhGzOzCm!S>Qn!eZ((j}hg1Mc{VSMy^_S-?O&1OY3QAIzs@&_$d<(%&y$_A& zR`;q@vX{i#OI^>}u=>vu@Prr-{r6CMN_2EV9d0GQGKtDEFrM51~w z1xn1f06zs2!%vxniYv1%kYjBVVix^{m(lb;z^7towX7iT0lt4FXxYDbP?ueH*xl=5 z*aR~07wj&@IA`){!_5gUxa6?_7y-YDb4!A=LfvY8g%ERB`Q!yVWX<&C(^%)F3c__c z0cAONC9z5_sN!a1Zu>Ss`AbL$v*V9SZsjhes<`UA*Li+zId=JkJBSqc5$-r6Fm6?v z{7Pz`>#RwL-h{f4d|YmB@Mft@(UYIvEzT;yR4mDzm0viyEsrCMmWvviPDj-MoR0d@;!Y+V!#*n2z24+ z5K8nNS>`?!NkxC0|1lz|tpUAq7`YaRHA-T|C*^DfI|)n`QrHWRKDgM_svy654dcA> z)SZftzO;C$%1pzh+}npM(l4bGD~(ZX!rGiMp^7UJmV(l)Aa-Ib72<5-YRv=r#4qg3 zoe8J|-io|-M3BYb$-$Pn^1s@}Vp-At{<2lx{7#H!IY^9*0)KeqZO%s!cw<% zy-nYviSkkle4rm@Sa`IfLK2TNE8b*IR2uVwiM;A)56H7QYOGmfbz1mD;)vXAHtHxS_YsF zfSt>FhK@6V<^}q(ULmcPMuC-tNOT^{!>fF*$DTM&oWikqo93`Py;A<=b9OpRy^nqh z8_5eEzI=6`4=M_8asO!6xLGDZriD-ol%CqBk|-P5T2DXAk4##&bF*>x=$#|b1jeLV zfB8N?3BDoas;2)bNEEQFd4om25bUzH-4*Aw8f|{fWHrrf^sff`4*R@#S_t8B%V`>b zOZwdL!xHC(p!ocBJ6#28`m)u0y18yj2@NYFwKF+~7I#f5T})XP%;KIQ?3}TdzE*%R z5G8zQ)r~Mtuh2O$Y@{)7dRyIiN4N|b*HAr zteghZ(F#Ke!UcbUODIe3QUVhu#HM!?BtT(=;K+W*G;>$nQ8<<)Ap+w;L_px)DYF9# zB?6(snG{48If()x0>AhB^Bm{izYd)9U7zRqZ12ze^Lb#gASu;uIYK$tQM`} zp!6Rt^i@Z=m&?`cv$R5%MW-+8#l-!ZC*6!A_Xx``V(}qI23i=#*p1{4VW|)BXcQLY zl<=v1K2N_5y3#8ti3hxb_!aTv)vw}>n`eLEnP`>l_3gy|fwvm3w5*^s8MaS#@Co71 z8B^|oH(_U~Y15VD(q?rUpDKJq^k1s{S1KByZJtP8$414=nE6b$OuFk=_$2x5x`Npo zD_TIQ06MU}5|oNR>iY_RXoY03p6J93oNxZpGUYT=!Kc&bZb z5-wn)1ImiYid+31C8l`9X30t2PpF?s5xbN_bYisLr)9kI^d3qjNOxn&?o*O#)`wg7 zP&hLNW70Tvy-B;~^ar`|n*cKV03L*p(Uw%y!_{g1!fviHDc=LNTB6{0XFYNqC2x0a@4` ze2f+fsF3Lk`&h}0VcU%~7$;IgKu&_oyjXz_W`!oG;$@*~0T*g75VWGrI~igIqV?D9 ziog^izy#PSZq+0*!NX9D^FamaA;*=w@+}x7F#CXw?~>$Hd4A1R;ahbd+qa@QK{CAv zhu79OiUb`kIs-+BBCg#_T)-p3D;`%~Vk2;LJIfNkebF2V!+?IWLh|5=N+xTS=3qoB zEmWkFqspcBdgZ3Sw^8^Ohr}&>Ec&xqQ=B)nOjh_*y-*<#N%owI9m`3Y+o+LG*nZfm z1t?saQSf*TzS^9Tm|eiHuHCl+zXG0Duxl4Z>eJ=^Nq226D1VPSak+6R9&UyQ1Q?o2 zPG)u6fUpMzI{p#7LYtHt&`Iqq{Td#tN{<`*_r?2yF;scKNwgQ zeQ?<9ejmY#q|7nxU2=pW&if&lgV5ZHBH>`reuaerGGynY;bAQ+sMt380iCxDC+{$F zj+!U?<>egXIvj8mxUWn&Nf`!$7UNPzp^~V88DMn5CkY9I=#xTUu5Bb-HZc~Rz#(Ye z=+sEz0-|0Ku&^J;ib7W03dC_Ll|KA8VbVVef_DdJ%k?XQ32+A#Z0=f4uMd=H#KZ~Y zApsk&<6DeN`8G}Ql7@d#neOzA&Rm{bWO4wLXZ?>jY_ZDHXst(a>2;A-4yBTA4dj_f zdS|%D!!H?}DKw(r?ak{_=c-4~2o;`}u2aMKx^dpZ+#ia0I+7V-^yT2j$67=xZUxKn zOr;eb3hSpM|oIq)%Ync1meTxT>Z5PCFz1egz0Lq-RjRd}NDbx*TG-HK$-=$7l?3GYWljhu+#r69CH7>?E zHkoOI$It8rN_aeIR>5h5)ZM`$JVt0B&$x$^ioo*=5p{**! znZ~TB>0t)lyuhXAQFV>t0CrW4bRdg1q`G*n;15Rf%@Mc5xeO%Am$6Do6KX?xB;AXO zEUmEljFYp1c?hhI-)zpx$yuez!fYSxg5iRPA!Qk;LDls*Dsn`=B}9Ytogo8vhn{h!16$G^fd5ABNNT{q< zArrq3P+>XX7q~kD=C$vkua^)?o*2YBSj<&(iF+#9fcqyp6u?WP>P+He=Bz7SE7{-s zHvpRJ_le{nd7$cehrgAaFbu#ET8+(t$BZ{uz!98iS>og^q1DaF0S1L5l-y~Ht3nI~ z?ntZ3{|ZjW3dsw6{keY^`uWYjiP*8s2VegI%(tUS%62s-i5Vy_^gaFJEio$bsltOm z(5d(kJ&zPg8FAnS22r{_nyinqEt9K;7TL}7^!R`Iy3Ty(HlweM zfT9G<-Fy~nacCNPzXd%@{=$x!a}ZD=GSWN-@Hbj7YF~)6!~ylbhmdICykB$kDwZ?c z`(tlDP$+m}`Z~UeGcvvx-4P~zo{>l&4!*=eQ2n4%(fY3DNI3r-WB=0-j@S&0DJOY+ zzoPM=EN4A$@=(ay@V8t`!ODOqY@2PWD2yRQhjL-xtE#3m>?zt{{1rkn+I6`Q)pa?2 z_a~2x6Es}Y;Tt&5D9eudUFa*iePAC1RrsW6BuK$Jw0GcRBGX%$xmUs=sp(yOEkVH{ zJI_c?9fj4Xr4q8_z#faEZ?+jiMXKgbZ>E>md}dLyg1;xM@HkgaQDEWR(+Xd5Brw+A z4soj>Ls)pxj*<~BK?Neu)M|iEQy>Xn zm1Avj+A>;#U9jmOGgxZ#kOxL=H48ajpaLfJAv2hH*k;%(fmY#-USwI+i~5yUDq1L1 z>#~nqS@(o1O(DhdF^Z;?lx^Aae~EpNN`?6KUKAD|iiUFPUlhbHBB;X@?qqIaSx0mH zb3bKU!eGtURqJ}v?~0)c`bd@F-yo*~)UGB~r1Cj%TN;t%@il@$n26|=kR~Y8tmw#9 zNJa%RMmI_+Z{(qd=E(v)oRb)7LHGZ(rTs%?djGjn6}Ia4&M=!)Rz40%oEY=;x1TW@ zIryn*|G0m7961%0UIn>>$qtsug@Ln!f4eBt)>bt#Hkk_WX&G3~bzvOHd|6E?&?f4T zX?4w?*fB`I-qn8KP9k9ZiI$CX(v5c812;7%{|1}mX_*K7bK%nBZLi#KbsGT-1zvfW zi~Y*I%l6I)I>Fg3k1F^s@N4Op@M(N1DFX{4%P+skb`p5-sE(#`E=T`@ZycU68t4-t z8*E%cpZETOAW~-Gr!fJHb-i3nPI0NP!ReKh5h+P836h>v?}{=JYT5FTVBLw?TmX|w zR$LkPTW<^cXE;=T(!E;{IYVhj-7ew(X;V|}e+_Wm_hgAM5)5y69PSWGx-PkVvC8$r z{jH^zfM5$`uCb6R5VV2O{PrRzy>M@dxrsPLqv|(8!G0a1tT6;8?H?+4yXFh$T_K`^ z;muv(34Us>umtW0l6!)blzm%Gy#HDuo~yxELEy*IJ8#zm7f~thtjAPhXjKu zfhjtg^8p?Nk(BE>S{|zV$ULuU6dKK}dH6(9!Ft>2R=w{hj>Fq!_S4+{0-LtrT4_iw zUB8gmJSg+&*u&8;$Cr!FxIuT1$jZWNottxLgF;$MhVC&cY8{9}&P>cw{QB-?KjUz+ zZM9Q=Q3eLZlaVcvFl2yCg8$Vzd3$r-;d5a>KCPWc(+0FGX5tnLxORn<{z8BKVPC69 z9fPUYjn+WpB|p{kEAdcf6N}cK-~usG?Aof_^CaB5OX>rhm~2Y{&aeIuHymI&>HpLj z9$aoJp-R%amS&QG4`vAU4_HOhmLF6RJ+j3{b&lV@w@j@pR06o+^Tc!tLet~?1>JcP zPlZ%(&Jokq=V6N_5SZV}T3ynCKu7|Oc)-Gk{J{THiJYHWJ3-_x`{r03HN-Eho&$W=W3iTkHmln4*FuaQ0 z?sGYc@?o4)BrJ2+eKT4G3~!D-{C+?CQ9!TemQWYJ4$?kXHg<2EDpkFpHg^fP=+-C? zg-xs%TA`F98=08}Z@*AMd5MZXi^7AiRhI@#-5!aaZYGp3?kXmT2k})Au5i`KjxG{d zd16QFs6}?~T}!Y68v>Hmx8E3yY-B?9n^4?<^;om?6O&cl*JZJwPQ^1%5mIg}w-ScM z`GC+G(TrH$Id?+n9QL=ydL|)7gjLY>YlujUciSTAMFEtx3qh1AFsakEgm3cV*co?c zdi!)RfC7_ohC}d)6(ZQ)+5AKY>@KU{iYCklBdWLi;d7XK!RS2t27MN6gP3(1{xzrH zu|eFYs`r4H!{Y^?C=A6Oc!s*G1LxJ}JEg-Vli-w@`cr^;A-KB9<3o_6Xi3sVfl+vhqLqgT?R=4LOas9? zq`1e%mz)phOO;2Z5^!5|?F^GrWM3!MIEr4=wygm84j=Jzhh152 zoLU2aL0-I#(gj^;{YQup%*j54MDL_5Dy#M71NQ9;gGv?%qYynRa9nUQMGTcK5<@1< zO?V^`VZ)B2G3WWXN#S5HK|hdEi~E(tj|JvOtW^Qda}{;Hi`)mYT|}m&BL07mX+(k- zQ#EykQRTEyr9^Qdy5YECfPaGI`J>q%#Ph4~3ln{Lt;*k-mn_c&6CJ*7Mw8@xOGbd? z#}-Ie$uwCMdCcxT1e=a=2LaT<_MteeAX=E{iB>)Bdq_G0>3cbwAz@nkSVcdvt=B`0 zRe&*$Z<2Ts`w*$#6;fXRA~6sIY@U`qg=9IPyGYoF9u}p13Qa@g)%WlnN)G4y%5M=p zTPNScZ&1mT^}?7)EInqfoH8PO$C>3Fij`vEyDvUsjFzDN;Z)2&dzofpu@a&15&Nco zkUsXB5;AQkF}#qG(uu-ZH>G-H8>Oz_#Ca|wkAi{OEgZf`2RaQ8cc@6&9~aLmshFb7 zBKmF_POgunam-3~4;wAfRd`Eegpv{TbYM!V`)8GXK6ZVID)c?SV)6bCa+ZLHN7wTZ z-C@vp1Q;(15vUFn^uYfy`!WD(qJw(-tm+=SKT=wm9ttGd%BZl3f0lRq>*D`14j znz-S^@7c{m`c^Q&2-ibTmeI}27>;>^XG`N4tTY>2g{6-J&xL;MjjR)(reg^_E$XmT zhAl@#Zf)i!xyUL~UQM~Iqj*Qx5=G_%wVSyBRl|Xo+FM5pb}g43$ly_wjF4>QwdxcK zp5AXQT8D{=JCGYD)V-vLf=B1_8CSjjHH!U&B5RmqE(r#OJHUq%x;j?BL{ziES;w-g z*RN=OK4WoMFG*>-)fz=pG?2DbhLF5{fWg}`JeTQfPSq@2}*pf0@fYw ziR`Uya8t;I5p))1pS5gp1R-6Geo1gfx4rUz(zMNI+zg6IRA^SG`X6}Yw5w_Ysp|NH zKHLxr?(FC2%Gb`s23h6yHe!+Juk^F{(9SX<2aGTYvKrgSKraf*i8Ot&e54q!yhzsY zV9oYzL5>O@(e#D0AaTHy(eOqxTgV*>1p+>5we=VTQ*9Oh{WisWZk7gk-sjEm*-AU` z4V=B0$NZ#&#nhl;vzn2V;_dx6r{75ghB%tJIjuNFTWxRvEW(~#Ru8pxr1DX>&Hn@<#Gw! zbd+!AsFDb*iZ~Y02QHPaCI7ahH)@W+QbEy}>Y)hbL5Cf?OS$lGLE|hjNvfu>8+;9A zx7wei2I#{f zMMiF_Eb?QwTpUJHWl6aaKpTRlv3se zxUD~gwcp~G!BmjvlWHz^2KACA*d*6 z*#b}a`do>&%CvHUVbz1>2J=dg4y{zevxF>961#0B2$G;HqEcHKEHy@EeYtRwJn>y|6?;FWwD^k2AGgI`hiclRZ&WUAOemqtp_t)LF&t2H!s?h&yH z`%%Oy&BZ}JtE4_r?|XWsAG*H)i=Rqt0)9-C^t(iNOS*U@PEHlRUVmfv4=VPF(Osj{ z)}?#IJTK0OPemGv1Qb6RwINVrNtIE&4rd8`0QQ~HNqKwoz|xmXBf*G}kSkVly{PB^ zQMK>uo?Gyj!zlH|Ri_+g!uOg{(a&cYqCH6pV}vV!|I@S+h{r6w` z0Fun`(qqaFk{U$!iYh1T>S&RgV?8WPlm_bzF;Ua9rP0^sjB$0@cPl8Df1G7%JL}8w)D+=+pMJ)$?@!*5m(v0kyJF7oPLs9&RoE5XuSBNgEpE!~| zCBWqzSR)~mQ=^8T%@1!zu5YY9_9qm+XWYy#Ko|wcPc-U$HD!Q zy|^9#h&eVT9k4fl&?jA)tY1NY(-8fS3KOO1Q1j{GJ*DxLXDyfL*57>V%U>IZcfVTu zl25WlrOSGUyH5YTXyUii&Nx2*X+&*S#@_GmSY+k7j^DGjIWhTQcN0oU;qy&s zQ*g(k$j}!kCG%raBF(GYXPA%D_dKeI2bytq?{!iw)>!lhSH0H&Lg3{Ekze%Z0#$dk zto`Um7&P;gNCNa9FRgfJ?Vb7H!qV-VnuPLt0^h}e=K3SwE~9DYCpc6Gi?(NL#x0V~ z$D{~9edE%k0qbo0zt;2kyGa{+GN<6RJvH_7@X(GwwUeuU$HFEL_6jdycE6r4(axX= z4;8S#hZ6T-?|~q2&K)gV8lK(H_gP(6lW)E2@8S!eG7uXpfDfhT10x2qh;$r_H}J8ry}pf;lA9hcxY^`QRxkG>2%0{+6yRh5h$j9t)>CvG{!G`o@=e_f%u?XL90Z6u3Uv zdwi6R@CDXF<~xu;i~`cbtutQX0{1Pt_!`dktSerOTO2UEuQ`bW%O-3OJ=h%1S15{% zoz1nM>@m(~)qb8l=;Vq+pX@;14nh{YF-WtXqn&HjGc@{5Tr+Ta3KtE?<9J*idgWxr zdt5&eVLmzY4E)W}Lzj@gOdd78lxr=^JMb-86~4dOYeum6#PCc|>ElV94`b>&8Havb zy|y-L*rtJE(SH4(jr?)dxVNou7%55zf*aAc0t<|70I~9NmW=_401sAv0YZ%R$GkCC z>BHvJG;VmMT#eD!?VN!}XIU>6*_jixdK4#U;L|sHhykA6+j*z)!GT8fea5LXV|7x= z0gd1_XdCz=7bE*AL5__ug;*ka;4k5pLuGGuu)i5!%oeI+TVQA0jfM=a6Q`@g1Kt+> zC2y7BUeqeh+E~XRWRP%Lnct(wsXS`!t+x$m#JOinMsBo-Vor`3Chp(wVvN_-F87E0 zEbz?rUx^=Fg;a}2%!JJ>@e$b%@bY}S>TjoNtto=K%7^l4 z7u*?Fk01`xM-KXfz3blTAe@KQ%sN9nAt=V^Q!PM<6#3HQ5GtHM4$L=Ejx>+DbL#NjNPWa`VA1>CGdO%@l%`SoCY}G~sPTNDG?!R> z&bLEnNBWHTCP(A6i(^*_sp6D}W!FEj#1Gxy=ayS54Sjuzj>FpkJ@-*b4OW69+t#+w zdcm}GJ<|60(i*aP81&8GqH{i=ENfOSjUHb!VE7MLGrf#6_s%+KrE)df363%PZnp8usE9Cyc-isK+1cqhMsjC;#iro#*$&IF1 zG;BTMr-8&Q&=r2eLPZD$!9EUh#~^&kH5Nyw#0+|ciHKFSa7M1z;~gPkpgv)`V(S{m z@us%Hh@`MYXFFu2$IKFgIr*L#v6*P6bjKpA90dsGDNc5k@!(r**!OzQ5a9HWmnLZ( zbEp)tz#?9XD`E)e2o8-^g)8(C^?2R1@q2b;e9VHyF)77h{Nbey)7txyG`?fukoq$@ z6y^xE{^BEI)m`my)igb2XdZ(nOw0v&rnWyA0a=PD1vFcekuz)g9 zpYbyb7p+o4ndidmL071SY_!P$AA(QaKV(~iE(CZu4@lOwj5)54IM9>U4vq#7c1%jq zn(ORF0dX%p0+|!rbutF|Uko(Bt8xJ8SfYJ@^ek5b767^vp8cYhMo&MKqsb9Nnw+Z_ z1Z3mV)A?QDe8~7OX5Ir|22YUWROwK@ubI4m-U-7U6Lwm+^~nK^iw)H^YB$AAWZ&TKguKPiTGPfcL}@ zi!Ap)m5N2P==B(4uMIq6epgbJ2P;JGYHr6+03Pi1f2Tbft%_hg?AO)-Wyhds+YFXtscxM? zJ4DSBM+_;v3J-DRVD!Y=P?q4jXwYaK3W78y%El{S9UMR_7+f_MXdqtu)e@;$g!6Ew z1Wp88M>KWDTVGUmDB@IP>9^pjV^t9rgase`RhG(@z{3M5@4&Nv2Z|r(>Mn5Opm9g| z22Hn7g$uuMzMc1#Vh{&$=MpPTq7kGex>34X5-wnvbM-4&4ve`){65`_qWkY_6L7ThKw+ZK zwrj-BBxrMPquvri>;l;slY;Q$lCqSns)9Sv!O#k(1B`WyS0fGD0?)4E4o;&>$3~MY z;+^~{hit;W+K^@HrK&>Eu>;TYzjch$ z*)saU*`^cdI@4(Yujy!cDWM*86g)=B2r4|!F0^C9OQ3X}O7~`oRi!U*i!mtzS;#Fji@10_ za+1Ih#LeKv|Ab%dW1?dogBr4N>EbJ1&d3_$T7TiTAeR?)Z>KHTk}+rQpiE8=yV~N8 zt;jKBI@f{W2gj-Dv!zAKl08IchgrZ3p_$0eh3o}>E&s7k8DuJ8|C9<}FrFYmtG^Qq zr-Ze^cL&pi7^x1ddJTG*!5c8RWE&|ci#(5~fjkIF+MOx33W_7PyN(r%5cAhN$SfR_ zlB|s3^>{QBRBBW%nAHkQ1WWL{Gwl6>H%prB1m9O2LX*eH4Gti0u(`DSWnqzZ!Dcvg zOQ6=cwZd1!r{$J7xYw>Xr`9u++XC?AHu6&VskQPd z!*XmupzkLZtFbNTC&Z46=0)(N*2B@`KH*G87e|E2=lmPhbEgCA19Q!<;C$}P@q2a! z3NZ;*?pCq(&{oLqfrwOkGY*sd`bW!3?oImU2?MWf8Atr;`Bt&FGtUea(tGsG9a^dg zUWfqBRytjs#^I~?(K1MWo z!9`wmd3bi%k1wB9^=k+ZIa*jC;TiCoOMM)RZik_Uf+q&5CfC@|zaHZZXpC_kpeW!p zo4y%v2e)R-N!Cr{l@ow7u?Pzew}F^& z$Kx5N7ODCkc!b~G7gXaUQ(bt1flqUYV!`5y)q=yvE@tqgY0eNKc0OLWiNV_I1aTx- zYo%j|RX^b$sGgB)^}|Glp*IYKlhwUIkGN9^UZDQL>o_#SNB{}O`U7Fl(J}14nGGXODa$ziNirA8sN5l99;5VceBcjW zLGmIo3&H3HgBwLyu&5ELfSuamH2j5KZ6UfY7eiluT)m3>e#EXhtj%?4%_Oevy|cS% zCKrZ{!DB5SAA_nLfruqe-}kOV#q6uHDiHTwX~E=kBJ8**vZ^*;rCZbsA61|VeZ*IZ zSdDy{(5)VD54UFA_yx@|apBgIwI}IawCag_TqhV~D3fk`51ZgYJ zw#PaOo>BQ0N+$Sn2U=&e^4+^WiAL@i^3#as7Vv=d67L@^Dez)@Xh1+`7mi~P4I+a0 zcPG|51*1iozoiG5^mNq+HoEvEiSY>>^`T&f-8ZA_+l>^8=7ERHvC1HoE{z35_3?Jr zHyEQ~Pa%oejInkd4HJ&x`9YwBT;$| z%_?x{7d{?y=kjr3*U;;Q3o`N$$nto7x_}75LwjZds5eisk7)$ciCzLK>mf~=%Vt5S z55}oW&Wlq66B!Ld1x3Inw;Pwf96jrQM5aCr>wsh~h`c@Ca=IjRT^KhtrunOz!eheoA_$2zle)IQg5CE*a)OqtcIVlJn$p*9Glca z(H2=%vi~fsIR$O1Wi;ci9nHChP;pM(e?O?2)*L9X<=aM1{)8(I&zn5dHypu5km1<> zHJ0N7bW{~#b9lyyBd9weK9rH5gqgmsoe3=neCCCV!}Ipe*mnzPdEc-_2>03Yq?VvT z-daIeoz8~crI}ovAGL0IB(7r^XoDD3M?KqIU?yW-eBOVVqwD{fvrOYvFae>_;!@BX zfqx;5+i|c`tlAH2vrld66?l;0nQV3duXkadipw(6FWd7H_G=HZ0ViH0?Ls4-eUZ*N z=&NDlsx8ga55Ml#iGI;ohfdtc079|dCvON}z!cUWVCB{wi-rtZTDz@rY{7(R;o@G3 z;dOR&<}$*j1volEn~Uco9}EO2|%ku&8O z%4TKo++4J`k!gT(O1I8;ZtN4;o2(ZEk@;Y}cohnHaV@j2PORe9Lwz*x1fl68tk)Hh zLx}h1jyxNeHeE<|PaNcm+7&p(Fat*elAU?4M;rmW)!V`N*aI!OZzdtW2EQx1x?P;- zijd$*d`4jIf8bl+$i@K}#+cuB3YgK?=IgZO$uid^#0a8Nmxvj2{YuVJ4lv-txMXij zH_MKYV2~tW<#wL=i4qZcw;Qe=5G3Ob{##mLMTm7<^T3yQ{p#(%BW{0VtlfA_3o2t@ zru^V-+V{|sA)zGecYoXxep^5ArLF{P3o-feNZqxic$GlI1|5S3n6Ys_0NC*gi|g>i zj=(yChIc0VXrI{0*rGY}Z{nxM6m*1t%9osD7K2YHS$lL)vQ7Cg-R_+^6pyknxg^vX zX@4QAUgD;$Uxw^XKnkv9m6s4mNqW;;O3~n5R-SHxP9?D%=gg$P) zpzLKH^?u0|9*lUt3jl}(x=Xv!ev3g;*PFQZN62X}6kAt*9 zJ78xJT&@8{B)GVHHF0|w;7YO$TNUS+ZB+loTq@BS~TmkX@i4^>=dXF z<|zZetoVJR0D@?#B_5qqe{cbNO;A?TobeSfLQR@jGNt;GHwZuh*9*eVQ#_sN+|Bpw z(jMn(N^too_lNiS^s(iqE-IK1#piNJo&wOra~!?Hj0$ZdCxas7uJe z(@)b$^&%ECduu`}bGLdYfs&5DlK1TYkUd^7f||?wd;D_55V4X34hE$N@tLt6d@7Fk z%ss(4X>0Sk58dY;qi`HG8a>CL1`)ShJ`XgEjSA%58n`2Xj3+GGrm=P$q1oXeFL3fT zow7#=#K2e3RU{>N+Tth6ZuNUeV))9>E!%FQLj~?nQG&>^$;;)1x}Zcdn#@xSfnNiG zlLh}>j#dytjv5~jfCE0^`2-#YQBkDLD24^A19@fYJ#=@VgrF4*OcruK#lWFe%9uM@ zL1*_TeFIVtq53yGh$3O#wW1{;-X;Bd3E!o!@z(<>^y(VocOdUg`%-bVQp^vx{=k&` ze^y5G_V}V?xcoP?D{Nx!A*JDZclv3$WgJnM>)e(H(>@k;9k}_IcTt9gE8HR;I3^{a z3j8LlKcH@u4`yw-TLB)arb;A>Qajf50lDr>b^)5eBwPOxSOS)P^?taKbl1f1!$8hW zfW*xA^q4Ew3PzpeTL3M-bu7q-oHc+<2!S-Fp6LT0mp*ss_Db5V({s~brBIFYnWg?{ z{K~sA`uDt=`&>UR0RML?myb;D9sefUE5kg<(EtEK9jEX`j5RYW8z#t~iI*p5Zpsy_ z*4BV;nBQ~={!c&(@NxnE&EQZUzg{w)LQy)>XBVEx#@jZJ2IonP8Dc_g$+jTPhqu;=WnqQ-v@H*ZWzVtHN#;d5AGhkH}_4ZK=kN%8jdp@vGF1ds$(`ib@k zgFf>fb67>`E2R-9eCU@@(+Kah&KE@|lrMu;n==&$!59SRdC8ZU1KOzvd!O8W5iS=! zyDS!7VQh9D6?5U4mR9y0ho%dADq_h4A){R3jp_Sc>9?)Wy$H$M+Pr>Rs_R zIyvloCsfevu&J4Oirf2KUO_0h#~Rw{SI(pr7L(tB|G=Q5 zs_2cLOvn*rPE)x&55zzG_=cbGV_LcdUAP{G&2zBWv}~}6DQ=3ckHXHuP|&00_f8K_ z3DI1gqQH@&gAWq~xnU1=II_E!Ye06|uHz-gLAy+zAx(L_f6nML{7P`&+1fqJTe(3zw13-hL2x1)i5{EQ02hb8?h( z3l_ki$PU0{9s8Fkwu%RN@351MwQx2od zb(!TQhpWh)Sf#8k>rimGQF}1o1wHv;HEWK1NBrXAn3-|4bW1f(4AGmU{=3{ z#ES8>MSnbY zXSgt$j&}Tld=Doam?WJN`_#P@R`k8{^}z;iSU4_Q#2m3Z8RvoZ^D_Ay>&|I00oW@9 zGBrzl#OwPH$~o!Lg_s;<$4rtoUBN9XOLZXgox$Pq7m(yQ`XGN+e>rk8wkfNb4!*Dy zB|+{ozV~{%9?^t#PFf2b*9sTwcBoUZdWPe!5DLY^!GMZr6ft7X>Y1nl!sqf4+&Sc7 zj|)>`|{s32zt!RO;exMLmu#sMn$jF&n?BG%?BZ1VK3|cvJ!63Q~O-4$|jQSit z=?GrCZ}hd(c7KytscFk`dK!S!gjNxI`_RJx50Rfs6A+!@aHvS|aBx1dAqRdUdDV3dywJw}a$Tmd{ljQ=ubKQ`>U=tIvJLF0&!r{Rx4;c!%39QYEkt82M)4g~Em z8j>N-XD=lLf`zFHb}()qt@4M+9PyE#UMPM?fKg!%4$lwL0yoFM@Jd9%P+uOK!hohh zHIwC@ZkJmZ5cxwkEMPbmhub2VM9`hvmg1pUud1$g*L%lK@fJ5qIqF+ zS}5RgJ|8-xC>W#4UNJbmZ?x_^fp#u#h!+`z#2x*IxQGTGh@gL*Ps_|$UfeM1AiJbv z=vWzoWJxi$`@l7tkfr3VJ9q^_(6?|;TzO{X_p9JV6Prj{KF`nRN184qMnt+yU^ft3 zxPCt`g<+?QXH$S&V*h z$4fjl_0JnYYt~FyTzCaS>)(>6FAKg;Q>gT(auC( z{>BAOVHmB^al@n;JmC4!$O*y+z(8=K`F8atLMaeqWytpUQq16v(f)v{vDI4u0mch& zR1XiW!0x1a;}gBt7+51wn|$5KOIaRAk5Jo<%SkUnr6O$cojR70TtSv zQZ0-pF~41B1tLt-@-Kd7^&*6f9n0XIvNilQ)aL?#Y$&E)uFv0f^a9R zwZeG^La%>cNgTtlErAg^wXFiGXB;1dhX+3=(c@4Md^ikAMiW`Hqdl<64m_dzR^O<_ zU3CaS^aoE9ZE@zP8OBTXVFLrW`HDoLk65+zt^Ps7e`RcGUY?z9Z5V!uWAsk&e?GZV zxA@w7x<$1Cmxn9a`SjAc{J!L8Fj=}wXI^+bQ2uw0$du1bapWu6H?n(%BK{qB1D9zk zK=%jrRr-b6?Pp#QV#M7OE~500{P_WVE~^xb4p;&ZvA*2G)X$t%OLyoW02W&^Qa>BQ zK}d^hSJYIYr2{;mW$oBx0}I7?_Z*k}n3VSB!>`Cv>I07gHr=qFpv5|$w5|bh>JTx4%WQ|HI4PB&B6cO6cT!Ul6zZNZl8Wajcre z-)kO7e;qd89RuzU;ylABm75718`Z%_B#i;~4E#(d+CXqip8Y3z5c{UG&olJxKstm+ zFmv!YM7X5i1eQ$c<2BT0CCAxZiD^m592q4NI`vTkw>EzmkM8Plv#PhB6Y?VYa+9yog8p~~0!8zu!IaY|2tWNBdJ$MD9q zTQNm*g}NRb2$~>ej8ra)*;B6qdE8Z-m2S}`4_J4o>{;tCPkFe>)8FhMC1qEVYvF>obW%3KylXsq8c z-pNlc+RQ^r3LCJQa7Fj(a6!5vQVlMPugwE=)D#&P1AYW%?2WZb<3^R1zu)7P4KRZl z*X*Z2HIq+pA{Q@~%(jY$^k$|0#v9`~6xg)~^a})(h1%Ag!w2XGW!OH$xjbO6OLF&G za&4Y&1Wji~Qa0cU_!o{v9m*{{5d6>c;wAbX=1@62Uy6(tC}8F9GjNAUey9i&cr_RA zM!NF;mdI5a4LdSE3pD%$;y6Ky1{fw5;&0e@CGHJD)6b)b0gVJPv^%}3eh4KKk;ie1qKJ1DRvaGopfCnS>R_;YtY2Q7Xe=W9H*52Cca- zPA7pFJSgtFh{!T)%X)sw8e9;WZ3Gest-@`tpncMwTr)X@<^?9EpkIz0pg{JX(TC}z zni9skYAd(k+WZHqOL}G9hea$N2g% z6Kq0@^|scFr}FjVfXt#l3`S-g$f*td1AZBuPhv9S{$0$eHIiYA%^2POqA>KnZ>K0O zZL`KWnXF1HU}6OVG!9)-bu!%U1F7ntprB}juwVFeHf4e&-6BuPD&?oAVZO^p+}#Sx z!TA^<3WP%q&kFGF(dWd+d6?u@)YKzCge;@oX<(2U>xe%mj$6$hfru%?egq&#)8#O+ zgJdMzlqsUAL4E&NA!|0)Q|L1`f5c8_ka^~`HXx-LYvGdoA-O;UpEkm)RpbO4O=A%6 z*mC^u)VIj&_vP1fk9v@s)ejBNj^o2UZ=H@F&JYH7FXMcZ;YrUv$fa^C2VFn&DG2d9 zR?NnhZCL+v|B5e|paAWD9hZKS9}<{JrmLdui&^+bNV4y;OCNxyvXdafChA#O+qDq^} z_ZVq`&I=3?k^ZB{Zczes^d`OiJ_Q{_Fo2W-1nrN*UfS?Cx&&uGpu()~>V{`c3LBd@ znX5x z63HD5{w-IIKV3qb2nb91_viR4i61bEiUOf`*?$m`i4=z9Eb4Cq3&}hM7rG4>QDDEd z4wJxbhdURb=|O~r+kjurzuwumGr)*$f>>f^Ip~Q=xX{kS-vo=80x537h~XG|<2@5+ z=Zgl^#t|9(NQI|}eOQk5{WoDK%R)};-Dm;e!HDZ|{YpGTk}RrSCW1eK0b-*-cy}W< zix1dXa)>>bAh7IR0m)ijYaPynFj#_Y#15%*`KSw<7hr;%SbI!1Z!McPL7C=xjpATB zW)?9P&|S8zKY=c)SR?xv3SnXZMfSCCc;F!Bm(!nIqZ8w>=5S3_oM_OOwVEZHGg#6= z#J|F<5NBa38=eJQ0wLLlM1el@3uHxbyy9O#f;PjHY8Zr{Et{+}+<# z#TF%U$!)!YqeuQu4n@&@C(t1N(!MF3Gg8*{h6nLMSQ+j7Ie0o<`P_QR=i+&?P8gH_ z2m=$3@{g-*Vu6Ql$_(=%TLSCx*?&rxPPlNXmurA_EYhV9{zTLx@^nEo500I`YyXw(Q>>&aNuG=fGmUMyO2jzUSK-zL1 zS$cYqaQs9EzeF^m#l`6D-I-`pr2q_Y7EKW|3-ht4_E?Ck4Kkrf*UV=m9elQ<6is9h zkz8+YYv3Qq?ywCzCfpPwaUXgU<58jM$pqPOzc3=|$dr5xgGvcRdY^N1E2mbd>!3Lh zng?cd{kt#tJDd|Hp>$bA*tZljSMBcdl#2)cs;|tZTc#ur4oa&9uL9mLq ztuTr>x+dQ46=9RSy#!Tkxd{zXg*tsKFc>q+3OetYu!;Z?BG}yn+vF}s=Xyz|m|i@a zt3y-l&fKO7xr|?(*rOc5doEFjSE|j zlh7vT-ww9jLu>Y=%Ic5QGq35p4{S4~kdcI*A)~R9+nHj}CDp2bt>h3bQpU=Cboh{H zhNyvHL{tenFQ)*I6lfHZT^5+KH*}YX6H7r@3>B{vi-S%Y{Qsl<2hqs_qBLT&5c=Z0@`hgu!o8oEo!6v;!^8w;p=YsWdOtJwXT zdHYGGimF(F2sPfytBp%B=<@#$YXeZ7Pd5t$QB=7z?~_JB z?|acxI~JB5te%`HQWQy|=~>QP;*vb;DX?3NWzaGZVj-C;DIY-%n4lO}&>{xQn+5t5 z8&wjlam$VS-7oYS?+%neR&XvHK7k$dKr6=i2)tJ>5JmIjG5>Xd5i^PR0uef0C7Ood z9Z{7?6R33x_n5NSiM#D!?V8fYah^uP6sZ0Zr_S3|BCkCkno2>A8&jupA08ZGW6L8N znntNYTVg9fPTt8!S(4aYZWxEdvQE$m1fv@69I=sTAy?>!%onuVVb2{J*d)?v347l8 z@~Jmm#HeeTrC*)dBxa3uc}LJ>jga9-`7UdzH6Nab{ljd7{P!8iG;1tuUaRKh1H#pV zaH!US#r@X%Zz1Ti^SFf$`0}%M(@Ofp{+7p7%IW;aEuDMBH;lOYo(+r?vRnt7#o6m~ zbf^po7MBnkSs?N!oA2%)%_b!^#ab}YjEvOsLN-C4q;s@b5BNg3Q7~^Pcq)Z%z;aSG zc>#?kW>RNy9U~;qtNv(}3`xmZ1ojR@bNP>zuln#WR8FkrQbN`aq$v<9h->6221e@j zxw167{43M`q5MSs@%B`y^eAnDaQIs;f!e0g>A#NMu$s{B!gpzx#^+EFGQOajIt;8~ zA}iz8*lQg4LAVUu`2O-j&m4)iTxUkb(w|hk2{>>Ew)M?kcx`^X4N0?@@ zN6Pvjw|vM*^Mf;s0j)zKlEAq2w~SBZHjE^}i#G`Hlf7|a7Bu6RHw1{_xsdC(wfVDb zZe)rJR1Hbk^=dx+N(NBVIyQl4K^iMyP%0)Vb7yUV9G8%M7<4ECAkMmJ3_6gXBQqVS zc|mSEN$7A*b=PQmpcfT$Da4uT0~NB2o~7!C93{G^(_jn1kbUT@!ZZ}G6UFf44)ur? zQJ&Q#6&$F&>=1Y`e3LK$AO1?9s=9n4F9(Au1Pm4qW64=fsV(T@=nfQmXZyb;l-$s( z|NR1|H!iz?Su66;9)B7jkVN%nd@?`;1y-08vESV&2G0WYB=0a#F&0EFW|C|x^6Ar% z#+dgMgN*!YFq8uiULA1>xR|l{+!2`gdA0i9EJpB(mut~u46l@)HuSKH8x{k9hD4#t z@p3+XKpb9kk)J@@4N?LexoCQ@?eK9c_GIe3kGi;cQMqso!F_^|A)_J5U?3%3Xt_W( zQwHkLMf?=EefJSlwO7=V?63e+>>(W~5HA~5Ar&^or96S=OhaYoy^~v)LXYjK`wIAp zsrUn{izFsyJaef_hi9bAo>nkn}IU&B6tf>BX$%EG$0OMC{> zhRFx(5JcW?&l6+QmAcyT3f}fwSg2} z{-&fu#4JEVCl9KZmZ>b+stfr_dif|H5ss4Ina5|aYYrTe@gA|`YXoAC$L_y)reHJ3 z7bAv7F6~S&ZvMt06T!`;JFo-$r^snD(K!>h(qvA+Oa{WLiv;BTFozG_C@gf) z78R4iQ)>tG&phQ}z<$&c3KXCJZRR@ER>2Z!eiL+k)wFmMhcDeSPU5L>iIPy2G+i~= zkW@yDaA`Xy>1xR%!0@p5CZ2EfBjt!Ytl2K$Wb?M-y899`b)h4Li2&WQGUpAPq<`X` zlvVNH0CTrwB#hOGpHVfkf$qnX9XwkojacYn$I#?~y0Ym4$T-4NL~8qM;whZrgz}UV}B2r$V87i?!qS2EpFM1^%Q24$$#w zLr*TN5U5Gxc1Wn`3v~!h1Z_|MxMw@MsBhS}uICN0G2qYjid${}SA?kumi8}{PjD5^ zNYVY#DQgdLn{Al3EB;L~tgR=E;%r-o%jfH7k=o{a+w`&h#Mb60;56dS`3-(3AU^B; z%ed7j3w+j~H_7HkF~3*GNwQ^6pdb_ErRq(Pd#vPqd7wxXaj!mMoGWq#?5Z28fRQ?E z>;{{ZR;GKf-^VDHi3MNW6Bt&Z6tacrAA}jwQPEQ+9bI6czqtu16yWA}S-Amf(5Tv7 zQd%E%S^Ro4*IKlOf!@oTC)p?nG9_tX_X^b{hQxY)0rZhpXV%Q%e+fz#tJ1u!Cm>iv zeg*y;N~jyaP9d1JTqy=t^jVxtAOeHtZ7WAR2i>Ndwn9QB?hIYYd@iy#H3dm*$Aq`J z(Ln4W_fXG>5Wy*&{Z1-?;wi#+z{E~kE19j@ooC2_a4ncS_$+ys1*l*gfd>oh`WV?n z`~~OZdOQj0;4oucg;oK^@i&Uu3K$px8_whcm%|z!x}vI*m-JyHB)IFpEYHyl7UeHO z0ChJ`<49r~@Q88m?&2(KGPMkZXj2dUT|mHW7k4S+Xc#LylQ~)%kpNY}Y13+D^T2?o zNr6gE+Gy%L5pl~a_YNd=OW>#i)!d~RKA@>_&(I}tXMnRu6Rjf{5p6v!byL`fgpm`XJH$y$`_Q3dnbDAN z$C4saD>W80NB!p3`Rf$@!oaF3#Ib{CxfV|bH7dTVi{eW-s?1dN{xhof0(4+uTBvHO zeo38;b^VaQ1YJ-1b*@F+mkJ}n3@>ISAN$YKFSr7_t>l*!zo&07_=SXwB9k#F)DEb&Nmu9Py(UzlfxfW!QC)}$QU&pLTYb5iFH!j8 z=k2$btfVqs?FQHjKw!OEC>a;8!#kAy9NOjhp;p& zK%+nl*0QtT01Zp*o#1Qzit}gaYevUi36R)16SHCBO*rmqBcyYz9S{ZmIeLU!up>hI z_fZ^P^j4JMr>83bJF*C^2!Sj=uh-;@oM}hIi)m72g%$mRmml4Lc05c{0n$f{CSmwV z#Iyi3lC)Rw7On-4?|?EOm9kqhiZyA&)3>5(F&WYkaU-UWFi{3FYZ;R7$I5s<ifUl%x zSviL-qUl5r`RrS9MMK&La$WC>jKwd`UTaGt36oqZZsJ~~jNyfU?8&XP6`3A*0vvUg z5n?R36#-t|9!;FF{5;M{sB0zs?F-w0`Sdefx5RV z4M||t&&YO)b60+_67>y~u!Ck&BM;Re+y{xXNcFF@kE)a3Nt?&KsO5?c9IJ+?c; zIyvRVHexVn3O~T*80cW*x1>GvGlr^;wq-jbis!9~2mV8*; zgQ%sy)h$Xk7J$~sL170RxZS8O3KQCrBC)ft!*~uX*{(pJV=bgvMEi_W+N>Dz4aa03 zPj94*jC*H`c#(uo~@X zu?f(>BY>jlSzen;IYoCtx^vJdG)maNb2YmmVa=}qE^!CtK=sfvv_qcU^w*r-DQ<9} z7ucQLCRC#gN%W#zsf%pCvHIm7K&rL`x)&(=c@N3v(2ftxRY%GUXEt@l4JRYf%K)62 z8t@FFBx$RfPO;4D0f>>bSP0P?;2_XA^$+=!aswT-vKjc&6!VE`TIhLI;Si>u{la8R z5SnuOhj1R*`%gIHie%!sQYd*@Qo(`Bf)#n_>5LH<_7UcX7zA!D9k38Uf3;0v47)*0 z5b(|`aGTJivRb7xMRQ2b#XJ4|J0@arqpe8Sul+tPpP~E#J(ebmSC6t6_YHcHwSU)1 z1&#ZG(~Wn7-w*Om@SBX28*0n;Y6~nF^!vY~<355=NoIe@M(+i#5o5fkRK`RE#?G2Y zf3NwKhZ}XAKqMixg@-8-c@I9CqQiHnSwXKqJMjh?hTxddhiTySliZW~9fH{lBdf|? z$ke^BW*%gp*)XbM(&Q<`1@N1Fd zDb;3S=N!cTGt-$ny3+8Kqw5Uu=hCodHA}9c&=Fe>;6cWflnE2v)gj6O2fzilf|)}P zn%RKT=&WBy$7V<@2zK1n4bBjtcY4=01hg@D`8D820LEUN&4$nymhO(@ex#T|r>ioW z3$9|f9oz`Gi@M-P!M~}Lm0}rnB6&DkfJ-p3rPNTmR?tj0*q=uo2^Hynf%z~G9rS28 z42q90?LdfGc;=$-I+l;aPf47L@`C6d?)--Jr)3rwA}L+{nzi3AS2Z{aV1ANrjnYh- zv;J_md|RY&p%bJXASmL4`xV6jCd|Y%BGN08!yU2lirfGM*Vbgh*F%Pj?f%SDrkppC z*Okf2GDyjMV3OIScIl#;;iziZ2hb=&D&5gFLqsGOyL({dn^xC{vFj{LG-#%vB^%oW zk}$XPGL=F_g!N=?;dc#&n!wWMq#p&i43oLn8*$V~v;mQDsB0q{(AiRw{ND0;+Qfz1 z{=x~r+f6Ig^={8f5M~(?7owpVmsWDcdxI(^J%xZdk~Q-bVzheI(owY6C=U`uDcDtk zPzNiW#Gx&GyY+$elsFyhh;N=YSVFIs`!r7p8b^DLa3sMbpG_5&eh;ggR>!$~swT5f zoVamr$|)Oq^Q40KOt3}d7d^Q`i~yV=?t^PVBzEsY%IDR6O!iv9+N{PAl@Vu>Gww`ef9SlYm44Y*-MvC{NSK$P~p( zS(pff^~5kh=879k5*HDCFyTxfoDp4N9p1;!PNNouOHX^Z&>uux$UUgw4+fjnR7sSn zIuw$8Q!fo)P>d;JiGGYSR*h{Gq;cqhK;u-F$s(VxAn@U)boC8v=Yc`42~hmi{vSNq zrpezjVoE*xYoc|^%B4<|B*P2i@wU)zj7>8;e59B+O9)6X-s=Nm+58ACiM)^x7HL~J zKs(iG7!sHf)7ArQ9daj4Xty>Gq@Jlab)(oe|mZX-K3imI&~TmFhB zGWA42*|0!)Oy`1op3kEeL@ulg{wGtu%_VnUVL!*$^p9gv4Fx~ubi6>}Pfi8*MRIc; z+(H5-x|5VUw6`I^SG1;9`cj~cYvx{MBzt?3zELzxkh*wiMOY*tI_?G4*&Q!`dY09W zm+9`yEk$1*W}na|$?OkH2!IA7S%~97W69jhz7Wz3EaSLwj+9oNN<){5jZ2*`z)3P$ zAoG(6W4niPk^TH0o5p!1@1$sf7K%QoKrgV???yTlhR>i3c5U^}@i(XVG}PBj5ua~C z_Xp1)gi`LH7hw*rm-*v4i0Mz*i~?(Mx^>TPE)1mORVfdsVdDOES^ZfS7`sz2$i5p{ zv|EV7OOB72E8@4-6=z})ilU|IbEM`mn^izDYCh#>60@^_7{VJR;u4|}(9G{=%#j7# z8Cmt;{6YU(AxPu+Zm|wH7S9Br^Qx+yk4oH{fqnhVnHYiSEQ*950_(LPa(m4YZ2Cdh z|IEn8|F%<5301Y@snY=CGfSS4ejA1JSV}L+K~EzrBkSA|S+NN^BCGa2!-Ls-pR}M! zu-ka(MSpHe0L@C`9yP0pW@n~B zY)~_#%H9pBe+NxcOqweHKpzIgS?p+YC{wMi6wC@ADyyz&@q5&dvaQm7LuT$!StRVSw9)qtv4Q(<YTaR?mlk{Yn` zTsl&@Cr;6&kYz5xe_^40`RuDbnXq3_1mD;2je%jqy+cMgGghEL)CDQ$PcXR&4QJS) z7J%Oa=Kw&pl({U^H*aM78|TpJL8;hPSv`M7aqv;zJ^iQeRQl+CEj&ONhTujVfiNX* zqjV~RmrP%}y63?~Lb4E@6Hk~^xOpR96@C$^tvAfVa7tvuul0_MQmmRn)h4|pM&d$FnW z%WQu40a#E_KVUFPN(v>twLmIH>CM|!iJN=UQHh(%CI6(m{Eg~2>#XyuR&HI9SqwGs zuwq6#+zHO>AVh|{AD`OF_OoyVE!MmLUm4_1>AamAK z-2pKs_IGyQXvO>@h6Mf}6j!aTiPiKF9kE{->7SH~Ako%f!8WQ~5{(m9xBwROyP=pB zCXl4KeQ}3S0s;^AWA=uChQ0Z3kuVhaWBwfo<9<;`j15%3Y09XP_pCe$#T$x_|U4?&OE@t2plNu9yF1_ zY&%_*CK6LlDTExJpZxC>QH^jFY`nTTxUTQ9#aLO^$ia^9i9zCR7AR5ET%d zDNDB^Ort=kz-uBTAS48Y9Q>`d?g!ub{e^D!-t7H6_jTXbwXSuoCGjJ`_8?R!_{ec+ zn87T})?*$99*QvPFFe4^J<5eK9#^X4^KD&SBgr({L-IRr zowJIi)`QK3JD(SWf$DKzuLSL0HYiM|XEFH-at3ivV)nERz~K-i~g# z%%iC?e2*@3?#3s0Nc1Je^^9VBy zBN)PTwcJTNB?kBi3~8-ftOEhL*uX3uF9(yG(!+Y&0VWh>*D+bEJ(0eD^xyfrvj!NaAL!!P-`!u8%d>g1Zh#3*S{Ri+J zaO#eH9|Q;z0>F^KtuW#&TlN}p=T?ZL)QgBds^zO>!_c2r#22FJ1sXJJu>K`AltV{= z%zYEo2h=UE70fPI3P-y(h(N9}&899;4}9m@efg(pRf_Ze_(hySoXx=2ha-7Sm$!tf6pib`&mrVXYXR7O4 zCi>p_7Zb&(Jr#$$!*~ku!aNS;l^ZEV=$Ac9yD1!(QSmOE;Qv++dmiun9dX!@?LX)> zW@u_(eObl=vR{!oTf-SfX&Exzm-#PfE?qN{=+`!bMuc;YBT``5hbFzp-8%Ipp!wvW zyFdn*>#Di;TSi09Xhf{F&3E{fk&URgb82LS~7cl8E;oX6=AV0x{TPkMb+E=rg>mEK!>DsXn#6!O$FMIH;xiolq5+S?+l?jpi z+1oiY0is7VJ9^h;P2d88=E_bZaQ;qq-E4hJBX8c+Tl4S% zW5fyv_m!Cj-XSsr&jM<-?J}m0xZh#+p+5G>fx`xZ41!J(QoCQ=0$eB8n1-wB&^Dew zdd`T873}p+y-hIB-CGQaQN>>+8*%h#BOHtOTi+~Zguz}L+Ww0Al{-u2CjZ&L?HX@? zo{9H=NZn(QnpbJr3GVK#^EI#qa-p}hS{nXYNN+`~&r^rRpMYdx{7~a9wbRToK_@^3 zAjt6O`zp$x0Z@+Jyvax_mj^O}us{`BNg0^d<}Hu$=!Xqi7xG(-XyrdTgSsYg`;`Dj z)r?^MI_*g7ZI|8J3XINkN#R_zeJ$eAk=ZvV3INo!6u^Sr#|k(#h!Xf0f9!BoC=3bB zp+(2cfl%&HGG}_c<-I*<#6#MQ_z&?E4Z~JtNUdll!1s!U5jY3FuI0+#AhRigL!;Pu z!55BHX@bxeiJ?Wt3r-GiwQiN(t;96-r8^?$g_ahYxm?fVL;-u)NBnxH1-`|9vs!<4 zng$1fbTM_aXw_EL)3!U${43YZ&DUF^&EP2W;5O>~JPmD#Z7z+CRZzl#VFyT~^0@ebZ%&y+XMaV0=xPA{K+4A>#ISOS3> zJAlr6;JAMpTEtjK3_XeIFgge-Af^oZa|{j~kY!xrMY`-icv0+4A`;g&BgB~3D@vi8 zKx-=eA*b~aoM4%lv($*0yJwU&F>8V50hn-HC7njRRg$r23+2{mH{SZ~r3_FhkT~~F zwk8FkL0YhLHRq!Iw(a7jbvzx{?ZN0bNwF>QEi!$l%IxyzG-KsCdu68hu3-+VyVxtQ zY(rQIN4J>CSct1M{RG`Oi(>(hVX~#gm)8=8aZFwx+x$pLOEi4E*XXcRpZyf)KZi<)3oUrzZRy@Qefwy1z}KzOQZga z|EsS3NlH)!Cl;{lgyZ};q%#$5)VIWTo)j>GZyet(OSw;&l*6h!P#s~*Q$@sVza|N>Xs3mh@N_UP}4+JK6O0{ zy~D&s2)}*G><>5xhA`vYH(^>wjM_-m$Fj}maSF0DDn|dPo!)vu>0^3_%(5Iw1!IOH z!=YK9d?95p*V-OJcb=6G77;zM@lmH~*R+q=wx?WRIq*=Diu?u^8k)kDlt?9_jWJ+V zn}ox>K3@hQxZYa=v`h7Uo$m6;Vcm{`%fk#As5y#I3nopLrZN~!1qBh!P?deuamPn7 zZ^dzCD5~H+DB2P~w1q0@VCCKdM;sN-rem`61?b^=7AP!_V9ZwO`?hxOqmvwV`#TK` zqDp9jw^^eb+rn9X!L7lv!YMlxTf&AesRxp>tRDESuIbK3Lx@NirO~G>Ghnf~(U2UoI+O3cX(&e^hD&)d3#WROP7pT@g{E-(Om?q;e?>lsj zbuy_t4BM!M-|N@o1xtYasZdA?K-Pv&#;HHFA-qhl2}}V9O02hWt5&miRlN@^nt0sQ z`_JHKWd9y9g+CV$o)L2RA!%~ifV3-ZGu*>xH?}2Xer>SFzQsIwZf-rQ=}-_ReTg|@ z3|86^(HGSO29Y*{_rL6n>mRW6H`@Gbe3ZgD{}RFuXy>fIp`Yizs=ee>%stG>Cv!x_ z^e4f9*f%+(m}d^s^As4EF?IF7H)t_Y2)aZXUS!}d+fwH93Ypel&l{d#h~g43pDb%2 zfb>JG|E4N3Fp&Ehafo0yy9ITc5ewgA);xUhb#G7^F!%F)v&XpKvnF&)xYN2r+7QO$ z0u|^p2?8Hr8N#1zVLhJ1n%O!sFA-}%diZ%X-ijEF<)~INJr63sye7>(4R;Nrh$2On zwNNh^i%l+4{Z;L&_!A?g|2}XOc*Xrjg1@9DU%|)A#hG#%!V?y zU=J7bE!US0@k&CH>`04+&Ro*-GYLcY$C+u0tt{;0ID9ENLcU)8XWfJBJx=$o`K`hYA&0bPf08+_zDys1P6EF5PzYh zC9v`vFk%HV@uIySDT1q2A3N&n+IM^R~7)m3;Z(x$iY38WBcfrv4wvId}x*Jwv=m*L~;wx+z zMvZ@}V|wO0cKK^GF|Kx+dgxBjM`gUT(HG#T(86D|=MOq;q%Z@gdU@c*5VT5qV~ zm?E7$p@vHca^1}EaH*I#dZup2YC_X%LH!^Ntk=qTO;heK+ULB*7$D3JznJm3p6p+M zV8fb*B64Pw7DohP>=A9miiL`LHuy&8`D$i3766`r$y`wF)#FyxplgdKyvr)r-T zpxa5GTFuuk3lewNd&`KAgIMg2so|Z%fZz=o&wDIIN$oQ_%#uDD&Toz(#<=fQ)BrCx zWX;xB4vlv|IpR_6Hw;QoiWsLRy`=BRn%)Lc@Mu911j)0zKB-bxc9&j?YBnB47!Maq zp(SuR@7F|!PzF%>udk$f%(%gCRZ7mdoFO%cSOV+tA%-lJ7;pTaklA{rZU^cH4uT(q z$8mQxlLWB4KwAIKK1D_Yb|EFeu##8SWEhb9;v)E4kQQz$KT&Nerjk3oaZ-VbXND}o zFJMp!`-pFH`C3L5`dct}!pHXjyLF9GlBYwSHLmUgXiqss4JdP!f6T4XLfVRGpn-jNLX^3>N4$nZ4g$avSfcI(3`>mgs&7E%%3=;Ux<{} z1CUeE8Z3%cGKqRPz(UAmKzH)0U+SCnTpmEmU((Mv{)wNoGOOQi`UNHPe}XhIFXP0t zO=gM}*$#>bUb8&SDxem6smVoh%6V6zwq!a^9A3gEQzQK$at^9!1uEhNcZtNI9HJ#D zmXB7vN7-rcnKtdaV@=UYg?guNO#o z955x|1k!PQxMqi=o5Y`w21g(P<6AFfFIIvK8cfa!Xp5heB|>2HOVB#zsS}5$*hFJV z(-!Y1qJC!Xu$_O?)@LLs9-59q=4}AZ{5nl1(-}yRgmxXN<0aB&2_&yNOg<34zcF%x z%C@E{0F|;!R4wj4ZYNGIoO*_r;Eb}k^)C9Z2%B|AYVzt{HGJ}o$OCxgrS!>|+7LaL z*<%m;w~)sHzCasRMjgE%c$##lR0T3IT-_ICaE2MK%UAbWN)NjW<13lG*(a8Hc!5R7 z%Zh>ymYP3%PhM}@FqjKHGovi+={&WJN|NWf>5%Kj{)^+o%P$H31ahr z8xQj*WN82j?_U4h)L8SjyOn=V8fzy&dvg#4FE3>(=YHR%gn!x}>J4hsHhbhV7;g>( zFOK;e#J@mYMKnbeBUkG#E25Ie+yWlhf+b-0RnsB80l|cjRcKfISVigPo}(5AH?g^{0ygUpA9$SlSa*<*)ba#7hwqa z#EHj{=kS)91?&Y$Fl@<735b)*^+|@Ie>+X%x8azcmF2;x`CWtO8LwPhdHtJ-xY+n6 zCMP_S6o5lrW+SBXP!a_GF1!Zt2_iJG>;(N#D3I?qVW0uJ!#^){xQ9TR>UmAfyV~+h zAB^qr+3oR@q|Zqs5OK!2O-h4L>y&_cGm?)2zcQ>ycZgyrk@MCx5;`?CVf}b=QT)Js z8l$6I0!3tN-u9%0H>(?qh_RS>;oEf%^{}TbqQ($>*nY2Z_<|MJG_6jARK-ziE{sq? zYrw~$0)%$>dB%gIwmiNPXd*)n#S#og_$=6IDvyiIP;kxmQf-6=vO}{j68o_D+F46+ z0`Y;-K~^HTYFe^IFj`QrFoin>rKVXbI2_~A1o~R7r&wvtV~*=+ida*4ppujYcjp&= zs&XBJ`J;w{h+SK4<3V=UIv#6YI8f$H^x)L3a*6W53lPF3MuKN4ug;9*9A7TIS znnspc(>z_&-z!g93O=r~y~f0!SC{fDKxtF*G>~0Ufh7uSe`iHCx(I4JX}q0048j(6 zzaGc69|J8E(;Gc6^_Gk+#<{}2m=t5^RDNoxFgR*p>CR{YJno21*7)U!IN5D8d zO4SJDRRPnynub>dbo;+ZBO9V4c5e<4d|QPgu-?wlfxR6(0NP8OHe$X515&koFt6pV zH*{w<;~h$-V<7Z@s+Kq)wFEW4Uatv5Ke?OqBA!c!&J*I3toCvpVuPl&*N*)?MJzEH zlKL(UHe53P1VDFcLlIrlxL1_kQw3j#> zU9Lta2|l1_7AhJ*jA6Cjsc|<%2G87(Tl?9y+O$*uL7n_s_lPBU!l$5{f z2xS3%K_gEDm$fcGRQRn1tKKZLTDX+{mz;2}HHTs3I}%o92>t^eLTehdxEc$hYS}gy z*H&Zb@YXo@1+G5VXig4{)yzzWhA_dxH`!y;-CXu-DaIk_f?B%K>J8oe))G}g_rleH zcdRZB)lN0ClChF4d|a^9Ag6IF5;8+@QG}PL>vt4*Vh8HXjAqQ7D<#z66v`j=qaY*wvdI8*)NaLox`OV6 zu+M$bSf*hUIMSj5b-ZSnKrpN`?f@P!nT+^8f%K||Fh!GB4fiM%gqcrJ1Oj!r!>L+w z8$hac-vvDeX0nYZy=Fh|${4CGALe0!XdE3R2sD3)Zf!k5$ zS5;%U$pe*`_pkECK`4AWTt^hq0pNDOBSoJj=&-VcA1Wiebq%qv$^h2^SHM_r;DkI~ zr}x=>BRRSs&N`M0-Y8kQ_}f$MUUF5N>Unkv0tUUJx-}@IxRCFqJ$*%6rXQV;l%jvqP z!8_%u!CqYPXFd3RK%F z-7UoEI=11G$9Q&^jIQf@z>E>Y&W~USezgIOU#wW^SfXfJvH)9l0r$aNXs9D1JsqF}BiN z?V#21D}e_sUIb{1-JH1^qkCOtJZWG9M}TDd(FXhq1slbJ{Uv}*+l8HGtO+#QzoI1h z8?nD(nn&Xj^cPd&VW6GfK{n10f;(Zs>+@FzK0mwGulIwK1E+u_44^obcr|8jewm&${4|3E8ypw z+Q5b_%9&s$m*QYxgJow7Y&;7@(sR$L2~wCCv9IR)5H_Ik40d&46<~kY)agj}BRCFo zwT}I12HLQA6dk3RMaBV&11x+>^~Wc(d6S2EVJ4g>4p8Ue4C?%s@Zpr)8Q>4|`)To} z`GmmfFXPMDu;Iju?dfjo*xfb;w_*;7Z9k6FR@O$AG2Y%b2N)n3?{m~JM^UyF$cH43vY+B}A+R={u?-LGv&LLaleUi!m|Soc%??}I2Pap{g&8EEMy@Aqwy!a?-egkwgEvcZ^A zA07LHI{kyc2r!xo<2T>M-(>d&eNHshjm~(+{2Dznflr7)7@L(nm0=jvYT~X%eG#+!bbe`$q3K82ASF9Dbz%-p4OWL65}~ zlH8d0?*#t`)S)@gufXB+8yE&RScTN3J34bD--@xdt;Tqe4caaXz)>FEX&`liIsy{B zu%-SEL?RQ3MvDTH2(cZYJ3)8%kOpHNGBM);T|Vg-Yk7HAj>|t5Wq9UIxxYe7 zhDTQGi>yBIB~KwjQHH)Z_bpyT_&as68FD4(4~VD`hPl|w5^RWOo<3FB&ly1cN=Xt| zhRC|YD&S|7U)Wb(Re!5KGI$XN@%a+ti$-R-a$^p*(;#)-rGs#H!bL52z5<@8(DxT4 z2%A}UZ>h;1AykAof+G&~SfrZdXEc!-YK;YDxJM6k7L2%dYYgEWBe?w_e zqSgveLk^7So6>M=-gfXED7D#o_d2D&1{Y+tIfszKc4YR9rM#hejdg3Vw%n>#QY8;( zQU_^qJ9yrYlf)UiK ztgFiJiNDa=XA+#i^!vUg!;7D8|3N0BuiJ~5k2?EeW9@|n)X;hl=@wrKu9YE*V=O_% z!aDCV%D*ymDj$Qzh9ei?|5H71i0ymt-C+i}Xbs@#C+mBRPi2xh;dyZ2bwLx~+s1ky z!6UF(ywF1a2>9c!*?a)h3$?K3rw{bibjB91<0+y-bk3T94v0ABF7uNqSy|CCJaLa< zPk?Mt-yM`$2losHC&t%f!(5Cy(fn-oQ_Ks{ z@+L6P!iD$$;o0sQRsmj8e;vO^A+ISXceDPM1*ZL5xh$C4V{u4{x8(IkyqZ|0^i6q# z=SBIgU!tz(vHlI{*n9yd>~HWYiZzQn?{PDTCtqf~&1@C5=va^vksop8A4K*lH^BTC z@v}bhf&6TYz62IyIJLq0ho3h>vZ;>m@;vG4i+zNu47nXgp7_}!D7yk!ZfQe6=)cXV zRlwO~o(lK_Enj3B<@M&&GM;S+jH|GL$4J=D^#U@Ck~!Qj(RfgxQeh7#Y=H)Fx$Jqt zXx73e$jWW6z}8k(nC1i_Q1_H^%Cz4fewqCZ&;%>*15l&}9!fqPZ;s!AbMGGP|`p^F*xksH?!=5P*iNEfbZVfLx1U3 z@ER!#P0w%VF8fZSzd-k#uU}6(xR4rRz8fu~1%ThYo9Z|OLod?+w)jb zK|ElOX>TEiDKW%|@v);!YwJu$XUdGu0cw9V{>WbfUAm>~`M{cmdVUXgBn*?t#k%>W zlsFP*hi~`g47ujw(g@E(D8&056e$sP)<|IatEWMEQb6QBcih~DQ_rjQ8~pWama#@u z>_qWO5VB^T(7OiP5zd0{I`UvV)AmcazrxIriJ;@i8csZ+=2}IS@u>|PMEkD@>$G|E z0Unj!jGc-5_Zk~s=nF9dH@F%a1$EU(5+`4AGC*C9S6nj+VGRY1w$|s<6k^AvwHGQ9 zEyD9=mYye-uqmTJrNePGso}urw~up97e!KMg^HwbiMY1b`R^e^arpM{SyUlYJh;?4 zxe&X8)G>eQW_;E#_psRIjqR4OQERFlA5ZV)dWC&l=WC7@^+>9EY3JGwMi`qK%UT0U z#DoMZPM=EMC2s&K`H3?-*N-9Q>G9jMpvO&4L-h;WiQ&amfj}Zz`gv4*ttEtv75@6- z9=wHHh8rKLz+e24|4)o~5oQ|9B$%awB6#h~!t={qXF%X&`G<1Lw0uPfO-shRc)(Klh)9R6PyZ2z;FEL7@ICpqc}X&p~3$ z)i~ayoc@weE_A*EmL57-wOd=wyGnGGE=ZXeaF%R#F^1vj-wZiT@{B^NOQSHz>mX28~u z%F4zs2ANj<7N^VqImQ7}&g@k{<&^d;5YCWyjxdp&>b^@kJE#Z0E%`IG`{jWkg?rbBkYLthh16Hl&_y;7JgxJ7np?I$_x= zhN@^{O^VKO(=&zy#%|;J)G$IsAsYqaO0%~prdnLQa}(P4d3?h2WOIPfL8&CEVRGrb zsOdeI{!0~Dz(znF1>Unbv+K6%iKF&E$UdX~Y6v_I^$hSH5+KZh-}kW?74A*Q|G2E; zVdiJ%F<___2Gd=HNb{Jj#M2=jGM$sVK+sGq58G`-Z=gSnHuLBUFWQlr>dEYRB}t{c zeormh^)YhK^^`3<5G#%gu1j1ds+=FXq5w?-a(WW`h(F^h>>cJE3cBZ#W^Up?Ko9Uf z-bs#zEq^iOV^k$kmIYf#s(nuUr*DksJq4wE3+g2Rl6z{%J;86|C;4@)yo;tM(-pM*Kj2Tx80uQ2@g!{RsJ+$7 zGF#MGZbMs_VRVn7ns%=524j#yBX@LQa14;|haGqScV>vgBgMGTz-??un8QqbDeNTj z0!38t+hS~%FX)=(FpA9>V0(p+wvLVWG(g^xnGCeni=qC8TT!f*a9+vzDQ$xMS|M~~ z%yiLw`;XY#X-ch%^>dChAH=cx1jw>j>r{`x30Nf$fW$c8(~NxzL$bU=8p3H_Yy;C% zAoj1V5}(oIH)wT9!b?x_kh4>*+zDTFUm1$w!ZS0DT6UBPjk`O2ncO@3Vq)3;Q%a-jp`A=4JC26wOVEN_Ih!sRIEJ$1TL;$XWU#uQ+;gq(0JXQQ~tPKApkE_g@AeK*H$xcA3 z2DYH6N%l4nKkCJ3dgp3-00XRf#p{cW4h!^gvg;6GlNw61E$JUo0VzaM49%hf9%HqL z8V~TzaSx7*mDFAqh_(fTqDx;65S?ix!Uslr5g0|{3rf!Qp13gOWqvxee_!F}J!g$# zG>Zb;d~s+w2}<~$?yJ>S$gOg>G>E9LhrIVt^BA1iQXB$U!WDtzmC#$(ss|S)I2f1EJo>R~Ye!g)>PEVxTKh|}l#(1mK3HG6KH1sED z>Xj?|#ZpnirdAY<9aIfH+%k;)T~1!)P&NYKm0_Fb1j`r{GooP*d?qCSz*)zixh|7K z)JPFV>I>!sL@3(RM}YjL$f{z$mz^;~BQ@*^qFF&SFr?!>i!KyBQ#9;>HN(jc6y$9m z09-)!XLt>svsQv9fz3aC;Cz0t@`M5qsUZ76CY-Bx%0(ak)+J>ZWEJ?y=DqnID*!kX zqnYQ$M_SOlHgj4xUrg)8zZso%3epkOc(Av7#X@2af|ZDwuztzBKhWwol!*)?o=`;c4e#UY zCGf~aCI}pnNFTmOdYuST(>Ml6Mc!pl9=En zYM>D9LD#kDWtu=UUK9<>J{rqKsd*3_D0GU#?&dMfPX|kJHi7esOC>V9I8p_`ir zjyOFVgttQPmA}UDJqMmnwT~XwU^dM1+)vr}!LI_$kcXwm(mK#0$g+hbsTFYA_g&Vo z6aKnoo?!%E-NP@b&6~d$m??9?ycZ5hj0FJ>l9Z9&jNzdLwM@xvFqCMB2wx;-cCUPU zD5_f~cd{;kJ@1b2K&M2+7(YBB4x{g-{GaWxvjVULLwgLwBV|VX0h${c87&6`NyLko zMoVFBaCXfzeBp~j_DDqg18OlS#h(yA2V@9U)JY@Z$$@@ro)=k$Q1=Kd4dTr#^p6d7 zAyQZ#u+;yFUThpK&=Z=1B94LqIQJe!kOOyD=-Gy=`cBQ7P*qb;eGGdIn9e7A#I!C6 zNAieS2b?URDJ934eo{tKYn3nMEe9uvl?llm1Qa+IDL1&dpvuS(ft$=V#u`JeP|*7g zoK~du81Cl1`$L8j{Gi6;Iin6?7?TlX9K7(*Bn{u5fCBx%Fy(;XpbZRKrHB@`t^Ttg zFV*vzdkF-`>?{-xmDE+AzbAddH1??gU%BZ~Y%aW3Fk)kb?5pXRC#<5PtbEPidx6pxCP!`B@kjAzGO*RcP$9S%hTaGMNsASjSpaMvGFc;y4AvDoFF*#4G z>P=Vm(dnLW+D8#|u`Z#+-RkLjh~Eb_75drT2i1RVm((lDv0xTi|B-*Ch^!9ufsa-K zU!}d11VSxK!7#e`>U3J)%#hpHSejiVCQ3s=3m_9DcOXF=Tu@*s#2UHErt_aLrrH;| zcQeWJtKk}mE>>F?%@mzBq*M@w3_jgJ5x$uN`z|obojU^$12IF#-(2Skf=tXNXf1 zko3I}5XZC>*f&l9Oj#W?KeEM*yU@7miN{I3H|wD7^n*i&yct;8zzb~qW^?d+qTQYg z>-RT^E#t&_>SI($(3=6o(JZ{-=E%-54TiMS+#GY*bi>14R7c%;iu2wngq@-R?}98OKekOPlZ9 zjp8&CD{wFmGs9S|uu|RvE)u(P2f1TFZUm8{x!yL3g<)PF41j6B~kRr}OF)2K^85;^H$hq&f*STDH>|uQBO!{r+S1vVdyc zCBZ2qVNZRGX96fVk6~SI$>1A3G?t_4^I{ht78*XN_8ym5>%hw%hNVZF!LCuwcaqx5 zEVj`Kl~6@K{!fg@muNV$YxEvGgk@untjQ@7E_WaMWQPv)Zwgj#H7uZKGAR8Idm+b{ zxFS$WGBeh2)YRYrv##?RC;sB8P#^mZXWT3wa07vrd8b7p`zLy`7-umTO28(UY!= zlEjH9oK)%SX{v@Vi@vq-)}SMoRYNWMABJBxP7+1>dm7+P{SNL?JdK>cgQYdd0*uSx=9_H%klx5QvqQQ+O1|X+m2+S`uJ06E(tivZtIkuxS}m z-_fgk<)x~0b53phF^OBLP>vX>mWpwU zK9i6h8V@-ZOV%=nf&0`qa(6`!)p1#&LiN{mJ`J-m|%2v=5J3LbtxXmD8oRT)R zOJ3Iawl_bQ4F7e0;P2b*Mpxo$Iy1hsuT7r|L!>IE`s_0u>;?}*-PJ~ za}|Tip)_vU_nfRkl+khnPqgfFTTDCGs;Ufe2{F0HfV&tz21TAX=efB?qdTs8*+)8@ z44)SLeu8!O&d%5G&|?iz;xND`^MC@W4I-_xB+Q(;*BX+CAhDC`C>(p+7iac|vjC39 zpgGbOLvlo3=I z5H3()N0|QZ)x8jzST+!HB_dOxAz$#*eA2y#{!(fZu_Vh_Bg``8wnVRL+`Tp@0o>Y+w6yJR8NnSr2QP${;;Y1!BSX-G7bo-LxH7^W&{5e3!CG|#sfjYeZ) z$|%bs`%+0?nRBF>a_XBcres83$x;Xje`qv4I{_2G4?RZI3F!RY>u8}(ZqcR2Nj{_~ zZE*BX;kI#Zem@m&n4Qz82L!Ee^)U-$AX zGm+&1BV+doRqaNvTr;PbY1&cH=$gtazp4hA-&9Y3-J8O+nE0Fgq8j>z>Ewxtn`KDT zcxsTD?pBhI2s7S=uWU<+aswj}!}0GU*ZNOT1MB(#{0;HRiWM`}OF@tD8<=ik4N&w+ zIzteTppUf|h>rQ23(R3+X#2GiBm zxQ7)mBs7{~5DBzUkg?xRBu`>g@s4Ahc7r%gghb1MnEe1}E8?U^BzAJ}m^l*PgPRI~ zar!3>N4W$t#fv!5pi@lqceEg{LXbiPS3C*H1RzzTGQlh-hA7{AM`&`$vR0wMEuLp zXXohy`R3!aDCD4JM9tjpLMb3w%BL7`Zr|J-(%G%aVJqjuw=G9}-0rvUM2!(*A>>m} z4{CrZzf*|g#M*UP)AdsRM=%>R_w1?B-y=Z;VsPQ>XUWFHc(EZJX-e6ds}=PKljyLY zBFvQGw54G0^ALLlZ4uP={CJMOMpgLEQ%jU54w;7Sml_aP&9Xp43}Tk}rZiAHGoua? zC&K>F_Am@}I8ngOfq7!}yzRo(jYUAN?^UmaI%rnl5*S5jNIn~edgg7oVlW-?%)eO8 zBc?WR+?%Y16C>P8G{AGdw+C-Gdikiq`7hgw`gKQrFiB(PcFCh}29Us-A(M!=u|MWboD7!)4xDPk^1G)?E4`1+ZIc%ts%dYBT$I>X#E<|lItZ}3P-UgX}) z%^6K-S8g`tOJ>3&i2HZ$x=pXO1UB)XmCq*i)UzyMqX)*-ee@-(z^ai<1#}^5aiD=S z%YN0Pfrm=u6HT-;C+L)^4;% zGfo%1f_*6@s2G~AQh?dE9;4L_zKO%>@~ImWQS~mpL^B)oGrp}h_B1n{T_c&XM1m4l zW)OCu-h^hb$_*TdGZj|`#BPz0>=XOf{HIXgMjK=q7*uHf&)RKFY@Cs&zF{1xG3*7S zd%*1d2j?62+@2}%zChB!2m1IVE2{SN;7rP^Q57pC?$}9g5ak<=Wl>8^DuBEPkhW1d zMfd^%aKZ@9ryZem1+eo3xEso_bRa7|CtmS0(F(9~-t<*OJM@R@2WOtzoTgP0{I z4?K>az0FXwr_SE@JKmq^ySo~)gYbVK8`cp0GYNv#o0k7&m?7vHyU%Ke)9Nwd9>7qB zhy{kqP-)?A84=8H9?f@Hj7#Xp=-15`RBE+MUYL^24^aB~{EQ&Me@S~YU4+`z`A=_+ zq2tD6&wD>Qa&v`i+HpL}=@o$aY8o*Pw;h(!2*Tm%F)NZ0oBH6Fl`V7MrJ52IIe@ez;5oVDz>z+f3 zIqJnm+YpNf0VTbc7(ONCb7zFiZj^FGCWBH&5#*Taka02d>7MboMicBA1>sJk6N}}I z9059`01Ggj@gK(&ygT^6F1x4@$x^WNM#rhwSIKl~Fz z$hQ`H9xd~y{QFIiY#A~g5TVug+t?~e9j#KSWs>IA|zx7f4V39iw zVLE7=?-aCEu=WLb|o|bndt% zU{g~vmjRPCFb(SMFiKFV#bGN|dE@Z@!Cya5;=F_V*caesy=-7jJzUyx^sU%0MiPe3 zB+o~YVa5m~#sQ-viyIihi}qmPw|um=Ci)*bm=IEbK+=f4#T;cAm0V>;<#W4dDgr}m zK8oSzOkcR$Li#*fP+EcOd1zM)rVs&!OxSOgc~5$a!=D{@O$Sx^>vpkNpmRIjt6&GV zO{~SH3(7Sjx`7CRmFm$?59he6z~29TgQYZ)`{d;pyv-|$OYsV-ibzQXl8Z<0ORAz7 z&R~O;{-GcP)fHW&9IO;3f1^Bx8YOhUm1#re@Pw)TJi&bv{un@*56-}T_2ZQ&!sgL` z1|LKA55sTZ@*tuY@7&IPIcf{Ep7!xVz;Wo(1m%H8-E%-_qmkP2yX1}H-1S4%fosj7 zU(i{ym#XF@l7ZghAR^M(LeYi`2W1@ABn#jRuyR{_c=JPHJ^&-1hYeXhmuY5jCgZ5> z^u1I=>h>XOiKT|pbPZiMW6q9=Z(WcQIi>BJ9!=sax=D1DQj3x*pO<|MW9QuZym z2Xd6Mz4DT~X85PQ=(A!utlJAMh%|B&ut~A|9v5=4YWk4*!;dqafQ+46<5S#^!$04% zIe@SNM7sPGMFC0XDNU*WKj#d;mC&4*a|RrMBuoK+8NM*BrKd$4%8m7MBW;}Bsm6)X z+fKr4ZA^;grkk-PU@WhdS&9uGpG^1#&$XQaU#2RSR{cvh9DPoVu<3c@Gye3dmPe9N zUq%A|{B|f-gBrnNAOJ3S{JrOu+ujVWcP2!EJb_giRV?xZGefKiBhw6`81yQzu9ri( z1vc*|Bg|Y=<$AB#o8BcVfo>OnV+ocZU%KLm$8nFsk~F;$lUyin>Mr%HFo_~412O@v zcgORFwO-ji)RJl`@<9)W>tJ%+i)lQE(!M@gPR&;;k^D|~m*#|=tq{hCI598<=$a_@ z`AX<$P{|(=NKba0dl<8j+UwUX;NyYgO$%=D@6m@e1`iTbLIZ|kuxJGU`7%IzUT>ok z)Aq>%jbo@r;YTPxnU2bXU89&)Ph1)WRZ9@}ak}Cl;XkSQHIZ04x?tX6>hwkhqRiLX zWabU$QrQUAYF@-MAhKc;#13kW8)6oO3EdZ?O zG1{$6q%gWJ^LSucgK7oD(>o2h&^BNlYA2aV!r6Sp{^n5jK7t|*wYh>&2ERE}WVmc{gKUwYx(H)E;f z3>D1RjSA)q2q>JUmZwIw91PtR?Y@x4B>|Bj@ z6ndjR4rc!0n^cCD*)ZM5e)uPZ6G?i|&;{&JO#o55`gwhP` zat7AK3t46~H{NuW-b+d5U?!596#O1%fuk(nkF87C>W!(_X zsO&g6+YBds8(9)!FEKKb^371w!F0lb*D#6^hKO2*$>*?Y))Nd=o&e(?B1ggzQlhQyPlr z;cQeS+`yyCK35a!g3}n$QRaV*L%+m_k;_l>ucooNj22rwS`1a=tUl&)x-U)#x*DD^qP_3*8 z@`s>H)j^jz5B948wy6j->8;o@}}QNf?Kf-`ft$iY%A|MLud51cOnUrGxd z6PysAz_!^o32=bu5XQ_Y?>nTue2u&FlO3#j;D-(Q73h+Es}KTpbU{f4pXHCo6`5oJ~Gq>&Sc<-cK!in5Yj7OIX~3~k9$)DBOl~z*uTd@!22!_p?yPQ zB5rax4)Dj4nW~)N2xmOXj?9iREMkw}xT<>u3s7Sw9U^k8ed7BY*reEKJ04)e355+X zMCt+6*S>6dIQidird7dLQE`lxSQrm8&s>8PKtmhHoX*&|56uNCJ*a)d>0F%ox!xqQ z$!Eh)Ur>aV6t?a_`i3(P+W-@}C~AKyk!z#$3P=vKq`55G>vuDX zl-{u-;d5#oC2-7Hj-vs?NAT9IK)Oxq3|u*5kPF>L|9Z}|15tIjSHtg%BllIo;L%?{ zZy>ckcXEY37{=`6;FHIs%Z%giRYL+n!lKfsDiCo1D8M(2=cH)j1UGWf<>3RE&g6;p zg-)sd&S@rAo1v89SC6$Lx`SgDyb`=atz}@n?B@GhXqx z=l8(Sk+=dzqEn0s2aP^)|J!bXl*VtjqPGXSSGY+y1M7SWvtHI))cGgZ+xbsa<5m2n zXZ)y@e>P!-P;0EV1nTAvjg=A>TC67E)%4MPlsLffPuhJaLm zrjh661-}XlnFId;M{xlpTzKXc>A$tZjD>Q({1uSrIv&bpt5&Ho+yJn*BCqa{)SSmP z*lsY#*^S(*{52@lcA`Z~bdtZ$;RtreNAK`8we!30mM!w*Hv|U5e#^akd z6db4&Q3xp6=sz0%LrVd`F}8F!6;*#!#-uRtdWi@W*nbFBwfBu|+5Lh0mSrWU-g^F4 z_QT#oj{3$wSmoWhWXON5{P4@E)86&D?)a~qc}d$RLF)WDqifz=>LoQ>M0LYu9sG_d zxr2zK!t&V6;@fhcyksgV?A*li2FnOwZDMK&|Cf2=h|@G4W8k*@m$qOd00VJ5erfQR zG~A8}-)mWj6Q*7~C%U*1p_to6+V&9{wEP_gNf?ws!7O?#R>oK(tZ6#FXsG(;JQ2;v zTPVR6OmT`~u0|6!)`A_`>iA+!-+>f50=C6?7^Z9=dKu6<=$yV*MtF01v0==P3b@fU z)&y3DgTfDoI;$$+Tk(^SX7WQ~s*l-W{SXJ53m7I(;wN=T|Nb}^z5vsE|RSC7uXB&$w_mE2zwE!Hxm0>n`=bg`eqGQwNG|GJ)SAp%Hu&6LUb?KUz>Xd+TTS+vzKY`PkLv zEn9(FytDL)n9AW)tpI|&Gr)mb6K$_zT+0}mW!|{x^#^|Vg!17n9@CVudWSuyPx8VT zgYU122=6slSs>MMi#HMJsPT_2dRrPYUk!O#Jn6_&zUL@Pwyt8y=bVLyd{pb-cB}rF z2gndNZrJAwGUIr?TFpwtiXW~{QsxaPFSXCCp^al@;AE9jA)Jbb)j_)*KXEZnr3(N1 z#8^&bJ|W!wHck@GB;xE}GV@!WyrYqD3HiC2{s9wdV8?3r$|e`Qq(&QhK1vQ|pM)(3 z&hN$_Et$QWjJ`hGlMg}@{Uv-%d&wjYq?-6+f_J;$k=grTXo#=!?I~-5iZg{sppA6_ zly(A2cNqb)wk=-q1AD2fa4~HGCw(8)W0zKCyw6MC1{Q2P6(xGun{lETJiWc~u1^`& zCOG|gf=jRb`8^Ph(cxyAzkbW7M;lIJKq){$WFE^mn0d^LLFjNO(R@!0-`1Rjp-E?! z@y%}1YmK9n(}=!6)1Bh9*khAEQpiQ@HI#yQ<7Vv)hNqEPuNb!~6IS>9kcG2z9GX>~ z95#fC$WpmbV_4k<_sAZ%4MSlH9;1rMBZyaYatbF7xt)QIga|XUA5fi4@0vBdDcTXF z?|50D^zgSR!>xeHs^u}|d?Qu*NJuG;Nt~i!#T0LvgpxRH91AL7bBOziEt@tTuW{lv z9~eZy^AJU8MQ6up8k$1@vtwhYHM#CnPUkeOJ#O>{uIGXs0L+2HU-zbFM52%H9zLO@ zykm!SS_W%vu1$K;#Hs-KT$2P6Dnl)NGNhuf%#Oi{_r*CfIC83i~eT-V) zm~0^3jNA|G9wSIY)uTpQSup#d_Z((~Q4c#Y+NXYQ>0@s7XDRua>9ac^Nu*M2OEanS z^%fabpw29iuO3J zgGlJTwh-gVRvz?`;a}zNw&o8}9&ydA$`Av3W5tpAwpsjp<6|4<47mGS%rWuMnRwXC zquQVETz05XtRu9ee)Psirm;9f%kn>DW=G)`#Hu;hVjj<#=)RiIm~OLtK9R9<$C-Yh z$@qijHKv+&4rXQv{J{$h&dN>!&z==nYG_)m9%`-Hl7!tFIj*C($p)KH&*3co5VF4p|-3JQ~{u6FPxmt2pI)6~; zjoyQ2ijmus#O|?a0{$^H7>{DPNH=aCfBFg5b8_sZOO-UiZiNBw_l$1Yv z8R(|c@H^*v1;DMou2Ya4K=KN+&fFHB{1sQ%q*3#2mfl|*2EDAhi9q3ftSwu|^V$@i zn^SPR^&IVYGXb6QD7LRjFo><^apS!fwlw1M)#x@ZgyvH+VmH7pOPJ z4}BHXqe*E!3N1S}GjZs89m*C%dlke^<=&KeG zu`*k%q|a+yyxO@UuCJ>Ix?Pa>;2f|ZNK4n?+v|79k{lal(U>sf59Jm+QuBgWav%50gs# zHhD*i3)e@Ew48sWOTvX=qgKEFaSkZHXrHT}s$;knA4AfjP4`se*EDyUhpgWJ>9IaFu#i zw)|&}YKx~wbq;rSzj}`@n_i5cV~ZJ3eemxZ?al9wgboQs^IDM^+5`PtT_qd=Va+4C zP61*CF-IQY4a^s@0!+q#t|3kUjTc#B*yr;a*DO+sSY@A+yQH(kpGF|HhZeq5KPjee z9doGg>zKJJP+r$T-UjyShZS5<6_J;o)d;S=)GPDAe=`RJX0D;U_Emp@jy!8=G?xOn z=|DIRuf;6XN$bi>O(+8G69Q^6YPd9;IH$e!+PKN>TcSFZB zwKFfZg;P;m7BnO!;TB(^7K9r{I|?a7dYW|(i64y{w- z6^0!2PV%^mK${doWGJBJf!M!f($4?Okl4T2jR$nYv}+#y47flciuH+s019z5Byh50 zN<+sm?79px>{2?AP{tX9(1ru?{_NeN{0(slG4sNuYg7SQPK ztKo*5qvaEN&9Eu3L8^yr&C!FyLf!(lW|xKU<6n>od?O8}6}|Z4=&We~WW;NT=+qV(5{4Zzc7v6HISWSr z7Z+q#%hy%Th$S8yfddH-rQfh|+r+nEzyR+Slu^G0W&)xwAm4H`J)&< zzXKChGBIg|_YkuaY@Qy++gIcCf!-2_H?R><;t$`Yg&uF9!O-Ikb?Uf(s#>NaHhyUo zoI|Z+Eo==JH*oAt-@lmf5=?h~FW}t8Ph|&aiF|^gvk}M#D&i2%fV0 zYQB+)D7_1r2%I0SJIv2#5=yNz2u1cUKYE@V@I*Fjsk{z8=#E0IU0gX>R zgv_ls12L>V%V-uvZQX(Kc|6v`-Y?X_Jh$|*3EJhNPoXg#rd2Z+=v*fmW8LkkgPQ7u zsrQ@^r6q+q_&cfD2k|@ioCBDu>a)G=7JB~J%KHyCj0Cj&Ek>WcWpW6qeN^&?It$ig zFF3y=rthcP)1)|25jJR<#z3T_pCG#q&-l7*B#6+xoh!Ln%&&(&sKaNbbGiz~$G%i4 zX(e`-98QC(k8l0d$Eqi_|B<0X0LMLKP%TPImfS`a0z!#tQku7&N+^9db{C@pqhrAC z;05ja7e0we$jUSoPr~Q9?Sh))`t?HoYoRHkP=T_Kj4#tKzwkOvtp0-+#VQ57c`Kba z471*adZ&sqx}ckk-2dtgjT)o1BlMNn8Nvp8E! zOX|X~Qa+=lzOQK&MZPS60`DJMb2VB~G<#(UzY->eiA$XOX0Dsz&y!$_Bt!%8a0z8; zB>hWx+VEdhJ75l^;whDi6UgYsA(}u!1eI4A;4av_p)+7OfQd4Z2gcV z{9qNk8sAXk3OQ~m96YQEY?X2GzhA@a-7xT5)mbaZx&<6=q<4a^{6ky^^q|47U{-gu z^sywy#47ND1DBWLx7}g0<~l$eC|e)Qu#Vv$Mi}!jc=&=zgeF9R_#mVldUSBm3HYDK zu&SUXh#fh>{##BUWgOsRG{S94Pjw`QnCldYfX0JUgAL~kR3!;lU>tNXX5lO-LlPIR zu=c8lEiiVqI={mU9+h|@N;W3O!+SkIq6mxb<%MOe82yoF$c2uobI$twR zq@Z7^CGPTlT&4ce=l$y?ECFFC^e={I@bwZ?9vLVcl1mJvypqpht+!g|&A3aaeM~)7 z!~+P$zf@@tl@avA;z@T?nui@EG*qc;>TMiA4)!AOMg8UX~{PirATaCjngQ~4)H0d2xEKbigG887AOG_^4 zt2D5jR3%i`hR`%CCFKCG8`QgWtB*SWlE+C&`n?BNLzrNIV_3zb3VF(@>bZlI|8;3l z9Qbd(rilug7$*VPQi_*wO$yXNFi@Y5e!L6cXHZyCboM!;CPC2K81^cdMsC9ht%iap zH=KnRg8}%&2n5R-++4*`xvWHC$yW2!$1IJY6&R3HriN13MvGJS816IDzPEjXfpb7+P0-XSBg9#tld|21+moy+rc> zY&5yxH6guh62@0D{{Uc9q2!0YIEZHNT3h|MQ2T^P1PU3?dXCE4Uw|6sSsp|sHvs+J z6iMRo4x-|=*w{ZbJs*rS38Qs;!v7!`;`9W6(vC2!Qk{Nv1tYSL3h0~ncCIir0k~2Z z9JF`Ie+z|f@<3RMYgQpV5rKdjCM-t_#?c#tlKj`TQYmafY3pUY55ZCQmlZOagrDj;2BCuTr_o>J%H~1MW)vO+H4HH-&A z^-Aha`w)e2_L&wfIheer&YjBlKpYJC^uhj{Too_Be?>O()vk~o!$u-|4mNR4a_55S zs<67!jG<>Xvqbz!)fG`E1kwF-z0&*4Jqd#;q8458=pFk6qxm;dTuG8fzz~``t zI0U5yu8_3P!2DyypcZcx_Ve*3?<-FH-2^k=Ys6pT0u3EE)Yu6gl;ad@t#Y(Dl9FH&l5{!wk*p;(8B$v!V%qJm zf{PFI>csZ=r|~A(!1cq_H%*y7aso-4KBRdJnFBrSb5m2=)CSJHo|l%5Eq#(_ZGz!V zw07<`kbP0r+9*A_G*n_^NZK9@;HJ@JA7O|e(|x~=C^qyBqc)ay`Z=qDEWIE1{rP*M zX$)@mh*4$~Yc!YQPeGQ2Hv`lYn9(b#Yr{g5FE7;%nXlO&e0);x+*%jr!@2~IEc?j( zt@*L5xitZld%v;)`V5q_myRa7Xkx;LXd>ReVyc%nDPjb*v`wsRxQwS`_gH0^V7&yM#q1XW#dF^TK7r{+$d2;}!-LIh zr&tO~0Hx(m0;7@rxSRH{3l4@sRc)POv>O09Xfd=qG%EF+x1|Z&>XIq^Cj%xfN_I?t z7ik)4G-%dR3#rd`v@$IfP4J8qa;1-{jw(>y;~pL=s2fTYe3O0L_xhUIc zV2?V|u53Rx-v;_1vJms+`oT7sYY>n?Yt5M-BzzViav|;9yUvYzJC$?^%d`zy!cZMW zAZN`t+;o)VVleiAST;PUL)g!!KA^79>T;f2JDtu=Awt*OFW1SCA#y)Wm>WlmkpID} z0KsqfJJQQz43|&-sjsEQ#^%dyW~taYj=tb6JRJt1cn;dU9nbM4ndJ+;a|$;oe;Xau zpO+rC1B#kDKCJq|Z9HZ9GAFo-6wy8n6Iwc{R_td^R006#SHuWh+iq74bfHLQ>Kmv& z5Kw4}v83VN>3OidA$^B28TnuX7ssL-00Rl3+e^~oMnB6HwzJV6uQHixqTj3gc*#1L z6hbaIW>(0hDXaO>_h+`7cgnnxYFtGtHViEowC15R#L@3dX$)+aP>9tLsAx?AnVtY2 z&}fG@#`B}hdVDjdt!CaAI#FBx*q3N*@^(WLgkKCT$zx$ZbDKc}XQ>8vLE{C<2;7p7 zPrihYEL&16nn+8x`JOd3g1w$6;E~zC*(?yn;a3*MXF0AFC|tEAz%mjD{38xAbOe_d8uiRzaPdqkx3k!Ek@i747JuwG0tf8VZr=Q0@Oo0xN1X|L zKM}ZxGu2V@prt# zInS3167GTytUBxP_vlU$a<9skbxvcRGGF6F-2LCT9G| zON+{ZHPp?A6Xv$nYm2UO8Z?dX2gc-d%#jrI75O1@e{9(T8r(Mie^kABK$Q2@J^uZr zX*QFXl*E*{Ri#p*l3&54h)Cm3R3_p=5zr-Nj2^oaSMu&Dh(on=^zk5 z;C<69jtaym5R|2f2oZ#d5Fp6*J?Gws_I>|ChXH1udG6=ld+xdCTv(BDJFQVw1h}{h~ zh8C2}bqb1prF7UzhDg%-qZ>(x7!ue27dhFxf398HY2g`I*IzKdj|B&?(m{{&mBc)O z$9eyDH0UCc?WZiyYgq9YRtsjP-6JAsxLZ*Kq-}B)rQu$!AiID{8@6U^p{5khP-+iYU;c zrU_xb+F|am9eSzg)u;are_p~U5pW7FM0qf}C%|5#19M|#n*(eEhS=4z)sgR$NC8$! zNz!pFMT3pk+fX*w~Lhr%e(lWB8SbuhA}v&VV6i zwt6j8()!G>Z6Dr1k^;sSm>y+nlk9Y8C2>E3*y;>bMH=i5v%e}gW7%jHunk_>?#Q@R zhK#enQ2R2Qqi$BRU@A0KV7rTv(B`#pw+sjkN!rRC4`lE}Npfj2$u&Yg_U$YVQ-2Ql zVF891or~Q{fg{v)o>EadmMUaCVy3d%EQ!)~Ytw{Cfwy|OKPGI5OW1eeKRS>+eScZN zr&q0qA2~4yKa-eb`*cmiPmPE>y~-;Dj=$5OJh)h?K^Vzaeh&=_9JgCd%PMG{LFPi_ z6&F`%LIgQQd*oyY@nQe6jJSLISQT2TMg3t3rux!oZ9J~-8d(vlBGG3{@(kpr)A2uk zi@s}S>uD4;f-2mAx1%6kaH`?$o>CT_6~*gzC;k&W4phK-9mo_rlGRvfx^-w-R0RnX zwv3P8>yY~@D?)w&6sItT1S^kY{27ts?gimu{K|6dVLqcYdaWKf5Jn%w<)X$Xw+Wu!grmMSn+9e=rFLj&LortGJ8$ z3t&oAzEN`%cF7v@JJ^Z+TP4P}5ett>ur;Nbs#+vvmH5WKFaf7uC2T`E<-+9*un5Db zXxFnSg>2HE20u`ajO}hWlbrQz3vbN~s(Y#Jh;HQ(MAoYBm(Ss|%fm1^J zg-ppy1`~-26zH@TzakMPB7qqH?l4~x@;DX`@INf8Wy{Vm(1eElDJT$m4?+x(K?8LI za`^yIZG_Sp1FYRJ6^Mk7C>W{6Ue&z}7JHR}tA(L7%dc5|u@fesb(g7ApnbU*+=vk? zOlNsp0P>Q=R7KIugCOb4&Irmg-ejb8ua40I1KCjq1h#Oe-t zM_s>ZqB1jH@TCn%d%VgKOkpqBmY$ai(N^^=Sj7AZ%=K?jG!xvMUAIv1O~bM9{t~-$ ztgkD@STZg_wNuEjOCOmdXyDdSS{BBCqF+R zHd=lH=ZL4Zw3eFbrwg7hKLv^>pXy?o7s$gU0gLQHq)ATv^*{J$3wBjMED%NMHuOfo zxWzXVg&DzeypbbU(iuNMO*}`#IgbAlm2{wMbVfuA5OPA-6+tbq1e2mSGc0E*)h2N2 zq+^|(Nl8BU?NAj%Dnc%&#{ehIo_I+D8pcDzdfN_N9~xZu#g*OcfKOn z->CTY6-`|8nB})nl*Me`Ar8X(hdh%Aylgj5|7t)lNq@wn=247O@H1R{L$nR{S^5ar z4!Ew!bzY{DIXIc70Xwdg;_0F9ARK}D^m;5c<5@74P4mWA6kEGGR0J%*B42TjDch{J z*Vm&qf;D5^m|Iw)=Ua(jWT96%H<3M3d~H6xaB&gJ4WWviXNF?kae>pVUvcY;HiwtL z1XPG-$gc`T0Pg6iiH`DVnXDHD&Lf%OYQ(M97Ou69hi?Eu zl(pcO@n2vb=-syAJ>EseCM%^VuvKF|nv{WH!U-ugT;1+_M(!EbDh3B#!H0z2$6C#| zS1o<-RNjS!8$LBvrU+=7itLC11v+`OF1;I%bglgDqkO13zXbSe_aNw&K>{B z%El1Eoyc8!)UUn3`M~AzOk82XGf}dKd{)9xyNKRS@)2VJ_itw?6AjG<`JpQ;gi(RS ziKLw!b?Zd9YrNB9jU&rK8GZ0%2NsVOgTy=pTeT>%d=;H6B$YkpIjH{G4>dA1kv4M1 zG>?d$t?;x~hv0252ZaKnp^leCTou_X5JAi<3)^g^#RABYFq)HAA;4Z;Qgp=wd@yY* z5Aw52+==6pt$oQe#x~V?jidq~{04eX~TL@2`S+6I`vHBP|j7`E~Efk`(;G=`pbK<;opVY_>>5z?bdj znw?SWgXImoI0z7tSqyz=xp3^AHqWbvlg1?B9Yxlxf;Xvv-D~DF9U&q}_nFs?EHJ}9SF!7B6%)mK9l=w0|HwfIk zIWORA2AehQZlx?sfhoi?ud4r+zc!NQjk-_*wvWIX{e624Cu^ZS;Mjfl$Ns)F3Id}s zW!%*7!njgkhmq^c4XU0u`zRs*i)n{6LJnilV8W1w!zpa69yJ%#$Y?2i>--fe!LUMg z&(X2i)&H=Kgju-HLVb_PDXiZ~?2uFZQRHU8!T_^nz_tafA%Ez_hSy2BB zoG6STuXp}93T0xMyP83TBTiTnTjG|2V2Xf-r1@zWM@WAa^?|5RfT`4iT^@aY@TJn| zx1^K$-pUCUx8CkBOQ8z-cHt+yn_TRh8Ak9^hG83SD!_FgWXo}Vb9}9-g*_80ZE4&U zTb0RiC%;AH0gKT^jUi99k6C|7W7VT~$mI2IS-x_EBzKrzwjPdaC4duODJi$_z z=j)j4U>%3zE&wGS-{!sXRWc2uivP$Al{p&+DOz`EaXVxu{Iw831YFPpQD#FjiAF~6 zOt6zOi`TmF6MUUzZGX^DbQ1oS@ioDh9?s-=VZZJQ9t0Sdn$IDO{)o=_-!39oZzR>Y~Z>DJfVH!A7 z5C;0E2tX)0FmQA-wLq$zL#vLr(x&gb%S-f?L8X9EfJ2V-Tw6{1QK&K8z$BJt>UxPJ zy&QBy3Jt2TMgMoAZ;2%#)Pe`c7l~~Yh=h53N8FcxRY zl;tkhA4#mg&a-L^7+mrI>^~Tc@oiqdM_2uZ;)zf#N;>o-VYa5=L#tR1?n?gl@o33c zn|>DikQ!i{`@}#bbapwOfTE+l>ySKv4uDEFR!(`0 z47haMo12gp!FT0mnN_g*&%7-*&VnqwRJs=XacqnlGvnI+zd|t{We%N2H_v+-x>NG( z{6#HE5=RaaK)8aroX~+uuTE@fWtG{C;Xm2;M1bz3)&v1KfT=HEGZ1 zA2l*u(hC>>Q}TAaJndv5|A(PYS(7e#oR14)s7$}P4k?!K#R34xg#9`oHn4?1jMh!L zwg0AI7`7W))#1k&9kBGM;VvQ|t9I-)3RBsI;({~(Q3gDlXaI+G`co$3{sgTr2& zR$R>$3K~XCq@Y7XN_iHkrBuUY*HY%W1u;Yi^#`9fGl4xBu|mejjctNd_?5(6l&H|m z)UH4|vK-SAWDWqSGEC4j3Qo~*JX9{GP8-ZViZ?*h1 zl%M6t(}DT{FNFUhkK}uyZ+wn6eTO+a7~nr|mV4jV9m6y6XY%a)PF?<#=Y=?qQ)o`w zve+utPksNyMC|tmpKI9Ov*7C)Shj@}K+B+938Lt%n39RJ@L0^g2Gf{+snYzK%b=r= z5*1b;5@BkV0}leR00cVxa_O4B`0ej=eX+uP)bsU?JRm^-VP(f{a^)F;x8d1~k`Rf| zpF9dwxtn2Yz<5G9Cux?%s@bx{7s8m}+$=-ytXYsFS!zRC%=IeXEzd5pmo@V81g$^n zZbdf655u^$RB04q8IM0r{XWc#(=S?vOAOSjz-+W>l(KgGPQp9;3ije8NbX4)5a{dgT)7?Z?71ww+aM?CE!*WHsGCQ;$qI1`$qzZf_8-xOLJjb1$@0=E zk=P*#Xt6S2hfil$m3#9{r<&y~l2VWeL8$~5ma7D=NHikpr5{cq;uAp()BP?O-iG)P zPEX#Vjscmox8~W|Vs%vK=`ylCsm#bcTl4kTkvl4C#th~@|{Zg|`t)pes$%{{s<#gyET;mpj|GhC-f z-mFdXD_2{cCZ!6tjAi-f14^!FOmHEJ=!z$ ze!)xiho+X^NBJrs|Ij^{F;;u;fJy?`_opoFcJ80z_%QmN)cX4u`~NStL*i#!w)&69 zec_jWiCPeYX_O!)80fu)o#1f_#!#{=1b5&5#6`?(s`7wwYc^%wXSsp@!2E`Zt5DO6 z`+n~8bDH1;L6L4*@U7Vi0g3MUym@FFK>P$v{qc9Dg@;o%{%kG<(Q#W&f*TJX{#bT1 zWm}WsiV94|x&>t~`J08(2kMEus@~X*o}~LAwTtf{4qVfy)3BEMkCg|2Bpi6clvv^} z8Xq(L62AON_#!xwc@IESPRiFAOD!tGFX&(zT zl+SnmT^-TT4TbCIv_zG3G28{}J7cVPJ0H$F8xKW5q^5?Cn3~LIbjetAvtwtz_fK)Or9%$IiRK#^>A|x5G0U`>lCQK0Sgm0%KPC|h#V!5hH?bq z2Q11J_UJH(l*yc!3s%zPOy-f$+Z|9ESH?VR!Ilx?c=sO9o%oh9z zhKy}yjP_WO1F921#Njb^U#*@#7hqt`+}LVtdT0{kwl@_i5L{%Yy$gi%mRQV|z=xn*ys|W%Ji)>wplXs>yE5P?{*?ojW zAdFauw1d70rZ$-tF+LIRn;zzSg-ncq@gCtGWL{Lxy>e#kxY@iV;U8-)*X572=Xu8i z!|GR-hsUr>d%LvX&PgZ8Svwk4EwIY| zi)@&-@v+v4A4}P2gcG#Jn0#Q7#g42m@(DmWq+FowZ&D3$7=jk8MLCFi4FnyN;PJ2| z6wzkO!Z~GRTA(vC{~eFKU|RuiiG8?q@hR!nCwt&W08zV`w03uQ`hsHSeAF^c8i$67 zII{6$E7Ea~?K1Dt9tuqMBl7K(1w9wYP^wGn%1{+f5MX48+`BbTeJ0(oq?z|5gRJ(q z9w2{7;X@7e`%r$?XounoiXO7w+;QVe7xcV6L~g$1Y5#=98+!G=-o@vGh+cp=DQT-( zwVBlLuwz}$5rHPK-9W!K5chkJhmI)iF7m_(wxIXkmk)#h;s!kU${$%R8f+#=s9$2p z-3xt6lx__P+X0)YoJUZ24&#-F*2|*|?Lu|;s6P6$YFel^r%P$sW#+W2L*dTFWOh0H zxTrv@<$<3)kHDS}$n6XoAP{K7M7cV^)Ob?9ld3y0)TagHxL!B2lcy^~Bm4~I&TwKu zx703_6l*?m7LIs-S@6L0LTjALkomf?AtDsNKe&WG1h{&GWhmkk;Tz6WXFisH9T)~G z#zSCLvH|NUDZAWCnk_`QBXJ{S@fLh`(Rm|x%|UM|-IXf6RHARkA7BecLA)z!_Yrw( zA7OuB)8S2xEqVM6umoU07Czd)Z2kxaLjKr|py>?e-op*S0cVEw+PB$^7Tfn*Nna+| z!XbPOY0Uxh-Lz*hKH>L+)|U%85FfP~3D68>O(9?VG>Fy-m$q^wo2^RA1&FaTjJEWj zt5G$<%=NhSQ1v)(0B8@fGXuOwx6T)Qy0|scO5iVfYSt(6H7XJG3N+k`W`;c-1u6zm zAHzh|nznclXcj=*;;;NKvkrofC}2J>{9pu6rii?agZW!!j~++$s#9_^o&4^)^kU!) zFr1)c18OLK05iom#9w54U35^w_wu{Ces={lqs*_FC&NaVOmePthAc^Qisn59JS~ww zSa>U=NFpX)b>UO)>#kA41l`S&^Z^|hQn;htEr5=04<=nI#< zv{rPX_h&k=bN_f2qH{0Eqan_a%a@(3YoQd5v6%e5?V8hRW>~$ z?k(<4*``fsr2aE*;(F2f>-hNN_ZV|^Zxvu;SH97X_213&Vl2}Delm)#mcbl|fz|r{ zAGBo)85&_&^!iE*x6#WVR1 zO?q+lL?LOe)lk~3&G0mIpF@5oc@T>)Z&Os(ZgLs)fZbVPZ2yfWiY;LXWt5W(zLiuH zLj6C$ItstT1dySG0r_+$u^q|l;4x2Bb07Q#BneeU_)ktGwF0(z1Uq!Mez2{x6dy1+ z6KHgYPsdjX+??UUJUB`}OHsHazh#aD&8}zhZ4nH!RQFzVd4*mX?pyFZ93uNtws?EI zxbLeu)^ic^tmq4XQUp17dxjrS=W}7G@Sr5&8aE~_VjBJ-nlZI-g?&2WddBAQ3 zDCBq>NVX0>zxRrkUwfWFJSc@^Gyxm+$GnMqw(vB7iWr(3j-=4sj7U$|M7AuHdTumq z-^&>3aewy{rT?9UqJ<=-78t;}#+}lk*A`f@t3E|v?`r^zX*?DUbeeAqOb7`b?!^Q- zosN+tWh<=dG)t(!m6Tuxtv4jJ&J)AHJ^d`~4CNwJ3qVn{+*vxkmAAn>T!43L-uOoQ zSG*kc*aE!MCbQp;A=uI5nK_`cu{HKp-Dx=NDOLr3lXWA@xd>Go7F!%Pv3g6c(NVA= z?0nc=jOM9Rlq3SSt);te8O!I?pulPEe_|22Br=p{INiIpN%j2f$E(nqNt0ALQVPBh zQFpG}>>V(D`#PTEXI1adJk3T57_x^V9A^LPnP{G{bW*)yh#=}%y2@NZ%~=FC_cq9z z2A7>hUk`}j@Ur23wDqh3?rjpX`VDt#x**|jR)soKWz!qE;Y?vtnftc&9&fDy;s6Vm zSzZSkIEmp=jDCq@GzQpSjETCHiv-|O2fz*ADFJs&%uH2?JFq-gyx?3)e3SMuW;!R9 zk{f#%DhTqmf&2Ie}UW3}IHcCN>xU*oW~8 zLEbrZWHsM&((-DU^x`K>zf!ql&n+<}p)(lJxe?y(GeKN92e}I={wDo`1cGU>@CJSi zFSH&TLI>Q~F>B?p?tM*s5J6E_>t3F#bGCkUozKM^vdTtp^6`U{i0>h3u-x-0@Dvrl zhUmoc#?FU7E*PWiMmQJmX8{8M1j{#7S<(G}C#XqO_G40qR;@47u@%m7MwQWa+11y> zYKihUwi$YhuPz|E7>`I>U7v3tarX>@VrC5Z3dYLe2I!&#%3!ksl}=+c$tAuiop$}3IY6aqpJ+Y2d8kaV&>1tK<%UmpW8op zxAs=&mODI#!~^Xn&j1)!utdSeq?ThH=rk+f^05A-G;FN>K4K>1yq zLLoz4&qT$?MQbY5A`Y@4VJi$KZgNkKJ4h2={At5hUicNPLy}0SK&jGggZv2lDDdhU z?^ZGkb^QxGLxCb{YhlTT-iSQWV2WmrN-ZUh0ZDX+O_O-3j~{FgFM)>N%GN;l*UIvQ z4{_}j)Dl_ORo-$6RHzPdSw-(AP2fltEi+83MbHB$tOd7sf8wpNPZ5YMv&( z6!2y&*{Y8!j*EIWp`LmbK3OS|+}WtJgud<(utxt0J&C(WIM{U)wK2%N695Fy5!L)^C6U;^xS|jvtvFHW#mob!>P3Sgx=`q&oit;9QKWj%Oap6o3_YN-oV=*PUUsRSXdnp9RaM z3-YI=EIwDwOSLP~W$72ua%d$}5{|XK-YJEBLR*Hnr93%C8MXGvuw9}&fwt=lO*}VN zVRVa&99n>OSyGez$6SVTU-6}vl2nOSR;zG9iMGRWb-u1-7|>tX#kJNd#-(dqX4MNq ze}KZ4s!eSAtVJ@@wM~_qAu@sUCesP(ejrVZh)XXH+kFKz3xs9%1~gV^Y@WEQ#&hwR z>$PINhp~Oyq+fXV6NC7-?!=U0@6z89#p{}eiU!qX6EDwIh+aeetle+Puhlqavru)L}tS;4o#G04;6WB2Nw`kh)7K+aTKrWxo;S3 zrT;*jS>$9&ERY#hL;iOz+G;@>jg3}i*@et%-czlgHw%x#zCT65#*E2Oj2MRN0KX_y zcF>>`laB#|f>CT!vE3X~Zt1}-0gWmZ9}oZEP2eih#ghkTWKiXcvGjftDzlv zUp(OR2RyCSlHjuY>j*K}4-uw7iNfj-u?O}|6*m%5fL16sY5z0q^=A1MD4Et{KG+u^ z?>ab)fwTl#6Nhi~tJ_~*B(q?8Q_Dm|5OLWhzwG5dF^Uhzw@~IvDBb@6Hrwe!g*42V zy+)(IN#TK(v>YfYnGwi~5gaW+Tv5qN0}#i}O8NHGeRml-7-6`d^(t0Q#ZwRPEp90w zR}=065O7kOdygMPl_Krr_$%NV;Z3e{Pdjo&9vfI!{4QKtK+}YtfSD9S+Y;0M`A+eu zFMw$(m`3t*n_(+19tdz?v+xJ#i>uonS1UJ36$mOpt+!}8Lq~Wugy-Mkhk{nX7?x@W z23cysR-vNH&Z_>Y-|{{k;9y|>F~Ng3C+X@kzQwAk!Vk~GPZc3E$hIX8TN*rPmSj4C zvksfNXDD8bx%l@T-Lahe9y%h;RcYU^5n?{LwRkThJ%S{RZ127WTNWZgAvGTwgM5i&(f;d%oBM=ILj@o>j3AofJC4S>5cQhK;IcAxPP* zRmT|5w$qzrsS(X4x+c_$k)sMk@EJ#EP`Xed0GOv;DS;w0U^WBH2MiwHRmYk~RM*vpD$ zBTjjR56nGb1k1+t!OAdq+wq5ecQKB_gawJ^q5|gL)mcHk0$Jn<{9uLvBxEUY{A519 zfvj54+L)WVLUx!YJ^vGae3$vc0~fP&lUwJDh6nu{#5o=#kHjhwOh?Q$gjVygfCC{j zR%`G<0J#Jqo(uoFHr6o?SB;z`-gtL$(XezG?2OOBdfw8#{G{94{I^>L$cYsc#oUGU zwLkC*^m_dj+rne{aVY$-1b;?`3V*?9oc|aGNweF$%a~NedDyR1A-(`Z+46P(?|=ZZ zCGfGgL?hUES_d-vr!^~IxfI0Jtvd59Wnekj)RqA6hdBp1-gyoX2$}J;iCXi3l-TAt zBo=mA02qjy^Qq%d>rufPk`*}qsb_)f+9rNfh%yiU;U1?e8cP_qzhM!`4owf4=atYc zmOPaIIAbYnZkhp+bL~R4XzP-e1Cx^q(y%bc%qZ>$!NhH^3MLNWk9F?}-e>ImCaUN; z>`bH!Q~R8*!Ftw#7{W*E_wU3l$A5L$ckEu<+I<3k0Mvl2<=wf&Ir&cM_=zM^tjmvj z!q)$?czR^D^kTLFQPBz*PV+Sd;cioCcejhHU5c;WYeKrhtyAM3iCe#HoV5nO4kj9h zqB1#)b@RW~K3tryLWY(l)`Mk$@2O43Ouu_$FR8~bT&$ZwNWcchI8$eFQb7}I#Qt2`#oaCcFOaSH+?v79M;~tsnLY}Qre8u&Ywyr`%c)B&A=46^ zi7qq%D&k$$_jd{wN=H|54h}An6-A#(0e4T_*^|7hE?ohPT3P-T@63It7GD}Gw~21W4vYL%P`nsjKtfS_6*@CVa_EJ znaJhmlN8@eQiTbKX)#0q@}HAlifF#z)F%= z@nY00*Go>xym#+8cz3+>K~MNsi_8&+YXHJ704a-#*WehPebe3&C(MT zp*-tG3TcueH&=rxb<<~ac-D7^E>@MM%GJS-2-KvIoH7)aT;Y(Qa zc}%xnfKH2G6gt3#>EK*Jzlu%==G8Pj#3qY0Kc!_6`5-ayb>iWYPW9RM}2M9mwVh71WD>zv1=} z^4SJniYf1YqPLz7q0)g9&=iQNI;v9iTb&*J;j#YZd2G20EpzP z3J7Tm8_cDYPzBx)_k5`Yo8~-DgvSzx7|Q6f6x>$(K~L9AE_gss`G7mUJ6g&v6PN%| zYHw)2nH6C2_#eEJ3fSNGH7A{W(Zt7ql4J(;9^A)!r{TU?;tci~{^J=JKtM%aFC+U( z%W>WnH?mwMQiDj0pdiEP@z9oi`(V|b3zY9#DU_qmh~!mVI!Q<752W z5MtqPxT+k259jSZ`AIxO4Yc1Fdba^o+F`21f)!p!NUSW-@A%fc%YFO*0l%;-r5r-B zd^*VB?LvjP7g`Z4g-{_ef*p0wx*ro@J@n}ZrJgT4^$(IX5LCOf0xcnqwa)yz3QWMn z#(iG(=BeyG7wiQO0@2dA?ZW?1sz6!vogxxw!v_4b$uq|d_4!VTj*7Zeu2gNF_da$r zrQh$sIxkMawPEFJ6{8T1OflzeFj-xmcPjM_uq8X%;#lE~8;bV9az`;ohuCy;y0w?|WTjPlO#CV&wJgAx8w7xxT- z1!;#<8X!Je6vCvY=C)a3K1KV07M}R%QM?m2$nAura8*!AKQZT4vEmKT89 z$sSNHkp)hxOIoi33FlmfdM5}U;ESG31pJ`B+{okuAY7+Y%v3;qt&p1OUHK5+F5pXL zDA6)6_yRO-8e69FL-Uw-w@Xfl0HiAN&9e3?fzsZhE%3F)L>elAS&Rxk4` zycZc^ktg`(Xdt}86NQXSCkM9xwskWq_hzZ%TUvM&4lZ3`x=^b}oXi5OA-5tJPA}Nb zIX{EQ;}&V2SPs;o^AVB~D$h_ZF#@As8pk`%W$ZXgiq%=(HwAx9R4?p8KrH+^3`-FW zEx_(sK<=4&_F`5y_JShqjQ_yn=WA&c%zANGs{hM0#M&Z<($vQ zfr{#FO$b4p)GI>OJG&K468@L+o-eMP`QpfzL=cHprLp~6Ex1K%^b5aIZaJuk=zp=! zdo$p??#+Uym-|nI96EqC3#9qcwRwow9uaY*QC&yg)#b$av|?Zz6kJtb>g z4WjJz52gAmC0H@oc0EJ&5-;aQ2%;yxWO!$a3XJ^zDOn`AU z*eU_kDYm4qg@7iFgk{$kd}SV(fUE3Wgo0#wo?W^ohi}jefeAsdf+{OyL||nJE?YWu zimGpw=dl!ouEOYn-NX}LjY*urhiFk>OTlv(T^v8x87d{5{ThT0Ww3jrk#e(8>no&= z3tBr;DZm}As5|Q{Hd4t6r0PwMi?9^nt)tlhBNj7O+pDT2(ejfpAFyhFwq>f|d7CQa zk+V-;W!|}mwcn(Gw^;i1v}``xdq@Zq{R!VKZXWyLK3odKIfxcGCQ}y34;;>FX{dV$ zP+M{gl`_$1?@b8`oFedE%jQkh+@>1cDT>e^t#6weJ=-h_RY4;V>e5fd~ zd#zq>jZW9_J21O(7V&rS3>tO3}kC);Q{xRMv z;#{SSi`o|9Pu*V3)KoN&I^@d*rwsVKXV_mAw2VsbVOI2{_YOm74fQF|2in7YoFQZM?7VcCI0 ziy5iI{$vsq&;`DT6z+uM`toRMyP%l!^pu}`7kLTCSH2~kmH~!5F>DXvOvcC}H&YT1 z%8KAi0_Ah~oQ!X>2fW8Pv%jM3E5TlNB2V$ZsZu z@z63MSJ#{kw#W=OPv*DQSnnkq??A^Sap5PjT7PRNYe8!He6_fcvVJ>~l^~H^<((^) zEX_nwszg;#k-~<z6p}DB!ETN1fcK&~XL!{K}*+F3rG`OOb~90uBZcpP znu8dK-Uo4~1X%{I5wMcxKiqC*lxhQ{4q2!mfxL0b^hJxTJtaW)fu%G~8{cn@Hoi@M z6mx1uPo3oQwf?)P#<2fMa?e9S2Q&>wCup=l8;-J6Q$PF*%PEsX(Dlg+qK*yg6z-zDp2V8 zGoWrFsSA}N^i*%@-WQmV66|@T?sYx?6P&7fV?E#(x$%bcXF8v=UTM!TdMo?!Ak~zD z0>21Qz$(#`KNNPP?rFRlUUi<^AnD#?>x8k7bmV2QG02_Ie2HJD(n(ljO3Na!6~8_t zph+-5|5!qP&?&0PW37l~SjrJkj%!z`Bp|YEi{hu8S`i3cOjS&c2CE3Su`2&%Bo+xHqG`xLkPz@yrW729WccsqKNPO7)smbn4~RR&0Mk~{hyfk+e})J^p-yca zr$uXUIW)t;15W1{4GOCw>Y?wXBAg^D&X~?1S`Zd!`s7^4eW?rYUCl}BR=twMb84Wa z+IvpU3l2#z?92v6j-wa+PU`xh@XGX-Tpc3?9A&}yv4lMY5)tm_sLkRB1!*d7QAt5> zY*8gq856chrq$v+B^CF0!6Q^41F*Zb(D3GX?pK&|DGA~@=CUZs90ld}diO6Zn3Z?V zG?XH@5*E=-d-(+0^g!$?@+s&CxnGzt>P6{#z+*2!ZACL7kT_Vx!AnB)#872%;0Crsa2b-`ET%4a-!Apx_8a?sQdpdyX zi!c#l6FaQqV-~i1MZU=I z``>~B93I5oJiA;3CLpXgkukOtO2YEM(no|uQNQ6ddU1RU$76LTzV0z&E};FM`tB}oy)HGk37y7sJUL0uU+TyryePT?n=8^OO-q!t)q3MI`P%=Z7zS-I=hQrYi$C@tc%DR>2E`4} z*U|l;-rUqkIZVK2FdM-mGp(tPvL&ce6Rd}Sf06*_bB6vH$3a3TdB4;VFVA*;v^5Sd z8zI7I06zPavS5Kt#Suedx6nvZFJ#DlLck*;&V7bS<<#yQ6OGqBtgC z7vyB=brmuMmY4+{FBJ`Q9DztyJL;me=s;D*m_n9&Nds;p0$o4l zL?>8hzF_F|!Lx#eT^KN;B-Jt}a!*E~7cW=YYF)s^c?Kv)#!YS2`9}w2&$qF;v8~I2 z8G)qJVLyoh4#2AR3`3ShIwoAVLg&mCq@d%IJqK++T;&0)j>oqNA6C1qWDv`xA4*)l znO`B%oBL|30N+WTiLxx@;%}~I>l0++Tnf!hK%7WNgl7e`_!jNMCm`~aj-Rsh%MEFN0lq$S~xh#^H$%H9Y25lNs(*4Dx_70-c0BTN+d4*<;J zOgmj(tFStV=R=9;kf9v~A#ifE zZF3A4cEN>zI_U(F;vrah;FHNq*5PyE;0h5oqNw@h%8?L!w}+!WbCN(q8&y& z^-BpMkbMn}iTVfGlSn*4F88ww@w_)iehOWhkj*K5i8s^yBMb+5En0zBs0Px7;36>q z^-)?~&kj$Xp>=Jy+B0yqa3o$|R*)iFPB_=47IbKacL!>JpnZRrO^n7}&k6)#C(r=B z#$U4R!?}P{1o1Z9ZL=1QkQky8MXY-^(l|egX{j2ted6 z-w3)he+>w<5>b243D&*xkUq1k*zpXxjsPg3LMLk(3%zs~S9e#YPa5FRE1f8{LIqb6 zH)ZoLgRP{6V13MHWk&}Ae>W`TgMEqf$y?y1$24M`x_X#1EYp!vf2Z~iQIM(;gmpbXnyT@F9CP+8nQGW>!B>IumQeSWWn(3pP zv@7cBiJ1c08mP7hMz)lERc@^z15`G=v~wf~17MB+ zl(Z4PGDXQC+4Szbty9cupzw?rl>34DX1+*_>+mGp`aPRqvBO_!uwlRO{kE?SdhhsS zgRm0F;#MjW$c@r7geyu@$&Wh6RVOh})A4Jyuc(X3u$=A!S060IA|cAX@JFPcLwh27 zgGxxlxhndWHh8{)94?LWVj1Yd_?ks>pX@TJzuKT&(m@R)_4yH6aYZ%FOL-^Z-9Z|5=VI_V(zwkTDMQVQWfK4$TB` zIWBF|^~wYjG~C&B8`8@hFsY{)3h^?$2X>!hod+6C zxzb!Kfrd)u$8?3Xs79gy#h2+o!!#g{-k;#MC_~5(v@%!Wi5~anbM$>fR-}FeE~7D^ zt+~(qMNxcSJ=~4(h?1~@_@dT>HLwWM%BOonS`G0&GN93eqeEA}=vu**i;gF)`XVX1qpA)gt@}h$(^00O+PDX_aLk zGZf@&*Rpl4%dbVUo~1n#*ZgMr-N+NQ2`KHq$SfCotJJWQv}7#PRak7(m2DPw1Y}f{ zU{pJNp+|s=;0oV2B-RUA#;2Era|at*ao{EkoW?Y9?+{KWbZJ6zR51x#A7p|9W!(cd z0Mv8_$3ORFCqW>Rgk01W@>EoorxzcAO8acSW7sJGV%T2S`EhQJ4rNVy9xQ|+D-OZi zW~Cxk-jN}SU*baX^5|I>v?@}!pfXn-uE16yUPER}P3+4!3i3~aRMTAwQWln+WB_HU z<|+rkCebDfpR-%i4Wm|FO7dyYc~AYC`P$+^1^|OwrG2W=M_5j<>1{zjAx6&YmS3fsMr}S&+Z6y^RyP3s zkjsa)=?SEH5uV0>QHtt%JUVm9bv46HswWYhjo-^3@G)o&-J7xQ7%1(~VTWd+xh9@0 zkp}~ejn#mAm zR;GBnM={fvPNDYH?QKmjm%)<<+@|nwEmIgbD7Wy)e{fw%L#f*ZA@?#J6$E0YwE(tp z>E(>2!jWhFeb8d<8;_-QLqGyF{xomv&4nLGy?5((mX!W!r+>5NH;#*j`~($=K`1@O z34GE0w@~MGC?(+wo#o2nvh`py)&@#_HY4fzcw7_Bx~}9#1`v<( z6gr!|mmtz`Zw>p&5GG+hnd32L98d7nvEx?pWGcG;q3H7=IqwX6N+=FUw#RRDhkrZ@ zLhAX(CN9M04cvYMY-ymW(2Ug$AUZ6EE}t9_URto z)U_<^hg`mapFNhsbnzhpRU6UihZJxjQ}H1 z1s&&O%327i@U6kWg7tE0$&W&XMP-fL4tVDH>Vnaz7Sa1J`C0&{kk9IQm;vgbRx>X5 zN1KFZ4$8o`3o_ajcU}Oq!plpZ{<2xR@UoFun&tOWdfmL^<203O3t zxOoZ;+wiL)eJ5K`BW>WXvsI=4i3*`ZZ-ellI>@8cymiJxo)SVJLIi3dn04H(U1PCZj$~Nsr>d_rrP9g=U4%sWdO2pLOr_V9iMDr$AI3S`c`9 zJEIJ_2uNyXJvdFwFZuZQ-!NqSiqo4IRgNZWo~4=p>Vja&4wLdJzPJ-|b~x|lpxi2k zlV8az_I8%xr~N18bI?v2#3XiO zO={}l`z@aE?@BtXVC`VOuBj=x0tGjhH3~ojYmO@l8@h22)hf>Nd{sm~e1oi%d9|_dx?-YI@ zMPxXC1xT_A9~Qh^Czpa+EBNezBC$rKUBDDgS!-qGHzNhI5SpZz>&C`S09Gp4pegq zuKY!*^&cG&E}t1{c4Ds!Bej&siOfBg@?i1q;1wHM)>cP?gb&0}Sr6LAEc5BRdY@)b z8W-5B$1g%(#}8((&1LM(qN3-09`%84JitJxG?-C5@L$%sdjwVoh2v%(<-=&sw@Bwy zt$_!ZGL;57Pu(@GjxO#te9xsghXL`oqq<_7nQbfX5JvzhulSiJiwchZL7- z`CSdm?-@4uVnwnz!C4TBP6>k+9r%@Fm`C=gEHxyUj zILnZcctJ)|1B)Cx53T%C^0qyF89+bgW<}s_jtjM-^EV^i$n7*9c?h?YeT^H$z~sDG zc$G{(+z4dO!nK)AA(Yc1rmze3)6UHH>>InU@GOCtTX@!&{Fjua@ENX#`hnDVz~&?) zxwrQwT2tpZ9x4vXGN7;DZJSz3o$Se{-7iK9YQP(rGGNc1W6)@55x0Tk5J&fe(Ow z@&mHVI5G|H1B5EOKF$@q_c&~(kU2wszSzH*OCtu^uf%eakw+5}q92BZ1ZDEFV4ed2 zt}7!#ma4#Bkx>-pd>;H1gPRosG@`{Yx5gMezeUoRi@y&?adlK!0cuZ9xG{@D#mg%K zjg$xuqENV`L^5p9nfLelkLt?*ZG;-ePy7DHXM$!x)1T)@eZHj_TFR18pA9JO`;ZaR zQ!>^7^|t0LRRuCz!a>YXXW#BRFYCbUrR$)qCkdyR`Y;efst`KX+i*Pt;A`(FkU$_5-v*lz%T=|e8xOg zmv7kz;6WmcT^4SM^xI%AbF4KP`+@R}#e-hfyEH=#Z;+Luk6)oI1?s#ISImJSnaf4P z5pnn7_()(7f!m&86O>3-rjYr@-V>Zh{d`?4Dd)+M0P()>UhrJxpEzLd(*<0Z{wdt3LU|#sxkyndtd`& zg%nDjTbSn@->`l|)nyWK06qXL+YA(xsCiGv)*Jg!1VS+Q#KEQ04cqKr|6?jH4HWy* zNI5}J)E*K!Xh2G7-u#vyWhwB#OzstO{s};0u#MsCYUX)zXG_JvI9$Gh^}z;<`5&LB zm*Ds( z610v1GNVw!I`jeIS?BDu(Lylx7{!XU$G9pqS;y6U$~Ij;;v>j6u^(M~m( zGS(C=r#2Zgul_BjawSH9!Vnh8%P(4&KvC9L2rCIoVtXD$E9OmMDuFBQbUa|MMLCJWz4p`RUr3qqXgY7zhY6igoM># zEb~kARby0yHY7De!^=LU!$AaelB3+G^IIE|Mst+Z^3!mBAf0!k zE>9^2X@Wa0*!RU-u9Ik%YWcm6wz?il{yo9gWx1XT#)&oREN5i96J- zp8~r=#=^asTl4S;()EcyTjb?nZlbeF+}zAg02sZ~WWx}q3@a|R$nROdOSu1C3mY+T zWp+022X;Yq7o?mdbMIX+)Ul>}s+WC}Po!P`3kn_&AK=;Yw7DlF=u<5z=qF{F2L7PD zMi2m#E-<*I?6~|ja7A?q$`caE6SfF)?Ef@=%Asa6PS*O2Uof%1)f-m;b1iucDby(;-#tSJ6KKqi+MP`TvB&2IvE5nV1R}HWDWIB>jRF|V40O@E zZpqh_Kev~lEq4C}%ngjZYn4Uw~>4r4#P>jmPEmC{o$k&e>SQMO=VhCU){*N-ix@e`_!zc<3e7o-sHR z5b!{zLIibW4(0l<>O+8#TKV@FjlaqbUQ|s29x2Y6Nx>>D!A~BX*8H z$TUIHn0HnN%~Nn+Eyr>d4)asJGVS^@lmFAPW#wT=iTykeTn0vyUyN8`V=2VY76YgF zWB!7ZpU9GKI_Thh2GCQr=rC+tZ_^||^vB2wMF~e&hf+s?3pAEhRb~(~u)cv8X-Hwp zqAs{Blc8)QCElgg&@EzEg=L>%t;gnzJHb#>cgXHZyvSeHd3UO{=bZxl6D^;}BS9J? zrF#Yc;vzb90LiP^jA5CCoviZCaJy6auryhNMWqvF1LC2`J)r!3xLhu#cp*u61Rkw! zm%a&H5*_n`PX}EJY@DWtMRk-`{gxsQpc)W7G`PX^O#slA$6sV9_u8Hh7$Wu^J`t@~ zIA%#+9thhq9BB6A<3PiZD)3p{8@T6aIm0>!Fr-LR zw+`n@;iV1GgHxo6LRI`1x5*R!+SfOZBPC`koH)if82N)6zsRo%@1EcV>>J@6$lO#y z#q->oE)7psu|NfK;ax2&i+#P0t8RnCB(1rE#VgatnW_qf>W4JX*EgfpbX&G$L7}GC z2hv-s(p5VrB9%chPlsQ@lVrQ&29+;y=(spwluPS(BRkhx?Hq)bmiordn;#ev|2Y;smS$xH<*v-3^#P&?{gJ9LD2 zo?FNsXiPIl6|Qw_s=h1w6PlgO*%QQfA@`PH3;>i}auLuF9+%MU9ib^n_ux}Q%){m8 zR8R-vl7s~H>}(x*`Tt;guOxH4{|aTxMn4e`eT4ZWNC|#7!}?YC-#bkLv!@7Cv zz$7ay8$BWdPWGRy+_S>_5an|&2Lv2>*B<)|puSj?TIz1C>3@XJge=qHcRP#T#bxX2 z3xK>`PppSiY%VTD&BUVqrx?ylzpmwTu|O4;`|WpaeN*I4nRp-AF-oOto3i#d^loG* z#5gqXv5dkWLG(aYqx_=-og+tq%i%4{9gd5NgJKB2ptm?9?b-gxUrb_1fD8AfX$tLw zSLesBqqV}@2gjQx2(Q5wbg-=O0}-v5W>2#kE!WnsBm?!KeN^1MXHyC)NZxO<_%R#m zYCp=SgY9LIHt^b-Hirr*B6F?5wuay74To+@NFDY2&RBka=fwm2%mqz?KD2krj4CF- zMLM0)E>EbkIkhNI2;AHiHXN=~f0}o;D?=C?slLVwp37rXAVmc1os=dXD$}ijCtJGZ z1#Qg^C{B--c^BM8@wah9b5?~zYI?8*+#|to=BCw7L;;A^vYd(PH)A~yU-=Ae&Z?Ke z16LOArlGAuW>+2B=Nddm?u{?P^f=eJ@H&;0X&%uPaflCvK1k08yhkX(62Rie<>tNo>8QiIM|ef|riE z483)9x(rv5c16(w%~eBXlL{{g?1K@8d9Ocw=((%1)N}#-Bxur5Es4-`p3J+FA&4Xg zg3~oY;}1qK4-(%5)`9e0g|!W|(HS<8cEM%UrTV(0wF9J+lb^5(>`EMq{bnYvN~rQ% zm2+!!z+f-y1kAUzkiUCrlTakeCOI0@sk8Tc&b?0neg9ExAPA_wv1_hw0`x5Y+VHP- zwT+^EEwHZVeoFEva|IW z(&KurQ4yM41Zmk*KmN4m-sU-Y=xo7qBJn=Ck9diebLrTflF>YV;A)`{_L7-U+< zrG65ZlT0ouiw=I0M~*AX#L=lWBoo5Sbcn&?ejq@5=u1D8x<3IG7buXHr0$X~v1?w0%|0-4}RM!dj>b8B|2RR-aWG$$!1TPRaI z*fzww?P}jk!n{hGZA^LFJYS|N2%5CcLxO@P&RFh617N&HbsCt~9J0Pu^;*)4F)Zbi zId2jwFEt? z5hxOacL}I)Spjh?jebU3E==AX#-O5PyhmB!QHrC*_}|US|4}Tu2s;o`yn(XPi1Je< zl%N1X-;VnqTpa>KWBvmdQC^4LP&!|(6vjZT>L9Y9MH^g1-50#VFcpRA(};u>hqS_K z>Wh+dr|@odwmuYgh?~jN=7G$0BNUAgq?{OVo--{}H!x$T-Vl(Q4v|RjsN7WG(7~kV zZ}z&@$A6SPzQ#M2YP4Kwf3dXmIaw{dc-zKd6bf+2C-GmH*2X~7dChN-gLqYXhsG^U zzXTZze5+x*ncZ0Zl#q`|}qPF4158L z3)@MYZlIesw9NP01-vV;4#eQ=S>^-N2Vi5PJ-#5N30%K86nHQA+*clZYmW{8r2uqO zZ|!glh2a9)L{i{PEZ@nBh}4IEiI1OPc7l-W>J-e;&T{>ci@0aUjw|S2<}}>t&fP+R z*&DvY!WrLaQEmiQ8Tj7j%u3S>iLr>7lA^+WaC*gGu>(wCg9^`th>OGIpXfzS%69P9 zx%e?>o+Xw*6f}G{xcngFawOICrPPV;pBp;q8_zG3EuOvE>gws2Nb)TJkc9dKUVquu z2DyP;XC54XUP*yEelj0(E@F-{mkbf&AAZwX;^-pih!ss?WQdk0gppWXBJ!-?uwI0W zwG@laHwmlPJ@z(@5pj1YY|1W5s@f58w_gVliFryf3ILWPOyJlszCc(&9)qOGf^Suh zH-C14e-qCg_*GvuLJL)(V%Qri=aEEV>;3U;bVZGvO35{$!-ZL4yR`{h6yRY1AbUAk z`53>xCWIxdqiG3WIrCiK4@`M3oJt{JmwY{fcW2hO+AbN4tIiN_Akj>jZgG)B| zvFc1H0z`~UhQ;qHX~KL8UKPrE8H7A|{k>`i)QP%7PQj>$D+l zzamgt%G#G~w7#RPuOXXM-@UHbTF%b24m?YRfep6~gUN&|URPZ;8dEfJ#O5SL*-Hkt z2D}aoU%uP=P~E5pa(pdDa@7e~&Ep0Zy8(y+{xeNR5SU%rnP%`juV{vhFxhcCd*$+D zH+(z#A`{Q4SZ`DBYZ9M|Hu=f47&VldP#Nsl8*2?VggW0~x6}f!Y{0Zh8JUR+>S2J% zlw_Qip_!1G;ibTFt|v?sXT&H4b0Z`J8L&gZSBo>m>DuS_98e(;I_uI)NoogAdy#3c z)N}BO=PMV|1?sugKD*_N12Q5*V3FXH1O$NW*udQJvM8a?wJI*tfhi;z>;Oe}pA+J` z!qAYRluNk*+JFC`4(fgZ`^H(tZ3C-~*Z7xFmpn)uCIw%{rDf@Tzuw0K*YqaT$51nZ zC5xdC->k9jq>BM7Z|`bc*oMLmC&27w9wGngO^5a!#$1N(#L1J>iTb71nD%r1Zg+!C zinZ%orTPocKB#`0t@&fqY_Gz`V{F_(AiN@?1oP^4c~<)*wB{HcU5jDO=GQ#PQQ8L+ zb`Pk2-opPQC_}1y&$Ye+4@aCPixnX>U#JHDC#4;Dd8`j@*H5Vg+11`3vFR$fc^i7s zPT8OD8vbiPLvZ$VjgF#L{8a)46K^Tr+P3B$jXv&)*U@_6w_5C4KN7TkTGNJin3Fc1 z2ExWS(x_(Fr_)8rWtx*|D5k1>_n0HFbAqOoqf~V_Gjv~sE+u@U7$S0-z^kr` zHv%xBrn{BQBOY*7=Wl!~Q#RuPhKGvZVPWht`!;?D)>_!b!u{%VN*EW0NTzrly2jbt zvP9je1KCwIWC6B$LZ^9P0n4s1xv9~M*9`_~Oja>%n+^VQ_qz46afNTSN(PC+FzE!; zknoa|K=MQP8e;W}RMxC;`~Ea9rrWtG0&X|BnuR0!2FP7V#DwiCe`1$Bgb$R%vfwf5rWeRBne!oj9gH+@lV--=Z~3V zfU(M|032r?ag-B7GcZ(h9NJ-V3KJ~aPT6tyfNAts zFmUm1pmbF7V>~}DSY8;)vLPwL6Sy1@C)Es=Dg#8;FV(m6B<~rl2Pn59xkk{j+!Mdy z1?j5J7rquNXeCQ1>;TIMRA;!sk6e*rts%dJB^<^T{4dV_ckqHB+TtJCS0_~;umpS^ zWUILmm@dy$(XiIu*Tz*yLIJB5u>=LCn>I)n^XQB&Ec$!Z<1&Xsw}yB@5q|eU$oVE^ zQK{)X{E8$kr@V*SZdda1g8*3&9fg1j*y5ZU^nm~9X!usV1>Qb*u5{Z;_DJzf1FjvB zx6V1wkwP{xP+~4mOb+3%=lDHHqMQBcT%^9%L5u7OH#T^q>2l1 z7|)M3X#zy^q<^xcyv&or-<{f+pb6#3X z2ywpT1Qkw#_Xx9126m4V>xCoxoLVAw#)n@v33U;b61`a)R@5Ub+^j@5__yK^3lYTeeQ#vLeG&~yU^4=Emt1QNGB~~ zv|~WY?XLLzZOQK@e|YW!z5{im(iQj)YP3?YB#;-vBSDT3P*5B`um@~;&^=-{TJz$# zAVIym4L-!vNy3Nl{rU+ zM%+Ac2J|yI)8v_kX%v_gAs_jBZdoRcUh4L7yf(2dn=f+@7@?jxHc)9j5X=Qr0zpEC zEpNRt5s`IVf6ViBe_8BUJQh2MkOq8w8#KojI0bGLssw;0bYG-O;Pp0w=?8o@|2i@e z&RtO&^NEQ2fgB6l#)n>4IWRvl0FZ=2n1D^;S@EkJTX5f8>1~fg3L}zVEz1fgLVw zG5`sKAzN~Ek@Cs0=VwHae206%md{*y2SFJ@iZ~+YFsj*!w8p8fp^RXS_1q$-F!VFa z!S@w9?45@{6GE#>#{zMl1Q7FM-sP_ef(K0pvdN*_Uz|sw3G6zJT2I<6r$C?hQrN{O$7CxZrC&8mM3M}-O zjF+AHJ?8Uj3DsKa;;>5b14^<-D&^I{@pg!&Yx4AccX^pHEzrpkww3asKtbG4G8l0_ z$w|oxPbhT(s}X0P5Ad!%Ma~^)vFvncgu6V4Jr3PS7&P&XqDL{vHp0H+toU>|fwRn$p?K$74ReISTX#cw)#cV+g#0h;ZW1(h7l*B?uKA)zAPXiLqMz+n&8Z$NTjwObr|5`1^T&U9G%^w=lmos0=8&`TK0*XL{2+m!(Lr^G5RB)Y z%DeIqgcOV8s)&YTNYojx)}b;xsX2s^(^a!VrgHWig(FTe*-cSWOpll#i12Ojm_wO3D?%wn%HZimR6yYT z%IOsX4FO4pTcQ*o6a<7!-?i4~p?m+b;050Id7k~*d#}CrT3)!(cwQtXf%Re5bP_{F z&cD@l2x*h;QEu1}vw>TF_xi>|{rN-`ifJ^$t@XN}mOhB!Up$@C`F2zm5iFz2@3ZRp1 zA#e=>1yoUi?t^0N@Ev};GwKUzQD_W@LFuPB3?Z&ZBMvX1{w;sgPvN_QG5{hefKB?Gu${`IFk8m_{k@5(mq%$+gtYVY=z_L6$ zSVX?a;aEL*ueaL`4TmcXX17>H%^{x%V!7YEQf4r^(yvEWQ78-=E?y@og@i-Nl8;>t z|6GFS@p215KVP-`jL1h^H=GCoZhjjDpJ5$g-jc)Y1HyMI* zh0m2=ix8+#|JHSDo1^h(AALAFfMUlumFx9DbzGfmYYaeVH2A#Zeb4fl^h$}dDmfC* z&(8b^o(hVWE0CnXVUi_V_@<7yQP!k)#Sa`Jp?I)7rVA<12f?xLk|n83S)s-B{8TH* zA`gzAA6Oa1$R*uwH{wU%9S^HOAR!Mf`$0D%&h$kp;xNSC1@$p;h*kul85(L>P^d*M z9i*ZHRB&u2Px|NZM!Y%qq|xNFb~}!xB(JQX$%Z|w@hON0i1ttw`%Z1RWG52Ji;N&e zs(+4Zxx{z7lb_=-^~CW{q&oo=KtHoG>>hp%DzA`sHb_*7so21j;|y!q@JkN-9w>oo zjeOsW($mw=VBWB(k0^>X&Cl!eh;ySH*s#A>Wp?732DVJzsYbElp%yp73=lns^pzG) ztC|y@16>bb+FSc+SMe;CZC7r~o5NMh%v&m*~1BKoT8C=WLJejK-M;J8&v z6H=0}5Y&-Pg)hl}4 zYP_Oa!*b`zS{Exsvu{XDLNY&Gp9kZ*O2#+9n+V3tZsS_$Y|W(1o%($@r^Ece{`ua2Po%H;QXd&C1pEm59SFg$P>e% zbye0F7{T9y<{%}Jm~}1{nN5N?L*T>O9(NHDG|qC2r-_M~Tt)`^oTMTPR;45pYs>a3 z1w)k}LY(75ny;Z2HLLD*oBp*gNoswNd%11Qs8^jZ63qnPP@PU0U@>uw@P9r|sd(zd zV(nmn%&Vr`qDUZr5i8gH0YIu^yCT_Ye|Jk@bc`tr&7udA_JSA`Bdq!!gP~;!uR?Q| z(}nChkps3h+KnwnQNf*_{ZpF4{tD#dyit6r_+=Cgw?+*w_DhkT!~7(VIH70B8smdo z9rgkMND3IAcT{4!Xb@5bZ+WSuJC#rS#Ce@j<6hhy3`U$Ea-a+NEZV;jw_$ z6@~f;Y^J*egb^DNt@?$4F6Q>K1rTI3vcR_cPe!92d#WH2%e2Gq@Z6kYwobIW&|hj{ zFkzSO?b09iY{mOVP8`CiJYM0@s#o-UDm(d}*@a^~r2N`+IKy9?g8jggsxtDhV_;T8 zd>pWi0_UwX^A(`_Y^+XD*#-Wau3CgH-jR&I)V^~jnjwk}2{|iH!pFqVh6olgJV?nH1+=D{>-T@AW~g=5+~`bpOMyF+ z#T>%4=MgP7QQb!pVnEBY9P>zT#k(+MHHlYYaM8iF1;ott!&pW{%!CTCsGJqihtvw zQF;GjEeF5!EheQSYQi6+xHrzeYM3A52h|YzQO0nJ2e^y)S{w-;5)E5Y%8jd3)lWIuoSTYTb0YC$F&QvcI#KbHFz^#4 zU9msEhgKy2bl&k#@Fjo=VCo$i&AM&QGo1&2c^BWDZ)sAX))Mj^Hs>E)0VJ0>$wO)t z9UFG;DL(YhfAF@)K0AP&K{*_)BKrS{=X8BzN&14UhM^1|3?@|U!IwPm8R=uwB%r#0FGmh;zZ+N3S=irX*BNxv_}ARoZB4AeAym>b4pETvH)6LCl~U+} zX({+&&;q6|z#Dxw@)BF)BTNJ(4a@m%`0utrZ_cy4Oa{1XJSB?(@b1Z^XOx2&yibL|ihN!neAnyLEB-dIY?2NGDRczX%4eqoGdkPjK8CmQZG zZ{ZF_W5>J`jdnraa2oTy)ExB$nvHfhIars0zIo`K_z8dHv)EK`#3h_Nh5@zUu?GoP z{ouJr!9BDT!HhQfad$HGP#ujCJPzWGkcz<19^yg%-4WDCDBZ4I>GQsa!$Ix^rY-9- z+DMNi&IXq3I=tpsGS`c?CPWkwNG5j(hb=m$pY1}j0Nq){Y3~C60tL|5rl;Dk{oGpP z!Et@0jV<%Rs?;orrDJ734EEFGsElh7#5wskwg63=GCw!1P8d*epb(Ca05eBR=lo+C zuid5#h3hUFQqJ8?nd-TwguI^_#tVwX8gaXcX$ZDrtKVgypd-f^i~ohO-W|pZ?7E2iX;jnI3-pGZR4#K*hYcaaZ|ADgI(1Pv)Rn0uYL2ty0r6e22o!(-LQzy= zB>86?hHPEW&g`sNsZTGmjdp!@%dO~chmHUBV@ciS#y3|crYtV|X8x7=@xMGB^61|Y ziKwE!*MMOpv@My=LM}@`3kx0#e2m8Obs5N0)5o%axwFbzd65#n63>Sn{46ga=w)HQTDws-9rLh$BTyr_4v^uL z@k>jEZlKsRLNIRUpU3y{1|0r_feIt2WOTz~RZ!g*7~Pv7Fe!Oi{Bkf?;8R7$Lyk+_ zZt(5+&CXBmCqcp}5-! zMEqfdO1O}RBl;;r{qU`cq-gm8SlTEE;PtN1O6{dj+RvrHICvvBgb6gJr88hRUCc+% z&pSc$A?&>2Fz5^W4?NK8U7>{GKODxY_!$h=3U@>KNTiIFVG*KfT(gZ4+oFBWDQvv( z2#53UuL`~OX5$e1d<&7!07p*)u#U>G{{7LEb_S$WyOROwZo7G9{KrTOc7Xc0RQh4F z;L%y~kR_FL#b`2HcAy}@7y=*sBH3(%;7?F%bM>MJy33EyO_>LpD#_|IW2ch%jIq-^ zv)c8k+jo7P9c5wQCutp97HV^Q=dh26_am|dcb79qwoz&m0L8UB zkt}G{u0{=tt~~#H9z$sI7MhB8TGMh1k-&(V!E(HYd+}Qm>NWo@gS4<&muwm=`fv40 zP-!w)W^DS0@WR$7TTCX*KUN|Wkif>tA@?y$kXkdHe1q0EQ)1r(%>jQiXT)Kn-_bfC z4U2Aq`9@6N!JB^K$Ui+MELK(5X9e`8L1IP5NtJg8W@at(mdUs3`5!y{;MoShDrgc* z?+kQCk}{4L#Dko%6rXpczT8>48Pg!%4frF*KYqW|mOA7l61WC)YQr9xWb`otcLu~Y zxbRXQ4|UkVX9qK1@FFODCy+3=uaLUvokL6+66G;JhhM_(@z+5rAsfv9u0mj6!&o2YUzRpv-4b|2wbr+XO-5lg4-#dIL}LZb5~G89jGzo>?8oHW>;@ zL;ii1g9KY{(8pgC^c|qsLLFu{4a*kgM-;%~1EL4T%1k(14-k1xP7DIB4is03*YMw< z(gR^6OuKN^N&r@15vvrv7*rLWD~x}daRV03gCH~j!pLqHoPz5qQr^20gJXuY*g5~775Be=FI4N^crTI2 z%E!TwL!V}!u)x%p!IWY{MOLRppvW0s#cC# z5aUsk$T?S7a%Krv>;t!p)q}S08xk`>&YV*Kr40V(tbjWHH0U%+GQMcHFnsP+TfP<+jntpj)`whp437c+SYhltP1N(_`)Qh44LSf7>L0jxj6uB8LgZg3DO=9&B9WWF?}w5}T2=g#{4=%@}5>Be1=4gC}+S zPly`CHOEniCRyQ38NcN8bK*i)&A$rm7!`HD&u^KHswN%;J%r`qDh}^OrzI0l|#2CEJt>GvXHeTpi0B0W;;=o%#*U zKO5${Q{ua+$~DDHyvK&T=;$a@h{)Vc1$Q22)ezK|3IJd)_Y(FFYk zp-~IMj6+A^?jN++2AAVxlFuW(x&hEJZo13n{O#_~Z~kM$W>gDf`I?8W|6c$im%F zg+h6~E7_2h0)tb;k;PN%XR5}*WvOjm5A&i!@kZB$d-DhAQ^wNqxMvh*PfLe_OakRU zW~SrDy#qwEa4Ru}rc<;JN!~tI|3V*8P+?f^Fl%3l8;h7^Re8;QK!4(pKc824DymX@ zIGzDj-dH*DFUs&$UqVS*HfE}#bHgN@r48bw-|y?@360hf%T&wSi~M|4#y2E|I6(&Tzh z8qd9B6;FfHlbEi_6qe_iZm>A{Q;rHiX=7R&96*z(sf1@2;mFYzPRFc10yn;x=cs=# zrMA5Q^%wZM_qXY$#Z8X-JXg{X-(?Qur%84Eu8~pq(Sh3;zAn56IPvwJH5tLU4?JPV z=Q^8H(iV@EChDb@_wCRr|T9Fr~Ci2*)Y z46Npg!}})e4^u44F9gEtx1?=xyyO?DU0gScM`-nR0D~F9=L<*OT~>cN!M43w^!R}8 z2lbRTCyb#m#ja%IBM+?=s+B$b3s&&g?mE|!7|f-MZRE=$^L|Nt5C%p;TjKoYfH1|# z)>gx~_<+3@D1zB5#>CSZXR3GX_h}$ykQ}dV7U!k=gAyB=!}bLIR?qJAdKJVb&`8~C zwULpurTP(zJ8@m>Pk3s?i?uga?_Ze1g(v7iEM2|{v*7=!r7U>cld|Z2?U5g0ddbh( z3ieM=qsL~5*uwbZWFJB(tZ1GneR#O&kc09jJ2=UBkps{b|BY>YBXg8QOio-iA?`;c<31jxr8*L>@v8g4BqElNZ5O@YIlT!4{ewabnz86`7W z5ln&8j(Q>av4NA7etE}n0V>|Oy}KoeJdA1aCjZ> zB&rA|CHGt`8A}BT6EPl~!)$xGAdkzEe_L^5c6Q0oC#Yu0598Z83aj~5yCv&@8L;gW z7-c{`0_q-wYwhmmSIjnM8`Der;C?XZBRs;z8($cBLKO#V@*YTXAUBg@fZnE&-IR%!1k`!ib37S)Bq+3dK|7zWTyF21$QXic=aX4Hil%e8TSJ_6eDtenE8 zEfKx?VYWCwEzgTND|H=CxbW+Y!xR|)&b%Md&&f;^%YwNkCn39Qfe_Y;PZcq`=)V*B zU&qWk_X!?Tb?oapaiY@+_kW;)qW1%i@>G}&ySc6 zZ3wWlZa|Iw&-~SpMdr7>Q)15XXXXbRkB*MEF%UqCFcnE#Hoq0=@feKQnaZ4x?G3)V zh9QwN(gtm7#*23^YoqFgDi(&^eMA{NCT2v^#2*cNW0XDq-`>ZljMA`GE6LBL+4&Q{ z^5t;HifXm90`18k@Z6$6R&d@aulAHpq_rW~IVp>uJ;?VP{*S^r_{oIDjfZ#{*ZkY3 zWOThoxtukofqc`D+91mc>0@*drGaI}QNWek8%**n`er#_xUCw{3b67o;fqJ`9sRn` zRVz7ZNfe01NZTZ4ttj(oKLyoG$>Md7#94*1inJx5k48me)i^N&2{xHTopTIQVZ6p6 zr||Tl77*`@ci~}(Dk3)l@(I1C92M3$=_tG;b;SPL_k8!MFGnvQ(FObwxe$9M#IftjK6BG!V7#t z4|M&4R6@Eu{&#aAg6zBgHnaZgb8^52CCF$aWz_Hld~L+$h@3x#+WugG?j9AIcoGHN z)`U!HwDwl*J*>dy=Ko=0;lLo zOsR5@KG(Pp+|1||hWsG;^CSE;n<=@j3<%v%Ln84l<&&J9J@^DS;ReNA^L6Wx(yW)8 z3`6jn3@E;>c|<(3DHsMklqC8lTd8Tb1vM{t;J0rL%**^=+aCy_=YM1foqiKdkghf? zw)xkh3yPnQr!wD^kPG+{VLs5U8vua!bR~Y=c{>K%HT>vzR?IQNmLEY!m)&Gwbo5_f z#HYf4@Cmaac(>~iie<*3XSg@|- z)%*15V|+^Rch3w>suw@~xa@pE7G0HMLGPo)MLLL03)OGIrax-PMEiuG@-#ZMG8dhq zf)5ML&Bn=Ql zm|+h{GZV3g9U)7_WpIw}Ruy3HiDj(n+W)v9wIu5_?`hPU^b_NMUiVc$5-f@{pWxee z9(PoD#MH5+4|QUt4-DDlm{knVrWJ7%>B~rxJ>i8kAExUy$c1!iv1^qF&LX@??J=*k8Rge3a zFJa~an;jI=uj3(f9X@P%_*3)#il4!fl4umT%DZ2W@?1NejbAp|7En7Y-eys05xp$( zI(J{rGm;X}m@d(;+ae_w&{xeEd1Sg?H{lN`Bv=K4(}t37O4FvfB@V-slBSI~ zqr4cd*B1Ej;{ONB-$<*3EElNr$xB!cxl@iOZ;5P3+fXN;?P1D z>_AJKAibz{bywy&8Z_eacYLb9TrLL2jLzBHJF|2&S-O6>)C$gJ%1Dh_VWTyCf_iDy zZUbg_PB@c#jyf(%gapr#2w@1KD3kRY*_e&a3F_WK+g7GS?aL=CGWT;VI_h2;$p|QQ zu;z1W3;@1|Fr4#uUGd76Qx?s_>rdJE!CCE}x3m1|_+UYy83j9}NMoMDEm7TnV|Yv& zTO9o&qJ{#aR%%*|8 za>@RAuSOSarWgxIQl=#I-FJUi(B?^?lG~yYzw2G`WhA~(Sl_5T1X)5&SKWrF%015z z;3&@&g6@2G6kb4D$FdXz7b?7`iV6?^32zvE(5SY}V*dd!<%mAowp(g^JP1 zkf*4Y@qI^+3+PNhY5>1u4LeDG*dWk!K|u+xBL&x*p;+VNs@3oIx=R@Vurq276tcM_ z4@pFyI$lTHh2&CNzR|ow)p*GBZwk`La>Te&u@@Bm5QCs2$PC(GDLqMYNcoSabsH~V zNQ0;xybPg9#Tk%$cSFpGqs=uEqM84~1&}9G*@V?FArZ_|k5&uE$K}-{bIClK5_0J> z2(1hwK9|lC8Y^2;Y(SF(6khp4hvLM9n;KTTBww)A~GUQzt@_b6D#1wtUp)JQdTH@Qq z1w{fuc0^OB% z8ow(lS9`Ra42caS;w-$|7hW7|ce#SojT^qDx^2pWlsYCU^qUJvx4>uFn`fxL?zz`U zF#wgMMGOqk*-xk9be?95@+f0Ocz@&H9lmiZ+MoJ>bA5mM5H}He;H`?HtRitP8eU$1 zON%GrB>`Yh$co^H|264`F)3U*P2U|EXAWGKY}Q1;&*~W{mX_bDFONkQ#p=jnUxh7` zh)}{{(4ph2It%(9cbtx7hAVo;SJ|-cr@clW6S_pT%1@Ve_9^i9Q+*J5{Gs_uNsAj40X`q1MKB6J}?cm$p(C@&fvqlCulg)-#%AsB6#H4Q6Wc_ga61T3h_*s z;<#5E`!9Rj#2a;HZ3OJZ#9~n!rH4MP>AKkrp+t^Mpt!THvjCl}dKq<&7Bv{RX-pyM|Nx!u+dX^t#7b>sm+pkAF(h;Xg?6 z78JdM=)}|GOd}il`fgk*OR5%Jd7|WWWV)Sb<}uxN5ODk2rk&+Y^e$|g;HNcDm?=umzT^wmf|2)1bRjecIOpBpxsY4zLeQ^>` ztf|{LH9!@$Y{}dB*T8I+UJJTrvre_>S2-o(QF4~V*ztnQKP&6Btz@N#EF0TvHN_X; zlnENM(?U4FL9NHwlMjCDKH5@IR7D}u8a2mkSvBqY3}K1i&YdZOcQ82%;S1LUTY!7* zJV!^ybk)2MELRkoppNCWEBLK!NIRp6=2ho-x09A_qIMX^g$j_WTn|5FR0fY3A&>WZ z+Xyy71(@l~cHu-=#b{i@zXEglPHO{ub^3(5qB|>6XvBsq zkUoZ=8x1`xJBa$8H0{%lfCk3{dI8xHD1n6vPS&O(&@8Cor#rL;#ofm@csD*)>m7cW z8!;Ij4>q$w#_8NFKXMwv~4j`#$A6i`X1 zCKCQ%LYPAMcr!2%ZX#lq`5Hd|91j218!F%9^Na%QGR2C$CF+Y+u1Z#hUQxY&1HwaJ zFkpOO;f)sK8(=CnBo3p}BI|3euo07}ABQT9_dWm9i25A3hf-ULYL- zQoz%{>IIJaIzG{oy`))yi}cD_1xJJ9YYc{i6GFo$e@DRzd(9$f5#W0q*S)|4Bgf^b zwc-XrW_ckq)}4>yr(1;w&_MngbdhP+Fc`ZFQl3C!6)p~sh1G)4=eYPW5k1rr1-&51 ze$?;83dLXeUWBLjw~sOEv6d|}lDr7Wv@Mv#cXMR@8S`(SEc^@g-Tv^>WX#OUFg9%) zSLu1lKMO9USbTn?cs!$Kyz&agEpQu;wbEP^^vuz94R8uT7hj6aU!9$rL*pTr<{~;>0V*p9|o1Fwbgz{!#$=Asrp+LzgWWNP4Mt! z9O1eTG5TYf1on_>Z=lcM*L$7Dhux$f39d=!bQd~7q4wbX=oWOn;$iQo3={Xt20k{Z zMfv@x#3xDw5%;cKUD?`d$&TS}8cA8~E;R7wwvMUv>aMS*!9~Gk2o9*K_xcq|L<4yY zq5i|k2%>w*m4GnKMB2KXOcB;pms*_lF<)ix?)E7Q2xIJHaSC5?m~|}x1P7~{BYe3< zbsaL^JpWMDbrtt{)a(!w{0j}3AB=E1_C*(F7f{i0dUa~4&xcKlnAVIy1+qDb48Ot?EhMS zL;-y!Z)>YmDIgP8p~Rj&_<6IR=sjo04W^i~B_&yX7*^xgI4t`_bfU-hSH9MeXVGqb3V|H{0-TO>ucAYoGTwJPSaq!YJSUD@!aAdc8qIhIlHKD( z)D&d4nylLx>=0|H&RX|4ACUg`XRTue(r6#5c!Tr`d!y7JHERNGdto#_3UbcWCX3Mm ze>{X9j%nMsdgXzZ22HXsnX|yZZFxj3Lfaq1o(51EB5^1mtYQz2MZC%wOEXu&05^=_ z1IO})W3qD;&=w^lOEh-p+*0w=^Bt0M1RB`g554TtrLY<(er$nbXyV}i6{%164wrF^ zl)<(|m0#7$fxo}RXt{XDUwoTy1uHa`s8L}G`y=FD;~s!TKrAk4a=YdbX?r$28b(YO z9BadnMV(}nvBv8@Mz9XQN^bh*?G0MxgF&CvHvS?G@ z!92Yho&QU(GN6pSVKo{dvu^4==Tt5M1dQn$Rl32c18VBZ@NJY_QD)y+auh54`cE{! z-6-Vnh){+Z4-EBVZ_J$TqiGbLTHF~ujKV4-A}|%^O~f))Pg^`);veSp6-hp*qJr1J ze{|h~7?y2DIu^KxMLM!f1k#2hf#LXD=5{SgAlnxeKanurK4-^s#I?~uK=hH!JRAyJ z7Zg%T8Q1LIQogun_A^jcLG_0c`8}z=*d+Brp=%4i955m8U75jS8f^N&W{T@g_Vy~Q z)F;^~g`=w6_!iF_D*U+NODqMT2Kf?evLC<~aA=x6#QLmhV?AX43C+{G$9oT0bnpZ5Ye}(7inFgE>4KCZn;8E;PoK2O~zhmVXB(p-M9IUe)Z#!?M)4 zDE%;e()xJJR|qv&ihKo*C8q4NG?wo|T$+cigZ+k-#!yoZNI~-jn*d&FlLah;sGDIwV>udp;m0aV%V@sa!6 z3Z0KZi|C`3`gI#-Zd-S#tt0+pC@qj6j@$OlWfV?=#F-m$EwVlq#R#h?NJWEUemH*@ zKD!UwtHzQqYDg2&J;{eA80U@?IT{zy*r^59`u8oAJC1?ULq z?QA7zdZ3mRP-PU6jNHV4aef*1#DM^iD4@^y%flDSs`T8?!j@9)kulRK%l!Z|7#4d8 z2QoT!9fvJ6b6oFO%ZkGpTA9jgX|ezqf1Tt#JQx-{(fB?~(pkWu`RTzt1wWP7vH4Vd z`whlFuX(z(KtgbW>eg7|#a$h5kN&d+LGLWGEFH?iMw!m^x>ecbe5@Q~lq*9vZP~g{ zu=IMzdR*HBGcP7ry)c%K0Ss_?28tkwas_J?-LElR)`s5GO_-h^u~*~RkJ@OBOIQwK zXDZnY`aR3S@z79lgAfnG{HqGD5vx(^q4OgChp(=FmtXZ?>ucFT<(3V`FoEr!`-zAp z=hgR_1>O2xc$Nfxo8;AW{!e`1aTq*c3PGkeT*&+Je>k^xNI@B!D4Nj%&C&KB7Q(Xe zGxlOzWq*+*HeF1$#)sa6kRi> zpkC#x8EW+-KQ{x4J6jhQ40uz~XL_KPhvOL-_4Hd}^Mf;bMM~_GzjTKIf$NF%+7~8{ z#PKSFDiofDtskFjjD}SPj#zz$Kacsyqst7{FZ*kx9WLJZcs&EM1423IPKemB;6r!K zZu^ZBelB}F$q(P@`KQqqHP-X~xAgLW-9D-Gi08f+%ZjG1kIi%;bio5k&YmuZ%|~SC zgmDP|4|m;X1>e`2@`b4}vHZ>MhYq~wuDm=*Ate_M0Gm}Sk-L#@cBSjS48L}o&qJmx zp#sk*{~4SI98nWUF~>Sr;Af_;R#agMMQJiD@l{I_V^0v}GB*bnwgt<32B+uh05v|y z*I2(3guf+%o9>*Jn*oyBnbnG$Lk0n#_bOTCkf8mJkV{&njk5u+@i7lf;31Fb&NLNBI zKodHxN%*i>Kx?vidZ}4TF@_9o+j#(-3oaJgUB5g};|(x|iFM;zbOh{{ek{T+Wuh6c z>cn&7 z;c0pZ8&F91X-H9w4|5}SX{3Wk6^+F6NSl8;No&ief#Qgv)E@zHfsWfFo%^>#ZUOzp-y*< zVHy;mCxYzI^EihEaF)Y5nV3Y0L9g?ZfeOcn%6;h@Yp#}Cai2wC()Gk9v)Sbs?4MNf zizp5sy>`3dR~&==2=zyJl<{mlSm!ePl$cB><~d4GVo_em7O#Fh&|;swW*8EIElNaC zknMKDxTeX^ZxOph#FXa^s6Swd*Uf7m7~pX)GJ=7{>5Gjq!KiTcpCJXWh~E%RvNs~E zc{21Q07!Zbfffv20rt)o$w~hPo~>3XSRD9S*fp>cos9Vt80rW!a4`Pu zjluc=<7}}{N)xFu_;0h<4&V()1cQPlo~v<+#Mo@KR{sJoeh60xUTpxQLhDL};i144 z{yoPfDNE*P1$aCt(1-9@R}ugUJ0Au;2~co+L=L;NERp8;_}7e{bJ8-TG?fzMi-Y<} zoB@gVd9m@pKTv|IxO&Q%f!u``s^Q?l0GgyCaIdZHC-`QmW;BIcpl7;OxmkO;DNvTjOR%(ZuD@%`|q$tJz_q11I2dpw&1SG;lefQ_5Dm4Qbt%>&{tY^zo^ekA5M za#UIa_b+%RllVPo?Z(oTFjmkg%Y$*ku==U z9NKmtWA1{{lV(V(=Ds9|uXEl5LIOBiHj#*C6?!A!LQp~Ww2h1JV8cp^<@oYh-RHK@ zXYJv1N{E34JZFbSH84#&ovJ|+b{Hmqr=>v%Bx_nL_a|Ij>!a$8io=KC-`T@&)%>yAX!tuHqa`&UqNV$4Ht$eYlhFB8 z@4a9WjoY<{d48HLF7!p#XX9^`n32)C++cXQDrvbd**jwEN4(w;fw2%#NMh@Ou%t4R zXdFMKX(~W4;Xsvh{*%`upurQoA#G`3G>hyxx^Dc>(LGLbmxQYbfEy5V94hDV{h^3&*fG3LHEqUvnmkXA;;;>!$ApYDj8f zf=pNi1>*pA;%$$?jfBZ7zsR?1VX$s)pA|wk8p6)c`wu;b4N~Xpa{2K1DAPX~UCi2a z1ArGY6A`&J?b7)VIMg+I-)3#f&`rpS2oPjP2-uQ+=(^d*FUm`-leqVyc=Aq|ZXpt2 z{JQ!4qzg*JcjSgMTbqp=NBqa3e5njY5gGQsXi^*IAn*Y?$_#I}^TwdYbx53~#JuCk zbw%{{?(@qp09l>k8 zM0r+B1{Visp`k?lkwOw+&m8_Q-gobB!n(>vYzN;i-yz%LfnVydHyuKi*Kbyt0EqEF z)HF96fOkz!upnTp>X(1toObm8*sI32A1U%jg{>)&O2CZ4i}9khyOP^pC=^QA6SXhR z1O6H0X1sx9&jT1NPppFL(J3v#)D~J{mt~YTUtSzqVn#{=akYz5&9=hdG7{TO%VI_Z zpqeHb-DPO>!uFwzmrcq>NCV`8qmD4$5{x#EHNe&e51?1nLH%2}B2Hv+*n8mLp*~)x z;`#sWDPd^*f5#!Pgpa=hCtL;u#(~T*)AIpmu`iyH-&DQdnK#%*qU)CG51ut;sdh)o z&|g+<=AEeqKa)yEtZzzP-e9lP#Xyd&!?1HrT`UTD*VV`74}H22-Ke*^DDDf^9Nni|E?D_NB{|a zC!FE<(mjb!;QKfucOj7{g_M_GRFMVnBPa~gd>u)`SShwFpGL3g@SC`E$;H1##7X_%uzNXz^zZp6Uz7X;_OVq0PX6 zAP}DvQygqf<3xD=1F#kx>`!24nQVnXIK3|nu7k8BMhR`fCmJ%%_U)|h*?~+3Il*H> zB?&XXC{477bsyAFJffd{n2-V(sbUEqzX^W3mFedkNe3DaeAo!CcsIoRf5rD4d{TK&My^^wT zz8;ks%G+&zuIh(OHH$DMH_kTgLC?QQE&%j}uoqf4iE*48Pa-)pHL<{iNzoz5E3|4s z7(v>j^^)SN;y!^4o4l}Rh=T}BP6LzA$hRcYu`MgYUa2wX2+Lf92@h|}`V;^SGH0R* z6!EC8s^|ypRih8qyb8$!eCFQ5!BRqia4EYk>`(CE0azd>?@q@S%e%+zc$g}>2x*ze zjzjW9)&Zv66)0LBrmp1TF+5JxkRk!ljiM5xh7a~BF$VEo!59THLDgfdwCw-l7PrTZ z@WX6ZHQIEMTCw3{E2mC`niDj-#e-F5j|bOOW81BP!7Y7O80wG}g=QPR`S63j($u#P z$f`8*mS&X@9@*=8Hbr;8Ife)6O-u>J%5y%tKW(^(4~7#Ff_b)CNyC-` zu;9%QOFu_MJmmDfPP=#xuP4Jny^QfMG;O^)dsiXWGEPLLk|dyDhw&W?nj9u>nXz41^qkmmqv_~bObiaL3QBs z$56z&|Hx7X5$I%@t3?k2&0NW)Fy*F#ilkPH0Rwrzdg69_D$;fCp6gd?Hc!DufqVpJ zoMyb+*}76>Cv;6Nh7RNt2C4&2bTW|eFiRi;zlXD3U{LAFW=*O4;13oHUOaNVz*3MX zFmT^V?xgW_=!NYtrSIvUH?up0c3DU|1Rqcd^7EZZkGKs(8*mUn{?)fy{BiDSp9<%ZJLrDnA>lCw3iBCJVgZV7B zaXVC(`8OdlE%=Y_X^5rnu6g~HXZFa}2C%}Kq@B|A{%giOAY&;c0A4gIU`V3m zm}|nNM8*CG*P+Igaoukt;b2W~M6PiXy=HT#{6Ct84kJ=&EONc$Si@n#(s5+5+C#g? zx~o|16G~BINO(~jZWEwGPZ|1~MQjCSP?Ovd32<@W9JuC7| z|8p9w7N6o#4g*uf7BsA9p@Bm5B+iayLANKk&Xb*0EeA+kM*OafBQIvW^6%FXX9U0g zkOhXLG*dBx@)BE0Yj^oWhcy0qR-E+{yv+y`&7HYbbX7y#hTRiE)d4C(TBU%j5^v!; z0Gei>w6$&Y!DX*D^38&c>+=Bf4(U8*JFLBPd>)m}G3(9bTmaY5J{Bqxc+Tj93v{nN zRYmfQJPbzoh3>DZ7LRqTPmO&i&ee%3-CK~XkW_D=pkQ^qhYc2b|5(8gPXoVvS3Z3=LC$Y zMh%qOWm=3wIx)%v4Z?GW5td9q%5GqF~$tA3}HK7(|9$xR4jM0MfDd@?X zpsvEP%Hw5^S|p2ZG}w;SL*Dn|0qzJTlJW&pm_1r-Fej5aj(X;dwOY|H)iTluul&O* zkmz{3x;7>)FoF?C2MUyM4<7?ttjoRhF-JApc*MZ~eK(`aoIlXdb+Zx#vG87v$*JB) zE}o*)N|P8kSHeLBzcn7UqPxmGk|Y93{e`u%PCZs6d`+W}iK%V`{Nia98t|kr3Pc@kK_f-Q+gv^DaEkw> zW&wEFf#_P8u6n)~>j7G5I&gLjTo!gjhY3Rr{cp#o$d5dN?)+^{Iu0O=&dLT$tf*ry zwCb3roJcLg;q4Qn|0~qKg}wWkW3fLV)Zp~}jUkMqgHU1U{3Bn7c~;p5;^sE6gP4?X z<*)gL&mnSvYDo(e_%x|@QHM~Jnfa3|9G3YErUFvYOc(q%#?kd=Bgj|S^wB+;wxM8z zm9OwRi{%P8^~@6br6u&*7#_q5I!a41NsBm;4xYj>m9v~OSaj{Cky$isv83hj_Ij@{cdT2eu)keNn$v<+}J#T7&+!qBxcof)ZW;-V^1^q_iyau1l| zEl_`aSku^K2+V6+*rrzp*jPZvCWvg_4}gYl&PsLG4M1kV2YlsabX ziB&CKjW`6Zn5h%+w72>Fs?iaq;sf_fgWag3gvog?MdW}a0y8xzndlj}2FB7^29MA@ zV+CMLG*rle9Fbj@ns@}+pF6)yLkCyh2~i2q_EQ9qZ5XZ%-W7|BgdzXi{4F%yRntS% zkLeYqm6{}o-Hdp|lh2nd1GL-neUVApAbttB0UW6~`>+h455vHDDXQ``UgyS*Ate<4 z@73{&mtBVUNpXej4jx5r-DBAAS?hn^4W6&_U0G@m&2Z;5dlO~34ZfbIcxFQzLBg-} zJ3PMGfC>}`!+E&)C4hQdXyOnTFLcdNs<6gP5@4B(?or`mjB#hVUF&A>*mQS1A9q>K$enSN(yL(c`3)-hr9Z9n6%;T1RR+bdFQLfZTZe3!a52j`ic@*2JyG zAnh?F#J(BL_w>7+JU#1(d*hbndYEPk_UG+}{D4hs-f%O1N2xg`h(;JR_n8Q(*TQ`1 zKpfh&*8t*m<9Wf|d`FIGS+1V>(sPABw>!Nex6q*@O^vdzs37hIXx`OU?~jk(#x*E7 zWw5S0eZX2i&v1BA3CrVL?VNQl^0^$mWZPf)A2323h#DFII`$|2;Ko-q`DG9yN3}|n z4(I90gFX*C|1VS?d>H-uT~Bak`zPxHz8Q!;ncPM=7IjR<>pViSIIq;JUNr%DaX& zWGEA%8H<8>0fPt+66gdrWGF&W!4F`4Sc6rO@ze-n=Av{%!*-;bxW%jw0pLw_P@ipg z_bo~DKmwX#&6R*(54-Yi4v)^Jjex>IUHWKkmN4UcvGVnq z7;)g}{qoBAt2igQ4kp@zwN4o~{9anm9{?r5BuIOMC-(zhU7j9dep@B;4U|6OMtO~{ zXEuV;Fv0@(!YwE~un#^`ssFEHvjiOBf$Rk`*g*5sR*0Pw_-C+MF(MBz$_)zadYQb_ z;PZO^G=!$=_5xj z%M6N1oJfi%V2*kI5iIbcc1P3=HT%m*h6RC2*ef@rWBrCKRzA_cSLF7@*3SsyOFk zK{t(Gr@OAr;^j|>BZQ7qaa*#XtD@jgRo2KYH?>MdX+R*c1=8mx{{@e_&g|E+xrrQ3 zMH^gvxq)I)e1bC9u&6srd*tmDwp;dgG3LKZySO(#KO-?f^ ziSZr0Hv{N3epGXPjUf^l!Q@%lF5r;+4_}mRR@9k~4UK3P_J6K!ST($ko@z+sRg>au zOkudL1|yHId||)3DN156y0~^=oD98L;vu=NNmuvNsN7_4jru~`lq8&ihsSx)6l0O$ z9p8ZZjWQ~M#Uq6~gq+_Y`)Z5?%HF~uFH{dZT>>!n!Rw1PJXWvg?08aBS`6h?!W9oT z3J}AuiTVIzne^2H0OS9LgC8Z(A2t#4vt1e!<^b~ylxS_cJseJP`2nkg%ZS^7Ky}Rj zJ@=;&Pqs4$jLeEf5pS+Ad(codx_=)%h|nryERg?oAg!wNB)RfG*ymTpcppFqF?Xk_DvJR|hK>Pu`)@LR`4 zF`~-8(4eGcW28ySiJiy{t|&-k@0o1B+k zHyU6#wQ^PQLn@F6L?|Rxm8uQ_*8@!`rMS@|L0Eo+u5qACs(mu5D9Fhe=&L3&YfsQc z=GA>Xv650?IOV?MtPwY_O4}OM5v73{V-bfiGym20iH7u7)}^C_!#%3qIRH`oWXC%x~l>p*F(JdjY~=^vW89mmK#(r z@L`Kcny9=KH|Ia`?-XcE69(^m%By`$0;6Y5QlX{rFeAa>qtP(Ua#7_R``{%0Y*W(u z6kie@@JUX}O|9StZB!< z%=w?@h^y#R#$UIOWgaq$lA8m2u%Pgu4=+<^rPsc7RCWMuhGyA8ggVU?XF8~?E%eV~|7^Xu;)q69IQgR6F>X3|3ly<+yP%;>jnxjJ*=kOh} z`GHGdXb90muIEKx!PGsk6q5yj9F)@IaI$mmx(V9=6*WX-3WK1adMF%p}64q^jSqJ`x?0N z1f%hq=iO1zqCTli-#cRUNdX7DBl`0rc%DK)fX&yq%di~!*(-ci+Wo#gFKH}p-l8B% zQjgSxtxPEwB}5fLt_W$41s{s+ilY)5$MbGkQop>Owib8{haweW+(sj6z}$gZ*zzFY zl`>6AU!#U+fnv@NcIypn%Jb;)lT9-&P-+(QM?K!Uc&zwiMsujO1fl3B@Xo;tGh7myoJ{=h$}Z3?HV6W{s! zA8?_YHWv9C%0`=;k;n(Z^n~@|w%wYXoFFkmC7u3vD3G}13<+;Mu0c`lFt4P*8E}{ZY@&M)bUVz!`K!-xwRDo-U>dD5Tg+VeBzna=#uFDU};O@OpNY1 z)ih@vYdVaTg$Ej`E?@yg>F8DT;CVGGT-xN50nZ4eZ3+%rLS~jdhDYOMbGRh9&Uzbl z9zZ&db0HW36sb~%OMyL^!9G?*5sJV#@r+Up_R%-8j%}&OQUwZKJYqUuI>gsQ0ia+q zEIew+8MBzEd9UUfbylqhiaKVj(kuf(;>dhv@PHfY-gW5{jR$XN{#Dg|{Z4}EjJv&i zD#s_#dt#npr%7~~8*B^q@s7AL@87K#j{vg>hbhoIEXNb`LlI2e=P!_qMq?@vhd-1aKyYmyEJI@a?8ow!FZX zoEh7z_W<)Fes*N|ofLs>I{m~lBkz#o_p6p>e^=oJJ~scVC=yA_oOjmn_asb8{~7Sez#*s}g|8+v!tjM(j#}VJRIvNGP<6EwmcS+Cm|_W})S0+%&_gm%}AI=ro?; ztR@lozRKU4hfvkxvyCy@B)#_1##Hn8h?X(Bb&HZlNu4!(DOUbj;9*MLH2Vf%@yH zMqBc_+9bf9S^>n&bHC}EKtsdS9b;Hvd|c;4!|l^cPo+dZ4vSywg&&OEuUApW)z~FZ zp>PS`xJ#7h0ZA%DnIj#>6udxm%pL!C#sXwTxpjnJMBWgBp|tuN%eyn>@}GnNcc-L` zTtqx?3w9$7g}i#>1CZ`SHHdOxQ5}CAwV~poEj2r`D3VBaIL`0A@BEoJKV2FS-1jrJ zZ&b%*s1WhEhm@R~SEW)>S|61uEnHcLM$H0;VfcTHJ$L8mDb};i( z9F{J*XpSIjxg_Nj_n~_iZm!@p(F30z$uzWMd!b1Sx2q$$NJ1#Q_chiX96* za3vBC%6Y@Zmal7q7QIi$G$;KBx2T_|Z z1ManKKOYRbF7aclAjQdUxUo=13!lSqN{BV`LZcS~YA4LFe!y#dQ_vBVOa$<<6+!Bv zYK;0rD%HM>^8uDY1NLFU%JhW2!)&9X(tbJ0cyr(mf`~*a#H!I(Hk!7m4Z!y;k`k18 zR%sB#&TNhkp>uGF)|Tg>1GhxM@0%!V8n)I+{;Wpg6xJ zZn6<%fgXC;SCu_uw~*8ZFCmc&oJt;YC1QoMJUc#8Xx_M;HM4lL{8kyYxLCN;Dk=gS zUYmO^{#;}9= zj&y_O0DXtIDnN5keGXM+UR7tm`N4Io5!0Anq=q`^eI>j+wi@q-&(iDjX78ZEMn5R+ z`g{|kH)~&WC*;dTb$Y)ST;t4NZ%a+Al-3G3f|J&E=*OwnN(L-f$pPK5X;0-D9B+9~ zDb`TfP~w#~w3X?}>jhOn?1>d}Uogd+^qWv?i4*Jc2C^83SHQ#3%|GRZYk&;N4hl$M za!fLqYK|x%0{Q<~oM$Jsr>KA(7Yi!$h4Rt0;fPEefw+ieW+L2ZgpOOkxDu?OIhOw1 zqFLy(h!IcZU7rw&g)Ut?95Ot0$UHr1ks`hhVg_6zX-ZDmot81!Koh_IRm8u2&xQ!X z4TzqhP7UNXOvBMJioe_6I5xT6f(J0ZPI3mmZf(BfETF0eAeMQKLjLgMjw3mzHA1fF ze@q?Mx@P1mz#+V{i((g={sVO1;0hua4NqEz7_nZ;i@-%k?yvJb!>7T?A|Wllga8{9 zQ+70Uy~a?z7HG+D^UDE846hvPi#~M2MYAi}B74{B6;?6r?$D|m@V&e+kDC!vPIUBX zcop&!046?3Z~}}1KLe#h*HJU61jr)st!&Ei_$)FGgI|tg2({ZzCN4^i(HEo^0qM`L zlNtj2_;+BX+?SW3Hr1o*D8-R6w!9Yy_X`~8mQnr?K`3aT;Dx`}@u@Z%AmyW8DyR6~HA)vlBzJL~?+v^d zD^f9?7Ju%gZ$4z^x&(cYEF{0PGgDz3Aa;Z$VT7J{pxN=g?}m@e4$kPgQyUV`q28tGgTW%7J_;M z2q_F#u&=7nio3(B-8sRQV+y4!^ZMRo`deQ9oDuJrEA($Op$UmSp#WA$1jk6k#{Rjx6iNcG0(o;O~FgaQ-=TFqE1_S&&j;7F4nvMejVwQ5g&J{qz zft9$PSQyuhXwhw-FcHspCq(6#yTm-g(}nO}Ec*-RB_$m4Vwjbf zjuS?mmoYm#ZuFGhbGvGW-!Tp}GV(Kxf|+&qPJvZ5ZVp_u?g2SsWCs@P1N-+OzQ=oS zZ|Zq4#xdqt`4I#BW=%V3P<>3#YP$l3VdbnS;61NAo3hfi@erOp+sU?>hJKZM2{;4$mmVta{tcUy|;t;=Y3y2s{q1$I1Js~1f%8d-vqz(RN9Obh7 zsG$_8W5r~qIWU97*g-KyEaKpq^h_>A;oxbF`kd?`HbscBK{3r0S^L&-Rh{y$ZQVT| zCA=31h1|+MH~Bsr!@ZL)*D|29U10Q%q`BQ}`-Pk|&bV7!^;>}4cbpvh(=$Y*VqZBR zGRfe@vc$krQYrYtllWACmZ@@kLL^oR#cA=@EKJ8z@0>Vs+#Fu^ZY?^^ za{I$374$}LZxveimt=H%5Z&CqWvUy6%A&MuGr7yXeVKkDp_0{M`87Z`sv-+%nAXpJ ztiUYW3J1i8qT$SMGfS)(;P;?&YivKRTm+DTBQCP^UH}Z;f$bi|n5z2*ueDp1OmMMk zPk^EmCRe4#uSu95QX^q@{Dj#1lfAK)a=HMx$R8U8|#XnUNFd_mL%*7KkwW97N#V~uZ?EJVta|OL z8P0%rZ8ihvcT84i1;c4>H*u|}o9v0+(-bXxBD^B-)NF+&1B>9=jf?Py|7W%PwtcLd zu0=T7Dcs!-(;Fp@%kJ6o%umla)npl0i=^QSr#a8m|L!1@Gk9A_4}4|N$MteJtN!Ta zHtwjp_WFov+OaJ-u%3e>1e=jj*@%IOjRaY=33QV|KOF(ReZ@n5hLim4Fy?a2Fb+$hKJ9DZtm7_J3g(;3^5_vm<=Aa;xcTylxA#Qe3 zloZnu0|W$4HI8{#yn?c6l#!H+BZ!FH=9E_38yNzj0+S+MKz3~e;{JV}wO%^s_ZPY^ zu=o3ZKA*MLv!3-lQY{5%-4~ld^hZX^#08Q%RnH5%Yh|+*nqE>$qY?G3KjQ;(m_S;+ z%P(ij7-zqEIZGjsK^qLaz{qN(f1Cw;xRdb-dM%7w5DdJ@^#g|^LP#)8il1w=sZQ!{ z5u+#E4h?H?5tlS>;A7TPv{rYL$v>=Usj8~Q2LpjbLXS3^+QTb#5)ar*sAPfG)0d?T z!v|fH)R03%XsAdA_zHawhEc>FTt;l_(4?+t`M;P5wp@e{F4PGW>VkKSQIW56D_kI~ zH^}-CdvCm5xQnz080b6>vCu`yTKe>g@W_zze<4WKLY{I z#Gntu$zHmj9h?IyG_eS<)5q}mF?~RjADppJ{VovDFYJP3Rg{GHGnKipwz7)F9=5lA zBX<|XjTU*)83uim_StBT@Vsh1ozb>&6^~L=exV%lN4i-=8G4%-0T@CfCu)fODp|?I zPmcyV>mppam>_CLv!xn{bjd+9S#&!4AZB3X&&nPm!@V&Cz!ojz$Szu2*DAD$Bgs@>4!v+u5_%zzSEd zKPIya$6OA{KD;}lQ()hTwc*I*kTF2UCxlC5XkhybjuNk~*nZ*TF8SS=cm0Z1^f-5N zAP{CLP>=W-mYH{x$d(uYm1>b9g_?xF9X5uyr%y*>_$pe5|&3VBbT@7EJ?A*htl0tWi|Ci@>D#6%&W_hM{VLLPl%?-+A4^A zgn7Y4IVC>+v!rh6WwEHlT}C)Hsrw=*FPF~A_&VlDj*}%s^;vh7A8K5AvI4(oQ{A?q z=_PMR@LvVB4qkH(=|nDpbSU_B|J?K5#~5hAjJkv@Ax?-Xo1}>OH8Q2UF0TU3INttl zv;d{M2gyX42#m})XT+!Esk{!?{G+Uw2J^~=|Fu+pis}m4IU;n4pi1Be5?2=g2&j@{ zKef3S&<+**9cp7ml)XMN{$8YMyiH0 z;EnstUceADT+gEI!I?ljH!!~@nIkkqlu{{5;_nWis``NtYwhTO3B`-}w{Z)f;l0{^ z^TgN9&Bva=G*VStOlro|ASgg!pu%lbiaq`{kAgMYv+vp&qG;Kt|zDbDK+|nASuc$(6bY zpEIXP>LbHBFpSfNxC5#=eq7~~>F9ycdUVavU{TBuObQ$yzvg#|^+brFsqJNy-X__P71cx`Fh07BUlkjNC&V9*S_OTe0OBGxkjj8y zXZRGUA%;_uj*}^&whC~QRfcfg+aEw1f+1iZ2@pt{G}~)*if|2)FjWEt(?F+xkhSAi zDvp8fu`7QRw6EUbm)>?Mp@wuX0P-^fLJY$f=p@1Qm zM4dI`a3n4$xMB$dlzls;GpRqf4DC zIhwyv@-Rt&LZ8I5g57r?|`<~+bpmE~c`Ns6&L;nZ8= zg@!+FF>lK*5HtZgBc<924j)t|+sbfmirj{YJ=OpP2&Me42F4c`QoWC;9zepH0XjN3 z+_`}$13sKD)xfr<9*lqjtX;yZa3#XS!yPIVJ}2}@I>TxqnNOWKAJr1>+U-6Q&ea^b zl}0a-1f(EWb20iW*SQPyBXR6TTd?c$?QWHi5{Xxk_Q-)nUGt=poSR`G6L*fx&AOCD z2^3~Q^VVPV4>7H1IM0-<%k>v~PNdg^QCJ~YEhuBkY>hoT%dbHSE|3$rAKFJWNa(HG zu}G$A2f%EKI|o^P<>&9oRyBC84mQHoLkjR>fLcq1JZoqt>bwT+E-KVF&6v+EE{6gH zDAs%SDMO-hC+H8Tqvl$l;4;dxRZc}^-5Xws@=s*UcqQV!TcMC@F-r%*2%OJ0-^=3# zNf{pv;!^VpCA5cDXv|`iw1`A&cdsgv<$M)Ws{22dib=N+FfOAzirrPlo3eA&fwb|- z@)F&Eee8!@w=)Wa2GNuG);Ob|>(z$w8LUwCIm@?Oo_pwDAw>h3U0#JISt!k?4#U+_ zm9NA~5ExK@uaout3VYrDuwocyO9g*r2$AqbSJ~JDJzz zrN^v@Hh2x2%dRf4dhP0VJriA&2D&-AiWxGTvAGU{O+h?|;24TvW8 zBSgleVv|lus`UvPC=+3c7kf9eJQ{3i&E4>lkS#h^0jR*zYtYOMog3t6)2)ZT(S90@ zWe-D)h+>ck$A)(=6GL|OArm-DME|jb7iaQSt5*K5Lzv}>mn4S^iI?IC3}arBn|o9_ zsIKk7AkfI9-+M#`n=t8ki>DQgh4?~g2IEs1mh21Ak45qo^5uT7VT>~nKWiX~V!F2^ z&eRQO#}@&jMnJyE(Q4ar*y07NZGsZ@QF$ljBTuusZp;8WC=W)jUnaui7_IB(LM<6G zRgLmgjV1W}s`Xtl&zcX2H%C zhvKyCO@BPZf6O*`CiR#loNRi9JKM4&E&8P}Y3Mht_XHJ&uZIC?p``Ip){(xv-*;;y zQCpM%CIoW$f?Xgx06%2g37KMLFt|#nj}Wn6`kriAgY<`wEzAkv>hgMPmsv;>54+5V zWQ#v)94R;Z)5VXT&DR8VZtl3cZu{MjcY!g*3OWDwY3XS>1OASciKY#`^TgjNa_eH* z-MxS7Cr5poSUlIS@S6k@5=wC2U|6$>-UDWrVVBLA0|%3nZ0rIuxawKL&TV3Y`+klZ z;#)b`2yp_;DZu17?SF^CDctonxfhEtJd4bzDvT`=ZctBAT!d0%$g zRqiy9Yg4l1@U@=Vzl=>P8~1kAUKhP0+LsGA94~BGk4X&%*r2PF$$N?F2oi9ki$(0! zbQU5XAXQ-7szM}=b9>TSf)b3ewfJ)^ic;gKfj2f`iAIj3R$&G%d)K4z7Fb?N-)KO@Hij8jQ5g zZI1&$j`Xo$7ZbY1%v@1;-53lis6Y}u-DN$+XPP=|xvP&*SmBdnGb-H+cNkrW+-_ zPrLsIptvnwsnLRUg4OFfBts#Z8}#hwEjni1?@;;=Av`7t`)azC>UaX$r1+X;hfU9( z8n0F(abLL_RKpC>s7!45aUW4Wd^@!{LtllAbC7fLYIJvj?#bs3*>82bty}Nrr|4$d zWM9l9k`8hQ03kq!O=wogzN-nqBt0USaG&@&smQFfKs1tMdV>vM6e?9af`XgIK2MLeg`&s(Ah3b~p=Hi?as1mT*d;$|=o%T6GK7Tl5s!Sf{ZFVn`O?D+P z%Md-nsY>LbKI%ZQ4(n#`gz`urKcow1nvj@%bD~lOlgVK6{6UyXZCMxumOAU?=aokD zjLj-EMusDbVLWcgRAD&8=8mqc5U-5ojp%H2F4%n<E5N zE{PH-Ay-Hll`}&47AS*?zDkp4Ode!OOozG&5j+fIHs}2H_dq=$W3{LuC_rF|*kNvy z$;zlXZ#z}YGKN+u8VltYoV#MafuP4^5F#o)8GoILVh^aMB=aESmwJw+a)fWgk)-Q4 z8MTgA6r5q%bd`Sgbk;ztE91Jvvzjp;9NI->t9^Bu{>%B*OOU@f*764!9^7r*2`0q= zbGMrT@qs#1&cINV_l1RwSfp{?HC%r5C-ws&&%O2~r+Hz!uKomt7d(qv zSmneXW)!?B#Y;f?YuGWII>?f7b{x!+eYQ^*kAE)hrl~;%I#PxTT8I()lNCLzJ(%Kn zeW5CRh6DxHx#%0=u#eP)&iuAEWqQM z_dzCq<;tbSC=__X(tui&z_bwg*@_VUqrLkbV@BVl5TmpN)=J=r!!#0CN4r(+->x2| zxgmOT@@`8d>}MsbG=)b>JfjRb8gB>Zo9IWVJ^+eEpi1r!8)_UmYT6mKwBnvYPs+ec zaoM!l3&%{i_}5v76tC!6I}2o>)*<%kQ%rIv{S6t3bT`pe27a+F*OAxDGXChk95?xA z?b9p#l%Umn8SQ%ECwgT<%0adp4sRY@F^DD_u6!3` zS%yW`iA4$ec((VOzot&;wYW)7Kb(yw2yr}2nkT=`VipfI8S5{8R^aeX)8M=9dlGpV zOzeD>%L|Qs83choxlle(QTOC|-=oU&v*sHAbh2-eL>r8}1~jB34kJL?7NEV*-8;nn zKQPYrRs{ZJ-AcE%3F@d;e9t`JqSV=LyVukG5`+~Ib~lWhi!(VKVqmczX`l@ywth$` zcwwM<%#5+Bca7&I4KNUaWN7n@65A<^-btyY8s`S=R|WAliYqVY{5)`;QaOqx$a#M$npyP z*8pCdF3&NeYe|CGRQF-#VyA!b-gC5E=c}w!+W%gtibwektGT;SGckGG8)75x`7|Rf z1ZW4WDV`GM1Jcglb#NCk%@$J)_aNcQI)b7$Pkhu|PQhiK0jpivNRZ;Y!%flJ)(AG5@;+M%vj9c0?wG}y@6Y3Yw9^kb(LyRMf*0s`e-ga z5n$@0ku9$wOG*1D^8AG(5$l@%FeV#`40=N8EmPk^)ao*Es|g;JKRg{zK~Cun`%!A8 z(R4%bzpH=g&d*^ufz3O94_cswyZ8N;*#- z(a|&X6A+Jf><<^eY4eNF(fVFcjqXlRZ>8aDKE|Zaw2uN=_iek?fCA z=LIK6)uk9wU4{e|cUjnkYvj((@V-K`4MFL7SOT_^@pr=XN2FY%y(jh&$mJ`8u$$#Y zIDAAGW4LqnULX_rGm{=FCg_wcSOrKobiXwq!M5Jz6(W2wyz0o*wNjzoed0?=5N%#X zB{$8w`$OevL(`nVVDJGa#+qr?pgr>qbus1ACL#S7DieUdI!>k0qg5$Q39n7M031jkY01-+zl6!AYl( z8dpw8LDIq1ONkY=djwj+%-K$+K>&b1^A^^u;i_ggKjrHQgK*ojXQNd)fj4x=0@U%_ z6?6h>>+_~KZ(LVeU$_cP25^q@kchdirNH(l9W<${zT-Lh;SHbpAw7kzqKyY zO<}T_A>`j}uO%4~^ytp!HVxf(yb^)No%l3gZ2TTKH%c&)3V$Mbp;k>wDBzDhCN7Mz9 z`Pt86&`Cv`;e)Bl#>2m|i*!=Sd!9|Rh;$XQ^#WocL+@;L2zp;iNA_KJZrPykQAm7+FX;{^P z{>+>MQIA_IJMRZkmBj>1DB<*(-t)Hok<0&9X0Fs*Byhz@%v~P5uYe9Th|F)HMgpM! zrV#vEF`UXy6dDUD_T#vwN~C5oPhBc>S~eWwrSuX?9|j2;W$5bi&4&v{F6Z7s*<#CI z2)+pJF0>Zfm*FQ-NMmkeVeV=^;CQcKWO|ys7A)@mbmK+R`pIIQpbVJ6$#dYpuVZL^ z7Eb$ZNCgM+ZJ3JMUk83Gj@osGKvvx^wK=_-en9KoYu`*{opA+EAMyW`#0Z2(p-w@ISMJb|SWHH>K|kD7_=+Cx4exbCTN_z?(h&b&Tn=Vq?MFOS}11@8TM`KrPZiXQr`^J6288b6OF^emWT-$;klWY@w zl0#jJf5gVWoxekhCW3H@m!!gK3i1NY1TQ7|Q@U2ri59#gax020XD)kiw8kM(2JCg~ zoaAwm=Yez}#xn`g5K3!j1qA4asBfk}jMUw6I3uQ~DRUYT#$>O2|^+ z51j�OAEu>*kj14BEL%86sMI9}>!ax?O z8k$a+y%Tc|iVb46SV~KbxIWhRAT=j@7Fba8G*B&Y*NXfJVK}6Hk|p(0t<1{j_fD$5 zqt#u{L=*8oNC@>qOq1Cs-t_jMJ7+E6H>=$`p)~l4oDY1TiJJzThoniT{gk07tIyhwGJWBX*aeo?Os0k}uf}7l0gD zayP1o_BHi-Y?V1#p0>Au6TG+&Y(gV{Lj=Ic}7vBs^Q|#-@GJ z#OIK%DkHGrM=?$S3OVft1T3(+KK?1~{~Jc1QUE>b*dBWttYlAqzW(JRZ(c2iI^kF~Pwt2PQsk#q*%lyP!B_(u z_zVa{2uB#q4cdG6hSW;vK~Ki_!(G_FATUCpis$Ewg_3*t?N(8&K!D&KEQdSlTR~LVQXAwzq8l3-~yDf2T1AT1P?!I+f6u{}e)g);l+48c*Xb*rz9DeaU zLk_&&UwSN|>mG@;n^SKPU&kR&d>#6%>8dpglcMyLX?&tXQ$h@<%5+)47KAB8>h3)R`W(bQ3gq>0VJw+3p0{5c!r`{O79h7tbh2N~(Tw1%{U|V& z4sWk6-Y-X9+kSt=vP~f*9tlrlY}SGzt50?dbARQzI~c^ z==^B6ATQof#W7?~;`yt}0+l&h&EZC4oQ>9K52=b(rLP6V#7>i1&& zNt+g`Eav7lOf+JEoW3h~B2-l{)}O>T*|j_l@e2n){6kvV99&I?I`t4l5DB`| zHhH2`Q55N@s_1;mO;13HKcCIrFhn4RU_XgXaCA)-nwD6eP)Jq`RZK059fnIzp0Q;-UN7eEvh#%OT5pEJTnF_aD zUz3DLg!%I)(YqR89@5tUHuy)h?Lg^T7JgkAvopiWLu^)Y<3PK@@bnlJ9ZM8%{hY50 zMxxM=-6f-im}vOPa`hjgyu&UM98b=#uLmk7(c`i0^;)gO+CqhmjkKK%TR;o@ zG5drI*~}`6+IO_Hxh%7_E+eRY!$vR?s7yFU|KD=UvZ8o`-B7b~-dbPGF*?nb0Q5J7 zo@ObVc**h4(AIu$H@iU?f0c*f6Tx(&E^kMgHqIYriX&xk*2{Md>P2($8|lgaP>x$# zD8$uKhCwgpbcwWiL1q`Y5x6lL!Juzv@h47>N{ph&4yiAwA=UPB^G~4FDlY?OG&xCB zr`)+}i*ltwg<{PGA|&WbBwe84e5v}0NxT(7_*>k6l7!2LtdMr|ZwNH+yR$ zx)59eyZsA;;R0y*x=Ss_N*yQeCp8l03U`5eLH2Z7sOY;4Oepmdzg&E7U`EmkmLWn+ zIIO*Reh9yikz1hYgs}7~1~6oc8SbH1-{@so3VsB{D7~pR#2*}oC~5`4Qx`%QGXT6mkfJFyXU-n@ z>v$X{|0}5igJa0gB6c@sv{-R$y=2CG<$ylY0d)2DQtuMq^g%HjPOzFqSi)F`(kpx~ zEiNXM&W3)1@}z=$2U@Qavg3DcXPAn73<)#dv51MU3ik-i-f(p)9Bcm^p+Ab?L9m6l z6G=8V9Xv?A=wyQpt2C$1&t5Z0bZ$>R8>ChxozVk^UudgqBNxu0{^!EpkNE-X`*p4P z5W`FFd0);9r8~!aV;!1;YJ300bq{|RjBIre;HR74P=g1t1~I?a=Em>X=PzhjB4h@x zmN_STIJbb^fb?bLsMw{CcAX}#eLMdzo*l3^>@_*5E;R^=CgzupOMMpv%D=LU3~gy1 z7!P+!>}wG3!FDuHYXrKm-h9dXitYM%2Zoe*GA*V#Zi-Id$4`lofzG+SFkl;SN0Aq* zcA*~IjCUk|{kX@QTkwbQN|_@BJEJ$?{zrE}^rqB}siOfkV2k$VHESh>0D%UDh!_Z~ zs`-4k(^_ev^-+Qj_ulkXETk^7B3XTNV8zufaT>Cmxd-tb%>&aVbb$(2)}pdXW;c)k zsGnry|GXsL$z`O40CJi4gQ{^<2E#j%PeIP$9P}}92_SMR`%)))(R;wQLUrM_9GEC$ zRD0$fhY&h{l5u#7*w!SibKPstC?yTS=T21BXjh9VO)e)Mn|_Y z7+(FhBL-|G);Kj$JR+!mX>dLVnV3wh&P}Wqi^dpcFmgkdY`%7Zxc51@d`=hW|cq0(9 z71)V4uM3s5%X3b{qD;IGadgR&8I+Q6kMekvY5Iw9gskCyrI;I(0;qnQPoFqN>yWft z;gy4UVtV*=IR7Zh>&xR$b9Z(1Uy{Wcexzxa@A~Lz7t&}w{|FQb0FD9cFEZnU9MqKV zw2Q-`p$^UEi0HW}or7KHX1~UTcJ(b#p&-@rzd+vM{9BAf`AaGqq&@>Av{D;j0om%T ztHNa)gLn33_PGHxt%&QL0xiM#W6AsI9fc%%qJv2}x{(;l8PwZKD>w$4g0G$1d>4 zEG+gU6H}y(IHC%Sh!@j1I)u(RMhFDmrB%dkktgb4JuR$WkjW49(aE)y7PyRn2%CLc z_Px#k^cR^rVjd|?a!A-%A`YT7)k&<5ekpvz-W)fTtiV)DyWcCHa3|e~UF2i2M}dI; zPjM*HKd{{+uP|MyUq8u3{8VA-(2pMw8JX-yVP${@h8`(ZTgeKaz#9unW48-<6>la$ z?E^mzu88n)?Gm2ParugpFhD(00JLMv4oNqJ%it@bzX=V1NRcJqn3^S;WPx5MH$3Cc zU*7pqQA@KhEXqywTu11mBy!(P+$ln|Rpp%!gMODSCp=6k=r_5zY{$biRz@xDQ@b12 zlY9bVfTdit(-U?F#oKt3?`a=VIZtgI;N< z)~21s7Zj2c12syH-tM$rU3U{OIWCnL_YIbGii|2k1B_o74U<-!)qK7lS(%a1MWv5HHXf%LU5<2Tzw9Sgc#|`Bl}d$Kk%g3u#t5`&GmC zkXjWKf1ASnVtce$H8x=#z=y#!$teBFnNO{Wj3(8JGi`>KFBp3Qw^Mi)f2Wl76V03& zVg%q@bq!59DX`jB2H`^2eXt}N>|9(pAhG#x3n>?aDd#D8fX0N{TWg8|IfP|*9x%ED zo`NQykR_(f@)7Y0;Lx+jz2YT6)GUlUD6aMO*jg{3R_o{3MPVIc4~{wzKhlw28{iuu z%K?_qKgEswBrHocx`~A0ZWi zG;w(`S$=JM7^)xu&7A?JM~Z@d|AWRltE%Hy%OCP*`*t7U zN~>-^rE3vMk{}?61X5s00j(jKVO;Zg&pZcn8C?+ASV#<*KV=k zSPSh-)Gj`4d!h2q={CR6kw8BGsl`L;`soW74uc1Q_B7g>fWJ)A~M;LcCqFs%9 zEHeBH%ZE}l>=~j0*Wv;%Va)RSi~@mohC72wA)oOuW~p|&7J#}_3CCipC}L7-XSL&yM%x2Xw)TcmSpi`|MEi_r%Wx>6Xmb(n-!QI0mB z^u_ry1N)nDHXcOADT#!Wx*wJX$_B*Mz@!ZWQP(Dw^bh>TNj%xmm}+{PjVY+_ zi;Q|?^>*%=`py6h{Xwb1E@40x>|7-L?AaIBv}!ju$G$pJE$X7cNJH|+*3(7*wwyb* zNFLj_pFUZkdK3#|a%TyLDct2KZvbQ&Ace+9z+{lPH|c-+Hp-gZL0oDIirwuoOTCt} ztXlfaSam&q^Zy%B6Hds&EV$>jgT?vPgMP^`Im#aK{QVGq5?odO7sRbl(JKQxP*@kO z&)GZ@+64wYQk-7r7PlU*2R8yXrtN!#D^cR_>!!YJYk<(w<`6fM%!~DNj{UJ=^+!Ag zYJ5`4Tqa|6WT!if#k+8b%;>~#u}ko{9B9XP^NlNli15)vW*9W#U%Qxs9xkyv4#O8ZrNF@4=`@cHT$-i+uV~259 zwKoMdgoEeK1t`Nl!-w1s69M12bA&$}Oq8A1`;R(1(RC>)pmA0Q)ejNsRx zAxgjsq6+~=))hDe;y4rM{$7!HJ8u6R(mD1^Kg3#z+#2Uhy*9{9C z&}I4gFLOzHfO%DWw#8Ye zf5oi>)#aygvd0z9sNW^#*~oGtBxQ^9g64z}iULgGljOE}jycRYwb1DRa(FC|L-jO? zhVk0=5WHo&p+b(V$(ENNvI~3^fW2Z6R&4ol&mWM*aSCyNXv1=DnAQ)*$bH*fhm%zI z5ZKjjDMjMArur1AF7YBk1|=xztgeEB-vHjUtC7jvG+pcdO8rR`RuCT3-Emx{X#u#M z)LF(v`_7^))B%3?U2{90MC?&mUxNt2bk;*cTp`qz-JZwK1|; z6h|${MPOlJw1@*$0^acwGmX{3D9brMZZ0-yc-d?C+T581nGo($oLqe=9e0s%1|iWV zVKs-m+i8#)3>PC_)G0CMT!XH2ZufnTw$KL2&B;!@Et>Mk0B>R3DR%DA_ zh5zs^#_l5J7S@YehNFs_%+J~a|G9IrPN6%7&*jjN_v#w3YK#f2zUrRe@_$W-BBI3L}&Cf7%HUDC(3 z#XIU!V9mbV=gC+>EHpK5KL18!wXiFCGBjPmko@XuXjo8e1$XQuYotcof(>PbgRmeV zy|)B|pPn)4i252xkT% zaXjSA2&#GGhyMXyA?LYafGVbkoq-_3nn{1MTIPan_G=_t3aYfaP8!9O%_Cn)U89Bx zp8{NX{KWik{}gVYg-+SRoM?SMeX)WP@bPAd#;uJi3(TU?JZ=hQZ4sXGe6A)0l3{__ z#v)(*9w-mkPF0IU+o_hFGrymneb>vcyH*mEYVrkvzUkl{&R(ygj`lRA9XYP_tk|Imi%*L1M@6RZ>3m*Fo6{ zN<+g;3@=CEwF^=fSjV41uT)Y=LvFpYVhq-1-DV$)f);Hr{J!OepnQ$Clh8O-P6c1| zzv6)=P}zA+-2AeX;|`{X1ix(>oPu-WI0%A7R+;OAcN=zOw&br5KYr}&Djtp;a>SLm z15F5~jNV=}EX9=L3y#R9`IIwKGe7bHVVq2EIwG?HIy8xeqBB%8RNgGitq|QFsh+Ba z@`bkJVF#wkq&ASZ1x=EcGp!U{qxj*XjzBgNEiD28-~l2HHx=^l7uO6Bt}W`uo%eXT z7Z?gLujEZ#cAF1$c7}v=J`&6>i@$}^>uB*NsB1^(-@;~9tO0SabneC2VN-Z&S+9S| z`v`WoBsQ3el85teV@8@KLKxNr9zmrO^PkF~l!7VwAM!Hifc>FQ)f9~Zg4P|h+b{zG z;~iQTSP@afb!j2J?6V?Rx6FJY}#_u)h22>>0O z-p5=llu<>ZEO_e}LAqUMQDhdUy>^ewvkK7aA@F1>E0ku=x7mmP^h?Z=(FyU3vn}_b z$XRO>x2vlYTG=vOd{;$cplk!i1a2q%kNKHjsS)bdMWJAV_Sd@zH&VT(FGKs57q|A0b^`8q`2-sbX$+`SA#CaJ_No6x zA{WBoxL~|}Q{C#!8dh4!{$tP0s?r^K=!4p~(Y(TPJ|5mzWm)6_khBrfXr3ssZH9Ly zr7jTA*sKF$0lvWpptec$)0aX#37vAle6(^yuBQ7411*AK*xz6@0~`gsU4{pw*~Mxy zSTxuXFILQxmlw=5zLDy6~F^_x2 zRml=wx>e5q)Ty*d#H?2%#>*tPTrg6SNs|GwbLJlFdG$59P+*KUl*{&UfGPu54yNE4 z4??zf=qQ%&C2`O0v)|K3F;vT?j|IKYL6{;&J_mUcRj2@(;-LhZeAOar3I4lafaz{R z*)g5;|U(B zAt!yb9IkHR|4HjOQ9e{S7?26N8tKY}@|NY_&m?uP9mG!lEdl<5n>P8z$>>$;6Y|q2Su+EtZpX^9&Fk|n7;WYbL+b{*_0k*jjVHF4oxb2% z8Onk*cdbEX8OUxRrsq8=c=0oZzWh9%lz+Nh^VhWIa$DZQbqz2UL43LJdg(c_{%Bv& zaH9HwOb?e2ek{xjYTt=j_Hc)Dp)9dZ6()&>(%A4 z9>@oQoT~vMdTE$G7saY(m88YifE0yDm~ke9pa$-TeG6oP5k!72sDBtQ;EyDXNX7*y z6T&)*N2dc!0lh_CU0`U&BpA`@G!Ovce0mg+de!k!t&?Oes=nFtl?JDwO#*-Psi=n{ zLBytLllfri4&PGOscrzclYvg0qB^epye=NT2q2}4WznvKO4H~ODJkz#gi{XsND+qt z7ZSA=+_dsl7e>2Q5a5I0potd@PjLPNTmyNy?5gZD16(gMcFd~zr6f{_d8o3G zKs#o~=^~+YDWUIeWY$-=?FBL{(9XGdp3jSjD)LWLEI*ptnxsn#O<=wlL5Rc3$mb9W z0TMRjsV=@E>SO&ZimVE- zlwzz>a+w_0N}W}sf)_o7E3$mX_AWj(rzgtX1Yb3sY0P+4qyl4|&d@SYEqm)89XASqZB^YmsYwP_(`@Pp7 zLJLNQ$o<5jRcwHWVKa7SYkSx*#Yud5zARjP9ZB_aMywH%Y)~XqIERqnY7R?bLDleS zYBti&jlU^}aUIdq!gIC9C7M z6k@goWA78Ffckwge^xD0W02?PY6NC~3BTX)dNVXlBu$`t1@y#YMf+LODWSs;;zUIRc)B$*plbqF+Q?&VtZxcL!k{~-i+ zFl~3sWH8Fa{e5Dz+*(++ldgbaFTcxvlt2_IrvP%3OncZ*q|s><13?F35+d|{kC#o) zV2V3xV-vJsxksT^g)5?J^cg8s1*0&$-BeT)d71H#Z(teDc+)x%>x>O3Ol2|ytjl}b z(eOt0btfSs)ug*IbEjwK-RhEeOg{~cJG$9ay_Gj9?uO#Fh?u2p&8+l24QzT5Lg>~UQ z?+=%2SO0fi?SaBm4WF2}riue#0aCM*NArc$nzo~SEYfnra273IQnBn0yqhAVN&7M8 zi6@3ktxXy;V$2BNZAPsv>{}+n`!fm^z7%Oz)h)`n*|)M&1nCK-Tv zNLrjeuq`M#y=9*teqTxT1Ff$XmgSTV-Nzw=34!U#{2ryjZ#OUC5`=xN!tbpV(`1c6Cl!5)r(Otz*sR* zr^Nyg=?cUybxv&QGuyp=xL!0wZVkv;B@!6u{_u*lt}2-_qBl?4h{^s3C8GmTQCRA( zqM6>_<%safbS|-Wai)^s9vXxAX=g|C7S@F4F$Ta8P-Uc3$BJ!8z99hJo*eF5Oit3X z?$x4`nVkntCh|K>TE%5=Ag^6UT?aiT?hd)yNmsa~q5{$*zeJcb$~9ECsWp57zyrrW z^Y1wyEq9_O^h|4oVN_Mx^G7aENEq<~`pzYsF@E=kXrq|ieFalyj9sStx6Ag)%`H5N zs|PscK4Sm>=5tiw%~{;NRzf=kWpJSkswu9tpz71Zpt>vVD>@_ag>ErPnx)_3*)vKpzBCGvbq4TQ?&PdyL$4ENj*d?6k*TrZS zYi{Y;yYkAPAa$_`m7SEl+INCxuQ*pB5pgM8`8)b{AeW4@wslPy(OZn7!}D}+IqY}l zf+`OA3Q^Y(84FAiSMUNB<&91|3z7xm1v=Bc4yc15HnzOmHIerQQjAd^60}q-#yi>L zJ075ku$32w7uehI+CfQ5=R(6ib9x_6AXH?C-lwhN$BnfU_u@N%ol!(GL~0yuClXH;@+-tpvw^NNJpc@d4RQdAT5EbECyYx)`>L$hr7bd(n!kGWgr)TLrfU9z7IJ z(zX@S1P8?JYCcx8q*8bjWE)D+l;ZUMC5tF9U2j%hTFm{5?UTz&JJAR%j(2NF{s%Xi zb4qZPw2K`hw~h9tP}-)t`8JOTRZw-0c>%<@cvUw=KjX|l2P(y$Xg(iSIQybV%OF?e zhD3%}5NDDUG5;&XIb^(5&}6?uBXiLQbg6Emp2$*op0H*@wDBrsW6B-AjeG(oJAg3^ zS%xk@i@)cSMSQx56&GlShNG(2X`jr%o^F_ZHUns&=F*fqPnzAvIYg$Fq`=O@5j_&4 zyHPOG*YCUd($BfWPS|{pIvY!Dh!;;|c;CXQ76pg@0m*Vev=6CXCZBqupzcMOKdC6_ zsTnY=z3sJ4Ke#0dC3m6;?+2ZU9GjQK+F(Q5F(}1h`VsKNnXyP7r(0c@!B;IuD(6l! zIZTbGE(rRzw|Tv!J-7`PjI&^qZkt~#{vDX(=g!8&d0gc?!|@kUbMt|S@R6&qqzZG=A`???dN`%2~|=736_K(F>8%(C%cr7V>=-!J0m z6|mP@(P;vxKwAS$m8M{dgi}6;Ege1SbNoT&c%D^LqGgNDCT^^o^eDrSt)&m4YIAn~ zy8ZL+>)gE~H8AA+Hp*OgbheEv|#$ z?U(+^e35g6#8uEo`WxUziErQs<^BUG-ZaOZKm1sR8$aAOee```RET6ja|k9NBE(7NWoFeI(_YQ-!D9{)-?cLQS2kw@{Y`9DDV z9mK^pNj0I%u0}IiG<|?!5e$Zqk6zleT`AlWmy8#jc+pQ}ZwvfI_JYWV9LP$p#nQ!c z0!}zK7110y9OpdPb&w1&|45~mAZvx0o2BjDXCjQD78B~M&xAUAqS+q&2NEsM_4|mw zEb^~jV*G{#ZT2X`0mS8+?qeWoveUG~br}IVn@ZAsnVtD(ll>(zYoqrWQa*UR!gXy3 zx7WY|^)98pFzH6vU-y12C$tV36d7_x>bWMDM-){e(b?rG1x*Vm*E5)Pf!?alq;Tqx zFZ)SPe6PcZ-ZPvt;i*t`AtNo>gWJnxTq=vLx-E$?yJvuE1kS9J_tx0?73rcdV=YWU z&yjDmIMl&VcOY-i$v^-QHs<>v7|Ep7ld*zeR6QZEUgTiK-Pv@5OFIIpLu+QiqoXRf`sFt{ysyi zK2&W(Op4o+L;+x-do9oLF+MUUg5PC9%NKR3!+prM>M&L+Yq8X+)opT~2VlAppXKVv z*JUpvU!{jBw%mkc^73`GBp^ImDlSzHmi63BjwLNwDEqiKdcIPgho{duep+a{2?duj zCwwP5pOPs$gGt2wxNzN5v-pQoYR5+Y(&giab3XNr`@*8YXuf!Xq!jt1^NWAtyDyMZ zH{=qg-7QYfFb9iqOQhx}>PyF6dZcEjPrMI4Cv+u7SN1C`dC~n@=)C&>tPL#V}l(sTPzJOKWIRML|A!1e`2B#oJeD83SfZ7?E3rLfj_yaer5s9E*a@~? zDm)8wVC|6%IB9T)#QDi(rBfm@s?j3&=Q7Mh&>T`iCrW=}&WC26KJOw}-{tO~439(b z3c0mKaf9$*cKp4KpunL`HQoA3=E55B91Z?DD@9mH@Zd6x=skbno@xZ>MP#CW(uKdP z0z+)cA(NfncE;+|Mr5|2?gsj2R5Bq)JV_k7+U=W9Jo zoi7v%QpS%aT-Ih z+u9+eZ-mhK14#txg+-CwuHny`OU6NOxC|jvI~; zZVaSykOxI}hs9-?%v27od~Oh$ZXm-${_TEAj(@Z|Q;dOB7PqLG?EXbmY1#YD8 z*izYJ_q?w1o!6;gi-+61uChw6!LV~8?^k|9Qy$prZLeYOmk*dRDQd_U;JZn3boiOc z(xu8g`+H)U82?)NBf}gWcu)cX*zAWJJ`??6H+W-%60AC8-ZRtH1uEYoPj^D9HN6BOPah%_J8nuV`LCZ z@)^b_QRoDNJ;L?Hk9eyvSxGeF)>?P~@=IW}BGw|>w!kb-8?}w+QkdPL+VOaK6%Ja7 zWvUY0t^+&SGJgNF*hgYark=gqt-%!*dqf+9ODN?NczBX+V$}oLlmaSeTxy^i1;a;| z8?YYJ@_#@jhWf>{SZ8z5O6IXXGYmr~f{a7{K*HdTG!^$lPm07o)-__RgL-hkL@)HK zQ*QT?-{&1 zbCslji8-5{-pAXgS*INO+&6bMFbKd>r^AXG$a_e2IrLwA1irT_KM{)BY2`H3PH|J- zPmd(Y!Y%<5zx^^~m$i-Y%L-il766tPY+e<%MFIL0G8SBrYppv_E-^Mq_RZ`!FGUU$ zRz&z)OazI2fkJ_O_D#+pe$eC}n+q0WoMW6uD{#HWA5{ULp0CL1fimpt_X@HYg8G-s zxYcnVclmEk1t28ZFt=W^#%ysy0Cg+$%(IcL65 zk&E)DR|tawrxk?+SStgDRdvbUj~@^~Dt+iJwQD z02L|zGE)!wmOo8=p@IZQ>=~U>0?EEo6C>@pVzQSXw)>{XJIZM47hVC+0(>krN32Rq z$`Il$%t5{iVKfWKz|@E2cV)3;LRbSV{*w@qfX=`f8dN;iLu1$QVjh0x95-8RVajL@ zJmLsZw>aDJ7dX`LYfEm$JQ;+_GH2ZDhe8Ui^Rr-T%8&;q@NS%A3b$%}gH}oTOUu=U z6dCMPo`^q>xb#V!Typ9~ODlFHaMLv14^;t}>pv?&DGz@2fg@&P*CC~w85XivQu5gm zG3dx5xr8gPaxB0HhF7i(Qnhv$zk#`5$(Xu?*VOftb)kit6<7n!(@t2-O!>(IU|r?y zxmGkPkUmbAly7JtSU-Hmdhh%r!}6*NGsVJ|u7F zgb^RfJWg~ikm-#s9r**m_~8!H{oM#ynb5p#=DrY{MoH$>r2y4=KYly73GhzSDc3*( zvC6UE>=N2nFTO+tKvV${HnLaUrGv#5u8+wrA2)$h*R}j25ZSWlHG&7*N%g~C^ZPZ> zijaQZ`bxlMOcAYW^;9?McwXEy2;iiRg3?5*p%k8P@&?4gM$}JGhiRs>0T~jF9c7?Y z7|mc*78!xi4MSYR;X5UE9M^a0FRnN5vN??xB?*ymuXqFl)AxK&E>*?n2WYW_-`|9JeY;@^lXL%$e>4}C;Hc@K7=+BdFPP53gI z?Zvox^?x!;o6NN0Tf_v3P+~{j9+!GDe3ZyTG z>TP%piUaTmp1@smJYT#EkBEC@?(%=(p&8zVDK1+mo)@5LLoYTz9HlAUln2Wos)qt~z)0zq14JLiz|HQ6Nbjd#~SNiDIV?{Y%cd zVqzB{IGP4yZdsC`+;Hx|>l_;4>H8zcRB&fB#fzMICc6URXme_`Lg~f;N1V;0OXjy_ zDzSyFVf+GZU^%2ECre8hpOFo&sU`4hiR7OwZ4MUJ1lnJpEj|f@(!S{qj?d%zRR+Te zU4+aG^PMi!v?B_z?7Ku5}%+D4YD)-WTi}Zz*bwDOPC52$%-{|(SC;F1NIe4 zmgh+y`E|ea;e5imfF9iL7X@l}44Lq=NRmRz28aY|Y*Nn0CgkiRvWJBy+7w_wD4(|X zC&9P(hE!uOZK^}Z4R7j0Ba_P}l#EKn%u2r}QcIWwmD~?CrJ)#uvZp;u)FooSz6_GW?|5-O^MmjbZ(rgm z9Eu=HqdQTSD{Zh7;Vva1y~ELiZJq&PYP=lG?{t-{t4Ryh@k?cNt%5{?_!EG;>i0lu zrnqNT@Ww(ThC~Ta+PnEMd{+FyjzZj-88)Lm4sx3~;|D#ijo*T9ZXcNfvgP6uQVIb= zQKqmauO)cd?8pqAcsUd8bGE zqk80Ts$1nix-@i-H;D`_m?fz-!g^f!#(l(loc$M;Nn)SY0K*@S z(j{foCK|GVM6w+*du(d66e&f0)bO&JiKBtNkTYw_t0xox`Px|GueDN5v%8zRcCt)VF?h zCKNKcVY2fM0{xV&#O!Wt!Vw8j6wg^4fH@HHMbp_S?~_dM&|i)S$loC>SYRkDii~ao zY%LjoaAa}2IhH_doSO;gp>~~?>HY)qtExKH1+A}N{AZin3^bZ5F&ZZ4%B0ofmd6+N z4&!yo%xn+_#W=r+$~HG1=#ZIz@873=Kh3-G*{I&>{M||4H1}-cr!sU)PE-x8jUV8S z?(GA(j*d*&4RBD}^&7m^&%;w4oeSYWi7u+*j7l;0t9x6c(B_G=Pc`J9FP9yGY_9wY=wge9 zhM@&m@(rBQxb?gbkLVC8NAI6y_WF9IIPUbZCZOgQcaGphpT)0HL}5~rx7~HAZ&{Ao zhKM66Xpv7>Y(OInM~*G*H!iPhu@W zSnHY1s+{IfsWO1zqigEJ_!_*%Qpv=Rm@V@+#}Jz}x%>y_OJ&c5M|8;5ZM}9;IVU)J z6HroNzE#AwkXPUCG7CtDsj_O}#pb{bL@Y-@j?}^4?9v~Uc6q|-eSS~zmbO5K!k}|m z|EK((DkpWCR*4tmv*wgBE=ZV#q9@`>;OBmOgMYX=%WU0&L=TDJ2H_)ZJ1BhMVTnc% zYu)zB-71x?Xut@3$S+9pHld89NCj5Go*71OiSDk8rDiga74MTul<3o;SaM$W5^X;2 zfFN}?s6xxCQxCblBuOJ};_A0z+Kb)dj8KG39DPh_C&7`FFd~PXXg(d3@4p zC>OTH{g#ywN_;=T$U&cZ=+SI}xnXZ%-;|uHU7{Zzbzn5$+t>@1oLm3NDG)vlS3Pk|EE7!y#vvF`u8#J#tE&R-9T{|7WMdOqOh+?zH&mxcY7 z#7ko&7gk#V^j%BJqJ>u&+0ki%WP5(iBg~Ra=Kh=Nh3y53`FjgIN>>x9fVXf3PiXXg z-&(`^2to+~v6_TW*?{*jV__X?20Rw31c@7}kF-vBNf28RJVNF$PKsitd8+^Zw1H<> zFhK|WDgN?g$7}p)4RCq~I@}WG=7I#PgX!kx)zQ2%CaqU<5OO&@rF$PT7SA>=I@l ze)=Yp!mtYpk0Y5RQ5|_K%x$G!NBmL%ye_gnv|D3NhwdeK8D+VV76ESsV&|AsNi!@L zu{PCFyG4yaC|b1b2fW1HStqd!_Fv;8eJ!9G`zD7*~o z|5s}*(8pO>1yqFmf;VD#w&a*0uP+e@Bo5JSaWB#-M2T#C?tr|+6|KS6?!Kn@c+PyhX04_r@t|mjEk~c-0~oasR&Z{U{+Sf z>c4u&EIToBm@U8eqNjv2VZT}wpujb`OUQJZBVe;C-WqrsS+@L!M264&;)NGc7y0@s z$^oDlP|L)UK`+5*4wVmyxw9i6c8+(BOt>AG31ko&%0eY7c~EsX@gF7jUEGMXGtLR+ z)e){?+$JbQ8dN_nLP-*ED3L@7Am%Dj9)f-o_u$}_Ewg#@l{Y<>nhGskF*M_JtHhsZ zV}q>-5#UZ~t%*6yoLxl>+B)N8C2K2#afWVFb1(?Hpen)t6O;P-(zaYK?jZXPeQCON^iF3 zYf+?$zzd|BZa0ca2nG+pWg~DN=311kv%+Xz;eD*e#z2MxF=u#t+!tLR74bUDoC*R%zacgKtarJ|F}T2NsQ}>%Lf1*q>(~2#&+I*G$BvJrv@ALu-*u}8*DM`EpDIXJm%fxHNguh__D3rFnd!j|=SbUg zvRt7JCbETqdI_A@TzJn6E~4Ofqs<_+BW5ivm+(Wb2dU_Vr;Isdkmyi=K{bAx_!r>d z*W@s|5?*Zx399fX6TZ@9WQPIr%7{MTD2dN6#Phow!*w9h+>`7$-$2{ZO3V|N#^d%a zx-Ad=E4B-XMEkldAHS!tStvYzv)AIbv2N&r##5K}%;;nkqNBROTUY0|?9B_F#WR2~ zH;*Mm1^}%eP81ewtR(Yq2m9$Dd?~xU)PCu&)2Z7928V&VSKA|jDZ~mGt-9r(n+UiA zYfB0Q)pTZGOMK_#Wl*I+fpcj)nKm+-d=1@uh2 zg|ZGo%^C1OB_>!|HmZ%2!GL%Ar}R0$*?i9(0gr1pt}P-I`%0bz-xvALTJ8#vqW-op7x7#V1L|g>XYt%3y{iqqTdy0Jtx% z8EG7d7(+^>TZ@e}eu)$SA*+Loe`e=_#WUzJ*A+wZp0{?Pi5!I4SV`jwa(#-Gfrr|g z4vGAHoeP9;feKGsO=0_#tYm&$R3|{Q5u{J$p2JY@E1?#nSwn<)b)>wqj+?4}&uEcq zL8NM5){iGCywdoSvtw7@63G__c?Hl<;}wgYq;utw$sv0j#`KUv%xdyCxjAzy(OBic z$Gx3i!`i=j4au)u6GQ0bi-{tasgh$zMFKE}CKya5_Yu*~z?Ri8{Y6#P(EU!7wrcGk zu1kW1Gm@A_&vCDm3U4B9*DH_#k^a;aVHsJ6Dmyw`+;UQP-abeX8Eq4k&u7!hXCO8$=%@MlQOXrBB)kBP z&Xa;kZ1CEHc@h><`A24BWRVwqU6I@0M@yvydCGA}=N$tKjU>=(rGT1kqhLE$J)_m( z@t3NGf+c}uV%`su#pQ-Y@$_``y#!?@-gV4C<%mk%OXlZ6TB>a>lWJpM9iH5j3Xcxq zrFoq=+&!NEf+jO6PRpRTSz;B8B#;v6Wyge4mB&(Dz6v4K2Z4@s1qHGE=UO9A{u@GE zSZDn3DI{+Wjy8B8;l;&_rpH4;S zfiC)){r0O%WqxW;bZxH#B|w!P3ym{^Vw`lMBD{Wk{I1Ts3 zrl2*(?8ofO=L(SjZA$GjFx=0S~d+A}lL?Vn?67>p5G3}*0}9rsE+C@5G> zaL{2D$p8s3z*7@&@oasp;BOb@~tc`$%TNIfx$xpr}%_HmuSWzs{|){W^6DCn=2Q5QUXtV)+kXs>Yswg*;m++ z)+39swcBon!t6+Au8#I~=NX?}GNlp>8iCPo8OA%EOQBSORE+pJk_jD&FUP0gGKkq0 zT_kf> zq$L%Fi2)*Rmc*;P*U+}n7%plr-lmtRJBv*rnlA=MXdT*w+;1!fad>oN#)IV#5TDGh zkAhgb;#4C4h-5#N^Xq;gTT{GS01$+Av>{=c2ulE3chC+G$j7381Y4zIk*6hLQ8rNH zPN{;@I#=_{YmEdoS=g6yG1b0szz63WHiqH(%N8*HcRELqL)HAlc`Ts9swMrg3vjU5 z`YEyK7RV4BkBiP0*Q#Mvo02r3$#%gD9mYyKx0K?~Y|<+=DXn3PHK|bE>QU@QfqjfW zd5MT0MEMWZbHhOMQf0=hA<+?P0B8MT!AKNL)5B(Af(zq^q8KAzV#rQW)82twnEZ5s zR!KiM@?K$6DDR;rd`51^QdHBRvkxrNZn0&{=u7nbKytEu1)l;L3Xm3N(Cho*-8iqC z&dD#US*a9=%-pcwb*$$4gli?MqF^_Rz3^p$cr*t^GVTC$7s-k6I{?oi+_GdQi4}N9 z$5J6n_4f$O)!Q?tN~NGh2KH+#xE5atRB^=tsrBt+ulZZ5~09 zg8g(dn8ouTodGiw$c))o2_L6$BA3CKBHmIkt39fPY zc)=XODNxzBIz8wzXb_7t1$dB7FjNvYd1)hZ%_@Uyupkf>(+o%dqT!)bueqzUC)1NT z&x_M#pHj~0If}ovv|LqMvO~H|kF1>KuuE|AcG)QB$6z24lG(Ff0<5O0v1k&A3MSch zvcYS@m*;of;i*uG0Sbp%OUIO{Nm#Yyn#FCXWeNJ=X^)~miZjy!16*`WCV88$fN$B#tq`H51I5< zI|XSm8@UWfog{x(Q;eVGlewy3qp>d(erC%^tGek1c3DGJG2Lt4faI8-T zI>Gu+r{m*$=W*}=kFWlOk6*s&U*szrsvs#K5?M4m2Bn^EJZ&6&-Q`T;871nEymUME zEB4=t(RP(E3k9rkcFx%O!1vGrL$frCO<9t4#XJtjHd&%f)M8Sjs^uoDp`@i~)HgwKI`-2BIijT2apgf#SBe7l$*@d{;|?7G;-r z4gA}ITAAu_9^RldbC|wfRJ;JX>lL9-F;ID%c17@Gwm<8j&lqw4Jgs;sl^@$bD;&6rKrkeN+tz9fU1UNtiWWzu}1QbtS&1ie!zDJEA8 zkAT2$njF)7N$_0+Z!#ewFoKAHz;|XWJH9}LNT_gbiBf=^1ceZR?^^5g(4F63ARLb8 zJm=Y;z4zK{uQl)#kqKyBx_=L91%xB}EXP++*|5K{_!nIE2wY@SHX3=iR&m%m9e;_= z~EU+DdMuhLq%Tnof;O135L$>kxaL;c>I|VH(XdLPnEpDlLTt_1Qj=lzP zTf5FN(kaH2o8@h7WR7B*pz}VLS8xBZJKc$3Kb;Fmz`>0*uBJPV&Y^sP`zQX`Sw?nz z!P`}%_waD0Cm!#g^^U|0Fh0zWM*h^5M$998H;6)JAvddtQGvmgk(o$)5;7AwFpDbG zR~_h@KX-M3hESIps7QerT;?q$3ah_D6(z0&WRHHB!hjI!Ctl{@gu_cgWl*hUcAc;; ziHM^IFjkECa-36(;A3lu+P9O}BDdnz{@y!YY1Ev#JB zP}4yK?w?^HG8Q;j~Re+cpvLX}h9pLyq5 z5^c6WK6P_mq@(9iundyh!%#W4m;M6bHX-n%@1i2^uSKw&uJf@y-yD3lXU~k5@Yjwm zOmN2klDB^6%KlWFA(JY1-NR8m9!#Y}D3y+<x65OHB4Hzg({p|L3?KQ4gjbo;%7@&y9^e%RhTyCa1xmT}<3##{a>IhxZ?Q zfjhlFJX>HmlGw|N<_oYt12$#&=mFd>f?-IlPzCNV?w3;sO8(T^l=AW(NkU=5#b_#c z%Y34BeU2|XWEm?WfgHg1xFv}OBBlOi?4BCY_OJtm*N5uyK2+KDO5NYC>NLCWykKGq z@xoTV@ulb{I~TmH>bf-L8NNV}$;DfT@<&?7qQ7l`WyniQzZ2sDVn!FC-IV49f;8i=awMAyTXEhoRUd&NF0^Awb? zO>ghHb-7~BK8~4fU*or6#|>bGabV&fvJ--MKeW$?lyy7?it3+|-YM0Io|!q?-}wGg z^%93=yAOX~2Y$2QBCJ(103j?M;4B1*31qmYJNLvztuES6`%p^f%Rvfkd7>C70nt*n zm2Z012+E}U%<070ISibYOHSG^6(QF z`am)QCK$sT+smFr=duqgdhqxF;l!tEMR6E@336Y>HR(O~5-r*d(E`mv3l}$VteN`F zT?8_fa$T|gmRaQZljp$~oWi*-5Cb&v6rivtWhVc*+b}3{viAbKmDEJUMj!o_irk!e z;ciV6ZVm>&j6AxFmya!#lR!F*KXkVk0ZhZwqURNDc5tsT+aY(3<7y)0yzc;T;5TXa zIFUkFdjh9}4(r>%=cn)<@o3B{q4_+1^@K*av9pcVw)M!C_Z*k`7a%lNE=3@r1Ddo? zG-Vb9g`>~Qwyh*AXV3qS5&<}MCFiXuFFocJzpA_ydSQ&Tlfc#1pvt?(B(UwLRU#JW zjWjyo*rWGMD?pMKFA3gx45E%+VP06nF>B>ye?PyB40L=7Mz$GVP|9k?jub`9WprJb z6d9q~EeGL8q;Q%j>v>H3nVIZZsqgR$V{!t)m{|<>`fz9x@u6ET176_$>=Uz^0D?l_+0|P{!SORyBoI>Nw(S0G|STa33aNV-glcxiOLJtorgvU5m zSEGJ0kjV(U@k@EQg5328t_@az@naXP5C&8h;s+QC`ey_cB&jPJdAGZ49hVUsjLGPN zN!pLUBH{hmZi5Kx101#~y z*6zjaBnW-YmyVtqOp}|2TPh{JPOpzuY&K@(S@|JMb7 z=1vl$21v-)m>ps~Bd9Mi+8{M_xfx=Q=s^NQC-GR?>$oN$@EB`)90vGXu8BRY1KT91 zH$rP}XdD$Ef0Dfe_R3t*rjFq)-}&Dl{qUF5a*o-4!jEVcY7{Bjr$)i#lqsv@i?v7r zLJMQ6qKoB>(>4g`mg+(o;UIgC)Y*OS7-alzWA(Wq{Sb>numNU67grhH&}*9<)jwEF zg0i4mhHH&9Tzs>QSqC@>7K0+-Ap>MMFD35ccfCS~7qBR=rEDCeqf^AC0Hnr_ia?Q= zpZhS7Tq~##B67zfm8ons%v5#laXGbfwf6*zqO=Zbw^W*bi|9GfWMp<@o>Ne}G=O`H zXrvFXO@j&yTxy(wafKpA#HBsy*G~2X&e;-IaO4K_xN^y8GK`9&MI}B zctDo1{&-_+eg1DHnFq>H<9-=mmbo)V%&qI7L4i>Qtgj|=Z|aqtrGV%7h^mV7I%C{?<}|7~Do>oOv0dF`86)D; zQipTYJaF1nWsk7GRffYx8%ewK+g;q<7@2VVlb{`fEV|8xV6TfBI&{8M48|Q2GYWBm z@5MaM3F?bVdj9+@n2%`(@q^CNUfUIsz zcS&#%qQ3NH2af^wP7osipM3rpN3SS<22B&$ZAL_i*I&)72Li{6v=UdFmOKFXX%ET{ z2W<>nd8n*`3VTZozLmVyk}rNH%pyRu?7>wc%X?}he++?@(T4?NzH;#*U+jqq>VpbXI{a5ard3r4#jQ=k@{ z`yZ+W!)o}?ka%0x{Qh@9qd=1Dw=nD>srsR8(4u_#N62cTSinM!*1z~KuAjuYfoHvZ zsHZvW6{e~|(+=~)U24NVi1tE2#=y91{CDk(oweJtIF zbJu97+vG2UNsQ(6L}>_n2Er_2$TNqp4^}RbX$*(xZL~})1kxlu=g#Bf zp79efrlR({V}j{X=^Ei31RsV&^-L{ zhl4R17t$9JFaWGQB%SB|(Gml=qqpZ%*G z%kTQqW-%W?d=#Gotu?k}S-l~!8BpnR!A4bUC>0jR!cD`ek5c!(6VG=C2`ZeK8#r z2xD-e>)JtMNe??5czK|(T*I$8Kl=w5sW9Rd_^16<{8M9Q)p?JXmJc+=Z^0q}JfVDy z(WAkhevlDc4Tp{bZg)ct!yC!I%p0vPIzLPPaXCH^CgVHw3eS^SVqbgsk~#1a<9!)% z#MnvQub%q{G{ojUh!xlXuEFRk=}Dws>0=18u_sY(s_r*D3DS+*S48&4E_Q^;PKS+< z;Yuu+BMBwX5i?$II4H=RRURtA1wjJ#py+Xq8hyBC9`klf^=L+mhs#NmNE&NO{6c@a zz&`-3liM5O6_|S{Ao~l{gDCPS#ft)0uFb)^O^(56SZzR?wK+ z=S6-vXOGlBvXXB??Uu;)029`l;jUPHSXlyhI<1k_YMCpBD2%7x{W7tZl=w|6#eIBy z$etsdfH>1mky6W+O)ec+bouu`V(_c0Y2s;TuCyi~R12dIp?v0p`B}KO(D)5Z8>eJ7 z+oh-v8!w2Did!2sua&YAAy6l%xh>5Iv{1XGIx!>SLyH7~=W1SOygHI^_GjNzNPP)< z8_&?)i-HZxNP40a=fv(|q3`Bxtd8PW zIR5V2IX)oBUDz31wFPz{JB#}GF$m9TpBFw+d(vJ%#y`72qdT3Cn-)q{Y(rgDFr4Pl zKOql>{1paoxS_*m0%M)Z}m+u#g|0)qpex05MX! zf^AcT5sfAN2=9x+5lCvTt6lC17?;7z5&8P4@+{|E7 zk;pk-em1Fn04?YGVWsC?-Ww1XAyzTl66su&DJ-K?(n&3+W2%@G_hL+n5fzG6Bf{K< zk}sfRsZ@BGnJc$w)NUm2Wu~aZ&Po7})3+Hg-&a*16UjG{PSzw$Rz; zzD7sFXXqlsIV}#u$|-*}A@l_FdiY(hJ^^N;5Q*0`>Sj^1rcnzMFo8jqxKfQinnU5{ z4{8KLT)urjx$mE`mAig>BH)-BJ}SNT1t%*$Ppn?{iOYbsz!KkEKUVYc9mSPLFCW5h zarg}#Ijf1^tWU|pjO94GD4boReCBi(=h8r=ZA)-g=2f;(7s4Nnt`?404ZEpW=YHOC z&aRNWhhov41JGTxXDO)^C`Tjnc^5*_{FV;sV4GHx+n@6RLpmk&O@l-zwv6z5kt_o}LABy5F@Y`{Z03ATEV_9T#kPckktYU%v+|8!8Au>zdgb zD{cFD*GCmPllgn+Kl7-yjI;vtamQ`tt@X(O#6k8sn`k)>U|Uapl7oQ7&l-i5EukaN zn5DRBr=qEngk$6V5=9#4x=1I0sumlaK1+xh()q?@dRQ3(xCn{+gUm$7I+PM@6Rg}vyGj?r|V zw+s2~SI=Y>5u3QwD8)Y*KAhr}n+8?>V98A^yzb7=Z_D1yhAh}{K$H29G4N=0)$d_t zSXtcIhhY@co&|GF7k)VI4%v4Aw#$Os34V(Mvi;u{8O0SWbmfJp{@c`6n)@Xw;Ly!( z2>!x(^)>EMZSwQt9u*FflUZ@3`wg(qZ~kB^1J12T6D{mum~E(%<*R!4N&((5NYI0O zN>&1alpLIu08j)N0z@i!-1sMramb6lB{av^#9&814cUO1uQLwNp;<(1?u!JZu8!C`aJihX1+o{9!=#w)HuMvK7y_P_MUMT+*GPO5 z2u(`l1xRNd!s$EhbW052JJg++;#r%8#qs(Hj!p5E44-mvg2yZvqVb_cXqq&6-*h$f zOOoIwsfY4f(})E}zSX5-SkTG{?~A%jSyFQGj>@{M@F_M#*_h8WLnY>yntaKF1#+>S zOcMZjcMH%$;b^YcE+UkcLHL7E^vKcAEbe}wfx*_lmOjAE zNXY`JRK+3W5Dg}AgCkr7noB;BKQOWjFPl{L)N#Im!b5~IQsqPnRCMO{2-hY(zSV3> zm2)Ws6=0Tl6Uo5RRI^7iTJYnMO)LZBAo14@Un#tP<1&R@^b8pCni0EhLYhKq{ImAv zVZ1AtQ~I$0ca5a^hE&s6xt0G|z~?Q#cb%OB+ZPY}J)QM}O&aM$(y8Q(QKiPATdSFxyU*`U&AJq`pjv-=x%_mIa zLwS;);4lv7NLk8h#-it2lmAXQ!5$8B(NwfZXlt;*?lW{N-fwl$n`hS$!UJ^+O?ddf zlz1J*ebd+0)G(L9#S7(N>yjNTZ{SEH(BAqU@V&OHeC~YHqBr9M+Kjq`S2<~Ulej=aU%CG z0Dy-pV1G5A(E<>N{vOCmiP7-$z{~P3)-PH?sW9`ha!XycYwEnqW^uP=M_1N6mMvSa ziPQuCfWoROz|Ymac6I)@De&AvNmc?$>VQbH_@7)?In?Kd{&bBO7A3JV%)z<;g-kqS z!lg4GvZHFkay5^Zn>dd&V?5V7y_r8~cz+t-k#qoVe++LZQeyA=h4I~iowgtd=mPgG z-sI^3h4*8|2R<}JA>_IAEtP+ZK<#54+Pn_>UkW+byJLBSVo!w+5X?Q(_X(r5Os)61 zy{l6(5T7aF)xHiBp>Qrup?BPxzEVtzeKU%gi7aH?G?mv`u619qWuE^(BsgU%Xzbl1 z8aos$W;Y%}b4N-|xn~2b0wqmVNQNf7{k*H!p)Q=BlhVuRqbIt39C;H_*V|A3)tb~- zsp=b?XV%gF_@hM5)bFrMwjWwAtH|UYXX3Xu<(wY~#Tz(B|O`q}602 z7F%>Q($E=TFA70eplH>QLBs9U(R1VD`_~f?9g;D)iKqWeA`-{(NvBR_OOyJ2migOH z4_~Bid_!YhI!tyAEVR`1vw_%F~hluQz3WE6K3%=G6vlFltW8Qw;bBb)S#hW-IO!P5KQ;{kTG&j)()jw1kULwLr0_DZK=Qp8BY zd4kUi#)IMwF)N4yT4bY|E9LKP=N*Ic!`PGhHZRv10=$}6Wk`^v>w?j&^@x_>9g)mz za9MfJA$gA`;FJcEbEIYb&QyY}o&of{#`9W!Z8nM`Nxh3cZisZ(Qd0<0CxZ4;dyV%m zwHd~R@DqoCh*=!=^?CN14%?D2^HPn*1ijy^o=MpW68)@CB$$LP{m)P6%8mF+g!V9* zj9Q4gp_b$P8>10#5=g~1p5d`a$N|J$8G|K$eTzgt95S2Q7+!8ZznRKor1q_)1t$Tq zgBQ8)d}ffbWsM z5-%!eW$nH3XbPwn{p{-!Dp$SF?_lkas$Gt>jK_CTr z3w;cP+{u(!y#qTZKL+p~OnJKx!4IA&FXd|D;a2_u%0u3rRf?>Qb(6m;G3gFk9R4^N zM82Rj^J<9oo%xVopU}p}4Bsb3umB|ka;K)YTPt_@b9tmI4Tq`%4+A z!I0yB0SGPXEDzNWQe4?jML>Nd%T~n;&k76{=WLaenVI9`B8p=P#TZ#`2!_U zdZsPTkfsE%`A;6vQTek=1bVrJn5W`^rXF&1)gw5*>waT~b@xZ!^;`NZkKF3Y)LQ%o zu(sD5lalpa(d4ZwY$QEwfc9fc%tH@yC9%1$okiZxrH2HgKXXSbQ(5!gKBwBx>5Ar1 zqb6jK%x+|hNxT0b722$oHw8=UM3*WsmzM`_ zRL|=3F8PK%aix$CzHs;^jR_zvyry7r!8P8z(XM9sWm6m4AugxJIt{1Rl9wHbT!q@? zt;!6lQ(&k`aTjvB@No7T{S>BK-SkTbBWOd2<9OpFO?UxRFD~%ZK^~)Y9N!##o?*ha z!W)m>6mlMJ>q&gZ$hFb@W6Im|4K}J`$Q0pMzPgsgU zW}_D@etEcISkw77bJ6f+a4Bak*|Ml>{MINkhv6aEQe5#8(8^1p^Bz~gw61SkB~N$zVZXrEEy@6I@?@KG?jJQebg&? z8Uvg~C*Xdko88z?_57%54~i6^fAb=y>F6il zL+`v-DNun0Nmje2KOOkgz6DuypZ`xm#(7S;xZstrzjh=rcsU_M*|W> zizs+|W(P5H=p}Lnh5xcD7%YHIse7GQRK!(IF{DRiRh*0{fOtr!2-pL;??gmT2p&!} zu!;|d(Kdd?jC8K~3ZD)-qKpNg!;@cDL$#?*D^%L!MH|)U1N;*w+u}^cQ1zK;v{`uR zP~|p#QNee%?8cs*D)OMH$JzqRdn^LLQVTT8T-UHpHu>-T8B`(_Qqzygd7g%I7OB?= zsN2KedZi1f%lIAZOFx~`n5EZ_MS8wMNV}k8!b{H7*r%TzDkP`PDhS=7^q5|f1vQi` zP@az%;2miMox-`BfysgooC+9H@Z1>fG&}FHg;}82{R@iM731F+Af@+QQLPydPyL~z@pkd zF+7jJQINh1PC%pfZ2egoq$!EY;cS02h5eQHr2OXg5uRF+cxt?aDE!85XQZ9PLz#*J z!$as7>(`=e!Y8*gxDOvFqGT<|^w7z41A7Y%5@IEOg3Aw=1g)*zndlRf4arjwE<_5`2z zjG`0vSVR;|iPWe{9@@A%xfH`BZ60ZgBjOu8z2a{^f@J~aukw+E1?{t6Y@GN<4X}Lt zrtmH=SoS}>%vh@e6J7YY1E5r!MITNJefV-yZfTIL{@ukTaxYokpEZrZLyuwgMHd-NV| zB513RCN=$ST%dUHVsQ@_2mZ{KB{aFbJ9feQNU=e3<>TQ46T!u_bi;E^{9~vxQ}qy( za{#aRPy7!L{GVXbrky(7vjA+rbd9LcLfH;tdNr&7dJtw5WtmEuJ{^-Fv1pCvMD>At zwCx4(-Y@$!>pBv*5#VIc;|?vd z`oP1UH8vD+;7Tq#WT{SI($yGjsU;a>QHYJjWO->(6z#emT zD94#7ytY_(_3pbxpev!1GG1*&3>jMFlyIWk(;v_gruXYaL?4{6V3&yLJ2j}n%Qx~$ z1!jC)TCrD~Q5XYrJcW_MGm;Z)+98Qi7>6^&>he9NLZm?X$X3IAg#AXq7lRXW)0GkH zCJ)cBC=w6+1IN44(3ap*j-L>AECM}8ly}-K*J~-!;`j&?{O6lRvo>%0^;liuL-#d%9cW)da0R zH*r1iKX8YQ)nS7nB3;@l@eEW#P@S-*l_-WYzZm6sfj3@D=cF4XAAyaRPhx2O~i@l&F4X^=x6U5dp^f%qrNIhx>SVUq6~ld@umMC z-aUA0m53CpLxOC0uhD6=I7*ic3=#2}9ml3pnRhXmewKFPY|C*6(lJ5^(@rt})V+Jx z%>|e^A^t&TY?qawo+*l}zMvN--JHa)(Pi-XK!G{+qB^Ydk&G_;APvn(@>_6@>^`j^ zMa@MUMTK4PlhDS|7H4DKs;?|c#(9|cf(y9mab_?#VSk%K-ZN2diK%n3y-G{*ZSOp9 z$sJJ|-7SO%l(k0%wt*E+4dP4I*gna&onp^+ojC|%iCUaO zqGH3u?#%Mz=eHDUW21JM?58|9_f^_4{i{v>!|%z!-eRn zdz6=S%rCT@+Ta$#)6sX@2K3kKdIe^tc;BhHN-;T<$`xg6`N2geQQ+t$N>4{15^7NmFCgX2H8bOTL_4F6Wg5c}gu+ z_>}-_QSrdu?^-BG(FxB4#0^dcW3$l)C&B&!A_&u6O})jRHw|A_eD7^uzsjhg5UHSsdDM4p zgY~wbuE1BEmBwH`v?+SG77g3JI+Po4+Sk!5rlDt@_9>t48yra&l?p#gPRw8cOt^q$q6qu zJ3_q`bIJ6R$6Xj?##XRdGo4V-pG$Q0K07MCFp%6w`oNx$T@?0YjIp&Pp*rmnm6Dj{ zYm9XB#6YljR7q1(6Q77&=K-de80z>P&}#-d*tb!pXTsZN)z%DB0cQNXTxQP$CJAf! z@}Rt@s&1-DRmOAxVzyQu!mw2r$E^A`l^@qKhw`OGHUe4T{NiRK+IRS}F&9XBIGO=? zYw8B5oCyz0vgi(9b*mqmd%}yKwsw|K-K<(@n=0)in~VH5lHWvb@53lN76~o|Hp4&w zx;wVc^ioE{;{LLbT~1dYj%}Nt#5!w*lFm;}*3QlMsIaXe?Ongd7g(7UT^B0?C0WH@ zS#W^)md09Q#xWa66S!AhXC<*+N&=+cm1t7STW2&?irIh6Y#gEuUNK7=5~*=sY`5*f z$#egUQQ~G?qQb8M!s9iAp9@zvybJz`XTr3iTRVq#&rD0}vc<#uM4Zgwy)m5T^>Pgy z-B&9dS?!a+&SVAmvy3(~oKn_Q6>S78vnU*oaJuG{>U$QlaUbJmJRk?mb@TvZbL z#e=Y-!Vdi&kKu4zbd6bz4^Au{9ck_h!mL@ZbSC@R_UR>O5lpZF$o_!^0HxK7 znh8@x9Ld$!me1c^Y6OxHS_Fq;>M>`cuy1Wyp8cv!J0_#18LLi2mS@2J;6;kK& zqtqkX-F3fYbkNb?FoL2QC!t$H(eg#kw54w;f{3jngTx))mErZGUV;s+5Z?UgWn!PB z*f5LkU@EI=%8JylvFoIk>ABP8-tk%6HyfQ=|AQ^NTZbpoN4Z@BO&m zLbF$>f~&PKE9gpMW&-h|VU422UYYw}Xl&S86*3Asy1S~6 zPWoQz5o^3oH+0o$C8K)X)4a^3VL8}>WFwSQTyK|G)CMXZ>KOp^^BLe{j1QMaZEH}V z3`@T(iDs&$OJm~Z<_e?3R)<>h#{!+yp*5mS9NqM|$P$hVH>|ztiTj21#Dz7UPkMwrn-t$wT0DRL2LPBljpm#k8Q)#@6~R0R#IevtJI}Nd=ed@Mmc>%ehc6Vj2R#Q))?uorDH6oYY|ytU$}y)W$%- z!us!FL5=BYwuG0H0a`VSkVl|QP&e(M;7UWdjxMg%WWwN>t0D_&Lr9fmhB4T{?zEr< zE1_5hhBRDz8hsJyh}Q3oe<=BSmT%M=pm(TDekfsB;jVKNAI5kf@`9kwc#VHA#vRq{ z7xS{22_R&k40hxRDHXL&?d3)!5X@Va3D|$uDxphp;Yfewji%*sWE^-;`s{`=V!*BZ z&48Tl(q~X0w4@yx5cUwgGN${^-Lg&(yzAMaSM*j-o#y#Ri;X-)N3742&gpt=&PN?t zja}QJ35oU-8+Xh`+xAu_{YiuUCt~vCzTtbJ`_0mYPufezb<}(v;>Nqc)4BUoP-H+b zXf@9?!T+NbGcOz7z~qYr7~C!&%4QuF()?y5Y339XN5X**hQ%*{D4F3>IJFq1`~z%? z1GjT1i%_epWM|M|XM#-LOP#ewHL}`W38p=d?N4@y6{xY&KT>rLOp!%Zg>1)&SnR+o7 zkY`4FmzxOVABDtD4+M2F;LuHCXHylLn z3f5f$xTCYRn{hL49m*|rKx-Ond!F(BWv_Y!m#EBIp+~gEb;Tq*;9|ncf5Gf}2@=?2 z|5}yWi|B}i*&EA3P{#>qqk=H45+ERSoCs!VE!+ENCcWd}-$k5xf|bVrd-Nx<%TO6XD@UAn`T7d;(X>-o{8nX0T>JGA3yma1>N$w4nLx)*tZ}Qk}A~Y`T^-G~H_&ZQu z{2SZ*ap56u;+{ni*|Xgk1a+U>^%dGEq||lf*+iX?!?7isT=@YoV=()n9|5d+mY+WH z{NL5z4pDP#bR+7vEqLeggQY+QtSOI!WiV@hf!314NV*kwXUSE@5aW_kh z(?e^NM=MBs0!%nSih&F-37_$;1+Iz-dJ~9)2k+UMxZtKvoQ?@`X={+d;}P z4}S5Q$9{jDSTrt@MIy)PX+Y9Bb2BhO_r_fRvNaY>Ex`rEk;D#KSx*s2K&!eSVh0EXL=3Z zk|HYnjs-EnF0|%x2!eGST2-Sy$hWF+7I4UwG~6MjcX!yAQj>@a-;ddYG2Y1S1AqbF zYJ>-sFFjyd=|M2+YPo#fFn0~1K1@mjmdHrA$s;=IGAR$pK9@QhwR|`+2NFhz&{ZIf zTgG6hqt`c*-oT23@d0^g{uJJbZTxy+;3BiHhe3|yu6XbT<1|-BYUd;t6Sog!uTeew zj%3SGH;vN32;R!YVyWX!6`S0j)gcM`25>M)W%uiTquR+N0?77lt}@}kX_-j|!wa@7 z0-_ef+8xw_-OzW`OxDb6Q{jrun^G2Rm}0-90a4T??@v&g0n{@QQawAwcxLn}UABC1 zLc8m28+JCzOrs^HB504QLra>a<#P$4Lw@)=KZZ`*xmTc+zzcuF%|*0UDea+c#Y&r@ z;T^GoWQ0EPtA@%U{6_K^s4;1(!NCAlb}5&gzf)eK*vK2}AF`EI2Trf~&GUu`oL%$K zEM}XHFO3UfONEi(tcCm$H*(4oE#J!W#8Q9m{zZI-|9dun3GfaJ+xpYUf$EwE1~|jP zCGu^qe0nPm35eo9#g?Zdc>)Yxc#k=2Ct=&$a2J5j7@v*Up@+rO`=LZ%im*gKS=Ci9*sD=URTKkWgY77}qHn0%iEYjD^`RI~h2=(#*?W7c2oT zt%^gCF%)J+6+DF|)V9JOnwiNruJBRbU&%WrF@$2jkbld*)rW)rjF!~Y)%@(B#mru@mf+rZIlP1(v#gC=P_EcH!Hs-s|Mz$z^?a+r zPc^9D{KbX-Y16#%3$VPdVHfOErWYG=y`jk9sNrzC9u?^ad4|hMlrPv132%*vdg4x6 z`??J@e#d^H+1^FLpbp8Hb9hH_Qy$&>$a&XFl+DhaA+CU7iyvf~86;k=n^68B?Qpdc zT@nY{7&HNGAF|lpl9F97O=ftST1!rr)f#Bz=lgsxQJ>~}j)C}Fd+W0q%~OQu%D&dH!JxqX-={ zP%A{t;0C`{dDM79wH!G1My=ut1Kb;%kEzEJ&4Cn_mRXZJujii|Mw>+0Hu&CJZnP)D zxDAL!wmZ?Zra=bi!s=(9Nln_spgA~7FVu7E<|My^{?-_4*vH|kC1~LOTEtp%0wK}F zOCOA*iLbceq@m+=8az)+Puj|szh}sTi8+e2xer2G0!2wXy>>T*q*$J6>1Blnl%^*3 zKOx}3{@Yy7w5WOfT&Ap!>9%}zAw|MK3IYXBYOq81)z_5BP;q*WE%L>`Gpews#xl60 zI{=ywI*}J;D*DDZ8hx;&6+H%_gQA%bpytbg0803AKsXVma?e{qHVksOOh_SToYiB= z(nos=4XsyYy%F=#LT6Q_(Rn$$!_#Pi zj~^Q)2#qAKJ;}qO#oFu7hKnXj!H-g3qF5|CwSWY8l&G2F>c!xMqM0?^Ya)^~jdy0y zc$oy?xJ-e~6@J4#(0jC3L(m>>BPSL{tyGi+g1JjnyOJ>9CTbX2(Sz)`a}#Si=!t@@ zhdOqapA}ZvPGbaGwxIhw{H8L;QRn&>m{sy`-YBY&vBr>q-5%>941r5nngW(ANe(c& z4d6`a8dM`49*amU564_5KTy;NNV?%c7+wy3LTjzizujHbY_uf`%tK(5wdcWDfDmAm z4#;^L?;6^^SziQ-`~wyNWl zfEt>K(=*4#0w9>N;XeG#0`dEerOR?J)t>@vOq zQ{N6XCV z8djV8TbC_Z&r(C0x9^C_Fbx%vs7|K)HWywc_&`SIQX?f%_Qe(c1eS^5GmJq8>d*uH z;&+bD5yJtNQNO;2b9G}#*j%>DV@51EI09Z9_>eM#vIvf!np1mZ?LWHx0!I9L1jOy< zHi-MMFD(z#6fzjFPupTEeiArHQ8R_YK$7kAfs8j=53Kbw@@}PFy@h7llig1~i@1jV zf*MGE>D|_t7JT@cR{?JjxB8;)vZC27&eyQ=rQ0mS3uoytw`K87Dub$rXt%yI;T9Oq z;ybv%4a){IQF6Zsca7Ia3_pMHZ}cykU|Nm;i^s0?Lb^qyoTs1q?RTh+a&%rpt3Mvg%wj)-hNT#hz7SFTDK-@3y_B zCMK65y0pQ-xGmE-a zRFfd%gMC!T#mtO)MGp2g?lBONGQVo11{lIbo=+H>J2Y16?WubGc3V>u_?gt60@ldF z@BX~@{${xcx<{cd2*8bX?*wE^z1e$G0@alZh>To)8JU^KV6l zxgUiJ%AnCX#)B7b+{AZgTFda4YZO_DuP_A}Z3h?oH+Q3@X$fihvHY>aC>SL_CrsjJ z_yVg4+TsYM;3X8PZDO6`V?Ou$NM|(Zb^DxEZ$MaqpB^f07vAUrUrH3B=*=CGB!Ta9 z^#cV-W2#BXu3LE<)wJWEuS|fjr_>}?80`oSB2y$)1c%0+gPG)%H%(k-nFB;}JCGmp z)6CHa2Sng2LbK9OwP+jKDrx+WP4KX+;0tNAi(d8H|BVcEO9Ue@ zp44U%Q@f!Zg_P|EXk#kv5nt&KyoxHcMgXFXzsN*54o8{c6YLYI{?FDUdZ){;05MGW z^;x}-SqM5; zl4eI1QSK=YWKeZH)0oe1hPfYJZ+vWk%CLo; zH_&cy=UWcX@f}?y86)_oh}BRYe(6EkYRnD^R)kFe%c{p@Fp^u*Ny9*ynd|<%w|}FF zux~N}LR9ASwwz*>=T0j$025qU!m(8^4T!q(Q4*6vDPC>{PSd6*or>jcI5DSE@RPj_ zd;SD1@ED)C0$!}Bkxd{%06JgYsXC;j5=S92`5Nxx0G9Y>K!h49$oI=~25aMs%M!qk zz~{>)9?Y*1233cQpoIO^$$XtM`h+yUgfWgQTs^rp%q{UGCG7; zL%nNR`!($#HP7t*H=MI0us<}A0X#dLk_cw|Af3dehgmKfj23Sf^sJR zcdn1s@@noRXB-E$1&zycudjXmwn($ZloFP6jegejX?fruy0i_Cq!>zsPb;gvzwCbs z{Dr{@Fw)5utN^qQ^g(>~6G?rZb&-~yif;Y2FxClggWLvm|1QGRks1Y8C1+i10T~oAF^ixW7EQlIj}Vo zq5Wu;V9(ZV;>rH1l2kb*DIqWX8vT7Z(k?SZ)Q*cmS$(l`iz@xeMTTkLihZeT^y)x^ zLk267Vz5oPLV2MFKl7mDjZW2eQ1a!^txkTTucb6bp-NmEV`vQ=!~7X!c9dlxbtr{$ zDoP;+Afa$OQ1d48M}$h;yN^}~Phg04S34(Jyd>%|6CpjEyjFfSDC`8r0ki$&kkfH6 zh?LUf<3f6-3}5~%79yK_#%-)~>~89ao4Z=P0g^}pc9F>dmQ+<;e{(Zt4kS6EpZD=? zp*^+zxP|*!FWaH0!f_p+*ef~ME&*5Ib(w47bF#t33FrOn(e;PClwmfzN%cCwkJ?&=-+U7Lslo0o@5NTGUB7-Ve4fY^68z{YQe z6=WRzo8d%M9l~Kg1Fa@WAAtQD3F^76tug8rdnf))$agEcfoXV6lr8;=pJ5>h-`HgQ z3L`Jx#4X$)1Vy@k0B24A%Owge`@PXpz+wSGy`bmgN0n(cu%G?$Y6(^1hvOUH2ZCqV zk|FT9RbW*Sk^sTtd*f7CMIkVv&YLS@byZM)0OK2AUld=+QVvXHA2j_uG;jp%b|kVD zs=jmwmfK(|9&hX)z#l+Qf$ulstahOX*x{4Hw}Uu0Y>nZ5+IB+4BP&n);k8ESp)>h( zBuBO`_T8YKGFjcP&R95&%eT;fIgbX8*dCNDgpk{QVx5Oq6gs+H0%5(Lx}6^eOg4_x zp=oG?kew;|fJL5ZW}y~TAWzDpot7>TTai}-aaS5 z$ed#xW&W*^&~QM6RRnfC@wHKPm=mUaR8+A@L)ok%5FC)Up004dvW$dpFB9_+K-VuI zcnYcn3bd4!d?qNJ268?NJ)ca%LGiO(rB#IGP7Wj&@plrTL=vknIw|}i^dH?>f0zxl z5RB?sn4NBEUE|ZriT&7w2GO;89Wsu#y2kr=5CtSgZ24(q71_c*O5GGPb^IqR=vq4e z8`Djkm5Y! zK3CLMfa%2e!8`2k8cutSkLt{-BxOq-6j|O7QuYp<&RG9K*npAc9vIslqU;^s69WDL zZJ%0HXj^ct#n|&Gibx3g^P6(d+;8cDpKoEGlI@(j^@Ys+io@=_1R=&Aup&31C zyAGX-qaYBrW;`)A`o|TD`7mTWMipSUVx(zSE>PmDZgahYXu{}afZ4}*&!K2?8LoE9 zXSIsDf^l`=L45stUuX%%UU>z5dfDE9n%k#SI3FrACK_$w7k@7{1ZXki&+j@vkYR;g zaYfr2aw5<<2fDB6XrJ2$lc$ywepxT~)bvoF@{Pt|qmkUOUAzX0FjIsa_rR?m)6H5E zI=dHm=OsP7U*O)&+aYX>Zf@QoWgCe6`*ltSQ37MG=xE~aa0UoNC-*x4=$coD$3V$H zfRO4ow$$+n3&Lr`8z*06kX$%*qxy%^+&u1LK(vVBc~Jd(&G=VCLmupK-euVDj;l!G zo$Wm4v0D3vAlr*Fj<5-R79oPBplA$+8k{9rqP=z7p66YRT5!iM^i;#r{l;?a8v=nr zzD)RhexNe2KcsS$&e+P#Wxh!bGBAVt+m~YY1B<|&{H<8zZYE@=3AH9+7bI1WEjrJ>SexOy^dvCPCbm^oK%zd=q2oe+ekxY~zkQC?gA?10Lu47?TtggYo2) zCVG<#7a-I&T@L++@>lsQoxRrYWF8?P&fOqo9Q=M{XoUuyaq)A@kh;onT|3_k29Z#| zPa7|?Ymd>1M7SO-VOaoOE=Fh8?mYBDU_s?YkNRG)-accIc6bxT0nrRBJ2dte%u=uv zq;dqR_+?{<@cP38jX{Sk1?DGXTm*A2v&J_inSp}x!MHvdxqOsRAdfAj3B0zwu{XsO z@A#nvq!GBSCULc8N|`O;*b6KpH8XC=V#dR_dYuNR#vu#;OF3PFRBs_ zl{`H#TfS|xMKG+Kl02CInQiXwwAVqIKuBzxU()vng`(Jbru8!I`$TI zCx5pEQGDSWm;oj;bP8Nf@}_F}?Q@1MyyBBsAKnI8FVUk*Ln8R*Op+E34(j*>>?4XU z4re3MUV(GCHg$Ua3K*k14t*yPZrs$(R~cbgA{cVOVr|FB!v-la!YF}d{Z!*`apACC zW657m3vMK32(^sm?>UBA%EMsAtnM8H^)RS4@OWxSx`|Ry-a!sCd|EOdtK+)tKP#UN zhMO_(3N+b1twQGBfk_-=iViE&e0m2_kRuuOJu@1(Bk&sFca$}_qAqf$Q>EYp0IWt0 z5ce3bEApe6T4DNy@IV7`>;zwtV~qWGGz9<>lv~kBARKG=N^0Bpsy2i^?A&HFH~dxE zQ#j6I8B0EQB+6p#7NSxM3%38nR`*{NXrK)Mil!>IW%9coSZ=Mejfi~sjmd}C8;+S`}A*ueq+ z*_aBuH~@9^;F8Mdp7}Jexl-7kKWzC>x3FMgbJasYb81!8W3DFisr#=5PYxi5AYUuU%7 zrZmm<&sb?9&X&eAt!|wcoN?G40sTPmU>(~N{8@20SB`*>tGc6h3nN!7VVx8i24%^^ zNXBRe%;0v}wL(cPhkOKInY}#rpoR5l3L|Ib@dooe`<&r!ju*!9yrC3B9ODRb2Gn?f zV#ApuqJ_h?!)Pd%Qv|G^z5i@VYeB`F_BD97XP-3M5o2x4Qmm0{`0J_Mq17NriKm{j zlLBtJjz#HRw6S6dVTQe9G=Q)abG+4H{wVqBIJgjY_USzXnoR z690_ShEqDX{1E1ozGrH}>PRjkslIfWK$`G86pPr$a>p-`a{Mnu=X}j(3D)&hg@P0Y zWiB=X4wwt*H?XO?dzPcx?tcos%yGVihFm-(Sj94UuK~R`mQG$qXAHJeqrMkP3qjXg zD<8uB0nMfzK-~uNnhhWMSQ(v=vvLlVGw;AjNQG4_#T6gn$Nso$>P2Ow&GXmlZ$Lo3 zYkPHyAs#ti=xJUz3$^6Y_C%C_CV)t507S>(SeC6T_}3IV0(K~ejo!Y>mR7E;wo$Te z9TT^wkp_lqB@6=8(FDl#39SmAdjdDd>K)I_uPtH$)G~yAmhg6a>kRHLd~%tX!SLpT z8tr|GzgUWSC?1wbQwsO1p*MIYD=+03`-~|rZCxq<4i0&-8A1JQ5EOhcW0HsUG3^l5 zyC0YtJo;SETrohXYw79I$0#4O;@f9<8UNeQxK2arj*p|SyG>%04QBD2zRyN(d|J-c z^MSxa$0iJ1N`(k)(Rc0lHG|e}pm<0htIgIt;F;5X8qEaUxk|MHV75jOf3ab3>H(j zZ816HXpM}(zi$@(07!@c)^5nMTfsA8ki_L>-B0%91<8KIlF!r*I$BqVTUBC;i{u~b z4Wh$_kVo?>4K=`U)^d$PpnGc2XwP&#}0&)dvk1~wJy=EB7t)UYklUjZ> zmgT2*)sd~uPAyhY7-=_`XF|JOs6p75;6Mb803%lPg_sbmY>L}3x%062jW;(xFFzn> z)CB&3dOb@sW2(lWO<(cW89p?1|5MO`K$Vf72W?cIQm}`tnRdPwXcv@#!i`E z@EA%o_?4BYx)Nc7X4y?66X5LI$|o!$&G%IX-PWszu?LAlB|OnblkujEzZ!i-i_`%2 zSFp%=IAU4iVK=l9i%qy(hx3k%yqm>8UYqyo1i^CKBr%RjPo-)HBIDpNl>YTJ-8-=x z#5H%Oq)SJc}T6%tR1zvqiiJs3vFYNR#b8 zAZms&gvg9gD1GRXFAW%Q&`lFHmUV6F;vGuQSOUV=u?hLjBDGrUeprW{`{46t`WoVR z-gr3Sy@(m)ZLx~LOegx#1fgZ=y>V$!DtW*tk}N;M25F2#hbRy`fdC(>AnFC~TYS$Y zW5V82bNhmRcxuivD4!`&QC=DGX|yQ`xoN-lqAgP#B4`YG-%*FdEhSK-YqQ;mDB$?` z9smoQB}g)HIt9YN0Vtp}hZ7X{zmfSeJuVr&2k{mF`)!{Cw1Lpa4dt=}H<^v@xA|Ol zisI^)b)*N=mPnkM)^T#9#HNpd;_aU>^>r&|$1~k!%AXH;piy15;05 zhhqlV7CS&N!OyrcH9GAS9tF@J3XiHn+^QbY4~)4I8lB2}!Py%q@0ES1$}HX1PY%pm z_Md8NV5;$3H5sRh^%ZK*!met=9WA^DF8O=rfz>3f^sC+2n&IY@1qk z)cDCV0}|TgjDy=+9A?-nrN+M#N4@igmMV!Jq{wi|Y*(s5yZ8pv~PrlgGo-(~<`)lVq zjE9EnXd!#hqoQ&iIEodKbcVox{Ghfk(Eh7RtWqwc?D+%#V2VKiJC%bx)+=Gf`jjCE z1c>i50_0{(18|q%amzG=yEk;##RyBjX)|GUs+C*r#6}z3!Q~dyj9^1k)?N4pwm3>j z3xWXQfqn&-U86fyuD|Z6ER^Cu0$?1pSExz3cpBv-p)$G^S{90^ZGKL}fPi)igoAi! z6-km3Ny!8qsz{TYa(p?z<~0xjsq;fPlJeQG)WICpvEU&9K#Z{UJ#m*~_M<hfFjN_X%HfrHP`J6vEgwPc?o@zyeZ(T1h(;2kD^ZWd2dHGV z7p0EwypG7KmKfw7j7kwZe684i@>t*SxsVD*yMJT;|G|og+c|hm<3CGleBSjj)s7fL z!)6U3$~_E)A;=)wKIe=H4S4gNLM1Q^fvnxE*~MJY(CB~>2Mr12JI@hkJC7c_c{|A< zfKCQ|>NiFVV#%=#>fB*?(2=EF)q47T&U*RMz`Q<$*KHBZO!W=4HTXzlg&Y#sNU2t~ zUDe?H;CQ=Ez6v6Pe}h%MzX}Y$xyp19VtO2tO`7F5aA8+cPY(AR@+!FBg6QE1 zG&#_`EqKhESvGzl_^zT%oQcZiuZsmyP@C;rW64CMAJUJyY6^B^ywnstyQS(iaKExL zi};u517yWP6ZD9kOaBKaH&#za+0wwO!0ne8DYgV0P*?c0LO7zY{f*oa;f%GCuq^B# zBLhzN9RPJ$9Ir0JS81lzo~SJ77k~dk@bs@=o^Uu$hRWEt*K@(IYo14^+wY-1cDbmJ zXlj*D>`6)d5!M1L1F8xcs5fC!XQY-fGw9m4Vq;x4c~Qk?G0OcwG@jAA%VLk1d9wR! zE#OhG!a_YrsnuFTT8f@&9M|gY$Nas?vZo|Pb9O;>?^XQ$vF6p}b|R({dmo#3~UhyO+-yiw(ICV@OUnyKY4ui z&$pa*4I|Fr7#maX)2?QK9ny^fI%tehD5_Fn?DVv;D3uWP7{fQ=Wp+B&d|rO7yi~|L zw4;j0d@mwW*Cc?Aiorn{C1!Ou*=ci23k_#x38Kdu`rjKZpW7TfRblg0_E zF4|`(PAjFkwDEWXFx#!_(IDi|PvYWJ>SB(s1}8xIWmqGRD9bUM+qq_QcZv>EiyrJ-j%qGPt}Pe(v$bp#R?Qa!@8OZk+~2EmtdocKxY6p2NDv#8(_W zx;Q>LBpO5fVbb@`9#-0IV+Fgv_2|9MQlkI^fYc zsw*L42&Mq$Ro?)l*LNcO;pQzf<{etr#^Kt(=t{<2qlT-r zQ1MtVm}sici$;1&YDY6*XQkiDL<4--ca`1{MEjn?k@Z`ALf*w`+E(n`vJb0Rik}n| zhUnYf{poxD|m6CT% zgmnN1-0?tCZ8C_*f|r4sztZw8s0HlH3f{it5HHD6ISO%n)ktnSu1?xPL=4mdGk1;N zL!>>4jEre{k4$rF&R}xoP+C%`1%+ZrfU^*N7Mv!{UX&I%Fdch|iw~%V&3tM? z>EY?(W2TCk1a6v0*Bx^kbeKXpa>;_&!jsA-lfE~y67#;nWVt-q6TES_rbsqoQos+P zXD24~JgD0S8@v*-TuW09q@lnn{<6Lo1sS($SZi`TH$V9*C5!-2%u<; zXa#}yIY4Rz(E8>vAYv?z=2b??Nt}yQbDDvj@X2^hw3(_V#U5WzLG<#GZ8rh3MOF9P{O@Bd}g9!eeIzWl9GE*}U^5{@ToSQ;#$ou^}j z*J)FnxAo6mOw|^!cuvbV^peZ36gydxU9meeaV>IqMWdOI@cMtKW|y{acAVZlWS6(M z(3S>go=fpqLr^*4 z0x|cUYZ@g+;pjKBI+$hRgwnfNe}$OD>y#*=J1)9X2+kX|qk_QQ6=mwjAh%yWY6xC8 zj?Bf>qQHi{LeLr4jILQ!?Lg7-kn>5rTEmtC9@GrVV? zUJk7!NG4};Rt|o`_QdaWPVL*Cv){R&*dnA2-s9c7X{0jN=y{|4;~(M8Q9BnSUx_eU zwluKJEG5Vztr^Kz$e8<#%9PNTp z@eTA(^#khf;hycB>a&}V#9jB{y`-B-yeXa8F!cf4$ema$gHYBT$1tvfp_4;yCWrqA z#ydhZY^eC~o@}D{rB3q?V_LeqMc5&_m#L^16mQZ{0@jMjvvzcioGV(?VeoIqa-L6> zS7KSuSp81|4`;L-iIUG6q^pH&T#fY4?pEg(Ahr^~#a?86(^c_r`x?_~v%qWZy!$p& zp}e2HJ8l<*U4?eUty7&gJ*RvxIs^~?_Lu(7<>(W2+wBqzwQUVsaVe|R{Ys-RB1sKz0U_?G=(YzY1_)JkJ}&Lzggs$ z;T6qzPjNtCMZyyEj$76nS(%M51K)gNDV;cGMC4ySS~&#xmDm7aR)7eU-d>+FtfTQc z6qPB#!x=b&NlEr9%3{_~?4_A2E+|OJqYzM{S$%J+&fR>xt zuAcu=qFW6Y%pBbsuh3@qU|X4EEo76N`y)n zi_egGK*M-$RY8Qy>|H??P{$K!Dg^Fv<{4SPh?9?L!OPoyO%y;v@X!KZ5}CICHux3j zj7GMtkPZdNWy5*(raGL+7|#gAfWGe=0X)S6cTD=%KZ35dm+j=7?l|v-jkV6rsT=Cz z^0NcPX$rkC#>-?EpD@S{e8p?x+?F;L5p#WLx?DAQDT;!+Zs-2xH})mt_H}++sYAf4 zbNab_Z2w}7;uG_5NHnac&y$$a7Xka>suqbu*>*Xw;J|+(+@`mLd1jI>i$O6|UU&kH02z!i|nZm>G&=iRV;Nw=!UWIO#j#rrO!6CHR)9A6%le7(VrA!$_*9@eH{9w*yO*g=pJj zG&h_Zwb(^}g(r#+X9mlU;MI^3MHw{iP}c zymZ3e%y@6~Qf%2u;V1?bO75UgIB@!>U%q{TA2~ty*1G>k)wc&!U1!_hJH5uK$z~`^ zOC6s?sQnbph}YzsN=bahOYazpipdBH^1d~BeZTMtB6yRLc;`V>KwzdUJ&Ir<5GF7s ziUM*H1VUcFwf6qfz4tE^&Vh5j=es|9@3q%ni^pNoTd(^*$wMk72bz=k4vfN{^3i)K zys-A~H-5C!dOz=|0P&^!rS>!F?@2v)1^)$)Zu=E>WQe{S101AY!vb(+wFpRHpGFG6 zm$M5cd6`Fuc$DwJMU)Stj1K|8?)EhYExOE@uyHDW#q_`(HWP$hQ5sNb5gQGpl@pGQ z>v$}iLfV8rYYrIdRzQa+l%vZoG5)wv$KXQcrw^S}+V!8~4HeqL)!JjsxBN^@-Q3&j z0fQ1$dmz?+{8s{eNpE%D?s5AV$Ha53J|aGG=KGC<@8S{W2T>-PlA-}jIWCqQed!st zh9b<`SY!^t;j!1Zgzp-D@g%m#yW?HCC4x+?`Kk8V}8ESL_yc%q83&+m50ayM9 zA9nj*A!NsJKPh+OfT1OuGtTWpEMDTAuboV|&(J^(yDByS$e^QSz8|*yf&>`)LCsdh zZBYjy5JG&k1m|wDko?+LWspG2V?y~y@aL4QM1mh8d+H0PBVPQU%J9gX|9aCZnBDO0 zSCo(;6m|@mbw>*27_<~5>&(v<9c3_y1?Iz`oy}8|9rl4#;ZhZsA%M4*pP8YA$wAHq z^(&O1dC_&g3Q6qgc-|fOBE->P6ezEdu285L`~$bkEEGI6fhCIk_C6Z@Et46DT86~S z+D{l)Xo$0&*LTUm%w`Jc^}sBhZ&lr1cul&a4hLhWqj;4T`1UnVt?d_f`LI&m*G#3~ z>*9Rqb$Yg;e6cer+v2+I;phtg5uIIQ%JcV%*9OF(z^M>maE;@Rwvnvjs`yq7pyS-1b&NL#Z_QKY% zHjn|)|2_U?;=vFm%^ZhF;uR*pCRmCS!SJ>6OU~lGRcLV?BXy7_FYqJ*EASvb(^8W| zC`MEs^oh7E?)#Ps1~S=-}ooKV$#%dMgE^_B^m`V zx;^qc?A0UJ7;??~mw_#dzZyE#NlGMlOCqX4sPXj1nT|_r#jE;q^bj|QSujJ}&I+xl zj+5VRe1*sd6IH)#g`d?p-p?f+P9BFo7ODl7iAB~Nn**-^xu5xJ~|5D>AvmoMkxx8EjHKK+Y?w{ej>mKj0dSkwbDXgIG zH7e?70m3D|0d-S9{$j#oa;sMjS(E_*3IrMqF?b|m#yiqq8l!@8=ujYeEcKVVjM7*5 zKG)oZxUn#8bO@iYWs^&1>#!jLP*`<4p7;|&w5aIh0m(~$0C?BkI(|E*L@4{=?ZW(J zI@>XwH%0&Y%!wR0;6auGHCfliI~Ji=fwoX|j%^bQ^vh0H@l&89J=sWF$Vl{k2q_4}ErtnjL}lQXQht-eebh_STh2eV&v08K zY|SM@6np_$(8l?Wf8;D*8vqLAuopIf{6y`xN+7v%-QR&7ue4E+tiaa+tK};?R!LS{ z)W_AE$`bIc$pQ2soC-@#TZQ2j09iKPl(WyEGwz=ZOD01WFt$Mqo6sW3U*x}*DE8y%fn2Z`}Qh8sxPmE_v)NXipW1DMJ1_Q0nj*yqY7 zBa;~$Sh>p&SZ`})V2MjL>YIY{HImIRlJiRzipF`=IXa5VQ!R>`vh$4lN7upRz!Sy9 zgUSvc@blb+u~3I3tp2khn~&X-irp#XY1yJDO866R#nLW+V6{@{4Ly;NzCjM8t3rm#Jkn2wdi1Q-c)+5# z8R9-(tWLgO;~uDOsG-&Ju!%7HbrDLU+^>`r@T9|nJyyYDzYv6?Uest|P7%sqmdF1A z!-oJ{A?Fg`LwsGze(ZGAyCtwLO!Wy7j}737IAsT*74f}-{TBR#My0~gW>FXy94r(; z1mennu~Ta)Exx^GXel0t0xIWcdz6Mpp1sj8U7Lp>&=H_$o zi<5l!;v&+Yz+wDd7>nPk@>d?Rk%C=L7lIi_F!nF`0N-$zg;Gxx-K&zlE)M5~Wnu64 zuMOqDq;;gpozl!TRg%*;dxeVgyrRuQSdpcJVJ zh~=W_yb1jM&W$h!@E+~vloBE(p3!$j)$T`WD;<}e30Vn!R0I1cb2|WCY-s^2Yy69` zZGqt$Pnot(G*-7@3g(a|cX7X8F2%?2@U#_9Pf3r}W@CkK08~1l;>{R}$U`x(%e|Fy z<==Q4G*gZOqp^3P(Sj34SENuysPw=6(IXrJWAIW51OxG)+Al4(k7dMrKCR{G7&uDd?iUX#!<;(VPc{?Bi2;->-vRI&7GpXt-;URDFluKaJPmSS*6oay znjPjP7I!DN8Zj80P`gGbpfH4!O`dvcurcoTtUc@yfkNMV^aA!(3K>CBHGuD^+>3Kb zR4s;{MQ;L<5*8X4iyueb-)xhQRZE>$zxgJEsThDvNk;xIT*%Jei0y**bMKAi9&q#I z_qvhcCRS2`C%Rklrv8jBIaL15xZ${Tq{#BGvIEsy-6&$xM z>y5Y2YF1xxWMtt9Jwa|J#V?} zDdAunmC!>h7?C4#AkvsZ=3PT#b`O~5DsA1EnFpEoMy-mX5mGO;{Bl!FT{~TFr1E8# zMGmU}5Otb(zlpaV6#W3Mu#IZ4`i?)JlX|A8? z(=1-5Hg~~GoPCa}CbsQL!#zOAm6Lh7GY6Gt<_l}#I=8PVk!8%SjJ=P2_s1Esn6`Jt zUSPBDz_ydpI>>GnRcT3RpXu%hgclbYLO0R9HZeuZpzuOe75j=TjNaodw^U^ens4@*<#kYKAlRBhGcfOXkm9SBmbO~2D3o`jhcXSKs( zMy;G86cHS&@!(D1CUp1E7z$yi=tnMDe{7N(IR6cxlXw3w39lZuvbP|C&TVnTHQ- z!dNPi$RA##Qfr}s$GaOe7Ks{<7@193hlwH#O4YdHe#sYEPh`0Qg+Pfo|3-;0*n68E zC=a6g9tae`$!1oR?)C##SGNGl9rzIdEn*Wn3>MgK`+YJ9_oMsO!6Ip)%=;a_JyE77 zcyE+;{Nn>^Gi*s}$P{8k@(24M+>B3V?#^Gp%j=CYzW{qP>8!rR4xh_gbB2)HWRNKK zX!i(as!%9et8>q+De$LKnc2lM4F(y!tx$csW1LGuM$bKT+kiWsfy(GQ$sHX2;sDG? zX#)em^uQY)@AB1nD`HNJPWp(!8S(qtg1+WFja)FPj4-F1|1Lm2V)O=Y0whYzzUefu z6Rt{KrTaDzbAg))F$$Xw^(lNS+O+Z4@yZgd@^+6&fKCZA=oT!cZJINy7;~vNJ#!Nx+J(&I=GILlpaf6^?g{TFoL3uYjGPZ)V3L zm^~xHw$!cmu>2g$5ZEYE6fc?=yEPo+^_$=){J$jcU%jLZQZ8s`mQPu-0g;rbDWQEK z=i7pu!{U@+pR>uw0+%R7B^V?gyzmjxbJdsww&9Ip7!NTEm)7Fm*K&EhhkYcD?^qTV zI)#A)27KhxsZazOMYckiXbN|N&g%Jmp2cQH6QdBXUUM8M?^95(;Uf<_KpfqBbnWX4 zB&!E~o$1!$#Mc7c%=7@GC@KT@#gBR*%&fZnT9`X9oZt^A0a-~^KQQCJswa%fcIEt7 zkS3;0>iEi56fvPlqwI)0iumoQy9B#Y)^^Pw$RHQzpsnos27XhFPgmx8+uEg{suXE- znDPAFx??0^zK*)m0WQ4fpHH5+LcrbAi|4I3Gp4BiZ}*qzv2*1XJPYM*o}6`IMei7u zMwj^+zE01c2YQ>Ln;y=vOnS@e4dUOB#Jnms-TpAFEwXd`&kQN2BWGX*-aj0e zN(kWjTocfubP%QwtRKy=@EOA2VQ|t4{0pq!kV&r+V_SwBS`l7{WqQ>1vy#oR7A4BO}u+TV^$%QcGQgrv4X`PKygcS2sBi&F2=0W9rZboM}F9W zCU*(Ct_@CYbl^p!c6MaiTSqBmG25hYAa3+$8JJxO@sRXlu8COU3$-X+0Vb6K%dxtQ z=p1(>SR&0~agU?DgxvBAjKRXv#j4i^h(ywv)hLb%S~^!4^2==Yuw(?60chw-QXY&o z76uau5{Yb!kYH8=v_QjA3jtbVy6dhtVeAN04?#%lY7o9IeqPCL3N?*ai1Z!0N1VFRQ7I7T_YUK`>u{{P zTzyZz)%D{d;Zk634EvU5AMtyI;g~$mCFsf2<{NTy&Wrse6iP82@1}<8?@Fbihom&w zj|VD_oPwM1H-8~x!rbwry}_!R@|ZpV3sApD}QA^dOe#=O6$OPQr)45~vCvWk1_O-0?;8f|8K??3=2pGZ_*GV5Py}TFb6XnElwZ5LyLT=>01pmv8f--9Z*l zqI=Mk; zw7JVGxfa~NvMyUz(Bw&fbqu4a#nEi&UxtTA&o#0VdJRSnhGiaE%Bux2Jw}!jM~jea5t*>&<2kfX}dJj_?}PRYmmx!rjwt9)nJ!}nY1!Q(~Q}U#M?%})@?$p zBxMDhy1=q=x@k6)i5Up{HuAQN4*v2G5j}iDGr9!P6Jjj3FNxU=i*cK)GGQ0L8ka8s z1Xk|_z`zOh!b5qU!L|tb1<6cQW)6T50id7?E2u>2ItPmM!lVxm=Qqz{NrHaH$!)g; ziOmGzQ6K^Cv18G9>=>SMYedy;EpZc$_1d}p2JKf>?TagN14AeCmERoTFobIeOUKyb zE11AX#L6V4;`kl{6%mOE$N=C{0QeZ!$bt?$4NqH8I0VK>8$FwuOW6}5Wiog1BYVH+ zfz5-G#nLWx*iw&cymxq>ftuUl;t2Lo!-m>@JeMinhac6)-+@j90|`gt8?x#GHm}%r zG~O$xsDhR2#Gud(4B-49Q!1$%%Zo1Kq`(_h80fy}x?cMz5bSLaBNt6Ww!{-~4WlQ~ zP87xc5?`v#5-n6+3Addfi1NzyhNdmL8ZuOX9=*P%uRj-(`i`VmRkTbYXq1H$c?nQm z@_EKuFm5aPU!<6=bwc|oQWY@4a(W;L`B{MY`2b2JCza6pT?Rw>0=e{yxm?)FU;`r$ z^wuf7U2_wBBe4*Ng{soLmr2nJU8p8(kV|FF2j*McxWs9w0%y2irES!&om8R7Wn$BR ziY(is2$D#e(bDt2GE6YsL_h)k6ObEvm?GZb8cMZHzzvq&$s%3>`-$10Z~)jKe-D@N z%0J}iJH1L(skz)F%lGJn$Ie1z?5JD4g2Kx%FF2v5x+(q;V8(BRjJ=!#&EG?sg%Ne8 z0vMD)>N4zw<&4`I6YY4n)Ge?omGo$NCh<|+dAv~&PuTuoZ(PU^VAK%C!^bd8_JHBc zUAu-Es%~-6HD)kKtw9+Rt%vJp@M>ltLd&l}_0qexj$7giz!pix>Z~6lsND54r9HG? zl*$C82e}ivgJDS_*wn#~2lL|QG_K=+p&XQ|gbDNsb6fcw!0?%GM|Yfo(K2?MM3$@- zMq;l#tYSq2p%`{hu2X_S-@qN|8?Zd&k<@iJCr)~Y=ZCLld04wV1b>1Lp@^4Mjm}!Z z9h)?fF@$~jS7n8M25?H;OVcK~-GsopZm85HmPMwe90$olRA5;MxlXy?RR286P>_26 z{H8>q-#QxM;ezyJf~t!84bj^S00MTgTLkuN55UTsG&$XODh|>Y9#=?wI=J;Gs5{h3SB04FS};BZoouQZU!D|I}2-?yo$zLse}w! zuoNqOjB*fw{9FY2^J2=z!LrPGA+>>pOhxpf7I-)K3uK-gr$D%6w;%lEq(dqtx`&3* zx{bI*Qi4|qPbiFD7xR?E&ei#+jkHx4GPgq2Xw~KhpUoj-ck#La3^Uz}m7@gqs$AQo z8U{}jmkR=U@o>Rd7doGNbm6|_w}6DnHYE~Q2sHgHrWpxFV=2Jaduw};pbGC z=)DWk0BLpmv+M8^==;5Nia&uJp{;7Dw}hva^B1C=OUuEJ03HeVzxB-GOz)+9HWg9~ zW?$ULEZ(3{uK!1tI>?ur`v@6mLX1ZU+9#a#Qxq|W$%H_2$$N)l0K_L^%qA8dma-|0 z{yL!T+@B_+6E;t{ez$sjHR!Osr|TN9bm2Y+%f3oagX1yR!Ao$8N+q9T>dP@8Zq%NB zJE^52ASwwpb@hp?L9%5#GuTV!* zY6ahY=Xj;943{R=miZbWIxqrFF&>RXEm}TrW!?e;1*A>=t1)o(7MMCq01flai4E0{ zGvMs6dkNpDJG&-_y%DBiwM$mO)8H!UE zkTYk!&$clfdF(`+ycsOGtqo(t>;cc>HTAdC@f=1W$>=;c0ZSi>oTJ!h(^U+es9-_* z;0Hhy0A~@kUvQa}T0}dftN&1dTmbWa@lLJ_qqG`0V!rUzA)@qc!#6=pXoyi_QNO?1%$p*&(dm-e@E191=pA=2r!f$UCmu7Py1>0qr>gJh@uZ zSYabL6I6oW4_byRWVYrvKbF>PM2hjv+_K~y+!hcMxVS(NbZW0ic?@ku-bS#}S`;Iv zc>aK3R(oFl8|l^t-KMhC!5%J_|K11CGO*+yXfA%m09!NRMNbhJAVNYlIzeYq7L6Nf zDcLbHCHPALB*K3eYuFt+4$SHYD+(Jyr2t`NB;pZyZ1sDVzERL%vz{fH9xl`J2_Mvz z4eI$}uzq|X|C(r_0~8leO>~@}^`Kd0U9&V7XgO|+p`u^a9`M+TDh_c`TfKJM*q5R@ z1wwnt!Lx$v6UUCL$71yON{m z_{kLjbraC2aTn7yj?$1?WD*bS43U3!&E+7vg<+~I1|58Z z0{Zn-=YbsZ20qbAORhjOO6ohzP1Vx5pLPpH4|-rD&?=4}Hf*r7$Um|&V?*OKdL^8f zOB%0{uF$K7H`vwJL&Lw3QRQCB!VpK)hvEzinbCed^JZDPckNk>r{g|=_Pzc?73Rf=jRMKx{;GxT2A^?UZLAA-LIl?;Q$dxUlDErY+v(wE8y%wBcdm%@*WA%meh@A@jQbS!J|A?BL{v%Ky0@?Xx&;Vr`>X)Vxq z#?>t>XXxWQn|;o59D`~%)KQcsZZDW4y62ReMLm@~NWRK0hp}Clii^d%(6HTTuDCqd zLqwUvKc~(gLht_SOkLrHxH|eF)SHGO2P7*c^?wCm4tdWIr)!0)oYEi6FT*}hXnT6Ii~_UneV2Ra+j3=PsnaOuQ!tJ6!Pz5#GeVuxlqtYx#Bu zLAK+mppNd&vx49svIg~*kTv|iWC7!<(27%8l;SiYNC+xDo8$<+N>Bb)qG7I5Z z{a&M#eL*$}aU@{`WegQY1+wrEpZF@!0mal1?^O*u0{r!nm+c=T4qoL43uf7WzXcU~ z^&}vGsk1AUZo-|q*!Q54l|ptH#b5D=l@nAN!e#={C)@>}o<`B}qIo{V*je^9p*_9w z4`IV}`Ju{9j5Gn>Mooen5@nNX0iK1E^|iY)JaKe=h?-`)dxm!*LW@4hA~J#f7II?& zVZo28ag*puq?+G>egxPpAS_8omD!O7$G8AlRxFFuuU2REYtMesnjlrnO)9xN;9E*x zh2-`@+fb<&>&g`TjMnpmkA|L^%BB|5B@q+533c)*kgm?Fqr1Kz zTx_i``VC|uN?+g6v%fa_30|_yjyF3y>dls--{8hDo&EknNA5=bQPL*pQY_)23m%<( z3p|bq6s&~48K|lHsJgCJkLMri(_6g8RH>!1Z>{BIveIT(jR%jBQTDvmC}CIB0l59Z zt3&t#ORg+0XcXlL5JaNMmZgPHZ{f;+3AKYpxhl)=Y;Mt2MHb6BnJh%YcRUKk^cOGY zCx|*Slcg`2z~6xm_#>ZF4n6uSU=^eNRnr=yDSJy5Z`O^F72ND?)sM@Q>yPV`JL<71 z0{K73(T%J9L_H($0`8W#c;jBQ8EXsg5xh*RjCGDI8o`?B>l$YUL4lJ1odbJwKN!V> zu-@Q7L#}Pon_u}w^o`&9aPM0EyMbC7R#yP?JE2&6q}gdFmmB> z!zEE~FC0gx0Do13rqeP$Zo9wKMsJ>pzpyrA>E8rz`J_36`weHP7rhu&qdRASWJD7 z>>?wex#%%*A4(p66jM^p5^w-xbTxqp-JVZ_P_w;Cy>F?D+EgC*F_kIIFBCXd%cD&$ z=9aae;CnBc6T{#W(${_c3*{4jW;I)$tK z*=8Y%EJatRWZX(0_JpYu7+OSHy4WnuFwhnkF-w6J^_^J6wWQqmIdAnF?YAp1b$FMA?_Q`PZ#S~ z3^Y2UEV#t1-RtqUrH6!S`^xw$WsyJddr_^MK8v8J(i<_bM3BlAix^t!0quJc?<}w{ zVBVEl>5s9#;xutN%a7Smc4pDo>R@%I$rxY3QiYfW4jCjkBxj93s7Q&)6-VNZ!gd8C zRb|T1;PI2_o9b|n13{Dyj{sW$ao3TrF(6Kkqn0{Y$f{cbYdUspP)J89+4l!&xUg3G zllsd;3{nbBxT#5{Xy^mBWfE=|mnxhUl;T{!Z>^zd3+g$9qFqK}Oodp?F{V=HKP-?t zsItsf&E`#8HYJ?rec=rR23oHOV}}UzWDqW(=@Cs3T$l2F>naw8u07pw%?L9)e|XsH zx6ya3$g@Cinwq~-x~}Gjf}R7~SmWrD76I9>ohTL*g+B%rNE8+OjDyzBGq-U9;bCOv zoRPLHaVTr&F3J*K#Xu%YpZ|>zE?g%QpTOYv3KLpW;vx@WPnc4;;JcrMV!?q)}J<#rBAzcSE(If`&^mFWAER&4OlR_T6{sBh7T0spAk# zVwp4UfX%K^XN;`i0aF?y$e}Y=$uJHV@p@1A6}Ro4kGO7rvdyd&C8jb^_JJV*R#9E-C81La&h+hGPiCa2;$>7+!0|;=RK1gXg%lN*(cktP>~H~OLJX8c z@+5NV4l^%7bz0I@qPqEZp`8+S_O6ak4C$^n(CtguB&dH|HI$!a(?eORxTmoSyxDi& z6beA2h_wKcwg$2$AnrjesO5#VnM9pf-4vmHX&ci18UB# zphyL7rPBR)BvnFWtOM=S4*mjd@9SFA{Gvs)FMc0@zQk?CvM0Dw13}q$%D2NBa~K<} z_6aqUI~ZBTL(%*0lD}vrzX2rdDUUJ4VljXpd!vJAE4CtlfB_>Ou-e6cI!TG80XvlE zUQ^K!0=+NXvi{e?8D+H@_g+mt_mFLWg`X9^;^`^J1wiZjtEk_j)+=Dm=T-bDP9o`M zMvfs0)G>Yj+?`lP#04+}DY_T?!aY{<(y_f25?c_q5QyCfN5>^;f-D*vQ`Vl?GCVO0 z8SdDRWzvEr4EOcQjC~Y*;nR|)%Qb~`GK4b!G z>y?@fi8p(+L`Ny1Jyg3?A;XU!HZgS8!cIUp59g`Ne9F9zYX05smA(h7g&N4xl=$UF zvL~=@XgFS9Unfhu#jQnFW4SaiabLtHyd7z)vR{Y(l|mcP3DGCyeptS%o|jjK$>@}f ztwqjq5yIBnm}BZfK>=|>!WtG+n!1#hiQ-IoMG*Ji&@ykBi1O79%b1yarkR&?Puz6` z9qh3R)TX{)isvv4HW=|}>=W1hXPxEnLWjT1Hu4jkK0sJ~-aX-)$(&c^dLf#?vOu)T zLcWc)t!P*T=R6rH5Me!ZF{DiBiG24^mvt>PD*i^tqoHmRJ8(zvwEKTUa8CuI9b)jr z&OV>`ibPSZ+ir#x2;Q*!K%9|yc`}zxl-p?#4}^qa32m9B)7gbs4vByIZ+Nxwec=IZ zqT>LXq#sZGmi)2cxXt1%1~fx>6oI-5EZ7AO52vg6F2D%WcQgh`ibAtRbLNa2>gWLp zp-BiFXvoAM%ur8BABBut>ymcm&n)CBvPaKmero6t5tyo?xdt>ESUCJOVb!v0O*7Du z+C&$C3j!5_ox*$MvJH|sV%6=Bpk1IUnK$QU{C}j4e#LGq%R*4k=}0;uW;!t#$I+l) ziJChas4gf<0ML*9LX;)iUU}G0S<$gR!?Dc!YLLRq*YhJ~Sq2eRKcXBWcy$Lh?F-#ACHQQHNhmP`e4$ zj8pqoLQ9!4W?z%|>h6u}r_kz7Rh4I2snXY7%1S_bCrlW6s~!3>98@J#G_>j-NNHH* zBasbL zK@~2o+f70m=9=5O?56yh*}iEto1svFV?4VV1s|;CL!du~13$kEr3OB}%>2Q;gxIls zlDBQC>t_bOet+x{d7#yOsYzP%$EG#E#Ty3f8aKdLg1U#LO+lpxP4W)k&U-ibUPdj( ztXK*<(Qz4Re+7S7P{F|-%F1wB@mm4bcl{uv4bF%8OF(?;t3KAdNQ%5#8>`zLa^^y( zRM88NpUQ{x_*LJWQQ|^{tHOm~1i>WW@GqOg4&q5T)d6e!x<{6oA%6_`#OC zkdwjSz)5;j@3>diWC_);W^6M>U{InB2$%RUV2f?-fP zC6(1L!J0GUV(s$ZKLU*c9(jEpJ)$v)-86iIdciufOLENlD{&1-u}OS?RoudSl0UMCG9xqMmp&w2{HBz!o_us~!7Y)y)YB?k401 z-{Bh?p5Z*=BLW(MGxrE55GicHYT;Kv&?1lCzub#UUP|;1o;>TS8+iALr4H2I!D*F3 z4~s6kKoyn%AhWsvegj@w(wa4?$Pq{{c}H=X$*M0qR8Lb}OZyw98`sHwIS|D-kY$_u~cNZI{He+bjy09E+|6Yp8`j` zjj;OWtjirLJr;{%R~0q$p@z*z%#McT3 z;68&-(;;ELh}7a{plJaPXXW7b1M4-%gfocRGDw!G;A`mv;wr_>ay{D{cKh-CoCNK# zXA=oB(K%d(E{Wll2vq!YOH$a2CMjC#k6W14+AUjBn;#`ig?hAd8wL#HCoB@nXO zg~JcNJ9r2V!GWa1;?=cWG1Ox{z0Ts89cdH%W$*bdv#=}-$0uW<^29;A!_hUNBM9CV z?mfm28*(Icej^hb%fd)jTbIg~JwQj-4Gb>#Eaj+&%w1`S{uN*tXzWhKrJqK4O*iE% zE~1Dg(bEuiP<1$}Y)uRxcNe1Tzyj2DJM8z+0z5M3o6^4C=z~m0u8`(qvI#l^L=0^! z)+qM2HHn7V0a2cq4o%e*-hA|j@qzxW#G)asB-Ox z49ZD=8K(2d%?{9?Wi*p(JIyo@CVsp*5MO(56EK2?NWSO`m5UgRhPL5g39Q% zyAo8SAt8@7s$3jITvxCthgXmUQ>!de!Q*Vcp+Uh>AMkjn&#gAefNNOHw=m|p#?Go` z-_<;tCUOx05H@ob1c)W}++x4y70rD|Aa*P-$xwKI7@mfVDU$2jkb(2PGpw+Y_-A`h zQCX7#kqWa9ZmoSK`+i~k%?!NmR}H8FHg*)Iz}q2_Nsc|AELFD!jvAqp=HwhL9+L{i zg%s7WRDDImVZB1w;tW1u`q1Np>zCneu$jQvfn*0oe{nXvc9-cTI9GYLkjbVPV7hkbi!6H&zA*T9Pa5b%#_Ee`AR zjUC~zWuR!FNz2_RjE!5-T~Fv|Ts)%RBV_S}!+yWSBQ*A9?ZdA3p}N&uI+R-68xSy0K$QLw|Zp7c=H5`Rz6Q; zPe5vm0$-?4*lW?1nbIpW11on483PNp5@CkKODQStR#b#y4uWwb!e(U39fi>oyw36_ zq|CZ5&r?cQDNA6wG4v)?cKpd*r3H4{s2`QlXkAur?H2HK*lZRg{DhtfIA(2!uZJj> zA(7hi!l51Fy#@m8#a9Ieqyldg?#3) z^MdceY~;KRM_j3qBP`3dC2ba+I~_rMKHRUi+d0!(SPeAxhV_s#=5Vp*JEE~tWxN?aW~LhGf79)b*q*SlT}}w zzD(({(81m00195hd%~GFOZZOhokcHX=*s@( zvLm7j)&9Q33A(IF>zP(fdqO-wt{E$#e=ZW=n(pqb!4@QPKY~E~p92>9agD0}QnkEd zg-n`)zRAqe?ROESg??jsZ)Vk_o4tYwGk{8*+LY&Piqefg6@9)OM24-aT%NSPOp)$Eq~(tAHIk{RJBlVR`+ z7NbO@AqHKV=fFf??K4o{B-=RMkivk4_ysZ&%RO8^rC${Kw_KNLFf9pGxS{F?!+`~# za*1+illKa*!m@p5OE{7fJ1LP~9B2-h1@Wvv$w7x?ZxIUVWMM#naJZ+6K=kumZ%h`CpXEydhwwvEk+QFS?B6hrr#06Qh z*)lqVPHv40Qzo>d`y9%iBY&{u=g!&}b^z4#KIK(K$s`fEIaby!oH!NJ0H>uW{pm1X zf^v*xkIFxmX6QdaaKB$3R%F9luJn7@1n^$wf{g5o(7#m&RkMl#rLoF_$?c=41q?Xv zF=*srMm&QBAa2LS!kG=QlEMzG-Z3gti#P1hBfOA~u3w~0*pl?7@txM_7wQ#ea)gkY zLAQ`KEZ^wecrhZ0uI?T!|1AH%N@G5WlOD*sG@x}@X{sU_j7FXKN_}fjiJl@F)pS*V zPI`01{m5;+=VxERv*hMaZtUwx+uMCRW^lf&Q4NbJaE?~pi?4G%t5ug&N_noiQkCm zOUFc)DFIdx5ND7Aa*Egn$D+tm!_tK7vNw;{!Qm6}U2JPIxVpBkmF;@`;`+$}%*0zg zvJ53#s<6K|dXcO+y7ok&N<^ru)bn}Ew?*g2BHU)SoWVMzOvV>HjsK&68m-(5C9y*q zf-~UGR~m_C0Ce%A`HpICk(A*Gjy%h~XBSC}#O{k>)gv-zg8!+ubBn`>jGI(aM|dv3 zz-JKoaCVc~uSzfF_J#bhRr5P|2v76#dv>Bnif}E0D?8Fg-un|)W9(~l@SW)`XBN*Z zUB{#r5tR@2L%q{vvLA*U0+QWk(l1=6_#NldO=(V7Rr})VKTgc}?rl@_#hU~ns1e)j zv<}pblsgZ)<6qhVmV5WuSz9eE;*#DPks+%hH_N)S`<_CqRo&8?)zCXH@?`PD7a9lJuCE#91`-C|ch?Cn zF4pZ{V3DrNoD7IcDip8CrE`R*QB$4((P!C|($&}T!Qc_FVTR%?uj}Q7LU0pF2kxa$Z1F7u^Nkw%16rPTqfbrNM?9j0%D9zsd&blM{t6%&ba-GpziQD z86$C*Tdq>J=QC!i%!CR8K-nK~AZ=)B9(0pm5oHc|E^N`0Q6+- zMce+MD%lGy)&SGjj=pl%(9<&%cuAIrBT=>#D6@GMY7)n zC`6+ox0l5%pv{X)E`{|iM%kR`X`3aINs#BGIq3r`->L(g{c?cXj1`y|g1v;v6g89( zQWRmr)xGl!8dU^=*`+L7*bpNJpGXLPLr%{2i_$3F#crHH6vlq=AZ6j&Dhv1GAL;UD zsI5;oeVv|s4COJnz~05!Ugj_Ob$wCAoh>mxV8SJhlO!h&Yv`|oWB=;l&xx#T@!c~& zOZSB_O5yNIL=TY;jBQ4&)A87Ul%6jbgkeq9EQCI~G%}`o?}=OiI)^@oA8s@0qZ!1I z9&dg1drEv)I`DOlauNM0n-M4@?Zs+Q{uC;~ElIj&vilO=i+~g1m4KF<`G$Ls)dJz3yHOY#XP5TqPX0AQ zeC}^xFXR*?2qzZwy$wF>^9#Ji)uZ<-QAb6m0JD3B3x-4C(uJv9VC31msSUF;2L~iwrPPa93l?1gp-( zSkD{*or!NWH7G|$y0ARBEZ7~c6px6ii(Ls?nAYutXfneEBFh&SsKp}6$u>#RLPv+s z=7txzFKA^xR3L)s!(zZGvGHZqhpBHCDma`8!mHqpdWGsHnTqONDFubg+_NqeyEQ-v z5!ud1(WXJgU@d*oh;l7~L*(oH`-fXq1kfUqfRw7oVx*h`ci>TlASPQ0tO9`-E}P#G zND!DWZ^{DIT`JnvZs;>-iuH#p);750_ciwsk>Ra=%Z6*xhn1yb`z9nwd)~$I-!^Rf zgAA`UMKx646?RQkp;(rJok?mvlUD!fyfMH?j&+ z?DEP(FeKhI=bQcGgd8}-eeM%{BoN+q-i^uMr-^k(6+j1|>yPvEEa`novQO#$Dut*! zGrTjKH_c+8i`NCP_zw@}H%G-&T76CX#osan{!o@Zn`i2hCOBL3J4tW3ZkWugx@7-B za3!ceizXce31D^wjjMPAVUZq>*jAT6?uyEg-N$FAhm?3P7mo`55_AjH494cwMfeIN ziZMYf2lsLB@$W>WaT6HyJR1^R|ff>cWG?CP? z;l$-^xWacR7EWDjh145@;sm!SQ5_5DUn@Han{kER;B046{@Ub4v^n{WjpP!ubhsubnORTAP6>Q(tCtU8{-&Fj0h-_GNpc!Sif@fh?F zzXR>=smq`A8Zu?iAFkrl;aawqclr;W;5Y5UB@%5d)-M%{tVV%C!4bgp6)N(a^Oc_z zhVdd&0ch)&)pbO_aEf_WG6)+k+>hmN8|+bXDUN6Dfkq#x@V0n*9p)?O9X1VPRLMRI z#=^-BL*9o?cna>~H?9y|HB}wcjzJQUGH9ipRm&n2Ac6*hC!+-Tn1F93&1@ z?*DrOL_$FhgKAmKsysXRj0`g&EW;D_meW?P9$kB!fGMa&lHo9b*2DY`=TcQ7@q(IW zJQOGR4a(_Yqg_IfbXu~RO5ml4OZ2@iaw)xbOI`Sruw0Y|$z}pewkkmv&^FD37#YaS ztv*+JgiV25G69|iIGt;sT5G?@Pq2r<&XT13d_Uq*>76SCb?e1ax8a>p9rMB7^Xt7j zC)Dwl?SPT9c>Us7)vgM=m{#rlks;bm){x-O1F2ZXXD8y8QzMh+=s2i5S2jCy;H+P$y(nV>F)g4=|Ke(Vj=sesjui{(6TvfLB2 z0o>sW+9kmST7|A>!w+7KE$?MIdJ5IA)6Xr)px%YYEs-F6dl`eU2@>o}wUE1!#X=6c z&FM@n=Of=bA$O=DkuSymjw zcPx_GEG3PYV=&J=DALUU3NJ0z4ZzvykLCtRRcH5?CHzWQU*Yi3vst&P_lF>#dX>+w znoc`SysDJV{=6EfCd$W|JFWnK0ZF33Q&Lg$reI|onbP*psAR4IMd!LuWw~Qo`3z60 zx{#lOg?vu;%(Pmq`IJVF%8EFQP4Xx6_`vNRhq1eXB%r}WA8p9rO8)7?EEunC%42-I z{f9e5^nn}q(k;2DPFu*%N1Qbsnd5xEWw*?jZrfj#h57Q%{rHOOb@Krz;)3Z#$3Ur- zQFI5h+Gbefh*&k@hnDL68ZhW3!VK*gMd{3)3D#I-W2D&BDgnt^Rn!w}Gef>yhnu%< z>Yn6;F}eK2@~5y&1X#31*XNc9H+_1`4F45$GDUg(GRzDJI0R2Itlf5|%d+npkeTEk zA&x4q*8UPaK_wN58#WZKxjhvz8aJR6er{3rJP?hA`PzSa%5;e!ZzDfL7&I7y1lCOq zpu#M(4K*c^-gS8q@`MstJ>sWmtYIa+rs@(YcaHa|PWk2LJf5Aow+_@(mEwarm+w_! zT@|W3i_vLHNuMxU9)k6MpsXPQJ1`823vRbde2&1tSlj!&M^3)kxK4-?o(l1J*&^`*;>S1>`s5OlQ~C6L`LtU^86 zzd~F5!U<@!NP@rqCAvK4m=(I4(Qz+Mgp*EzcGmO0SPM9g9<{RI$w=M~w{5Ig>UH{k z=!5!iHBY603L|C|D404Rd zp3RXxMfxYcE7)!zq_=xj5(O`piu|R>yg*=LZ!m97*cuC9E~@FEzbJLxa2Yj3 zQ&oZyGEXah;!W1i+b6qEck8L7q+&^!WI-E~n_n9lt^1!L#&Xf5CNgZ_MO>i`C2Q>GFm+Are2NL@$8 zw!@TlcRrbWf+Um^5Z76|vDSp)?J5XJCNm#F%eqXe##F^_LN=$p?IMTY`*y*-O1?Lgb8ljdy$&12`?gj5BqtS9kch|l2F^GUVI zjj7_;26k1t!QgY~{Cl&Qi$bGz1Mi%!R-lia@QD!jXKYnXZ<6{C z(o82 zJqQST|Bv@!bd6hu!gyF2lnA?#Cv_i<1&~US_CyX7io%edr2N>6uBQmiAr{qczW4`E z$b~VEfI!ORDZh+a%Dq%ww0koQY%eLER6X3*bhg2}Y~`ll4EIQi(8|t)f^;Wx@<^P( z9}Gj5lORz-Md#1PJ-<35;cpT|1qBWXkWOYEYQEbRs;Xm^f8@k$o#`Si_bQ85D>LgF#`6Ap~(7Uqk8fIGl;dF7O8M%Gc?6A#nxBI^@-P=V1JXHEpLxL|f=;HR&L`njlMoSByF)XLmgwwxuqz=p8K<59~t3p-QvADl!z< zSisj`(GDA}&APV24^!Hq>U%DU1%l|`3GAd)bN{9ZDvRjm)eMXWu-pK%`W6QFyOTA= zKzRd`7VO2zhCPUM()Y}W9NTmsFVVH25u5VkM(4*}43bnuA5xqWrTbkRMysJS{Pad$ zNro2{&_q8g@F;Mn?~(?qt7{PfOPT>7j6@`uc^qWkS&*S*u(=|&^NM!uOb_`|9NZrC@=gej17s9M#g zg+T$M2(=qO_2xlvQ-pv87H{7;r3q`v7338DOoR=#Bz14_;R_3$JAr2l=&Xwc2F4;2 zA%!N;@l@j3z?^K{=JY$bO*3tYODN#k81`8Y3p~=`F^pnAl=)_c_r4sy5=+@eHJwNN ziNSz5WbPS2*b;H|B99K!-2`okKqHj+QiN~%TuN>N-qyv8@r|nMf~H+^lj=Hyw58E@bz$hFfBcxSG+>) z*c*7A5n-8EB##R0j_c?6BeelTBSuf_x)5toNa)d=0ml#;uI5 zBRAsky9ukxyUZN|K4TM(N8V;SM)5fVT^~4hM!7KxYc2Z>=NS+fx@4bk%^_cUcRXz_ zH{BS}_0iJVXxigFf06rrd1bYTLg<2GlK70`CH>Tn@1C1;QvEzYfX9D7pxh<~^Eg6_ z275rMmiItmfEj9(@~*hP>^<5`QWZ{N6z4e`<*a@o8?=Xvfp24ayrVNSaSGmR_RO5* z&aSaDzk7R<-(*UUAlal z`V=|+8Fn>D_Uw;RY9~vv7qF&H;5%$FNZW<_hdtINDmWB$v4%;GI*?yx z&ClHJiDMiL3I*1C^a*-+;BVDT@en4Gh{sHle3e_1g&~IIyKoV$om6uo+JjUFbSI&~ zy!5~$rzjD2b}{69MIPipSqa)BjE#>b08k?0Fb`$B9CsTkjV`)7*y;XSehc$M8$Ml1gh2N|Y_AEtuuRpWwfs^e<3z(( zF$w;lykPYp{9AORqcp~mzlqEwQJ5w|zvByU6>pZUsBKAav2nsjA2l^uU_J^p!?zszb-rA)f8YN=Aquqx_RV{V1UT{kYBLkdKauFPX!#gm5) zvYhBkEE|k^k?+UW6P{o7UKEB-DOtTvVsOOCbEgHB%N`w_`GF`1Vo0M%oqFLIQjF01 zC7T=)wUr8ze}ml*9MCFQi(Mp7>6mm83TL@LI;W9?vwK+z;=a-Crj6<^WqD~EZW5Gz zC3$!lN^*^*x>5NVkizCmZN(bu1Yh0*wFUlJ3V$a>rO8G{ZH9l(y+T+BbuPgQW2wV!JQEHa zS^A*~DS1Ju^x95AChQD#%%z38U`LE8ye!El)|7MxSCeFMyoo1K;31zh?O{Fw)xuu< z<6|A7N`gC8XrS~61o0&4=yGpy>(7^U^3>UVftCUY5lGP(IZG1G66!^O9;o}fpwWsJ z(w+PY2{1^Ea%GnSib!ohBhoi8+#olXxa@gyc)a{k9}`1LitE`Yft2O~@&XhxY&CWKuhG{k~!tvhlNm z3j+ZQIpbNjQJApnc>(c+cs9VNOezMt#PM(4U!R9}`p>hf*2O$+^!#q0z2 zr4*MyRF&sRt^#VTQ&$QDuM?$M(0sJ;yZnu`RM4(rco>_BzK4_FTgy8yA`LYm77Dh- zULo+;`TPCpT++e;b)$;NLT~8rl#?Y37+?==ftxU6=s-U8W~W1x{eU)4L0=nKb4tC~ z-e6>tcrKE6By>0gq{uFl;?3b~+vX&7#%+T(& zE+TfWZaHWV-2~8!O3Ykw>)zYChsRdk9~vWJ{76>}zAN$MHUi=i0Pw;>hb$OK?(VC1 z=qlE67J?<_@5SUrLZP|s>4%5eW zf)vG_&6^-%AwL4ZAv_5HPx2Lv7{M{3FhFWxaAEOzvP8L8C2N+H_Q_C}z|HwfsII<) zD0y(j+o(2i5Ui@@Lvjt)FrxE*xh}Gk&U?jXg4BQgPPSMaW2klZdK4UmL^{IpG&JbJ(%bBdbJib)Fc2v_ z(UZ-sX{C23uPGq-EM*Bzd8$CJQatrDF%rD40xb>7Fz<|XJ5-IH3bXxZ6NSAO9)~MU zFSpD4XVlIlX!_Q!6u#ZM=aReHr6*MGgQ)JwwxN$K=7N9qtCp)Q)cR%~+Bg`i8)#fV zKI~mkQ zrCGpUUDmTDaNh1)9m;~sDkDM}5SX+qs@_Nfgef{8F3l%I1SBlsU!ZP~rKpMr-RbYo zMla&*xwvN9}*8mJC6hy2G0zclb1}VJvuhwg46%WyE`Rs9W~+C!{g6KXI_t zM+Qmo(NnVy6L(KRx*pfz)*GMrUr1aB)Fj?513QC)fy0(l*L`0C6wQG0l^X`STmcku<%GlcN`&l?Y zhG=`hYpVo(!01v*u2>ZjI03FmSGdp4Gp%Ag4-9yV2~dWiO|$~yzP7cS$MMSSL8kTQ zUzz=eA}KdB%KbP-;zaa}kx8;Xp> zD(V^q+rP7(QazUboXc0<3q6b$;k9BWD6`t$?sUs0;Xe#QFU9kgX&vX~zF!e@Fwh}! zCpGhrqPUAvSUo%J?*hXG%_pmZr3k==zw#DvzuF7E2!wCz&mynj8x8M-r5S_q0*Fq@ zT{7E>JUYA}XRj-{8}+w*RE?qz53vds&@Ob(elfHLCb-d2^ddl25q<;=gN52B8o;R0 z@ra)N`iRVYc|_m)qOw~1n)%UgsSqR2zFJvvRkGBP+dI=^`G@Q${#3u=9mFJ3N&s~4 znPvU#bl9e4)B-U$*Je0hIGfS{OJJTH&d4>LyE>ry+b-_olJ;wdy z0GcbhYlq;_Jj*M5=`zMeYoELqwFnMiUKm&Ub1Wl^JF+YO6K@K0_EO18 zl_>rYc!sLw65qLkP>DF7xz*YG$~Sg=dy%uVlFv!#b@}N+CD0S3H@wlyGkUmx=s0?M z22~^YLSzKn(^)1&O6r*No`iKK>pO7WzRBkQoiLl3x88%#aB`9XR99DmZdyUw(wQ>p zxZ_`eCoj%&RWI@@hI5ESlS6`hJ70|un-h_{P?hE^OCt>sru@$ywm3+y zMii1Mj+?{~zKn>0fGrEY%JZ2kZ=9p%YhPaY(#mF&HAGJ6vNt3xhx@-IdXifD!b#aaT_Hq@O z5fBf$`VUG@B*pE^;j{{iDjx_7VqTQTF1#e@kY!4p_uqN`C;Dr3LmxKvUKMx|xc;p3 z08SqET^Mw2iKL`;8Bp*Y!m6RC(_?BsAzfM?4B>^2_XoSno}>{mZpZMEk$LNsaM_f~ z9o*^e_kWgYE$Tcx(R*~jA^%1RHL=!!CDfAz2cJMa2bgE;qT2jmybCC;7nO^Y1ojvV zf6xy{4HmU@Ol85QU`Pe$I>^%?NrB}kAdB`dZ1_E|0sML>v$I|tp%6tOl5fBQ0_Oqi z!ggOYsHjBKEho1_7HgKxrmuVLk`3ZCmcDuUIfhq6mox@kLX=oKh|S{sTsmv+yyG|S z-;@p2GQpniC=1*ZJUl}t_FTWubpppvQtT5iNCgE$I13J#AmOf;y;_W2NdUqsA`#};y z#T_#}h^62KDDeUE;HLv=mxJSNP#gE6@D9^l&|qOF9!d|BP{{#TJ9-C0R_UK17|~KD z|3v93oIBgY(mGun#8KTqK+^&@;AnX!k$Ea2x)-{elDL|m&c!20_(mbJ*_Xen+ni`S z$P9RE-XE)ZbYe3sgfH)QW?lR2 zNmQz8`626~^A01qd$92JEHi@y)Lcq5W7{{)C0I0#?C<8YuPUm3Tr%*#>-ZM~ zLc%=gB#JxjHS1a3P5WoE^b2)#X&p`YFSMr8H&lIgku#JQ&X|i`!r;^9%;4(#Jc@RA?EkwMf=1;(ak(=ypC5C;O{{N%`@l(B z*opJ1g?@5~8qC)VAr`}-5XKf9-{dI}YHz4#W8w>ez#dT-LoyhMZzLJwmB{uZ~&b>m$QW)79Dq)Nn+!O;3Hu_le z=jzJq>z`q)4k~61r*5PzO#ck1qGg#f2B-J2BOgVeah6$yF3T$daB7B=--;1NRM zdJ#;T43VR4i^Xdm1A00rBb=WoI4GW44fCuFT*;CA=k8qNec+}wxtG=Zac5wb^H=Q5gm1}7Folb1OJ>7 zi`51z{%wzI@*QyAln`-#FTRaHTzY8$@6A#9m;v!S=4wy-u3OS_WhF1k@D|YtNal%x zM3jFcic}rT4tbd9x}OTQF4->d2N2XD#4($~mWb6m#}oOT16HGi4N(|g_9#>kzL0dB zH4v0wdD%9ZH^&W^&>^^jfwQw{e27vAZO58}b<$x&B!KBBWnS!?(k69T3srb@;!e z`G9S}aODbSP>4+Xy9g_sv=_D#62fYLy%5d8s^ali=OQK*)JEqA*R>V!7orK{CbhD5 zfn@@vAOLs;#PJ7{>OrSagWf=>iRdRxplCnsb4>ZGV zyn6$~jqikU1L8`i@?#B&u5P=h3dW(aGK^BrsKO5b44HDj0^FU|9hNDSou_)RvNfU} zIKU&q)cvseB?zKQnWsbw-v$2H7=Z5cb~46*lavdmoa>{7%TC6emzVXfjG%NH!#ot2 zgKaPgEeN6y1#3d#+DtWZ^Pm?Z@@`%}e*N|HMxco*C zD`e7?-z;_=5?4Uo0^$Os#M?Pv$*?yqhSoyEH)VVw12g08{>p>}DSyHHU{9L1kwCmD z4HhH3r;$x?6w4~OTys$H4fnS0^xQ`(J2kho_uiBA(T3Ss>FHq{EWxi;K69Lo%PG=_ z#MA@`8NX_quT>-USEjdINEPicnO@2qu`1*t^xAo97S)^k-Fz2b`+=_`tVATldcV?J zDFm*+FrD|IQZo|t{(}&MLfm-7paj67I|ikSw5wpu^NnWYpwW;vYqcNSrWD~SftaiN z8_qrp95?fcw;%tK0e9dDu<|hZw3WD;KIufc(#1ismZ-8bjl~-DD%dSX)Rn@EP|h7=ZoKfPboSq*lVB~y(Jxq)pz^`6t$6> zI0}zs?cNGEAb~NKxB`%%J#qBdGK$e5ZW%S`O(I3n68SJZ2k8pdhSjnJ)#6Za*65ph zdh86B@3@bp<*FC6Y|>77q3*-Lluq(=FELOEgAt1jS_vP&3B!5%;y&Ty`A$D6%J#?+J$}w3m_mv(8eJlXq87EebKC_FBNTm&7ttoz zici~;bj0rGk8V%TdyevZKJJ!u(@Dwhp!aBwox_-@%LhNWL*S)B*H~1DdL!wq-2xFQHnTsM>A#}C( z1M25E+M+ZiyjA79u6dRu1zAW!wCb9ZVUmrt5+$<&ngjYE!xcj_0^+o+WZS<7sa^OK z1T=)WXSTfey>MQ^+pzjBfeECg)GNdE|C##sxT?!^`_Gwan##&)C>=C)yd*JR`Uz@- z+{!zZFkT4(JyToB`vf9#5zds3Sz3v=!3#7=ks{D+L<0oQnaOesLZ(2dz$sA@5E4Q{ zZojqG`(@_*C2scK?92CgpXXW6de$;pHI`Ts_$3Pm#1gqn1*upUq*=T`BoH!BYOKx! z@t9E)efdc+%F$GGWELdvhDir3qw?d;25;_nCs}w}iX#mYT_!QLmX{fX_Uv4JlD7bl zB00w+*p7Xm0+U@t-79JiF*VDj1t zSHos1SPpcog`S=#exMkb3WbGp)9drmpP_*sm2UZrdK~(OGMRW8lRsvlbXVl>-#~`Z z47XP*QiM$$6YCY{21yYpX`Mggv;Uz{0-0EkQ!52zjNz8Q^vei-CE$zSJ1UH#JWtje z&OwFc_dMSG8LsMMn|=ZfLK@-SO+%jNu13cm2|~0++wquL_L^_ovj6gH-gTudhv)2Z za0GOqE}VN*k4!slS}O44Gt@)%z-txU3Eb6nqVF^|V;h#w%^8(W%?SGS#XrPq4--zi zP^Y@s^G~~6(UZVO!gV5^1T&z599UiGppNJdz?8w-*zU1R!Mka-TD#b5_;JZ#4UckcN^=vQjWjx=B89Z0s8 zpBt|HOJ^7`4VV9MivLbnT96h^LyktL)3w|vsN0beT5N5kx&M)@rSQ+sTTJZGC`KZt zzKG@zaV@O$<~kF;Jl`{|2e0vR2jZz7D-DK+3ebG`Sn;B)hd&xEajs*A$>omisVoMO zz6Ste$DH?{vX!uB{XPTPg`Z0Rb{z!SNQLpKJK0OK`U zaQ1}SGlAiJYppNgVS;K9QXuWImcuLLniE3a<9ntoY}+gxpolYjgdin6ams*$duHlk zG8;g4}fd_wc)me_DKleKMQVPOS*6yWLr5y0*= z@q^l)!_jSZm48CQ^$qJAMQuvxu6J%_ipV`Rk>wtW-sbS1{gID_IQ#*&V!1HAC6m?* z^2!@0$OIdje&x`Ng(i*`x*0_3px=RsQ_L$rb0Pe4oTu?(^#Yw#|m6Lytt@ zGUFXD@h&ibQC9@1;K5(>!i%P&!!&*lSq83n!rRjLMHb~9C`D$z#dfn0WeC$&aN@|3 zamV>Km)fEy#Bjul60Zi3Tadya=i*J>Wqz3mxPgw5EB%1M1FnaR)dXhGy`U)Y22HDQ zi42uc86=0uGY9kLVq=<2YAjwDatSUoxPb7rSso81-$2rz&jlSD#!xs0$E(5wC4xrm zKf;hxT6eF2j*vof{l(&Xvy?lj;SGsb%8-Mdd2q_o&vd}k#SPVAgfZ~-e$x`x87~@z zYd}Y+-i@I?KxLJfIuKImeKNePRa}eGU)Sa&v>dZHBWxLJ9Ni9m2Z)A4DNFu{DhZi{ zK(wi(6P#$D^f$UUdh-uH7Sy721>sC&DeMB=D%{@Rph4PzniE`D=Q}_dpmNsx0>dV^ z=9g=$IEO+a|NJfMbl{dC0=iZ^9JcpTBAN|98p;CtMZl|h{G%Hljlox?oy#X=%)xo< zsC0a-Sl$@?wT{g|zrwV~5j!i$ui&@p1hoi)u*<=(07i8sZ#dtVq2mXiuzvto4t(Cv z1|2i!Tr1Bnhm^`vCh48cR3rSRyRZ+3H`;i9!9aUl`jm3M$ozIcmZ`CW6l3|8c?m^>R6MiLV1iu1T|7*>J zN+F_ECR^;z-Xwx)R>_A&eC_5!KW+4H=~O%@@z}><9sDMw5`nm?q$w}yJA%>ESM+#e z)CMEu_yeH$vkZ(+o3u|tVIUGb45t!2bd8pH-HKL)nVx^IxdFK838T48X-BC?fV)AD z8=TU91uXaw5iPX5hUsj&f!Q;E4?(pcev)FuqSqij9T(N^NMV2=z<=F960C+>ii&Y| z^ieR9{X`m=kYOMA1RGd$`Aju#Ae@!XIgQ$`*8CXW1XNx zaN5;7VZMfd!b$nc>}aFc-1_-##u|hK8_k5aKdrrDwksi>aT^11-e+&RuYmDnZasmh z$({bPSp&W2u12mWalN_BlHHnw;~Y*2YQ@iY)Nnz~$&E zV+}N``Y)361BPJss>~ypi@aK;!3tOBoeUvVgCJN?ZrkmwY0sNsWsP&dBfBB7J`U?Uyy`8KaoD<;Z?Y32W{SGy zN|~|&`5VeO!vel|;w~{B7vndQ=kR;Olc-xeKiStqnW?mQb00cHzbWs9;T}k0IhkzL zkwnVvH4h4*$gK%n5ybs>&z;#LG%j+P*t{N^PY$Zt*nC3m@VI}lO z&2}mr$atl+o!SqzIXynz0@c5mth(%aMgD{Qp*8=8YfES!seR8J2$@v8XC|VEael%` zM)*f-%C^cX@~Q!fC=Nl2IkTez*wCeARy969-lCYu{Z!s5og;FRL0f{ zD;YF7M9p<9YiBW*(w!Yw>`Ym>(H95fYp#dyFJ!DG4W(nt9sO2Inag#;@YrcZr4uguzXCh2M z(OK3v+fpa0@N*5EXjrjkM>jNi9%fuq6N5_GQsGqj^AEf>!`XN-~FVv{2zld*+UskFKl|(KAIVe#N*OZD|Vg5f~0_sgR)!youIhQy1{B|8vE5as5)|+dDkbCDUl~ zHfSD9=Rjni=?j*t1xjkky2S-(s5MJOZ$|jc3a1CKwMz{98y9EzhuI_gJei z@k#+5-r>UdB%RDrkV_Kh7U0GE47LjUeXu2amHi`1KTF?@=6w zZK;Q!>WuK+O3wC%9K(lL6}HX@f-U~ah(S2Znd92u)&D{iWxr7fFFl90i#&|wg>AtiX5rhq2oOhRU2!f|9?fE#DKa~4_gGvwW_r_ZMi#%HsNrfMh zATH_Sq~<@`7mjyX_oihja3G4|PdRV~K&p{ePMUN0i<1l*`-mx{#gOHSgb#&RcGMZ6 zw0BZpCXIG%;&%_HOyOseP`WD9NUL*6DKi8`^Z=*^ML}-?X903YXp4y!->_=41ij&B ze?*VZqYz*&-y0eyImM$Y+9z)MN~;VpeDvuHD#267-5@k)oYUgszcVth?(6p&0)!is zBw|6<1KkjU5e6`KyUfj8e00Xi+13ZkbP?0h>TP8@^TWcVDng?|RCN`ys#bMoII%|4 zd}LC7&b`b3Ao$QUItDBG>Yuy}C20*CCH9V2waLY)E(TRlR#Q8~G5f)GgP135s)l1b zbJ83q83ACEgB2ch>5gLELg<2P9aDQ3myRhZhAxXvYdDxpmvsh9hgl%Q(qp)W$puWU z)1SDs8(tA_K(^${268H`YcOh-VR4jdz~Nfs#V%5jVM}<_ifz#qQ)!F*QZxsT&z*g= zb7Q(MXlHNYUtbX=21U2~MV}W57TA{G?s>_TUkP|hH9{6F`SG^uJ9}df8lJ^?M>XHm z>h;xSy(0i^U_I^(d!+E0AlivAadXEK4xadKz@Fsq7Gz{x-G#FSDs3%{yb{;3SM&oW zzVS;s$z&~h_5uWk)cE9BD#{C*{c9B?*(*O-+LEWXdSE5bLi-#TVyG$Fl9vgu)+4+) z;r7`{7T@qh$oF@jc=|bHr>QwcKMVnDh)8Jms+ESaL^J|dl1N|lcWf*5{Vz0Xd)dvB zhq(JoSL!g}?79&yM&Dn93}+aQ4cyxhyvGDpRUQ;2E(GK_HA^zyo@7QBrVx&RgAewX z8H6z2K*DcPiAN0yd_2wm?6q)JfPn{iE&Gx@9UGj%O5!RM+64A8$x)q|S-4gxkC-Mac2SPswAC%vgzm$?H@N%Q199X=>N4hI@qpyPG!3OB=9jw3Vo?2q!2gSC*ICTmzHtnN@iIi67|IFMqX^e4_V?NAxFaKZL4 zo*5Ba=KN(ZYE_&REs)q7lo;yIMGus6PyEO$Fw(-xy}Njt6=gpNc?KrBr0<%3G(3pE z&@uz*>Nl@*9gqrt<&UPAU*)ynS2!ORW5mqwxCC1^em~yF)AzmaVR|Fl#Lm#nL@)jp z!jDG`Tro3%#T~{&umWPQl$#xMsoSWBVL*PuY#c&^O?-+7?=u#~FFs)8iYQF4h};;i zUp|uC9lBaAYTPV3MJ5!*oWUg8W&`Lh&MbVT_V6K9!t)>%GT!XR!^O}ywvrO-(KVx2 zY?0>#$OOC!<+Fif$m+ii{~0+5bK>89&+^T$$B3B2FlCnFDn8asf&mCaLYR|1IRK!DDk146>s!9ERpq|S_U zFg07N=c9D(}0mkdM21n}d;#P^5kZ`AXqRbaK2pkFok!f}Pla?P_I2e^2>OXJ zO`9kIOGt_uN%JgmDI6X3VLe{aI&t@D{tFx!4vdT@h<=L%(fl&FwvX^lU5k`S1bC2@ z>0dch8gox(nm-9^BSX_$sbicysjs47oRWVqjJP$$OQr5;9%~J;2eNAw3=RJSO<-eW1-I!hBN3_JZ$9{q% zTP8{-E!odLu45Fcc+5Lbg1|t80heQ^VyUz6DM(@B-HbZ6i5w4jOw}evUPlWFcF^s$ zsst_SL?e89tyRgr^{cBHlJXUe`TK<_su-4-Rixxm**fm6PLQmn zHp`Xh9_?y6vw*>&)+3=DY4VBeng_9K3s;tkoMf(IAV!x~)I#tJ7m1`LFd{LcMzy^T zOv*Ka5=1c*jEBv7XiB-Xm?BaEw6O~++dVA0E*@}>tJi8tpGVAN<6s{|hJ%DS4_!aM=ofC`C z7-Mh+Nr0rKI@Zt2by+&DlKzDh@g{E%vTp71;unS4rUc#68;Cf2BG85n5AAkQjvZBo zi%L6`=>V|I!0By|@9~21wHRAAm_*F`gst812dZg^cv)b{slTr`-=uR<3n|{#oGZ3=_vx*7@Ux&FnY_FrSXWUg7aCw+wL@( ztHFr9-14?D;^m9d2KlgZPfdOZV<}w!RObWD)!mN5md|m_0C(^KLOOyjTlrFEYgn?e z0NjKi)pt0=1X=7s-U%NgDqakR!Amki^^j_1GIN`~{}bnoKt}`MsCEvvkZ2#sEP$x#bRJ38wG3Ds0^4 zih=V;O+xUYy)C7j1j@gQy=p*CSHzeENoAz6gc)Qza6!BwRzEGsICn3)9*fln7OlA- zd+uc0tWDnL0N$-h@D5e7w9s%k+z0$Lj{R*t9S8T z_=LuW-aiC?DEzIQNJ}&``g4y9WJ$#M!R10w#L}la-J>1^w%`oT{=pNM`2g5fUgf{7 zS0e){B!O?pD72h<84C5hAdnn==?dS|uU^@8dQ7@mX@o3YkoXdC{5@ng*F6s!JnC-C z+Ba5bEg?T00#yA3-U+Bv)Y%NKaN6V%9+8TQ)q)1A! zjx+NVe&DV~Ic?r_{)$J9` zTj=ZS11vfOBMg7WX*6^Bf@TADctMqiQzhg;4=!&&Pg>}nQ6|^YWw9h+QSPd7dNs_F zbFrccg)sM~v_K+^f-6v{rR0_i(ET@9)e$0*F@=P^m%z`qK_)}M&{2!T?)1RXIIcq> z-iMp;G6cZRBTHX4{CA+K8z`^L;%piZo1LMx?2}Nn7y?8~S=l`Ui93Dfz+B|JThS6W3K>37D!_6L2$O2>R=K= zT7s+!LSz|n(!~dq@j97RF1w0PgD9ov-c;Dv(Cs;kSr}s_yt`7zC_!E$|lfHGRm+H zH2cnAFPssiH&__tA-`S^iteb&frSB`&IbH4_ zJ@t9D7PrTqQ>9rxLcbU^KY1y(Aprumg{lc@N!fQP+q5Mo^4YD}Fxgx@?tt)d)DnOl z#d_NxsSSD?fOkX6M(CBR)j-{;R5bT>R3ejh$Hf@EhVZvMe$K1YFS_+bT#e+YweVa0 zWnl7=WEOl$I1J*=USdz7OR>S$p`@LCIQRg6-+APVy$~AUmza6;1EbA+Ixw99g~OrE z1(3bhnC$8$&3SOSfi)E!b9s*B!?l~|aWsZpo!%ZSu~y##H$bVjAu0!#~>=af^>X4ZY_|umt|Y)$;Cx9z!~iL zq(0xD*K5tU0_p7(DBG2~S(zG3FACo)J6jKi@bzx6FX(*S&KIhYFdBHST0z>+FldWB zDA>q4^2`LY-W$^L^HXLWDt^Ut){KBj5KU)(dklLn)C1@4eSTX5s4z`@>z?g8iZQid z!J_gjdVWt~hHam8sy(YJgNCtc&4xWQpXGskYN!FwTR$(0ezDJ03|shja)JmHoG-X+ z!%V|)7+JcNprK!MqWQHYr5TmR-YoU7u}VysujSX}&u3{Qg?wpB$$1%;3*rk#Jn2y_ zQZ>~&`(U4pk=GPpU#s@bmC3KHXcW)3kloa5K_gbPx}9*{&Lx>q#wlY{HEsYn5pjPE zv(%VeMC4*X1Cj6_I-{@5 zBz+JT78F|uTr|!OZU)k(*BMxV@?%Kgd>WDD`A6IXZA;U8JoUj-%EVJht_flop!RI6 zP3G+$lRiX&N8buuKhUU`Yx4rLN(C**uXN_C2n0#-(l!0p;xciL4!5ltW*7xQ=k*b3w!qA=s>_4A(!}?to{}E?SZHR&=stIgZC@B6eHa+AiJloo`1tf z&CHc59ydOAET} zd`V_*WkadVJcCzcX;Pfb7M9Uy23J`jHT;scd?(zmfhfFcX!crl3pFPh?6}wu$q!Qs z!)o6NmvQTO511o1S?j`v(lN_2nxlUQ>s^Odudd)#aum_^ym8bH=Pa_8}1^BzFG z@55Kvdg`j!eYU{ZgL4=!eMWq#YV?N?I$@pZv2=g7FPop1GeEjgD+ z()W;?DyorS81~T{@jcnfHZ>ZP4OhrONb$(f{oGbCc_D^h4A=eqA#X#pE7JKly7^)} zQzvF9!}x5`W|Lz^h6$h|V^cLeho);dCmhe<-;gn}P%g%!0A0-wc%8|a>#uxI5PWrM zMwI=1Q)He~_uFx_O9%@*}FqmecGf6ytRg8Sxo6_eencLHczgD1vv0^>>3Jpcj(3A1Apz#1ofPW?un85_p=M9K>Pm^Rn$5lX zYn?`J0l{$d78X%)=kf~e5Z1QUWfoD@lCQ-FA2Z}44z@|V(xNABA z_SQ2k4lVnFW5m6USJ9MO^2?%?^906_3ZJe4GFmQ8L6O5<#HNu(6=_t1+4so3dR$~b zkyNd^rp+vG4)5FIWf>hao2|T8!7s zbN2jFjdmM1$5;G8EZ0LfQ*uVKXX=cn=`xS!scHslSBrgt!}7qI#+3}PIgJ_98rS1a zGZvUL6Xr20T8cJ5#PbA0a@2FkJCi>~RRRKeORPO)x0d7Y6pk%6)*8J@PeIrbiD)`N zRq{wT zp8Tp}d4Af8((~;2MJ}r;|TAOog8g4bYO5V=OCpj3s)EwlWO} z@4yBHiKu@0ZbmxZBZ6yGRU?&KkcPO}3(oaakcI$ajBhSfMVt@f zJWmJO&@OGa1sdefc>;E@m0*%xvXgsW=LOJm@EYhrXvi82`4xhjnh5u+^UZ5p&2Jlq zgcJip05=n1gY`t7aZ-qA+*fQ&G}ghM(`(EMF27TTU>>jQctHtJ#v-JJ!t$DN zQGs-1h7x64{JdOhz6?(cz~Z<#5uNjv{LUzw+fvr847z_BU%c_JhO`jtJ1BZ=#NQyV zEY2`zjs0g`*VcyhdRIgEYoFi><5zQxu1Sa> z<4EmObj0{_ppv5jsOm@Vb>I)X{7M~6{6h+V97cS{MavhKQU(c4OW~R-vkk8p0bL-d z#Y}WjgQ(zhRL4{UH%Eh%W(Xwbt8ZzkqPJ&ZOI|QvR>-;vks>f#oZ3kFN>nJYhL8FQ zi3$~5`g)^)6zDe*A1U_(#%rD%eVK}|7^tqRtk&47XwUKnA`yrk!+iz&h%d6-e}4;w z&1ax9t2L9SWz;>#>0~0VHz+s6l+=Kl1Y88zYX;kSkN3tmDq`S8=p3Y)Eii!?1V_f_ zwFI>{>?fgqAqJG-tau&dnGq(|)*&ZP0GCa_|%(n23rchHMSd*29J!WGDxN^vP_fTPGXl90udvICelDb@S57;*m=bTxq06U#g5 z)-$?tUI^zc(EnM5e^J()0)rn62=M=&_)$!V;}17617VuRr|$4_4vG&jdFP9h%sa-l{4>v^8NLpso$j3nP3E{ysPy;w{S{gL1O`U)Sc2Ol-p{{lv=h{meRJB$n#4 z;uYSb-ODWWrVUbn1pETO{Fg>W*J}FJva`s~uvoA+ywn;sfg<*QXc3TeXK{1riZKm9 zya;fR;o5Y26nxbAYk(CC3|i0&>Jh%KO=*>eT-4?rP+_b-ZOif6Y4U`qqk_awbNBD6 z=T(G_rH?=&JT<{MN2e1sVJyb4`6@l2@{;oBQP}Mc(b2K2d4R_*TsK$>K0Nz+XCOFq zsHsBFVbu?dme_JP;&1c}#_bQM=*p7iPh&oTjy|R} zD%=KP-J}1EfX!cqr}E977$abD{)Q9($P=>J*^U&Wvd;U0(n#a26^8EQvOYyI0{I5^ zdN+cibjT+(Z?;yFAOVh@{M0vX731uh2(6L5w17O&KsIffMuw*7d3CR4@5_G>|A*4z zOaZ9VTVs8LTIi^OE@{&}5GJ68@S%7RaB>O4mw1VvaDLC^R-%unwWq*23RME!+v?$m zi?|aN93qIP8W$UD!L-j1CvZu~<5nRg`X1tX7+1}igCLQp)5Kgy!!o>f-7-;i3e~xK zofIHwgbm+5Avzx>w<(!`BKSK@ZLnp4v@^5y+kQ#(e;QgJjivSNVSA%@KfXYIXuwg3 zbb}=Q48Nj#`z!aEYy3lt-dYbK+%O>xtod+IvfD*QH7r{Igm1sNcg24y(lB{@aFw2| z{^7NJwpC!^ z45V|faD<5d&{A1Gtf}qgFaxszPUH2Kz0Sy7?ady8dKk4~WE5|fcL0);F}=0VIvGw% z13rN*;xj-WTK{WvK^~e#w3lI3;>;Bcl?VFo&{v)n($Mfl1i$p!<{yvW0WAS<(dN!( zA6dF(`|;RH5fVC&_y$c$){h164D*q*1ypo4(Dnpqr}@Mp`kzg#t?sFilR)&otr`SlSUU{X0=VV3KAs z%a9+3@>9#XzX#7Wb3wrXy;7zRvV)N0e8nWHDsnoVKFp`ZcSq`Wh*k)-$^pwE6n4GK z7tCwup++-g(8B&vql&KTl+|ZtHw(N1!%ZAkrw2BOGc?SH=>|4>8+kFT&8C+P!Hu`2 zuTr-@Et#u=sM=G+9yjHc!MU$JhJu7o=9K}XRwiJN@-&QN@pTRJQGHfRb$=G00vaM^ zvH>FcW0r#X-)fkO=v%wa+IL8B8m(mx9AqF`;BAjOHg@()ywAG)eUC$m!2VuE z=w2%P=;Wv|b1Oh9VZ(_8!cdu65ciAAV#88OH3C-$#ncrVl zK5_--uPX7};{YhkYTb_ykfF3!83!=d(Rix!u`r&f0uNIVrlPvhT}c~zYjwqY7{M?h z3wWfl_q6B05QPB-kuv6^@PakQ3<`A0)1ufcNcrLx}dRx(f4D zU~)K@3u?ZN;NNfEdXkCAFgN&NN=>mj0u*o(>K*%eTw;RoE^R)Xz$9A&eYOnI+fBhI2>KBgBen)3bGD|zy52x)m2l{~Jx zZK+GYbWgZhJsMqG{$DpuP4M4Iz|B%Tn6cm`(G2ifYUd_v#tqwA9wP9m*AqP+mSzMe zj3UHKV{QnLwt#T^oZv{Ar=^CxNx0l`ZNhQK3wWh%WTN0R9|n=4m*;~0q*vd$epz~U zGo=>#|1jY*v0>XzF3t*R##2C*ULbiSu{$TfcGCSQcvZG%g<0N(`mtpR^_UT1fo8_( zW)UHv9IvFbf!fMwBmm+|y`p5;{3CAsNKw-b*ZvDOEEt(6n04dZ#j4tjre)QP3kC%b z;%eXw`!P$psFHIcQuMXrgoB_=OCHXa43B(L^o6t=B_p_&Je{TH2S*pf$}IrJtW+wy zVwUeS96+pch4OtyYYewnxLT{{>%klq;q%J`J&E;g-Wo91$Zv>`J=hrO#bPgdDXa|E znh6W;GTT-jVHJ8bq#o#qs=socBw6%ngP8|m1-l+yHa|_khK$6Lqy7_!lsG0Df>GV* zLy`1KV{6gb`&V2J*GDGZ#xkZ_Drto|Ijm_ySJd}wL@mdQq@NAqny7i1?x{{ch_}Zd9_6;k@DPdL=`Oz zkO8&BH^2FKKqWA4lr?!9CiA>Wxdm_2o}i%=QfNYDmtK8KSa?Itlp)Ndkm;vrGqAZO z}s2&-H-dvx5*!_oyyU%b8 z?EiY_MJZDhEHzt<|E!l7bCC@1krw?5cssW;)KLMRe$kDepvQ;Uf2`j+V$377pXl^f zk52CmR5FY;`fN@>xQRB~eCmAL^ilkTdb*}qD%zD@3(xN>RtJ`S0W~Y`2umQdUQ6GE z-@TLi=hQ1^)xC3<|L-gotf}mxM}dm)p+e{kh~$PKOScWy_LkoUT)>t5vP|@d?l22J z4MUZ@>Lqkk2fzEchxjo$UdxXsn@G8E+yO+|bhiy#P4^g|?F%T2t!PsIuDlNhB3|lF z-#L0_EILGtri9S>dH-??zh^Tt>UYurk%#Xv&IawSzv8d>MfTd5@UT;a1cpNC5z=>& zO$2Nbp>SZ+M`h3-(j-= zI~hH=bO07D6cAHW6`P+rzG<;FtfCrsz+9m^{7Sp79Y>l}iB{OIh&^58jQ zR;Zw&uQ71byb>cShA8Gmh?$HC56j6ps_J_axblU?2Ep#q%$A7w`ca()7?fjxDjOVGyDcx@|Jw?uXI|sJPryP z;AVNw*Ps$SeQ?m3vdd?7sW5sis2RGpH~5+st@+BGZwfMlW#cDP3*F1JCu}9>O$!V( zZcnJRp?VD4Hcge_PC1O7ntF8n_xy$yxui-_qEDj4qiE@=jj81AfiD2^2Kv(Z6eg@_ zfTkATssj40X+c+!C&e#HpA#zXb?CRc3k-ShG4qM-ZCYQ30TZ;OM)tXSTZ99&EJnH|ZI(a`PCDAB%ivU0~IR%uz3m2_7sIpZrJcH40X!GS0VU(lBy>GKLq=U)+3n zQiPVAx*+Rg@%mcc4TG^GqsZRU@mp(>1q|t4G!EWTu1jn{){#Wq{;*7R*l0||%#^%C zjn>W8`C!}rtD6FT3EqoT9FjZsXlwvY6=W?OKM@EY5Dc!SLwA@z$^{sl{fIZLW5-p4eQ_F9sH(@rg#M<;})W0 zW3;{P93*N(%j>VB&c~fWA>GQZ2Xy4ulzvH9OsQ+ z?%{QpH$deay;7jBCCV4j_8R52Sl+`|R9jY+r3&I|daKa?)s510mr;W^;y}qop_rk#pi9li%_oZQ|a$ zVxYXxQkv?*eNkMIINu+`2DK!Z>!V55pqJNB$p|Y(CqFU7F5?NGY7E#gf_#bv-vVn( zcQdx}*kVC#%KZQ#D12PkvFH{c^~&6<4yhA%racUaVLi@h$@+!fk4%OwuCa#DpZsmM z;_7GLGOqmFZ{_hb1dz@Dmj0^W94fwwM=d1MWu(f~h4&aR^2KWsSMK8yD_T?9NCPo! zIDGT}CMj|Epo?qvHv&@)jqFroC%EYN$~ywoghh8`8lSP&o~|?r`*DQycm$cSV8`x8 z>rudq6tz(IM*c)87adCt?~xx-67}UJG5_f3rXVW>g25p{#TugL5G!xYv4?T@mkE4vO=EtG||af!5HM?nC%RB*Qrx z|Nd+L%+?n_Z*gNyEr2!-2|93@0F2AbR|&Vs(~u$nV3TCEd9E}Up&>)C9U4U`NJ?Pb zRp%_-Jw;1d9bNyGK}d0UV%n~IJyN4A?{)0v8_#K6FBE=6`KC z6B4A3F$iHb*m)r;gA3+Zl>OW4yI-;%g#7J%3sldIltFIF(rSoX_D>uo_yrZ(4jFN{ z23#A?LJqESJBdf^?3j@uSJil91`feOU^YBfDYBDgjb5MF$Q%o=2GFAf_(q4TT~5Z3 z31vprht1mK=sfb^A;9hIub%v_?!Y7lpL};W&n+Bh_wp1g0wRo)bxqPs8b^QB$iA+C zk)8QZ(S|524WT0wGoBZn`~JfyglN|~;0<9S`MM-2;xJ*1*HWTm__}f_HC~Y$9NhiVxlI0&b;pIj(*7Rs-A9`%wLGmkv zHm@SHjZTLUh6S3zYVTOR$gd`HAdX}jt-DWD!3a~okE+JB7ka80l{wA)2x}M4?U{&F zh7$R~RTv3#F%B~;u+7eXlz5mq=-7}lP#wgRK^cbPJ@Sp9X}-8ORmIe zwt!M@>}t+~5yEQj--6{){Msg}(gPY-!U!ulRlF)plvePR(?5QrlsE2eBo}=~_!;mD z#NvwwxzX}$f9ac!!J}@Cbj4j6NsBIJOBf8Cw8br3i4Ho3llN%R6esO|Jj@A3ZD_8I z?0M_B@8R*LxU8N%!Gm`M5vGw_94N^yQy~n0-#Fa>BvuSsis*&!E6F6yh?iU)oY&jU z!eV#CvboTu;V}%v3mQrQ*Z;Le2%8;B>b-rdibY>l?UbPAu}WkBJ!ubxG+VP8!p6*6 z;3Pl-S%f3Yc?!M{ylT9cR@3|)>F@`nCd^^tKe-ku6{Hwa@^DTF(HM2iD4*kRG8lVF zphf1Yw7n&1!n!CwGqv~1`GE)2(F-wA_cX!{jbbYE7}fX%kU@MRHIe0O?tk=OJORG~ z<*qVAEl+OQAXIO=Ew<98PUsq#_G^rVq3cA3P*Pg;l1yO|j+TQvE#fo`d?mk(cfk!` z8nT+1R+Ca1*!xO;U!rZ7XyAg(jNn~7=X!gL86)RlQx@|^q}_G<0!E4?QQko`3>zP- zuT+U+u|;e^uMGH^m^BRdPlxcc+f*sWMN4fN_Vb@mYFYW8B0dzPgz#F5D|-iOIQmWd zk~P2RfBQFMmzAuZRRO(xXt#L3K>w;y$Si5AQ5h8O!@EY3Z zbS{C&M8913-hK+KAyOo->lY03-c|x$74jP%9oyPl)p^GN)v(y&qXE7af(xEZlS6#_ zjozS)W+bAbcC4=4$M2!j^HM!J`!$+IjY_X-;gfwI$3eGFj_6bm~)+{(=cfkoTka$2meqdhzSgn;x5nf|P zg!HEyV`Kx0rkbW=Be0iXhT|AJGZ&Rq?>DAU*z#dab!LQ!mkcm}#P298fpE3CW6y&% zTsW2vWXA#RE$qzme8C~P0n{N)r~bCdiOdP6k9h7Nay@DatSVFv{tnNDgRI_&_6^Qi zPeKHJx0H|4JxHd|bqtfFhpHH6*e%7sIHImj40!blZ|STLoogr+046SFCv5T>4e^J? z*E^Fkb`eOZ-zLFs2)i%|DT`@7@xIw)13eVZS*}hV+xUu7^r^#Zry)HW-298tf z3k`fXxU98kx@ld#zvtfoY;lCeLH|($tZ93onMGI*^2V91U>2G-*`DL7_ifDK-egR> z7_xKFe`zA2UG|Z=SH=f)RI|2Aa;Kq>Qc=7i8DZ@j)h=$qM0EJf7dS3M8=~nI#5h2( zU~@Xyn$$s`p9$HcISh#Zojv#1G+&Y%*5Sk@#f z3~O)BemS4x)bHGwAwDOtdvAtPV~;C#;hAZgAiH(m&>{{)m>)OoK4C_|sb0HQagjxvJs&-(xA`L>G7<~Bfmqmv2e?D6tq6?D8U^5;ywvqP7zML-(Zda{|XigoxMj;|7Nh!AoYpWlH1K z`Ekj{DdVAgxR1!zysuw-eCuSzYw}_IRgfhBQCDBY-iGnfScA9t2wz#;r8xCOQ74zh zi#8EXV9UbQy_u-44a~kWIGoAnSm_TQyak&aeRU7wgRYDAl?M<=$0wn`$HK2|ws%A) za^y^VH6$QR>I#lh3q7a!X(JPxlwB@f1+LMaJ(}o*|Lof`&u2(R)TTVVkXHKbH&N$( z1v(MD^vaum6b=r5Q;vY4U12Tb!sgk`N42jT$#5-@N?6j74I2T4KZ_GFlHw4-t`upq!{m>iaeRc&8 z=Kv?qIM%~(#Zh5rh0ODPwFwV7Y9}14G+O0{=?R4^#idD|eX|yHRp+7RH}-}|5o7T^ zhojK6Q*;l!vxFBqy)?;v_Cw=TgU-Eivsp(iaZWOpV@((mFlcbv^%##(Bf&?rLS3D2 z7Yh`mQ7+zLHE7`fDTaiJh@GbzDP1`+5%fT2Adl%yT*n0qiK8ko7p53Y(4~Dnt{q$E zImkDu5sR%eLj?MeoSAbbb zcX&_>S5Ao52ULkF@3Xnhd3Tt3%qR-Sp9!$L=T5Ns-VRbteDWFnIj&=%worv;Km8c! z9n4CLa+=j89Sn|Q(*t$baQ%CGRUs1)ircv4Gis}9Cif{HswXgKpeIS`;}CcnQ)SHy)xy*QK}hyIrrggxD_KzD_IC?9ES zek**Id5($3;6iR^Sw)BD{U@FlW+K}di*gp6TP@zOr)Ztb;(dzwxfgOhAKu0 zh@0Piuef=`hyRPdJhMxEN5IIV`?ASSLUld`YtFo}yMlrZs>h;E0;c7}339qk+Q5ZR zm~>&m17?J!9)1og%fl0A?9)P^;Ub|~#Q+##IkSt8jjK(ZA1`GHNP$v?4>6RHkxp3^ zbe=~(Wz6IbUuIT^`woAx2+0Xw>Wf_3M6Fe}XLRvC6M13nbAlTGO+^<=oKO`408TOM zpt-BmM4M_n5a`H0%Z=txaA1nH3g7;vu;4YV6?^vK-WY04A$0=T(3F3dJ>w?w>++Wj zi&3M5rk>*k6zMPa*OjB~&F}~yvbeF9;S%?(mo?7qs9G8Y{wxx}ZJUJ$o=v`4K7#*5H=@pk4d{BLJC8O&2!&g z+99??)ut&DL;!iA={-FMLUm~0+fa{>OL!r=h3sm-DQ9nJMxU}AUNXj(f_ORVp~u71 z_bitUgj?X%@|a3I!=3jZA9o-{4ccEe5B|08<{gw7Wt17pSN7a8#msi>E09fB8bh)< z{=Tb|p><%g3uy(t$j~U=l;jz| znxNK*Gh=(|@1LpDCpstCyk4ALn2!kX+=Q(cCLmN^LLW>Tf+YOiJo8w~2ckCC*XG8Cw}4cpdc%a(fkI9J=nHPmvet?0f#V!bV=KA zm*N2MUs2gafmV*7u(~Dg)^e4@H~P z{JdWjs2Az16x2!h96Tb^B!M8l7N2}w?0q|8UiqBihg2y8lL3$cMSDdJK^1vsM%=aP z6{=gJm^{5L?Sxqr4AW4FwHbpZ_T*DWMjM<>o&&d$9L;>fU}Pq*V3V^){15-3_Q|kGE3Sy~8rC**P^>ki}ki85^QW$s>7Wv4D3sY#&=jp-#7-;p5-? zz29Vn*USY?X3xwL%CBXgq9U;_3M0hk2ovb*wq=eF3OB#6fSSq&F9#b|0H~CnD;eI&4SOoIOTX z+oEF5*%WIXX4V+u*$eV*PWMI8ILh^A#Vl23nfW8aP`uUc1YM`h(LgA#Kh4yYb@J(`pSnvzWsDt^=$oBGXfIUrUE8Ur!~A+_GScy4 z=G4X(5chzxwQa8H8;;F>wr4oyH~pls6wbCIp8|hpG<7fA1eksrjz706nG&4K>oRTn8s zD1(G*wIW@V8Erq%2U+Qbom9L4&D|boQdz23=L&Y6u~@O^_BtqChDK4l@Z7HrQ#8sC ztRiMRl(6dxiloWj}cE;%mt; zP79ZR(W8XbBr}|*^dlhd5hKKo7nZG*--j==1SJq%<^#;8(m4@rxTT6~zQ?pnt)Gx! z!H8%uUZe0y{sutZ`I*v#rJnLYzM zY&WA%XPIEY!X6}za8@lmXW=Jju2!xteU5b{be0UL2Gzx46*@&9pNJIXTeT@S95QtP z?NX+zeX5xZ{EAOAcfV(xh`P&x#2h#5_VD&OD6X(6m>ZJ({(d2Z9hs4a-3|?vAwQx9 z!ljY#wc@+5Q&Y*6xb5F3n>F;gz_3*t$@v;SZWAp>jb*uqO|o=5?{pgXfaDbi>LYWH zmgh|5ae60yUUbV9{?yR)V-vrc$G)@70*pa5tvz`RVs>f^m6|z(RSF1L5PI8^SD|0p z%k~?+>atb9)Y1D_ZGveu-|US=Up;A{I`)tLW&h4k*oqG-6ZGe2`NS8@+3&JLcG&Ux z&BoKib_#6Q-|(XJp7hGs{b1p_Q*sHL>o|`#wbbCU@hgCFwN{3g+++}wpBpYd~UKa3>r&il#zk@0fK;Hur8UxIY zvX6~ceEJy4oijKndxU*oj-800{tqDvU~eo74Tzm;kUgk;ZotciVTpn&?FfSkvkLw% z39NYT=7L}|w4I(ci12zd$Hf?J3%Kg0=;Q_zo_&hEsyn-9_s}B>@qHDueDP}Qw}Gc* zd43}3*i+k}N6S+wcA-kc>8;Pt+A{qLW285CY%zldY+6!h5i}&J>;+yuaeXJQ&67q1 z<|<-Qpi#RLL@cZhBYDGn!N}0KnB%gYjt8NkFzg3saJck>@YyK`JrvXF4KQ8pG~)#a zAr{`iuYi|GFd!@5>B)8bBEFIxN3Ta(ZpfQ8{0(y9lq_1id1q<+~e-4 zsj9*v6t3iYAr*D82TL%8x{h^_4brU}n>Yb{K~8w1gRw`(Fe5KQ3b4}`JgZ$NOzy0> z!v3+LL?RzHKW)h&(c1%byha0!?vtr)NPy-|WKx>bdihSl8R zbt|hVB{9byM=JoIM5};G>k=Tb0dsiPf|SvoDk4QwW9WYCe6|Gt@5Fz?H56<qx`5+Y9~hh1TPH8SUiT%NcovW(z^`#0%*AkLa?9= z{R_^oA42Pba8^}5TxXe=Fx~2!K3j+Neg9x>=$A8U&3(*8_}=n95H0ES%kDq8e>s@4 zN`VP5`Pe)+*(E6B`4K$MyI}-_rhqX@4?};?E^_~}4q#)1LFFY&cuLH@fHzf_Qtz+~ zXVH?AeDTtlvzhrIUtpkpd##G9IAv=*n8Pt6o}REOH_%s+n6Hy?!tS z6z2EDkLZ#3PN1HP2xS_^NCAl%FtL%L06|ZWsecyB*UawDV22sSG?CS?G~Y;QVr=5N z3c7@HV`E|R(%-YhBX}I8^y9P0`LF!T0$GqNu2k_n@_&8$72FKE*bhp7pdA=*9IW=N zA-M>)71Xl?Av^hb*+gOU^O(WdOQMP?{)tb^)k)ky1{^*lFwcyt0_HozS6mO`_iO6r z%L|_esy73pN;OJm?*rRWQNaF?hsHr52U=n{BMiABg0?y8v=V$}>w=GjwjBc z7`AU^9eXUuQwucz5_;BVQMaK@+`iUB-~D>Ubta>Jfj<;=pj9cF1QG%YGMkL8wDLvj zv+_^Pr8pj>uRMBBV-{^v;qM#Xl5Zzr z#ZZ6GLnfvYN*F%f$dRQiA8-2f4gdwN3qBEqDPc}{uUA!u(kqFWKoWX zy`OhyqoOz4D$KC5`?ac^eUkcWo@$RXaA4Gz?jTu2@HlBZ2{}gu3*@%vORY~r=KKv< z*D8^p2x|EOpV$CL)V!76Pi*Ap5IYntoTdy@8{3UCvx|1AOv)ObIQnz;RH!tvFV!oF z`xP)}CDn%}_NSX|pKxzYj_iCG=m?=7TG%)s_knQ=|S z{Ek&1N(>KS=g!!ht-i&I5R$GN_{WsVAgK(hHf{`<>&fu&zTL#n2^|N7I$6iTK~|0d zcq6L&+9<;Ua>M_zUgJV?{}NT_60P#)Q)?Nd*PvU>)xn464-zKs)$p$Bz=+eIRDJ-- zOE3s?j8%;d!kfG+^m90o>g?XMU*N=y)}BAW{DZ$LfBOWe4WM){KL-ZHU&Bm2Ty5p#OJKh{`KGj6a9CQr^RDTTrNBovb6lVwj_#SQ zNdBUEyVgH!vQS%SAYd%k=2gi01FUS#psc+^?j79Dii@y>wS4O+3Vr&q_AFw6h>BxT{HW3Ei6onnIfEow z+~9G6DXR)=;e4u{Z&|8ecc;pDaAG1>!a0G0SZO>z==z?6V)$v72hzvvmmZF6EmmYb zdx6!UYG`C5<1Iz40Pe)Y0~`{$N4hOh~L5x8D&5U)PNG6SDB(bYO{_dyF?vDr9K-w&3eqP zr;Go4JwHgJ!HtF#iwF1(*E*WPfj}sp%uZSCBPU(RAl?E{H2~XHYs`%E`M#uHTFLJ` z^|?WcAjRsx#BeCw?cz}2a^+=q_px?7k^3VAX<;q8dZN@7ce~#{ zp;)^aFR`BK_N zAR+|VlI!0f-nXb_I)9Ia1LduDxICMcs5!S+X$KLr0sM<=D%|+vIQtlGy~hRpTU+FN zRLf8t8+?r%_kVtii>FIv9^{s4`(}W;-Yi>d$FPh_t>HoKZv~aa&ffPAzI<81<3YBM z^uINYtm|OYA^y1qJ(MSX8mmL}3T$lf5X-`_DB@tT6=g10kY}i5^W+V`52Y*tsV_y7 zQ8T*v?ps!o3=-N30ee~yG;TPT^P5!*ngs?d;6c_)?UcLR39(ypjn^c~G93qTwr52^ zN=?p{HtaY%c7l2SL5GEG3f7R^phtOC`rk}er5$Nl^hcY;Nj75xOZNT3Eba#e^7f)t zKD?;=%hwyN3LQgI_qa?aYwk4dpJN*sD%8>Jpb_7646rBi2$< zwaGh7PBvZZU5azTk-ct+hA5Kvo2B{)x?7{d-+2PX6=Sh<|9rULDt;_}1MDo)*c^?h znpjzJM8643OCjX9*Yl47!YM9o(Z90~!Iwq!h}>XQ`oWm%;1ESb9_h5{=SCi@Bx!d-Kqvv7{@;T?6A;@m3~W{9?-sT?^!OzX*7-pVh*1H zu~&Y%*Hz7v=77T$l4xhdL3g|sY_^5z#(~RwLk>O}3JUXtaTM09uw+Wwe3lvg4hTX$ zidO*r&Hu;K7jAgGqu1^IVNQn?Ao&kUH!SK!5DZl7mZo^|Un~l!aBY$(1)AV#yw#(2 z7HF+%pOUe%KsFPAu<{X%AM7!LW~y{!+M=DO8uk^l=94@V{`lp8FjTs+NQUt?_Tnd; zCu0wPOH3qGLX8l=M1|@MRWPQMYi5k(8Haelp}l;eO1|y3HSW@>9G>{t{4ADw02Iz^ z5Oues?DGzMlQRJELxYJAgG*M8&L;v3cHh)b`@uxgeD;CfnS(} zgX;~Zaus*F4~P60vdH0e2R!49bB5yuIcHFOXjz7oCu)PD#L>g&frh=4VMGl1%=HI( z20|8T1xvhbw)ar-)#U41s=b3S!DeqrA+NBIb?=xZoLcv}IO@xKenp9kFkAIz)_QPk z1RG4H(OM7^5mnROw*S@rPrOueuQ(utI!{>(%ES=TewtC* zPD-_gJQ7&Mg6IgjkYNE54Z~n5FB2H9zV1ieFSb3bo^raearhTM$l>2ta(c0Qfbpnq`Lv(?kaGG`}D5FDN{wev4|vDaD9t0wda> z=Z3rHzw>1U#WFcYslmDq1avGAP=#9x)x$9E=$YSQrcUVpSlnf=6Kd4GG64q*?*iG% zNZ|~A5VB=;(3L+b*g&r(n;MBVvd&dxicgvJO5b634#-NJv3k-K zwsat;Og?6dinRvbBw;?h^xVjN5XWaMIPrXfL@>%%_P>duCqA}F*#T{o-lD6KbV>c@!LEKyji|?m0|U1tE^Fp}#fa*s zod9D*BL$loYWdNoRItUCt=}9GkiPn6Ykc4GKc`}|j0%3JAs#W_Y^GsyT{)3z8Sm7U z7^0t#eS)#LsQsNIWx9_FbvC^%wRTCvFto{@f(a!m~lVg*; zpQQnzN8SgPF*Bu0G2LEl#L1mowl4+D#1u5oC{}=RdE`j1)<_z*p4?FEj@{+u8BYAj zz-1W3DCyjVE?ZfxK@$IYd?YVr66K0XmZeR?H+04~>xzEM?+H$x0@|U* zl~T~i-Y4gN?;$)sVt>qP0UU(QZa9NI6vLDiUg-N#_eLfojQSX0C2gNbDGXe|#^yP{ zL~tj~IN|)38mkRoJ!etTBfgShcKEC%c#@g+9ZLr@UP9K*aZTb6z3A4rjPwmS#Afbj z-Wr7>9YX;&1SOiq#rwTEse%2FaqP1VUo7(u5bZlV2T?-?ToAq;GLno9z_F-mE`3-# zQW#QC!5b6gV-`RfvT&ymT%}_ zgwujDx&RDt)O=U}1};R>;5)>%t8bd#)#kq4qqCjGjaTP z(CQ7iT(DN7dhplo5VxpH38>U6J#g?Tn&@J5rW}RkTd1Y;;%*&tkvN4JtS<;Pf`H~C)~DNGW8(amuLhJ|ioe~>)L1{r{F6%~peV>F%H8E2@=&D7X- zX2fbsjf^}91VE-^eoD7wUHZxD#=xyVL!Ev%&9a}u_RT5rZIM^2mjILGg%{wofejeG zbf@y*j1=F;c}+J~=P!x!@(mk8p=GWlvD_H!7^hRsV{~zQrf>di63xN`A}l)Y0TY7P zq|&_hyPmqTyp?2tPg^kx*oR7OG16zbeYo^XXvZQF^P54{##&>|Nwr?@q<(@JdwusJ zPMT*(GyvGaG*X=a30``VpKA2#Z_E5A!Y>=yRq;}NOCbd@yJ(uSVaIzz&hdT4xDgEA zmM|`4Y)`#1PilB2EStLd9=rLlXa-U_g1*z8IUetcz=tJr zEJ%^X+%r+C&GDPr45o!DSUP?Sbm zB^{^xX}sQA>3J-P4*oDPU&Nmo)YpHMs?5r`O`B|h$M+Eo1NX( zEsrcyd-P2=|DyE%Xx577MYfO0(vEIxV+b$b{o! z3>@=R>KIU?q%(e=HMw@TGXU)lz1L&%M&uMlBn&I?y7rIHa)A8GKO4R`XZ>W>`K!yP zCF{1(+gm}bz!g}t4crNQf?&I6UUyc^lx^*CeGXmHaeCCFtJ+i?>7@z~2Qnb6A-t+H zdN(JGL!FLozV`TV6m8hXNgK4k*5s)@`(`s_iqOR&GtH4+J?Ayn5#7Hkb#hs9WJg5& z5TF^Y&loG&2_G(wXpFD;3su{V85H%U@g&pXLx6ai4&(&FjJ(iF&w({i_B%RlLuL{? z{lXE8{0c16VwlrTf&fOk9}myl#Ea?=yUZwxZq_(WJbo^VjjoP0gow#VBV-6IR7)as zNx`8H7?0&8HIsZ?D^;x&?GPOGv5Z zwHvnzU**SR9FP_qIpV{Jax04TS|M@-MT`W&6u`v!VXc8YbKvj<$dId}vJ{prrOX&g zSD=^K^+`YyAwkZFc+GTmEi^J9;4#RGcT|->^-s}-wxq=T>{~$Y~Q3^)M(0>_-AQ=F$qCsc)%9E0l>b<+lBI%1z+1UfO~iuuk`MExl_9lg$up zhzid*a|AYGp+gm)Sy9CBxrk$7E|Od}EI=5UMq6v(SQA-nB+o4!zk-#3+j-)Tvq%$= zWMYup&Dxd4sX%8(&;qa-B*Ty|#yhH5t3S5ldXj?1Wj06@lXkC#YvDl#R`gO(xm4*j zWEzpNE-c&9Ft2H1$@f~7bl8Fw|O8jXD~R2UFa@7S=XfPc~8 z?OMfuZ6h)&*U)bOGByu-qJ;uNaO!--L7A(a-*Hz2i-2HbVNf-rzaRpx{LjqWAMYE%DumPZlRC zU6+R z8L{vMUI^E{x9z1GddQd!RzsvkL%e-MRkGDm#{h1I#$}{%#A+dfDDU>$jB2)nRl5xK zdaLIBfs-bjujg-oHeoqTWc0p_5E|u?9pjrjMAy-!Gx&E#>sy_fP@zCc{l&uORm&K6 zorA?NaQH@f8G>Ky?iJ};$VNp25eaA`2Q~lj|G9eifU3)Md;EOAUFMih)=)Zj(Y%aC zUOGwUpdcmQD1{MA2LzoAB{gM&WD9aRT^#eSc*P4e>WGM7fk=qJOk?Tx0#gWt3Y-+7 z0NDuvA-C^Z>;2H2-|w#to4q%CfA;&n&-1KjJ!^?Fv9mRe8-;4_W^@p)8Ga9{a}}O+ zlBr&G#A2GGs+XLP7m+GVjXut8O5#cMFhRg!+-(*x=*eRaAcpy*aog+{eXSu;6RsZX9!9#TsSU-1T#Y!)AW(j16sRQkt+$ z^qcieRO~1wkW*m~fDgH;pE(~O|6Ejn$O9QOa*&+tr=cCAq|fL-Lxq5UMst@OCRCEc z>-X`$b#?i{G>88`ck{C~cYDSZ%je3#2ArLz3!#;Vowl*+it$O0A8{xq0UL^$sZvHq zdjjS%*_P-TYQ&VJ;5|02KHJt|`W?wuo4EC{A*QyIl{|JB2kg-n5B>sqBGm%`s1I>2 zpz#N?b3aESwhEHQNR51G09U~Y01FEy$Vyo}k*gY_f(pjVpY!h2Db|VRAZTS=2JZXx}Qsr$DF6kZ%Vz(_fUVL=ly& zB*JtzY>s{C=9lyJp^`v$guN3|vp%J`3u0fz!Cy467hInTW+O0$si8a9WQuPQgIXdQ z>~zc)<@7QP49fvzoiSK}r${aq6zZ<&Qw){D?y+Vce4d#69C!N0h)&dQO(fwWlAQ{k zzP=5;%TzQlT4BA8sDGGW7zS?i*R<4j`X9EwRp*u^zq8%W z044CMw;PqtfjguYZ_j8Ilp&0>6uIjH^Q5l1F=#{5BV4nPXjb|NX&O^btvU5< z{s`1lp1v`xr`84)07@Ih>z_bB`GIlmZ|>%}q5uHWF^(+Qo~fnzFdE3KF#&E_boejZ6H2mkO)_9drp#Ft3ir*8E{>(|QGKjG}{#c#;XsfD!`q^nl3 zdkpvN^I_@%*Ek=scmQUkz&8^|LElZOU%%@aH$cc z=zvCd1>Ww%agArU2@`~l1XZy&EDh%QkbN!L_Jux1f@6Y$Ysgg>BIFo^1p5SG3B){T zNf>kw-D5_~D)9+HoI> zhqw(3`PkUDITwDE%iJKOd7V*cMJLg8*WFQ3NQlfc)Z=pPJ}ONNft^>5xh+1U-2)Q{ z?WRhf3hpAeM>R5)ysVr=40}lKa@SvVzQi{$^b*v>M8@dc|KS2tyH9;iMj<@25~l2%i=4k??_o6v)|o(38uTo`zyp!Zzb)tCf`0x%p=Z2+@k zmR9u3tFgL<_!V~jTsH0l1RJpSg=j5)hKnrv)EFby`snP0j!>P_Bd%5QB!^)acoqZW z%3|imcV}@!HEzFxgW1$YZ$qaki@E;woZ**9M(2_8QLe!tPL!GuwzYM+vHEibddMh6 zJp}oC43FCm>YGM_UG{`kc((wt9Z3trH7GHW=1vM*>0rdh~Vt5?bFGSYC zJ9k9Zu@{U><8rRnAbe{KpDDp3g3-zMp#N%wRAQ4$B5t-QN&}BKrlFAjko^|efdcDFW31R4n{6$O?&vip01Q5BJ^q)$Kk00p+f2H z3~M~G(HY+H;OM)5IK?y}uV#?)=ZS7~A=$QY+O}j{V32p#3~fX&`%YWv(M4CZzhW*8 zLkcA5!GtZ!Oq%Wde(zzl(O5tSq$%cYhu%0VX56biIx zNu5qC`nr2Qo0?_U(-K9l#ltoj4zi_{KK{@o64ybH%FBmqZdA7mKf&q1;xkS3aYth+ zWpk}_RY%muArShi*)gi5&w=je*Ht~g2LP8NR!5Da^s(~h&O%+i73(RVLW|lI_bdE6 zxDIfwF+aH8XZ?`5s;2Ygwo}`pUY|WlQb}Wx_00ZP{`;*n{GsVSRS4~-R(HQ}q=)E} zJ4WEu3<=|o_sJ!(2Q2QxxYrtuqJx-BSU0_8e%;)J&JvCvqeM{=ez;x``_;`zsjUie z=G;ip13KLl$zV~_R<``V$w~}D?}JNBO=qA>gFd}YK|eN?1gx*QbT2|bK#i$V(F%er z`yRTdMqS(Jw>97ZMrewWgF&xx0$0!O_qp-Fk)^!GWZzkHCpfxCVgv=vaS3jO=J)+J z-6+}sUQm8WA4FipBdrO=3J+JNII^XJn8n~b_~;lYh5Ifaee>L%A_ghn1N|HIR?yVp z@d3h%;!>LZm~_ghyPvM1D2~dt28~4BVXjGI-uX@UoIk=&E(0J&86}jir|XRTd09mg ztQ-)wPrqIYfhvzExUOop4ffP% z5mkU4`zq#m@$J9(Er*`_rWmWAp43EE&uVqTzH z)7Nf|^O_c~TJN|9c^I5mcoe~QFiZv@cPyC8^UX`fe@X-C6{n?{`FEr4+n*V!MrMGxUC#jSC7uL(JMo49U`P+Ok57)9^VS&=zo?N z+4Fur@9OA)gfA~p{?I7R0r2>j$0*n*4x?h#}65~Mf@F->gs40Uw@U`Y=X zXV900et$K%=H=!el?eVa8r^}qVZ4SuFn_}O3a8ithZJIv>MbT;5DTPbozbw0xY{mn zj}Dix?0Nut`a%bpZvtSGKAF2=>;w(kZrOP3Y+{~iSeekNk$ekaNSQ8RwNK2!uVA5jv}Nq*jTeG&;;JRfHq(i{x*K3^3)_5YAHYulLqNiYu>w0vYVfD|okp9+b472~+NKtkd&Y&>l<1dIw$R%s$gI1d`8n)8_6 z?w8yo;Tj0AOi-pJ+X8zRgm8s%YB&ZU1T3CRyGnXUg21(rGMzw5V~PwVeT#+%}MWHqGXsTa{qW|(`vEU_q^8d@fo97mUjz# ztgk(7xD0=RO|M9e@(|83ezHW?s4vEapn3+Y(ndrNScYYFecTHQWdM1>?6+p@3orQd z`^+|i=yj8*$S>Ha}6rMh)cUw#)rYVi)#mg7yMCXb!E0wk<}Xhn06g!B2z(2ItasO zl0pYf!#Kc!FK(1h?wz~C$DLhK;*t(X5cYMaYnKbmV-;X|_CA0a5s(fhHZPqi(o(|Y zxyHWmovehbpsJ%k#eOls9P$N`-cajBOU=*5CqRJC=u#Lx3g6tsdWN5PU}XODB~jsv z=|fF~jL6rmfj_pQyyc376vTnfyDqPoByxp;!rVqL)1Ys`+lYQZwQ${NdQIuShQxZV zqxwa^j_k{~6)?Y)RRzda!wq(8X^&#sgio|{q{ht51IdFCS^~eHHl}mXygc86>DcE5V!ie8y=^-?L;QZ-(g|zBzTwAZ|RD`egRBqaWmTzxK=z&mXs_+}(iS@>c6sd;D1q+$s~O65lCIrgea z`Jk*`{zDH7iJJs79N3$ZbCDG`SgM1bGZN}ca#906jN!G-u7KN_(MZQE0TUpYam|;$ z9h97D%K_FyYP+l7(}*$27yqt25E?8vAx+lTU1d^3MR6$BAHsD2?FQ z)pK}i=TRNbwS5x7sqJsy4-ZN`oeF+9>-JwWrIXtK**Hi~sfLL8f-_ zc3F=7ga`K+Uq=jni)XL}1US#rpzx7|t4i=7K~pM{k!O0=W%NL_U5wvA+rfhtV)S+f z#)}!fg0Q8tLzMuc8B6Vh$9C-E)@q!%7$&_hXwMb=U3+8BaO7>CjXAc_IdwuVqA^OZ zee>ZK{55uyKt?tD?S?t)yjND?7eGgmUX z0{}4}V(9!4lpOpCA}J3imhnJ~g&-F#=TO$~b0c>TK~>0ZiaET9uR zqp!W`$85eqhKNcP{ubUQx94|4TjZ zu1_@m%}8HOJGb;m_XwoG0q;D9Pb&+)PP2epI4F)$Bnd`8Hxjf*O7Chb6ao@@jWxoT zm*vHAxjFAS{^S=ht>nvo2w!7S^RXsBbE?mU>*CqoQ%Y0!#92^!}C_#^kLXOlc6^B*G5WBfL;V& zGurh6z@W`FXfzly9!XhD&|}_iUVohwS>(T(kxH};UTi!?izXX_pta$4LiT}M29$Zq=YZ8h(FV8$jBMg&7-h{6=8mkhU}AYcJw z3-Y~~htyKiTq3Se?k;OT1(&B1$tsx`1<45%POCMyDX7Bf8{LIII9U^3+53&f$3g$k zi5f&Oq&{Xgs{!f|4uCh%8llOgmSycIcr1PMpua(b0=w)^@Y7XvFN30-IuZsf7qkZs;j1i7>o(9$J^Do+8HAp#G5Iu#)@r(wM`D&>`h8HO`L%x$H z&!p)t=V5}N0?Y5Y^2V9aD~e0Nr;a(aUmz!#NLD#}tbZ^(QSc5H8MqIUU5Zc8@Uol+ z(6Bap(@61ImH}>$LOzfE-FnnG5$CZ*aE&M`wpy-Yx0uk6vMdq*ux20NK4`GeTME0c zDAV}ccmCHUe;eP2Weblk)Z8bj~f|X?|0-Sp^oF&^h1XpPP6?hjP(m&5h5yF0IJq%4MMX0SS8Y zcKud73n;}ugLxct1c}2_XeTePi>V$%01+uElnPM`K+w?5nlE>}c(l>DFX2cj^w5mc zSHMJi+Xp@h`DL!ONuOhJa4SS$Dt))f1&uEv`d8{Up>IO8j)sykWXNJ~{w|x%DUh24 zI)KhqxPuQ#uKvU!-Z@w#x-BXQ=iaFu`!XYrgAU&O81`cC-HAp`tgCYX0w6!hQ!9!$ z2P;*L7%cUPu<1LKUH~T_sm$ne#*=pIBLX4NaVqNQ8U)4?uQo=LPN*Y`N|y~?)V(=f zMj0JGxG45Ol1j-`(I*``ne<$ z7ix~f9fiuLxCmCWlMa)%1V&PcX(FQ0XX6s*~{sM9p91 zk!IutmNlQpy(lG$n~aCYz*$BTBP?q>gp`UDyk0zt+c9yLSjKJL4Namc?b=k}E29m_ z^@Vl)zc>q7tGj)taoYBxeE1*yeIpAL_o?Yv;hM;)a5H#ZS zWN_BYR+B7l7!D8~E->S-fCwZi0hE}~qWw`#h!3YMNZ}7VIeS<>7gAD4!q^aE{)f_5 zGz+%h&JM@6A&RQakugjhiD~;0xrLMmoolv1+h0| z8Sgye5!6~B>Hew{70%1k^^Wuq0Yu`iPeg&E#pW zS!y>~;6mcI(|j0aw6uq4aEL|MYNjy?WyI!{eHy?6>eSRxrKZaZwJW~;x*v8q4Sh*l z)l5Xi&K9WyX*?9$r8x%JP;6q_ zV$%3jLX6VbQ(NjQ%E5usuMEt-cT1&C!0#duXeNX130?$Fqg&Bu=C)5Zx)Oml=}Ls* zl#6_*E&dauhO%CS2@D3}L0FE>X?jpQ1e0sGiYIX|Lhde=3sOT5b+VwQ-RtcLQ?OFPsBpG6+K7?3Q0uC;5s^n9;CCENL0{ld zr(%1nyX&R(Ef}(xIc(5^wjU6au)56@hT(iusEJnq9Gc+y zxZmtFV=ga4HdO#Ye-nrN0Xcm_HeINK{Z)033cT|fop83h2)aW&hb*|U!>OE~s@xuj z8TOofCXL{Wvx7>pa9q_L$v-_j7o``7mA_E6ifHQUF3y4g^JokC(RZ6Qi6uuiA-Q zj@8IOKF9+-501Qp8hI$%8O>T;o_fgh;lc%#!8l9tr|>gWA;j{~>4QuxWjD}e0sui< zHXyd9%VD4lD6(tL@bJ`k&xg%t{Dj|LyEq40@UX}}J-qVD{s;_~M=w*A_$ zsUVx5D@|N`NZ#@{9+XK1g~<(*<7WL&##IDIbv6Yp+^=H;7I*rR}P}g7NJwQV*36#)C&a!odo!NsXHxAmy zW4a1LhN^(REc?NH0$eN>QufvPN8f?gC}O|EWRdF3PH8MZP{My0Omcv14dQ$U*t{23IhF zHI7cX4f|4VAvju%#SLoQaX@aF$?Xi;$32!lK08@1I5Q$jn_B<=M^o5@fvkaHTx$<9 zP_@k}93wWL<%08znfDeX5T2-F4y4|o1uyd7Pc#OVW_xW8?FS(tc*JCnX3}k{?+r7! z1QlogwNzYtNi3gL$JXBJ9zWYWbMXM6dq`ZF%wbO*snDLPRPuwcWUlVzeh@$+o^8J| z`r_C}7??l@$oF`hK(fOKNfdn4>6+3DzMEcGH;|7}wr1G&nCystF zO<^i}upo1U8Vxob_;8@WE0;a7EonnJ`A0#d^+DF>s(y5N9S9B#Q-wG{3$MS%k-zr8 z!r|}GlZW)M??93RZzk)mp1bki9hbczx z2nuamLZ8+wX8TH6k24m|&FhVohK;&z-X0m3l5a3wQL?CEucs%BTYobNlP-( zZKC@Go}KP%Z_Lf3J|BA2FTL^}FUiJ7bPUL29?iWcA(Vz)=VXV{D3 zne)n9jK_)XnnYw_r5rR}e>R!FuE`sBng;ch8y9_>w2tEmu!!eJ< zumpuK^nAFZkcEzTgnKHe;WVy4?~3q;rC$w~zVPHyv1Ryl>X#x7n+XmZo zfmpA#PFW`ET)|e;-dO*r>LF{k(3-{#%QDDpI!_+s7k_ofF78S2n6rBE~YM~AQv)F=Xh zigxbn*gx$?4U>*KA?*MMnN)=UYuAy+&2sYmRy z9DRig0BYKLyD{A`!ja?w)Z7gcQwXzx1(aP`RT5~j#Ln#&lL&vu3yr%^z&J3##MXal z`XULo8`!0-#%}}g0u~oqvfFp@?7gufD05<`=nxA-Q3O(I3__Jj^NhsN;#d0e`%E~^ zM4jGm-F~Hd?8d9DdCF%O~aK{+(@+fd&Sg&gZ1+pKB16tvWIzPve3_OSxKgf6j= z>LHAdy^vuLb4z1cHii(#=GH;y$yh^ivk|*<@@iAp)x40sEPQw7N&!-a{JJyvXE7z< z5x+~!JJ4~j(7g*9oM!KsybUU(JY0jHgqEE?(`FE05mJor8cgIYWmX?2d9rc*7~2G0 z;qyS~F!2>K1r20e3X|@Id7L#BXv^1ftOfiqzvwYTo^^1n2?zn%*(kw^_V&ZJ9}DP- zZJ?)Cl+-cH!=9Exti9{bN2wZ;t}ucz z%H)gP&Jo%NE942E7JdlI{;bV2@|gSeh#cR}_GiqBUlANFVd$quABHn0KNyNv(P1aP z$to%oTnIH{bNo4fi`tafH#$JeGAw%=sLiW6GhUKnW%+mRhgz;4GVfpTic{w2GXt9m z@lX=$n{2b@K9DQ?E-C?(1}t-1QY%^GPc3>ll422F5#hYWsXN)9zuRl}0Fset1il`~ zO7v(=M_eCm`&rb7dUPMs`(PhnWZ~8i+$8V7pJ*`X zmAzfbgfF?IkF?EW+DD|Ue-s`Z1LHF8V3emY3mbf4dcMcsqBzxr(UR?gxTo)4r5Ri` zV3BJ@o}BB`!;8Bbq8Za<3l#sUK7pU9GiEHWU(I)KOng&r8IaD?(lX=CQyqu|@ukp+EQdxx7WS;!iTAE_g&zjM8IE8Yb z*rq^WRr{&XaKmlqGwND*FwTJO$CM46sf)L{Mc!Ef5osaSmki(?>v4FLf=?R+y&g-~ zsx#ArF*Y}RL!K6j_;7u2gS#VJ4=>G)C~Cr7QS?UQIYZjRTg1DS}i4zFmJaa zz90dgD1YkuoI8*YP`lR1TRK}F0BJO68!{l(Dy}GLlL;kQTpYsP2xr&BJKeCMPL<$tze&lSn=?O z_eEZhlp?i686M{_aOgl~%mm;vpC_SJEs{e>cMzv=t(j=Rf{Ue#iK$NR!;Y3^yBuzqA(68j0D}zm&TUcpH+%%8Nz?mi2Z3_y)B3c|iRw?xh=p-}Z=5(WSMIx@0Y7{X z=^LAji`e*BMI;?3+2m_V>drpOd7q}OGrV<(82?-}Q*tMEO3yr7upX9&pwc!9nOIiXT0_c~pdFw+Ch89cVBK?Q5{g$eDqlf!Ed7=F zo!dE>IyH0&6)16@R4gmDoH@Mb`tx%SOpP(_)2(jV2>8t3vi-NBO|3%6=&bBC0!A2W zv}y#X`ZM3Ngm@b#i8qcst)?i`vIjtN%D@71c+Fz4{jX}M@x4mk52+(mpWJ$l(iX-P z16r6llge9pIpKdD!{vxkG3JI{U(WeYK9U|Ov;qYe@6I&2tZ}+`;hdN=?7}i9QK~< zDft7X#-J0VEN9%WqEQY^@pY$KCAb5HT}va(AZ|mA`-|L?<&L0Cr*QqXzk*d>&6hA3 zS7sRFffwm-pAX|1H69a@pyBG@IWuy^$SYhpfX+P%pT!H>1Gbyc6_g0c+c%N)fI`B? zi)C*3T4T?@yL;6*tXWmn?TEo*K;Sc!w5noC4*x=4X;<5gZdd_@AaKysl*pc={#M|Q z#m1&)Tv&wiLN}*yb31Rjxh=On+(Tl9pl`jJPRXU*b%Hbuk*LB~UAJ54d|r_S{KX&2 zAPE69WRnUdcunJ~c#a)XOB%zi!D#lfozw_5Fc19%TiOHqigK4*N<|povy_LXMW%Z0D8=o~uIQA#W<5PXz*1p5=mvCCzd-yQMWTR^R z1=G4LfJ7=6F{(c!p^FqmI62i`0oK;|?5zc|;c9D1Vi0jgK@Xs3Xv zrZ(@1xKX8&3p;Yafk)za>ag>Hn_4}MssnV9)0pES3y+z5Y$!rNahqX4w7-)PkS%ox z-zyjHx%Ph$AOs-#pM2TiiLZ1^0Gkk02sBsgR+9HnwPO77uQHy%yGqXwv13K>kM^?k z8~xLn|0!E^)BI;zxSI>4aP?4F!7hOC*`GoU)Gt9oos00bd09-Hg0gs>G87}y>C zo&HlE9a7xG^-h&G1d>Ds-TD<%EW0%*OrU1MSboMXwr^(+8|rVWtI%9BC^h`=@}{P| zEXA-g4*jU5a&hc`>8muoC8*#bvnOYMLp(-pu`RyF@#CpaIPFP zI!I}8;o%0^&K`QmbQiPAEw}f9nGXs#Z(o){R}N$}hs~JvJBHCp${tJhsPMKJqZdng z^ILR)-&rYVPspOq)!YSk@2~GM7{M>9So}s9eF%k+F9!|RD`+2p$Bn4xTZJ~-OEDfBX$_6W!2@^L4@b*^En_GmwU%)Y-ns(&}3T}@!yP~|hH8{D& zNBR%YUpZqF=b@~!(;Mp&<&@+}R91l-$y+STa%53bjxg{nrlbt5K>rQF`HN%u2AtsJ3;9TaG;ixkIt>>@6@nZLI`s+_}VuUKq%U$0-0X2dsG8CiOUhJltrMC~r@4Zhm zLl7B?ntIZZmdT6QWBBLwR!EhDh2vvw5~AZnqH><(2ebm)(!g~X z7zJPuV8=zQz4ea^D(E?CdDmQ3(Cti_)iS1u%vqBR@*!c!$LT*jwA>yFOKcqL9Ey1aKp#fk_^G zIVSPR#gQY8ovzUtz(L=36hpMYGcA9P98u~>R6Ui!ka7T7m2f-ikaOcRM4^2_r%!N@ zfTqbTK+H`8Ho!f_Z65H5fMFR4MqdTZUNIC@VnLIc4#wP@=EqbcBxyv+isv2c`q*-10a?e=^XtAo&~yTkq!!s7sV&Tb?6 zP+4|AtopdCVlvZWQk0v4aoKFOku+b(zgFXzF=ugo#+}F&fy(8aojh7SS)oOEH|c2K z8ctus5#J4Fa_jx=b!8aiU*UvO^(OS|#TgQ@kNMqYm=~H5rEzv35#t8HRFHpynaTuy zw^uHJD1p@!yv`?L*27%LF{@2G2~esl*I@Rf%Sw~m%7283IzHx9-}*%CE66DH&7eR; zLhMu?oNafCkBh|PSmXN`j0>^d<_7KZ8O$^esJ=sP0GPbD=9dcb$&Dvr+l3`i^;TNP z@d|t|)j7Lsd@!4i{#3@5GoqP8c1VqcId?b)rDhN}P|N8$7xM3DE`=KCm-`b=8xIA} z4?~?pdu{P2d2%>KWV2V5NsPpJ%cLFGP}91s0_& zAlyKm?o128!=5DW2U;Q%S2ia@jWQ+DtUm|BXpGP7Xz?k@)O=z0p+2f1$N`7E+Z{|l zlG5tyNIc7B-otaRnv+svNU>*W1lrsF)b<{awP3O(D9&!9uhq?2P28;;dVSQ?-o5q6 z+8}rLcDPQgo>8*=!AXpSV9eihjK5-}400Xz?lJf2XNazHwSo>ppNJk(A7=AKcW_4A zSLbu(goK|RA56;cemrh?+~K`0KT6hN-o(3=b*uUMYmfYX|NqG5V#A!?e5JKV_J+6o zn+ky!*7vs|!cFb_#*z2snhSNaYR?E9(iC0ZyPY;6lyhSjwXYrG?Z*BiC4oKCrCG8^;)w+fJizq_#OGu7TyH}8mJ>U+8JJr)2vcY+b-cWkq z=A#<9HFD=>Vq?^>FTD<#%Vc?`*`_XObH~=gzi+U*f%nKko)gZzMhedr#Kj}3Y-7NE^L?A_t@;-YNA7BghLtQ*mtz&fm<8ww!31nxx$V9${ zsn_|>ntw9sXxlGkiXqGXr|(Nj z>!O0P007__Uasx8xa#tw2NQ$6H8{Cp^&wum;svPyy|FvD9*->Q$VeMt$5`MJFA*GQ zI_wX%*0ukEJ26d_fMXWVEXl!NQHRXweVDH>50%8wC-AApp3B>Y*TTmB{BOrgrgG~8 zCTklrjEDqq>c2}@8TsLkUE)DxUw^i8S?BlENN$hpk*R`0h9Qlky-KV(d(aX~-SX0Z z@PAwb4NJH;#A9u0rJ)yuT%PX?qFI*M*GZ{FSX1fLZeKbRI6AKL^)a^csVsz`UT3@y z;~o1()%5AO95@*hj%lb17DFi+Yvmejp%5V&0#hQ~9Qhl;liGK)v8xp$j)-IFzVBhR z1`X%^Zt!{JDL?yzP%&^=z zardX#)yQ^Mx&6C4g~Wc)r#TqVT0J9t-1~p#^(?kOkEj1a)T?a(ZBG?EpPW|%brlV4 zRIN#PY89tyf$$sN5r`ox#DEZ%hU10hg-EkT4i&0e+)&kB`A^4L8nl#wrR2tT%lPx+ zjwzK3Hy-&AP#fXEvi!Af&76aT&&k10c~5*(`CTQZI3Enk76G*sJaXtE8D_vryO^fb zcnhkaP%wR-%l8lPn_Xb2II4_FGj`NXu}OJdSxP<|yesuthNu*8PdHlE+KgF)G~4-aHXjTi<}@otf703rq%`a;VMo{K)I#> zg$x!$BczSfMx2?eXL>RToAWw;BsF@6HfTo{?+yb4%+QFYi4jd91XAO>oo)Fm=!8x(r|B972?jF0-Bnzvc?2?Lq;zt zKA=&;tAZnl`)=&DQ9kh(7-ge^TrT>L*JiRQT(z{`^*^b>bgziN7%9CK<1wSfv!`s{ z#M!$=MT*Kgx`-e-4*ADDCVF&9=gPS^UIA++|EQ(=e>b*h2qmzvY!VF~UUEj&R4aDU zt-X{k>9$+|9$s@eI^podNU30=sC2%w3eiW9Li(5V@DB7fcd28lN-BssfRy)!$XO(=YJ)NZMDT8&$-@3=%4%F1@Vk zFi8}E{HDnleSiVuUu6`@RAXL{8HnQ8x6+=xM655)-(~67R70|eT80gf7&dKt$FC&x zVALA`%%8N+&7(XDFcZf$RvDt!tv~W@(GC4(h`H%fUmp8q@>+MUzh%$8jpG@-!@~^{ ze1-HG4u%T-=~|gF(a0*tv@h-grWXI{ssU)!f;pa?W#hLJlOS-yu@0&u#y6vFhok#9 zGxqgh=cs0OmWmw|v)EW_qshXJjld+7cs#jEZl;OiI?o;7L#$?pY@ACdWDI{4mO`~m5XN{`#*fGIs-Bw{)H(BCOrKib#+;n!5KXa< z*H~ahMW*3E+(_Ny7%^R0jMpJ(L~yBpPg+ILq!eJ{_=(t}o7z{UIbD}`yi7^u)3Sx} z9E%#o!|~Vc9i1GL;b;cHW2d1Kd;pY@J+ID^gcx2AMg@i)r}qig>Vs0iso8U<#ikH`y0{_S`pOx2tn^C7!KYF;|=ChYXD)OFL?)L z8hXEkXmd}?EtO;2Ujh8Xic0^654LKT;Fjrr4-%c=OmG*9D9A3~dy6^+#ruS(LuGve zTmitznsvZzE5jj(_eoDM96%w4h|*;y3WI&!6KM{-&qxy$lmVG;ZZ!D~0%EwJUW8CN zg(5Sq0WazbeGyxN70c3c^v60%c1=#7KA(dG7*D$X9p}^0_4fW&y(lxnkzDu@0vIsR zFK_c=`%KB*Qlq1tkOZ>$mKXO}y5$81j7R{eG0L!A3r(^I7~fdGhn_x>BRWJ(NM`N9 zPZf@ejVox!Ml(qZ6IJa!bMH>4{}dbIc+uV^LeL$?BE$U`kG#s2A6TP#yDoW5%-)7o zEMlD~6RnP52s#lntjhXl5`DmsJ9xaF+Z&D?w;(d>oyd&k^^kqR{6uS>1Z&Ynung7m zwUQXd5a!p+qr4I7tC8tYj=qCc*8Z~-Pz=BYxKj2(s-iPdo%vp%a zqc$qP-w-Nryo;FB={{?BPLUyz8mBf)c0*N#mkYgakGf2zn%Vxl*BB#>n5NYT`2_gl zb21}YD}m(M`V2fS-qSk_e@U635jFyW>1HiWvcYx0c7qpZpiS7v>xN+ASI4Bz_n=2x z*W_WNhcySYD0J536h!=G9DuWwPEnXVx|){5>l*NGI%r0`C-yE#DerD~Gr_`j$fdAM zSTmy0OT(JG+oJvq^)Y687mK26cpDCq6PIMzv=kbu;gu=Sf>Rp#zF&>_xBZZ70fAHSjWk#t~8PvLB_6ptb-O|gxU zyA3VSPj*sm!KHx~)!gMTlGkhb!-#w~g^wf{&nM(-2w_h$Z9)MCO3AMyqvO6OgcFlA z(}>Mm1=l?nX6-wJoldAt1Yg4C&oVgFMucJ-9iDEjE<7kKdlJ3kXgw({WePInY)DA;=~H4+t1{nYBd;mC(Uh2J8D zhDrrZwo5Y8n26T2&)ofhn7Nh3nd?>09l=cJ6MX$y&jb+mcg|VPqakKwx<9=@neT{D zamTxVqUt~RWr0+!`!0k>8%x8kK&q;9@gdaB(^bKp8@^=q2P^t;Oj!w5y$4y^>ba&)~})F!8EeWKPA`; zKs_6%I&L~~W}79Bj;J!oeyK9AQQp|-BI=Rr;bKwKJsZC=gYsy@9Gt}OX5lnX4tCGh_bZA%!2Z)BIr~z@FWd%!1|Pp z06Tlck{}gbH;g$)-!$nTA?lev7QmU29KakCm(n#;Wz{k`d?@ur8k3d*IV*N z)TK!IEU{q?KR{(dwFPT{!cIZlb`bvExqY0QCDwe{{?Wix!U_P6W}bv};2$f$u`9TVtnM69Ju{vHEJvfr8MB z4z-JA)qxTn_8SVHojV{)5io2XvUf2$_PzW$o`Mu*luu0AFPjO8#P3meiP+&jZKQ{p z)`e0g_X$X)BTGN&K2gf76hNi!guF{)*)_8N(yDVBc(fnIDTIQxM8aas0il9URq+7r zKQF-!o}t|f{W>YP17P9I2GgzT?94bLJ)a`!SU8ECjRw*YP*eKyA^U!p&5N6!2vU(zw$v8-cW#*AqF9B-b1}1Xi1K5r zL0QlCZt6({2oNny#4Ct9z%|$b>8lh`)!}W>-5vXoec_mh$!AIX>uc|vw7VIvf_-6# zfnyh)HYGMN2`Nax^1B)obJUo zVETv_?F$*|chnXN36y{(a^v-z?iuR<+t26=V+^2|K)_t^xwK){1s$V!3*OROADd?y zZI+$E9Hd7dnes?(!(3KM_6Vd#>fR#<@P6yLlZKS!6aP8iQU1>}mFh*Q?g|)PmkWRb zUqvAm#W?a>IEoT{(rxJPb=R^SAXohsF?JYX0GT?~4?>Snf#~biJf`Y?eC1>3{ghsE zkZ0kf{85RM@hGH6M=JD?;y@q8sJ^>g()IMF=xXdDX+A`scP{M%X2fcq!p}Maxs^EE z6uc_OS)Eg&voPp1n!3D3UD>y^!BG0Yc$A_1z^nt7;l`m(?uwY2wO3vC zAO-_1aN-$YY@CbPeC5tD%kec*YocT9+AA+qdAtHJ%auQWntnaA2HI>KHS*DRTH+ z8?-hm&X1}53!N5LHhnT{1sWI z84u2`fKNyrcGZ-z-XKn?s#~*QlMyhZGJ|%)JmBSfIX=S`0;f$yp>NTo0s?Ki>YR$l zQ9aKKD4CgRlo>ioD1p>)U%b(4jX=ceJ{|R|ryxWJKwx9;5`aep@g3&qK6ZM-gvUm3*Dx@z(@ z#CZ;;-P0}Ra`{(?=2)vJE%iw}&OQ(ml-zt*nc1!j`l^6>{`xRN2qm4&0_;=pK*rw^-=fO%k?$P%>yfl67k|{7%i@+`d%hhfxPwRsUC2U!bkWYK5wD1 z3(A%|s9owwReHnFU^Ihwdn*~QJmt662On=VQKtKBZ{#7s*)?g%KO%)sIB`mN#WW>) z)tdrO1_+ucH#3SJB&UmYhq*0Y!Vz=9`epev@AAE>J!bz2D5x5G9Wb1FTSnWpag7Ju zB-GHgs7rZL<+I-%rX0<4>|t1IW1~$CEp3-k)Yz-NmJ)Plc|86fO8jqQfP~>If+5C*-n0jgXE5=Q>P#(sAz22@f2{XvbHJ({uzPE&P!8UOcM)@f!7) zVd?_yy;hHxzj-CSur=wi%!cmaZ&SFd_}s_c6A%8zUip0X;Wz36uXpe9Y_z@B;&Aa> zC}FPSIeFdxhnw||cWn24ygbPx9x-=~op%6sK<$HgI$-@eVyUxrHPO!8uij-zbBuA% z$T_}>aS0+GO!gX6R*84kczSMXTu1)Xg_JF7Te;Lq17zEGt=PqA{{5-?AwJm$oG6~J?t}Oq-taOY~JD%s&!+@3fA*O$6 zWqfo_;iK`WFx@`L89v_=7QJOlM=B5OYF^iSz4>Lh#S7Du%m+y5Wv-*4n4DJEX zDB*>yxVF6Z9HEv$1;mw=LQV4p`7Y%Pw38C-ey8-bocYO;9k_+SM;)c$m{b$Z>2H$CqHXWoA9P987@>;FZd=7Sv9B&LM@1;FCcHU` z&NqD9yh%E;U|a0jT%&8*YpD`I(-_6~9NJ7zc&#Ic3{9OYR31OtiVd$`!A5=(-*)1ap*K!Lr2d>b(i0DJV)K+MbX z3o*)whXcHryNtOYB=frKt5`~1lmfFI#jxhSsYF5A`mG`Hvt#IM4Ju~pYj|5h?fJVV zkVgRMBDg%0(QbpwoAY&&KL^VJfPBY-gE|lNW-ohZW*SqM?gJ@r5a)Yg`(s4~6|wz6 z9M3i5iA|?2L!XI}t>Cw}lh>y5(d{4qZW7Kj5HGs34mF!r1GTFPb*F9BmB>=WLPS?u z!2^`Lj)Leid6PSrvyh@Txbi>a7#fP2{hra8y^vlp3#AyM z<$-E>cZ5D1xu5?%0620~0tzfA`)@(@?e0w4)={qMyGfEf2cgN&q`e6vvV?iTQp|X4 znVg`9o%@H7eTZzHjmgC(hkq-4L}ir$-i4isdH@7 z!XL3WaS-GHbl{gC*#fO*RJMDeqQOT#^>y6b zQUoLmmg>dZ>T-2XvV1ftD;y&Xo&L2{rda5kdg0y=bqpX-XlizcU^39mRM0fQpg`)D z?GI7E!u`LYzzWfK)FqueR7K=}t{R?$fnw*FflJTOuSVIDg!q58<3z>Uj{0sfJLdG3 zr8eYTiDFd4e0AT4U12THbI&Fe9pFaFervtS%e_w(vY&>QxJi<#w#-iGAGewDk3ueP z!}kC2ck$9|&{W6}=MrPLF{tbg%s;hVFW&|xwyx@4-gB=Jn+O_o^F$8r0jfWL5A&!F zz;O#mQ=EtAy zX8#AQsP?Y91=UR)@t@UxAf87javno1tIQmiGVVUnKzu$<+Q!XP|5?U6Zrno9ifqyD z`ubw#Dvvd#w6KoFgw`54sv*Ne_iA|D3s8wyN4c2{00JAhgk7DZMqF#v8Ib%59)@D9 z$PIYu-yrfzj@t58Q=V%A&9EibkbJcoEfW%nyEmV?ilL@x4D>V88) z3fOo-a!zY{>%lz^a!QKG>;Skhnbxlf*?72}MYa5f`chYo+ z+lE~-3Yr51km1P@t3lLb(TiCTekI6d;vZN@ZU4MxHhJg#@bE836#^GY8p_bVV zG#Ayp!pBP!s)6^qoCnRCK8F2jgkxaqlsgaiW8u$F)iG0$cp~ZZQT%EF?K1Y9$-#yK z52fg=B9U?7rB<}jZicSv^Fr(l3IgqE#x!X55$BN7cAPhrxB zqRQy#=3u0Q(JBnMjpp$&^p!xjo8fVH%5!&lNX+W*sA*jhz|jkm_Faf_PTuySxdbBoRJf z)uP+3_?sw1cp-2AZqe*iXkHx|7gbq3bBmN2Gs$4+mt*FQfo=sN>cLOZ9!C1~(JAn7 z0VT+}18_~sIPSZ6;Pst=sDOsq+-|!ih-0zr|9)qPO^?6#9xsaU3KS7Wv8uaHmG zA0$W%*34U`J0Dc}uC%F;;0*eWAM%o`z2AC56_)6N8>_H>8ODy>Fd}{$ zBF2OLg$2P?Rs6xCz3_6wy_TkV4q$e)hPi2O4Q~6Ni%waPmR&ug<&oueN`rstcky9c z&V1s-Pk^a+L}<u(^{oa+fL=Ag`QH>wQlQ zLKg`q`MQx_2*2B$T;^mrnwls}3(W9c1s!CGitDJt*nz@Fz0(`*3FdrbeNM<49#MbG z0v|ox-f=Xzl$+7snU*HVX|lA_=F8s~`U`j-y8r&aS3p89i|aJxa_Bm#Lj(=_>|(so`Q~&c!E&woH}>h3yXM{FQ#`C$8vh%>GzUQL*zOogr)yHQP5k`6Ul@n zu$998bvS@>FFaZP!0mpO8FbFTM66>L>mW1=UkNI0iNlpoI30!n*bU4*}v_e)FsGNqZ>qsoF zfayz6XpPh#?W}lg5*;V6We<_wfht(;w^%Uh$tSta6L}xmhsAnRCD}C)#Sx z-8o(YqYb?I$CIKgHHy+4y~OdR$;S4>{;J_rEJ$tL`2i*hjPwQ8QV}IEa2bv1c3VRK zjB5wGTz*Gb!g=19lY|UswjIsp7rr%0gRyxj%QrTrf*gSlx8Ve@%ocQ0i#3YJRuM6t ztk=U}KwxI*>w9DPz{>T?E;j7rCX(2sS#1WAy0dhFwq{WQL?`e`-X1MES#~UQ4!oU@ zoijBZy)f<|(EQKn_Dbq}l}o7?mRWPd!>jK~GaZdDJOZDtA&1XwERmvlviFQNv7=xQ zAiv#pL3s=3GziXveVq1-_)5S{l0Om54!}m&I>;IwG4&MC_QTMV1Vo4MVq7|cDnC0X z=F@-|>Ns3=Ye)omTRJt+P%>Fn)p1g{`(}l(C-2=~KVT7RZgV zDbO(-h&yR3YDXm;%;Rli^l%?&$#CCM`%mM>W%;8gL4jRK-Hu50wS*C(LlmY*8nH~2 z{w^R65G=2U(TBJdz%4j3kIw$y9{Y~b)^bnVZE0-wxD%K;NSp>GBlid#Oe3Pu*M4`> z4h)BJr=|VAIW0#_3YAg_Iy8tPWa9(#B6fj;x3Dz(m&9^fvFWls?6vx^saEvx+6x)xgLIkZF8afLSP8u@=A?B zKG}6i2fZN;LWiAu6@&XZIzcbqW+haQ2!&i!DdM?1<-FcoM;#s|`Lbl2(|f!!DJIlxsUavMl(rrF{2k8Jg`m65A89X+wKD%ekFqP>g#?mfzsx<2KbDZv}4YL7tK#s zh4X=bfvioz-Ma1e(@9MLh!1)+h=fF&hKt}9a=*M z5(F06D4tF}AwLK*M`g4V-?W&-~LSMWqDg6w{TeY`4!fX`}K$tx4`8 zupXhmj$#TgNj-JE3szE=(p-0aZ%qo)Nb{YZH64G0?;b=ZLU-*s-zBRUft-`_x(3u& zcrBw%8BePUu?ReUL!zgBA#jUTpjRL!c4Dsq zoJgrlwdoH)pp6;?CZzbls>F;UNZ}L9Xk3#5AveNtTRoWGZ-w95=_0x{tUy ztxRYOyr?iX!7W+~kNNJAIVk2UqsV2)0@x+CE#zdRhrTY7{0fv&^Dz}` zDDlS6%_SVt4m=~LJ85ca%GgNA#xf^CW(R^F7<$tH(CHM-!RQ&Xj|a?e``*`Yni7m_Xty?Hv;SU*8M+SSLL?VWI(%2F-4Xa}wy)np%kP!rhPvpQ9SE-{lB5ZkqY??Ct*vqPd?rdiOA!Lu;{Tv&86@!VE%zcQ;A=7sJRVW7x@sj z8^z8VR5C%mm(TIdTe^}T7+Mn`U*Ptoe-yqxk+}V}gE3kPvz~dUhav$d0%S!SgPt$! zEz$+BX+XrNg!jqvrpt9+ii#{a1#65Ll18dKG~^r%O80K1gP03)uT*RrJc?rZ#_7dt&2}y*JYp zf2K-3>GU6gNiz^N^jl>n?|(Id^6hvMQb5#A5<1^lb*QeP`Bob}fNhd(+viLGP}*k{B|ZJ{|rpKX&P> zS(AVC+3(xBZ*S|k9XC5aF)9|xf~W6^fEfM&OTgkmR(ybmnL4sxuKb`?#MPX=Cw}5< zAilITCAa(7nX?6qPD%d6&=5-c(EftW-&Lv0c~$e^mi*a7_t=u zl~~~?*Dvn6gr_1id}}I`eiX{$;EK;+PRrEHWrsR#om0Q@^bHK0$WIK5ZC=CN)9Cu*dfQ*U zy+GUN^%|cTTNYmXE9B1AccUtgfNYMlo>f)y(csvy1%T9bKtUm78OM6jMgAkklNe;` zT~4dJD$m1yarmF2BXGW&{YB44G(y58kQw^bWk3e}giZUe&QDXKXKzOPkU)c{?}1f& zl)d3G+&&N}x70bru|TW~kEVYcN|jv(t|Mq=ZphWWLi$#V>qF&nFGN9_(==N*x*d+IApNa8#inS zOBi&)j8-9*(}V+w;p6DzA|mFE@Bw3&xty=us!`&G&yNN-8;6v|r7)1QwEyrtuUxfh zM~p;Sv<~yroi0Cg7>g{8MvVM-X9~I~3!w#%iVETNH4ww^t?dASp9R?@dPb}i#=jA=4gJu}<_`y#>jTji$9yo~s5g+S2xGZ;`T}QGU zIN|e|vE&ar-m(m@iL=+s|yPH`lxr z%B`B~$rMJXGIncw3^4+l4XbP4fS)fCNt>JBc!8)fN$t2qe95+1bDyi^q|`4u=w}Ft zW0362B!YOTv+U1i1aQInv0jt6pzBvxp=*4wZ8}!fMQcbJW5+PZ@0d6c>1Ub-+!`i!uLQII9N$!!7FZ)6^1gp$JVfl5%m}hz1CpQufBcjlrA_Z<9HlRm#ve~&MkyWp0s99=t+OpB1LuPxr+ z;>a&+;|F*<6yJZN_E{`(z-YL%@D0z60@Q~DJ=EnAHEU1qe#(IG+Y_NHuecWWQS4*f z(Yuz~-xgwIysqgCX*0TG9@UeXKmdq8jA_mj(!xMP2+NSfgr-u=%R*Ga5KgKFuK%o zNm0!F79?*5Ic#JEGPaD6)8RRP2T2bR%Xyrlk(rLF>lz#sCHxN#2|sd)uD#*q^p*y1 zz>gqd7KdfrA5cp3J8=EP4BQ|K<6Kx05F?u{y4qep6hb^`OgG7rsfdB)4zgwZKJpQw zeQ@Z>in-PDPc-+ATTJ^;V)>acATHj+Gx=xpL~m%n(y*-^3<$&3fGiUp?qBTXdH~P@ zevMmXvHwEiYZGuvTj4|#yD+QHH%b&f;Wz*d1tNAHq^K`~ObH#w5<_$sd z>n(clR6`{VLGo8(V*`aBGNoPuH0-d!%L4)_x5-4j( zZ&WNY&E1VRPlHs?5suTo(yb+D3)CcG{KkJhb`-isI`W4swJNN9D`o-9&#h@w$Sxkw zjfEqCL*L-}7GFfL4MqyDOJgh5m111if2BUqVn>3f)Cxs3!3V_Sm3i(Q8H~ zvYx*6Sc<3-_{5|>5W-Qr7voi!Z6YoKYlxW!7fL$*=tSJqx-fgF(I2|*u*f>L%jraOcudPxI91j$AsY)04T6cg~@2sZ{Z6bZ%23V%kBSfY2XC zc(JbPKSV`^KgD_a z88!X{HbJyoZPcUqS9NC`PD5r^#}}vOvE}?KA(g^g1WhuoZ@|l%aiPvLj?Mh5+$n%Y zlDQUw?CWfTDL|ST;v;q+w)J&cP=T$;r+8ceEY0X#<+DSo(ST%x+Mz*}fw^*qDWn9T zx#C}Ldy+f}=ntdN63qge58a)kZ|ygRzAnqJS%Zo`w4evTbtKv7-Sz~Y2c3&usFI$a zP*(6ent^Z}9$1m#N?<-nwNy$okO=IWk9ruUjBr=Thk!zpA@>=aqM7P{K;f+NPTQ{Ug<)qW-;O*m@97py0n$VGw31iI}Ak;08A-T^#$(ecU_{5RyHawqx2CAJ~z90-i5S ziIwej{i9)5s18k^8@Bw?4-w(Ut-~vR{$6lakdbKUaL06wGD6l&$TcG$ z0k=X1w=|vtO+NbmTyB+vFZ`Kn%j}@7bsObY$5Dhg2EVs6q~J3l;s%bvQI21NUkm3v zmoJ&Wj5g(@VTK?TDIY3?u=J-yRYRPIw{*urj^{<1Ed+)M|IHR8kH;?OBnwBEfi=`f=3AUGW%iNgamG|k9 zb;TvXW`;WryHXo#kOCA?d6Vgt0bu}c0Ks45%9|Tdv5CArjx@c~2HifChVHWf!YLa_ zdXqrl4JueKv*aDpBcb4u}&6V%y~B2$5^yaUf#3$U+>RMEn6>_R;Lz)6I-H zmTD4O_Du7PQZ4X4N-B{wd*ArUm;9qfW(IYw>EYE7jzHViJyv81ixE*$9ROcrarmU~ zKz{L<&~YaOUZlNGeg(6THs5gk6&r5Z7^;Fa1$runZi{`9Qxhc+A->}%M)sgLVf3gg z_)C~$t%lLHhUNRcEh@#(s=t`!daUAZ2Di~>AIDpR?@6ZuextsE@?=6XJ;JNRR`ERl z?V&(FdL4qtW~i}oUDFI3pHrSt0NBSZT5W_XtdFrnCb9@JPE8V^*vLjJqt?n(zHyXF z!e%Nu+C7)wrA~2=jpA}ZR}BAOlDDS6Fwg?iAko$7fo2po=0ZET2;cQFSfkpD!Z1 zK6~rgdy}}aYBMfCJWc$NjvnV*p@R9^?lmz??R8pnLg7k@-88h!c#8ZIt+0;RyQLZJ zqKlRkAcn2>ZsW6{4`ZMR05#H>uteXnkr}IMJ`iS(_f&w#H4i+|oywmrjU6e!jhRhP=H z<714<`mR^O0akfwbk#xG6G?*qIgbC?`U%E5+FDN?P_%M2BG6?CXL-Zh#x5PFEWQUh zx?$8#wS9BEIge*44c$7jyf$H;yeAd|jq*H>-8H`rWbimQ1A?5-OxO};FZ34G4M))Ad=dQH3AvLF1KgriVz z*^E4!S#!8hF|8~=I+J1vSpNI#*ogv9`v9m#H5;XNej_g>Z63Q(2iKog-qyaQiCd%n z>CA4@`QA;N>Sjm_=->W#~)Os5+YG4UpINJ(?!zFmlHC&t}sX29IR%;l5N<#O{Jer{vN4? zH#=*gK2bZaigQy6NNiXkz_bP*k%c0XO*8Zc6-l+-I>N|d3W-Z8wuvlDnP?0q5}A|! zEaL1JDxtb!$6VTE%n16t5h>=}cj>HoIXcx!E*c%BzZEK=Qip{@E_A&anBZJyhCtRTeVDfpApJ8l3Pn@JTvE>+FN(PL$#4-rZ>2dRa>|Cp^ zE5<}eU-~aX6oAa-I!alLf<}QB+u8-QYZ?kk^h>tQ#;+F#lG4IG`w(>?$I@mbDD1+) z(c3EI`xxkc@r%e`wiRm27dG&xvbFDucNvm}DQ{!Ty$NIQd6Hg-+;e$o3)u}VWd-3F zj}_73Pst8cS%TzyKOEB`=io>vEsEPXQw@m=Bl#)ZYT(jYK?8Ub21EJ$y4{BQaM1Oy z*g1&F`8D@QZvdoUaW}uzt<^tV=S=Y{~3&eEkNlD{*B`E6*{dAHZ9yrC~Asb zkM&7)bQBSxO)+MI@VN6hC3n!xCLk`gXOEHJV20iFf2cWo*R+`)FTNrNA`u;~@>+wK z!(*h)Y@loOl2YBm08;`)SD(N?K~E}LwNlSxr|xr=S__ls{gvq;)K3JUBmXXwqS#bh zaJ*^Gc6c~~y(TI|T(R*L2zhK>*mo!$Eo#}^k%9QXJWq@KcN;adCUQ3^p9wlPZGB1C zJHcUH@Um$Wg|}$oSgNJ+P~tx&Hq1H8TZBNkCx>lP?ioaxfwUBK+m z4F^w^svz5m{qZZEq!BjVRxs2!DqxCS5W`78v4SSFyO857n_&UZBpqQV3|*Q#UiPF# zlP_n+iNVQL2U3A9EPX}sCg3!ImJxUDeLjGyXN!Ejk{G`Cb0##Rc; zr|3L1auaPvux0fGJi`dK(94<_O(QSf10xW8OvVGSf7?a-Gk^oSe0px9XwdKgA6g+F4DjDF0DBC(qxAp$AL<{4O$XL` z6#oXR`+AksBcgiiNMtCsU;8v-**X1c(yu8RfgDoS|e9c=}2pBD1>20}u#GZk!zG*|9iZDiOflbzg zJMQfXqcpKFc~FEx*Qn8kycYf|3q*hf=^4hC^LsE$JcaMfLcm|oq)sgW)bG3fBioEt z$i`7kcl5b7+2Ul2?G0fT{Kld9^;`bYdY!a`CQhMJ;gtbYUJgzXfZK2w*9!TDW4R*C zzDtZdm}&>$%BPlo^1X-)-M-%Y&5#t#xeE+M9W?Pk1viX_y1tC*ckFO8Sd0E(tiU=L zWBy6iBBsjNg+dk5D(zWYXm?4*1D)bvw@GHBVeZ%NJo#%|Lo7N4gp_|l=5P=^Cbt|B zhOo;T-qT}jg*hcylu-eKeqlkp!EJs#9 z#Y4;h6agJQRJUt4oR%4Jnra`10g8f>c?eGT^)$%4S>cq*UN8zoO#0#<`^b9ZHh}OJ z*Nuwk<~I=Li5rB`LG~J?`PKeh8Wh>}3f%6z`T}7MR0N+gq zjeSN*v{WfTZAO<>EdA|{{-S8~9U@C13a?khtd}KEIjT+8C8>YT`PXcX-R49wVkO#l zrPUD7Lvg-UhE(RYa1>u_kX<|)Y3ASl$w>CCtqiv8eA{TFjr^oKVD!py9UVmvCk#nw zD;{siB7R1j50+c<8tIDegy?`k37>lkVOU?XVhF9Uh25@@DRyzP1io>6fciwcDtrhh-)ee~qR=*O1O`amybE)1+?yp|17Rcf zIhTDZ?A1;5WSiy^x&vA#teVh-UK%_I>!M}!4@Q5&8CtL*jN_!Dp^V3t?D6EFTS5qB&>W)G5=hXFbGtL&OF4r z!oWEord={1|3<%W2IK#!;7J9{L}#`Q=mU|_I*+uij%H1d|@b#yPPiZJD}t7HX;)XPf$f-O9-?a1qTl* z9%cAtcY0o-a3;{GxQ~;5b6t5~=yN_A18s7JZYI(+9M|h!?o03b#+;yH$<2g=%*aBt zXRWUWys#}~hp*{!ktBB$Jfp)o%>TfWHA7GwxHo~xXff}GI3>2h&Q!|6u|eUR9uz2E zq;>qkCF2`~dWSRmikUN>pBn?xjku3#gG4vq8Ji!+q2;lwnChCo$VB^haIpifdx zFYdm5Hw0aD0Mkd8S6V=Ng|Kq)Cv+|)O#oP~H0_%4@xYuOvAHThKmrr$iS~gRfGIcL z2@TK9vY`*l_ZD255XAfOY?x20;PmPr5l4nQTBZS*iBMR9_#$_Y=bL|3+&WPVrnYvM zN%@G#MFNpP-h$^yUc%?-VaFi0!<##nF+v}Mi&02BubRj>++ zmER$Al-kNP+v5`=`h!j4C@S;aghcBxC%9x9XSeR2rM6y|LqO^^fotA%~6fS0WY{@ zuyW6UF4u=8>ZicmnV7_v%4)U@1j-NcAaOH(ZI~-QVYf;nl8Z=|euGy>(}dXP16%eQ zCxV5Cn-B5oCEC|w?2jRbVwYwh7eR_i8;6a;zvM%PFFvSvoEi56LSVW{mRnoHp4S-0 zT0DRDF_+eLz2~uL?V8qU4bd{iIrdDMGPx_i^kOGtVsOKpHQ1uMcn^I_!HC7KoNoF> zpEA$j>uXOoRzIX*BgP&S^;{2VbNLwJYgg}HW-^EoE579=tu1JLLO>3>&#)RGVW8Zk zG8kh~>o=0}Uyzzv1;(+$Q01fG5+VI#ZZ{+WOe8jSao}x^e@REq-fYu!DqIC{0%yl@ zwy6N2IJ7enD}O9yNS{D3kd|g9^nl|l(%s2_K>zJ!DpnyB%>5caW30)eN-f4g!iVXd zTjqAe%(UB-n^b-_`gvk-EnPz}_Kn6Yl7(ofJz!gw<~8zb;yMT-NDyKLNwC0bJ+#vz zw))T;1m3HQNQv-{sxV$5(NTXv^EdgIcUfOy$q^wtq1_EsW{d-3>$#mRrpSv$xodj# z6duNrjH*A6P&yFD`S7vroIO(e$1V{|toI7qbv(wd|4d!9XD2kWg+yfZ4PMrow(~iA8VC@*DTK zwwcc`>_A5Opu5(F!MXFt;+O)=u75xgGxmni6l4dWA~dZ-)sh7EFx7bV2F?yFh>t_# z0bQb*YK>gaaG?)94sQQhmbDZ|q|Xv=c)YkG0HY!*b2`+Zy=cNvV(s92)>g6@8j9*D z##t0$GqR{6IdLlptm-LM--UIz@-cw^+z!BRxIYjZ-5`{0*b7#~jmz)Xu&6(F3DrQq z=G%BZQ`|k|PDUfP^24?K2%KB=$TYG4Bo*vruRR=6R%ZuxLzU?w3MW~P;B0*_LQ{ni|Djoft9 zj^|OEk{+cb&+z6Ak*_fR0eqEb^Ol39=@of^YO2un9zsQPj41^T;Z40JzC8L$K@g&V z76ew)p>NYpP~dXOO$b1+xNCTiEt27G*$}$SZh5R2E^e0{{~(zaHaD2qZC(h_a3cRa z;JoW#-tu>x#~tB4^r0Y{ND%|lZuINQj@A@z^rvtF!E2|la68i+I(1mWBMCkyAmgAv zJ%21eT&Js%{*TLV7KI*8&=&?P_V`Fk1i&{nFJI+2x~L9e%g!-Gp|`jSWZ0Nz<={|B9>xAVLrx75yAf56`aUd|S7i9q<*y$1&+@ zCdHmVUil$*8?x}~XzUZXHKnVy6fya~`2p9vU5Vp)xCu6yXT6>z?Zcz$9GjxU{ZpFV ziP=Rk5D#Is1ofGxj2RA8w{;n-9^!5|RvQKgfqvln$>|B?a)5ssayV%TMjQ*G(-m*y zb7Bur8WUTKzP^ALa-!ODks7j3`&m$#kOuGAUg>|8j;i3uAXFt1`kF{Nniw2$J0*MC zFNRg@$-;{a?FbGQJbXHcyAP@=nKkM71`yrEqk<9&WVMwRCOfT z7>S}9n()0s{9w!XTGXz($pcRD1hi22t6g+8n;Qih+2I%M+DMzr_BdLt zdFZL~=Aa}b(2xwU+IYG&6YGOem4@Sy`K3PE9$c7K*zeU(@cR zUz1S~b5;I(kXKkzw#Vyg3~>l##modDxazJ~8P$fTBM163^jcl{d2yyfG_$NG3b|q3 z6ZUB^0l|Sf+E!CA7Cxe|8b%yK$cRH=&1*6$l!3<%df9=9l$W zg>x=TqqyJ(pn`MfI&&_|J0ytVHOD`YNrV`B`)&!hSyRCZkyw#&dYsk%!o}O&);m-| zEF>h?Oa!jQt`NRgxO`vQ1m%j?b_!KDO0mV&0R{1(>S4d+21IhdfPmM+?k!T0=gNv9 z;-zmFU3k)JCH!WJ0@~s4T=A8*1W$9d**~}~#{gXHA7F_LxB+7G(IOOeyB<4Z7)0K( z=0&zyC@^nteb{9kHKM#?+slo&xVPt^k@8ENohes03@}6-u@sA7V_WGkLMfY%;~y+F zqZ?7!U&wgZ7*94!Xs^RaOZ+Xp^6nUH9nM>gku6nc6wK|pO+#wOV(<%_28TQKRDeRX zqfQccP0Fz}Ib9W{-jer5i}9LfOxe`hodL|3YhXlN>9tF?QH! zUN~DtkH=qxEI3ZpFIs{!!EERc*bGDmkC97We2s1V9ln2}+ys5ILkU{Yu;aER7x6{5 zCOXRU0+s09eiY0G!~&<(On&W-oEBgW(q_H*8lTbGJg?D!defUyx)U$#z~XOO;B(46 z2yy(49wJKsBUTJ^U2hsj$JRS^&mHyximJHk9H=M|g5VHzyd`OQ0aOPBr>%Zwq)BoU z$XYl6g{X5iyV1;vVPK@geS6Xp_9Qs6uw4i|fNQ>ohm21s)yv)G)9G z46(JYD>O>B@x|o}4THxTzq8`OQy|#m{+OYsT^HaP%!BN9e)MIV2|r2pJhU2S*k%?u zsO9Z*%EC1k6{I|iey8>-Zde;vY&8@v8j`Rm>AP%%F35>utk}}I+?1T;KfeK66szmkQtKP)Z+W;aJV6l& zKxqxmh)Foy!ZTNc^NPS`xgjQx1P+1z>d{Zc;1PI%)q3q&1747k1iF!N`6gT5J|$j6I%jpK*|z>sYDUf-Mr{`ThGxYe%tLXr;A3bE zcl#o14ENW5gYY6^f_Ah$#F_2x3$ZUQ z0#z^rYrTEg%8)xOq!xEMMH8E7Xui(Zo1A4V56UsyWOSF?enR64Yzd4=n9?f8FMKlp zj~OM3&cMMT_5UJRBSt&*^beM#qmyh5cksDTw;wI>1IaoP`r+lO6sYV@YC0qzy#Hs<8V%l z6>U0;Qe*Go5;m?;r9~=vA2DeSz;WDlGcyP~I>6v4;Bx@) zG5MEVI5sok2!?GR7!I(=vZXIcV2c-Y;K0%K#Lgw?FO;HeN-|78**_{Td|!+PdGx{@ zXu%2Nc#RV8i%5Qi_?6=hbQnlI9AtB3W$wv~%BUuW{?G?ro&g29lH; zdEi3;eD-U6i2E$oN8%C2a|5GeE8UKD@B#QB){`g!S_32*c)K%m(1imh0_p#{E}!+SykEeM@sjvuD*Ppo0r z#e4S6=i$8ehN%Es0v%ovK#uF0Q7QZ(B1eNmcpsRha)CC}(iYDxHy~QVhGMFR_dS#b zSc^XZw$oCi86qw~{#A9Bj&EyM_U6}wtdmMA7b6c!LhPbt%&~sbnP%)Bj+Xgp#;F4> zJELt4?`C1{5z!Eh1*%}j7!yxtdL7QN<@_P~Q=CY63x~dC5*;#hUWqXY*c|^do1EAd zlb}|K<#b`?#=%#+9-y_?eKfA!43b*{j!+8n+Jf8`MNhJ5Mk65bmxxnEge%^l4&;O6*Wal+JTN2k2VYIxVYyvd1jPv`$p{#u#<7e!9UEuc`VsU6XZCTe#?6f z`X`He*dts*Lk_>%7LsJU3qk}sjRVuMZ&_ARMj`+GXBZ}P(>l>h0x4s}=!@y$Le1TN zkpD*Ljtr<#C<)wja4jFX{l2M7KROUvU9D|~j#hh}_;}azzTpKyi`&r17sib;P01J- zjLhSI$j{5VPiPOJ$9&1mfrSQF4;(+h0KkbGl$qlHy3~&ne!$;Q^X!j_wE z+xmf#D@PB6A{=-*j75pn+u3svGhJ69w0McrfbEJk|Z^Zq^$7toV$(#smq@HsVx_Mte~$kwBk4!T&pwlG=r zG2ZdQ*59q$X``&fmQ>Tca)wR`KS(l4{j>c$d!ymOvGC5Q4G-KIqsp)6-Mz_r=Bd!? zoe#!uts8#;W)d|I9^;=RWzys#nzI`FLw`Q|;5n`dQ1H8I-{j6}4oc7N*Z0(umblS- z`^y6}pWMkqVBV<{(4xaWk{FXDm%W3(B-#!(@s|Vg`^mE+;`5C5u$?Ju{$JgR15IOn zrGzBv1%wt9y$>&d#|+Mljy<2Bd*^PXjT60?s6ga_s=~aV3jX*aKF~aO{-gddiNEfJ z*tE4S@+s4p2>4sCh*U7ALlELxi#!AU5GVchVe?qL)u?KCZ=vRMKkpN^c2D1L%mUwy=G<=ohCABn1SyfOo+P@gyYV58f;oIg|wxF7+Yso#4s zmR+ym8bPv^xt+}?r_ne8urY-QukDDLUd}|}kj)LawIgGm$t@qeQ^$RTWI?jZD7U#| znhItXa0&`R+RMS*AQOzRCSc)nLUm1?UwWIzDvS!n?fb)H)R*u3JIuu!fgg+9O zynyI^r>BpveOg?p&vX=Cpq(*NtzI_yMLZ2G#w*sqQ|_|;>0SX0YtU1(Cc)8v#@2|8 z8>|_t->+7#Lmdfh+{0fiPVR1r41E_{5_K$B2(XjWDxTKNqt1_kR73onQsKb?y`EQb zo~3jRV-p*_C*d5PJ}bcs=coldOYghxc0`frcFrTYqju(3(_EHAF?$NG1XNSZv_*89 z$}CxNS|jsF!`8&0wINKg-U!pujvgCiYwB5ppZEQE@by~lsOJ8TMDv2}c?kK|JpA;C z3r4W~%>5%Zht(F|SZA(y>8dqqRvkU87(8WsD!vj(t1=YW5@`CG8TXcN4lgz3)#kV( z2DtEcGk^r@#}E>buoVAsTSWCbI4}NFTc)8*4J&Sa)4JMqA77<~NEpVZj+iJ$p=l!cIOq7+3CM zT!Dc4lI?)$R_&Q$G1zl^zKq9sq$3ZZ?v?dslg`%Kl2r2Ye|GANwohtT@ z_-F`fl$YEBmxAGaEzou;?`0{@Xx3dR;&3R_Dq-E8A}Z(yvEqhdFzhVPW0go+kDl7H{0EJ?+Q)7uU&1lBud~JqqC#}?gz}LtG0&88L?_a zBmHR*8usi1mX6TSEzkl5g9v;5-c}px@&zeI;I?MA$}P zI4rlhS^gfTGAcKDX(uNz#Qc#qGYjt4vVMCk7ax&jw}!kM%z;9DdZ%q`v|CL;gH=y? z=%@G%)N)uWA-ejsGEl|29za5D+QI+Z)3A80)9PFr+2HPP`$h?4+FOJP(G-c|d$$I5 zAIstJAnO~{f5QEo$Elhs`$;L?vyi=4FDrc{ds!`kJy$H(8>;T%eM*%N)w0F11!Gp& zlv67z&QG0_JZ8&ANxyZi#tr@D9iG2uhRDx)$VsuLULG*BXY z3v^laI&hu@)fLVEF;JKbAq%1y{OEFtN8m+Zf^a+t z;Y(j2=O~L3|0Rk=dGkGyXQ9!*>Dw7!O^cXuqJIo80entV;V{``JZ zcrD9PYa-e}oJ9#c339QPfFKaq=Y(KDX|6P(i9Iey_~R(1ytn$a)r&%RO4~}p=GG=7 z#;K4rDKZz=mNBLnca<0Q;;|2fRNQ%cfa%t?K?2DRlwsZO?!-@84fWmgj0&YzkFzjs zQf19j3!6eNq6iKtN&IY(i>e|smZmxYCO}U~cs)g5zhD0viSj?Y#eQoPxbDvrE3X|# zHB@`K>;P?ZpJBFP-zAn>ojV&caWccxB*4n;D)RV*9Unwx2I{~4mIxxZGv1T{3Uc~P1nTtY-m06wIK3a-KNdquOSRfQwu|n z#q`Ee(_k{1EFi*gn*jhdoTVr5G;VVJWT{zp{X^4&ICm%mi#KXCUd>cKH;J({Wx@$^ zIoiV;Rc#feheIHpD~pCf`Dhs{HN`fY4q?At+ANU=XDbptb>K-ZwArUGl@FCx1d4n8 z1`ZrW<+9np+!FB=QZS`E0oO%eqmRD{@_i;<-4p`GZcio(1%N3`OT zEk#{$I4jghj2tIQ&=8eLDbyyP9+8t_C$C}}_|%AGK=u{v=uJz3e1xrajc%x`uw?Ru zcYt(b3o=@i5s%$B-P+(e=|@8${Du4qDj=3n3=wrsXM^^p%_A2m;nBP&*d|t}gCGxY z76hL_l%1J+W4F$tEIqqBbF=Y#M^BskWp)O_o$|XjiHTu?RY@quJDgIbL~Ym?NWHd$TCFHW z(uFmV#4V{zj`o~9va!s0o!UD0DE!#z4LSo^4S2B>QZi49^(+9WX`qX_73ORLbJ_sS z;9ziw!1YoY%nI>u7Z?R2Tl|U-W&PwpUsvH3)eo_S3!QyreNZ`(BrFzn+tdVWRKel| zPWem%5w_4%zG&DJysMh=Vv1T*mTQKltB@T@a01EOv;dQL@4cEG$^!#7;mGee2%^mf zeX>i<_ghQ*7o$H_akoT!sqH6=J2c<@5O)T!V5WaiURsG}C6Ypb%8mM3cjk1~ji||d z)TPLE^TjWOrWbrZuKrQH6xujOb`SMpXCl`I81sJ}1xM$FhJzS3q^!*KH%wW{1V=o4 zP^T3%;v1-Fc{wERMnY-BL?LQmhO6Y;DFYrRH8Ce`qm}ewpv1xeSMuxaT?^tl3J)s4 zI@r*P3k!OE6_;%fhhQ+b3+^jip3&~yz$IT`Y@s60w1@8NG#+HP+%Ahb@y#B{((xKQ zX3msq*mP(d^ICUTe0U(*uvJF?-7+cXs5`=CXj++e;T=ngOLqX76EA(oXJh=l=kv{# zf>D!riQ5KxtvXLcs2Egi9rS3G>S`@L@5*;RyAzEgTg)DU;Mo?N|mLn`$i+hYoPUM7-jrM`hyh~9g?_p}MPp9g2ws-C zryGVeSP{w8lcuht4gVZgi6ys~0>a{bfWIP>&+;ic`hcp+?S>pOH~yv&4L0o}48dtO z7uE)y%5IWA2{wz!UNcG53FL_61|r8?^dd&inH46YL^haL&0&qnOYs~HiHeIZ|03Z5 zqiz_@E!+Mp4r(N2oO%nl2o{_WkN9LxGa611oFE%1U*w4BPqWmKa{ECf3$)ez(npG5 zkrLqMxrr!-%kl%G5BQNyH1(7Fu2sw+(pU6&>YIjntzlS=-jfErF^Ri=@|*vm@7>N1UaqtZu*VO!rjOGWVKdO0~0AHh~VK0LC5`PP1B$cWen+Gc1=G2T;V|rJRB?y7h^1a*65`^dv0$ru zEg9(jZ{8gt9B5yu4 z2-Bo^1&q}_0>&)qf;AZx6?X*OZ3{dgfpYW^naS`*3>xP%=Nkp1Ovl%aP{BgOA6DTrsCcPGJp%pDX5hSWVC&&+ z)r-EDdc&9$de$zVV@4BkbVx!Fs_4QnjM?3-l3T*OvyJZm#JjY6$6ITXw|me!+gUR# z7=1pwp&pW#8g|HVY6I2)azv|OL5Nrx{1RyTW0C07DmgjNLL9~o_!1CUOrNl#A(aG-;gE>^N_}Et?LLs z9y`p-J(&}Vv^=cQC}HaLewzmK{hkf1v}%N~_VdtPcJMwa?!?xR?b*$eka%p}#d8#7 z_Qu?~zCz5&R9G*1X?0bjl0XKN$*DqUHI1)hVb!v`+SfRFLQK>ve>C$voT~&*C^l(z zp*DfxX^Np;9~joF_r|}M0iWRfm*uhXS^XRQqtcIke&;DVZRvb3(}S`M3fY_zIY^kG z91H>Cv^tpqcjHiveyxcr9jc}BA zh=1rdVVQ24Sgzi|3ONgtqoNgh0(bBGrO^x=Vvsl5)jW0L!W*Rv&6}-PL6l_41L~5J zGYYQ29m$B(?i$6hCmay&Am#}ewKzqn+Uq1<%vT{(x93nswo2$iukM4jQwTs$=(o?tx5}^-9o#%OOxLGiR-RwfjY79 zv;ZfU8z$t7-pEhum__5gRRMCaq8sQQ%ctlKoc{|>15=tTfQ2=x zi6A28WZc@I*MKU5?QHJ3rrjLp1fw({m_vorU9K&{aG;8#(r8&}K*^6qQfkuQ;uMN% z^kZF0XBMbCVxx~(S@1{!V^Glm{}p7MLmPZsNkqGulP7NoXO#&zyY|CILdlu&bb=k% zVC@Dow?PYUQ*FVEeGzl=?bvPS}!!<;HNf(Abne!V}E6$ALs zjLJnG1 zc)@Q8(2BOzI6w@*w~Hyv(X+uf5b zTMY!(&EzpyBXQgNy=|#47!WHCZZ@n3`g`X%v_t&>$xeRWe%Y==es}c>M@>6{(|q{5 zRcN~L#{y6;pN6{F@+?mzir1$Fe`KQNW z3==Gl!p7*QsF$(qdX=6Fr$(cI;p59~RQwHcdQ&b@gXL8!uNv)%Qsd}kjeE47`nZ-= z6Hr_WygQp7-svg8&J&Ut-Bbm8Z5-+};SB~)y>;##UE@}RhnJf1jPBbN`_jD_v~2O8 zc{WfNA1--^GBs%k%!(=yI;~8v2*kB)t#=a0vlT=~GCaz8V7Zh>1 z@48D@Wf@Q&qh8!mRQfXR?R96|3&vpf9r5hKdb#OhY;{H#QScjpvNcO|YolQu`E(TU zLjjEcCyAPG0ej#SS4NavG8$Z~BvGkaPW=O_OkB^g{`SNwd);DF>)REygZg4oGB%svPCb(mww2}51fto zJr}{GTje=A7flpGt!Rmt7ntcM(aUjqk_ih0`;@OWwA-6`(^({Q0F3MMMX2zoK7+4Z zPmX+3ZnZhT<;Dbr&28A(yFbGA6^o}`uUkzgxMAoK=wfmh9tjMcG>~}-%Ppo|t0029 zcQTAeXKM1(6ZE#93GyrKkB^R3drSqR&nb&LL>j&_6*Pl+=AkvL{dP5*ajs=NW&05# zjNr<}W`X2(=ele86j+(Y)`Bs6io-lH6^=MnZo*IuvO+VqTMt?}7ApE-w5-NzOrDl! z%wUuN;slMP9~y0V{+%lxUfXC#fj}`o&YZ}_Fx3Ssir0O`i?zI1vD2=zQ`O#f58luU zYtu938RhMH3{D!7jxfyehE8^pMgXueheW>N%|l!kyAM7t?#WWHgoi@ka7P6xL41wK zFFv;XY}VeJjn~!nS@Rb6PY|f;lg!qFG<&_q;UN@k_^=DT3{?WMT!br5SjKdFUIUDQ zg31@=6K8vF);~UzW;6B0lJ$=(9-9}=Bp=YndY#$OUTnViC@u&X^38ddc(h*J_9PYy zI5j^R{ldD}HftO#G$hy3p{TeUBI@f7-{$^kKq0TRJ87N}Xo;Cna$T7lAt?zE2!BrW zd51Ix%XY)a+(pdV-(~)$eDRkl&JQv3gbVH>jdm9m1Rx~`;taZogexJvZP#zGzFAkH zgbd6{b7SG1nEp>|T4Y9-&|u3&aZ$F^p!$w{8>%<)=YoYmhqP8Q|?i?4wTW)ortB7=Ap-$bC4=je84idCXfH4mtOx&R4 zbtx{dm}@$E78C>ovv`bWhB$rUK+SCgO@N9g51*2?lV4Ote8avg4Wcl7n@+wxp1FE) zB6xUlqooWLp^xtVJjZxQ)=l=zKfq+7FT`VXSr~=%|6ZcMovD?6FZ~VKDo$F%lgAji~=C%TRv0<(eSegI( z%?}Q|@D)JG9hi`!B*xXQK`>Pd8ha_bSt`nfU5BNzjKaMGR)l!%4)Mv48MW8;S2h_# z4WBnCrDF{e8C8nZF-G#sb$!8j$Ye}Y)6P?-9GvU96Q=d)c(i%epaI7 zs{v(bEEf%tFblW_aFoSRgkFh(Pp~){@Kj(a0>fRTj;fgNJQNl$c;}&_+Ctn zy_+nXdXq-nB~b)&wNlG#aQUk?-g?ijn~e898F1>-8t2oC`=f&B(fw(=b4#iC2* zwOFtOFc;5UtZN=Ac2_s3TQOznzW9PVxDb);nV>FFV|D)i(Kdd2>={ML%C3JHun@)Z zY)b!s?vadn4V+)-*%=4w`_6A`H(4r{gtZ-tAq`>-KhgDkgYkILXxkJV@*u$a3=^ie z(pw4?CpDiJN4tLJ;GQ%ips)iBECkLKzw0a_val-++^5^n%NMC ztZCaeX9w;a^iO^G4?5y(xS-g-le8;42d(&j`1g)rf-zCs8wz`x|M4gw(j74aE!RO@ zyC-Dn>5b+g?@$pGi=LaRc;q*3JYO14RP$fs3U;HZE%x~hjo>z64%JjMYZlnafTb7v zeiR=f8l`=*nw*YTbI)?mZ~Um-@=J@^V6}zqe}<4ek#h`eVGGT4u|S8j6B?@DAqcI%Gt;tzi(H?W$c}tWyX#UK8L&%ge(kPK2OMU z!NoLig_a=lH*IdAf-z@KSW{53K5GmK63P-KJRWavTM{=alpA+#_<~8vyrXmhVvYyH zszd@(O^%a~yZ&c<%~H<7!}Ltg0^4r(qO61@Zo4MC5B<5Z(04&OvrAO(rI<#;0i-hL zNiSd08*ya}3IPq>PGdujyNPFjBw{E=$H6twE%Q%6!$OZhi59Y95IxGOz}IJrp|FlD z4Q$-+TzM3`HKs#o#A754&M!+f?ADB~6t_aq8;Y;EAeZNQ^Wd%>?!0o`gOW!{&V@D3 zFskwWLVs?5#q5=ZcT!-5hP`oNJ;eJ(fPiU+hU2HX)aK+YHHxpc3f4xb)U%U%CLk-+ zl~RDK)Cg}XW@_=r*yOK!{hAMho5h_%^hW$+hyaQYi&D4COdC3kz!fKz4o?)UYE04; zS#ij$nhPogPO0EWkdwm6`3KB?^OGfEAe|o{10s>aoeXAP5V2A5Yw?$=DQgSgx6DAH zr}_c^0gh2wdjh?q_QUXFE91u~i+UPq8t!AQm+(3&gkncg?^*I;WqXb+eLk1Rol}gV zMJ)7rIG7);yFF~*)oeU>MoZ5t{$xIY=SJ}e+*?jTfqQgRf;mXY#+osigKxrjfZ=L& zW_c)$auETmr7;GWZ61oW#oF<3?AjRR$^TIZ@@yiYJ(_|6#bvy@aRml#{AO=xwdmvT z1)m6-$|oa|Z6XtP2AHQv|L{rZ1^|;_YAkb1Xz6aXmPW6KzNIc!7qnSKp0EZ9%gV8b z5Q@y`Md+oNzsQPkJ%;;u$-$E{-P_U+{7ZX3;fuagb==*Of;jXa;4PrjG{`27L&rpL zADG1ZRfheAej8Z8UU1hyf>7flq!TOr;yFC*?l?zRXYz02I-l z?}s@htR7(ZllGGQ9&0~{i|5^AX5Q_txw1)PEZcagAV7o%&j#z8h?)9hByHBya|6Qx zB8o&|83~sESg2v!TgG(oxZ4l<5&y58E5j{dzrzTP>$19uH3F)}G|iPwdLYxtL{ zsi2&}EkrB#ID_ax5(>V7gd$oFC0Adph9na1`;ev32jeq6uN^d&?-l_fWso7Ax5F8V zqDXQeo#s;Z8K&}>s+B%%WG0I?LF!&xD>S6wC4Ga1q5#xo2A=F4C#}CZ-RC0g5|Ezu z)*4>HW}w7II24Rp#hFF@v0vcqv!l;3xBr2@-KDKW_KZ*Ld^C5vM=L;izAW4v|CRY( z2~K}SY*?K6{GMny^-%}-i$`vtI~t>I0%6=ryK|RSq)i~x0T>Vl-p3_mkArfA)$A(D zje=^Sw&c2iHcv1ipg45`gwy=DQ~#EFUwU&7n&zJft7g6eg{0G_s?uB*`Q(luO03%* zgz=5%cpukZe!(=z;nLjO6g-=VS#UHoA+Y{2Qgk?o>)+xOjgHB6j}^TEHemtJ=RSku zEF!1Fm78zcLaVIut6U^WOU(WR^9e4?_NNW(z45pvcXI>0>}NU1ff+q6Bkzsp3ER`q z8_ic0;Wq}^xR4;qDF^g8)LYKVlCnptQ4{(3xQ?)%q`WQU*Ty+=#s)$TRED6YuQj#X zF!?eAkK^}e->CQxv(A{GH{I*YPjdb5gugM7BJ0FD?~i|1x^sp5p_ zhjIcOQ#J|p)ximuk}X*F8^zS?tgky@dAtK}cBY=||XnOP${%;D#vKb9w^T0)m?Fo2G8d7X!`x+YV({)4ejCDV#DVD#d?*QY48D zbZt1T8A0T4RJY_OYzQ#gVIbZcb&XlDpHu+LVo_zlZG0BeXL))$gbUMy>A*L2(A_X;Fkl+}^t_AjjJ>a_{o2?`n zc3a(NveY0WNBUm2bjxGS(kwR=2#9u-6BO8%3L!imOmwNWF_APqS(LXmfPh?;V$n9=z|6MUT zMIk*&uG?jFh~wd#0G!y2THxoVh--8*lBf}SU4qR@I_icR222aArYh-F2+_v<@q1bH z;6R1(!hMm*U1z}pfhe8~+JUk=PK~0E8h8;7?e*uV4qcbB=QWec?-*r_$0x53mWTDR z^`D9ujyi9?0J-R>vBr$B3d1`2?+dnr&;`Ff+t7nX)hBc23s|}KusXTrH}b@Rs4X6E z3EaS6xi`bb(%lD|&HTtl-$6>m7#VV^x$>e_UL9>jlraFyP2(809grzj){+G7rDG<# z)%i6BFdE{bVJ?_cOvqF#9!241B2Ps})`KpeVV(KUw|;D>tjDBr38u{pe^y`(j2(P6 zkq1Tnfx9EIHU{H)=r;jK zIQNd(U;ERbG0u+%q)}iXvGSY4kC{uX786=rm+m*fDg)BC-@%NL3Et-DEVPw{Ovr$G zdQ0HnmZGCdTJ0M!jj z4FO@&Mgxx=`oLI3K_iqL3+RC0dDCsj3?nvi`n(-T)sPd|%ODl6GK#ui_|HyHaMOgm z*FJS0k9p*=2P4SJ!*Q`HRA^t}i0Lm0`u8wRUQ6%XxyTZWIFi#8}7=A;W!V-v`AD1g6 z;`WEN+&GJ!*5E5v$JKxyP7Y!-?#O^r1Sg@&G*D=?9fH;c_%{yb$?KlxX5Kn1Zv;=J zePXy0Z9}LdDm7K3Uydx4M|5h^>kqD0lFt+`(NC4_#3^nLJ0@ohNXWSTAjJjvet_7( zh85U%s8#MYeZw^v`#EiS`(qAYxcRx`n$wuEFjH^^;-T{Q2_PY`;=%%cgNIpzacVD1 zIm_(<&Ty?N(9B6@PW58;RXd{*Qj8OhPxESPSRF^yo|sDn`heSuRD?pEmTTh@+8{8& zw?mMwlNf+_WA?To1Fr1?ky(pF-;sM5Eq`O;q1ITV0ks#!@u55;2fc>WBYg#d6fe~Y+lt1%NCwI=|}Gf=h9 zv2s#76T#__4KbU*?2#(V%!%sZf7ZukX@tS9RjQmkFiyZ-*KS`Wfd?5#3ZDM1uG2esPN|EKKY1KI1j`vxVpq`PH^^Q1trb zB$Obdj!yu1Cz1!9EF@-*H6}4WILsyUE(c9WUll5jN7d)e_snyvFx61IZa7Yf7W zWnS#Sf!*IK*2L`?m<`pSzY9r?eyfa>g9SSvYtUPt9Wq+l5nx^y9W|t(sX4x0f)_te zt%-J@fkg^hF0cs*TU_nl={*~edb1g1hjJ^gTDj%6QB^eQ2UryAt*(;~s>jH*Zk!nC zt)#s_+9;c^o#kt00b2rp3J_^(>%rZ+92*1E%`|S^_F$iN*nVg`yT_{DKKLdNl^qye z(*Pm#STKr?wfl7&+q>vngdn~md7FkX&o3^~r}8Xkr*MsAz*%r9Ul0=t6|&ZtGeoQ-=1>MuheXby>ij+w6x6mp(qzkSM# zl4CtHy#skps^>C`Y!+z=D7YdJ0<91RG>q=ur-B&LQlq?r8D7(M;sm_MNPE}Z{rO^5 z*oJl@@xh(gSIasI~0Atbq!M@8(jB9Is>C<9H06~N}1WxEN#T{wGS4ps_;XQd~o@! zmjEPw;#f@>9s`nNn*S*BC%tR~Ua^JVc%Zn3CO-uA<*_J<#jD7_o?gyy1&x*PoeT=p zVg}RfdB0nsF8st1+g~RCwQgri4OSQUs1uLMhr=ts86LJOo&%Xww*tKY?^IEK?yJoDO}hF*`(j2^{mUINZ@LE9&phOIG-O{lxqSDC9xu z2QICQ&`=_fosPk^3^|ZC7hlPc%HdA4JBL&#tpn6Xt4|c3e&M1)7%g)+Bl}y_37_qf zO9o-Gkhc$?R?I(aVWa0evuJR#)uyN4)}3y z5~Gyc=Q}b|Vr_iUa$C^ueaZ#r&$*;+xAq_)bI27IgsW{;8GNa}m=B=mH<(MT`6vAd zBWX*?1q|o8T<^Pb_%50YMjc5}4pM2%|Gn4x8D=I1)pu2>XdLm&-98s8b6r z3}XLHzbrZ}pwKFe-(vd>#~cs)P74dUaS+!yeM^l)E2RU{CD6Qf&r5t)o!#qWM>Q!6 zq<=0h-1Iv}`UuW)++_$rmK{yiP^EdHA%Ee0dM9O3OmCo6@EfTl2By;k!4c$;PL&vJ zYVzwv+yRd6aF9?PW%9u0r`tjibPzzC9V`~P242jVJ3wRB$Rs63QMM8y$P{+RJ3Er| z2NIKT=!}{l=f)daZN_%*P!0kTFGnPCZrCA;MUas<1f$fTz-SKgH>?JBhcGeN$MZd8 zeV7_Q1(z;mje+k!FK(*5-GOi-11KnO;GKH(q+hy{5{&8b-k$F&K3>trdxt4hktimw zN9RaDUwuZtVZec^e@&GJxC4*l<3J>4D;64_j`?D7#~&|tXuEhT#ra1;e3q$3=?i@_ z5`u~}b%R3cRJM_Y*#5FqBiu%$8yxU-mXRskwcmdr6e#hIj(sRxqGjSyCUOQkM`y); z9c(Y|53uJA>#uHi2d52+3vXIeb9)``7;43HKBMdJ>eZlJ%RAfWsxj7PVO)y0mHV5Pk_+mdiacRACqbDJ$`DNn z!*}b7aRU_Iy$c{BfkkrYwqg?%S0F|MApP(>W7APe2_&oSqKt*ZPWXh?0l4NV1lBN5 z#6%^^h<{amWZ4f|j3gw?MW)wlNn3`NWEe;vF-;N1nw^yv<-2viGt?NPOOCN=#iue( zv*S9G41sKq`XB&2ej~PD+1gWR>oa?!B+Z()n5>lyIFkIrC%mc>Il`Zq-~%cWf9VF| z1ae*G(OvNTGDBCb9%pS5goP}cTpaMEbia5|A+|s&m_Z1e1%yDgoc6>pJS|N5B=&U= zOQta8yGVw_g7osCL?eh4s^N$6N1+jkmxMJ~o=K7O?nB~Ip3rzl?x4X6Jffysj?a&~k@;`6W$uV}( zTHTgy*Wewd%WA`@=jBpVk1j7VKRQ5R0=~N>pJ4rMU(P_`X0G%$qXES?>~F$;IEmK^ zm}c48P2}{((dKDXg8lF8Mc^zCj@IVrAiv?=Is-`*G-feBDQ|l33SaU5yhui`$2MxD zwfkL{7Z^VCuQEeJ8YMardap2sQTD+RT0E`{ZhFgq?{g~J!C!Q*PpjlqBhsJYdSYl( zOkpE$M;vq$xS(}1;SDjkB(}!R!#oi;lgCL*hRE4x?6Z6f7Jv-UsX=yu4an2+DS=jh^bS{)JRx!HzI}BzwdC$d}3tl(d2j2)q+cTVP|;vJ-DPypg~;y3yK5Xc9AC z6_pP)1O^P8==YXha_d?5WC%_$tUzT%;9V_Ptm@7n2jc*T+Bib#ghy<9l3^=%=t^rW zgO&+qvI20)QX^35zRO_Epj>DVvGftn4Fx=F7fr`RhH*MLR%`gMU30UbNugZtm2{dm zN!$K}?APJIqY{ZbSJhG7>IJpLzxzCi+LAH%gxgc&GfVB2q?4GA|KGbXsAqx#%7ry2 z3Y|jvb|7KMI|_v%)P_vN&`wIWyffw}{wz)8_Kvj0jT=_-=?{|9m)!YDREfRybmCCg zj=ZC3E$-t}H6yq)1XU^s41SHs~3>Ro7ddS*bxp;xTVX$O;m{QKf$Xr*oYuH`xu%8+vwg+YRyX-&D664)i)tig-;@W=RrFzN#{`()eLSVj$YqqxkR1gG5B<;Nk$bGrBR=P*HnwzKnM?8+!ER= z6;r&}tA64E3#xQC*_+eRr5g%K=ImC{+$@L=5X9{_SDNZ#uo=*JQnMKD|>{NjZME;V)m_KhwOERwnd zwDbG?I}QKAaRMo%Gi^%)J0kB8=-Rc_KdN#7m)Jl{Dl6={GBGA%(sR_6T5pIEQ*C2> zeiytGm=64ia(K>ltzOXF-(S@c3^>Fqk8Feq4T@an`H8RU0pAcfXAsKGR_}@POiHkx zgy=fr7#s+eoqMUz{gejH;a~bGv*F||05z}0e?lpvo>&uh*-02Y`D6)k2spr%6!m@k zOI!g#g!f<*mye||g=_Wj$UJ-{IZayw{31jo!2dg2*kr-&aa+-YrxJ<+@qRzxOGMibW`dcnH`4ST*95fM zd^!-qv6@rV-xZx4DZguEZbp=XTz(GH6qY>g<+cN6lF-HH253JljG6#f+gU-uU~`HY z*7dt*^`PR)_E;ed^MAyW;H-gz3v)>p6hZGn)RV<8aW{1ic`=|`*6vbdFgAoW9f@Nq z^Vjffv>#_q$8MZ`el0)#(J*5i5Gx}?W@(m%F8t29SB659ztxZ;JTc1zq$PA5Sms;n zUJ*MM?W7z?LExBLUiq*j%29iQ2rAZVIKgyjv8UGP7on7Wg=Q~pGDamle!?_xtHAS= zR0JY$c0ZI1LecA`S4k*JmejcEU`gSo<7z=y+I&~i`jZjI3rPX*E-Op=*}2O=7mcr& zL~vp2*Yl)YaI8H{Di9X~&ajzMwQOI0KUu3RiLPdRyQU_HWr2Y-PC5RQ*+G9RB%Og~ z9ojLa<=3&v=!A@PdGaq?qb3Yu(0ao9YMF2toh@(pg`r|%|K&^GxUl_oWmEk*O&GJ` z79sLc2_V9AM|JJVpxq~KAB6MQ-zQ9m>q2BRMf<;NfYG#180q( z8V0Va+4my;_=7kTJ@lzT-MEHiS2&!#e_C0f{c9YEDvr`eGHNZn99_-rQ#CY1B2_>< zJU6lUgeDxXHD-|?MLR1bq|e&PlJd7t6H>>y3*1t`mnb1|lOIbFKM!e@WdRlXa}%Ed zSw%dbFd>Sp{_xwCAUR)NVTRsKn<=P=0R6quEy5)#<}X`+`pE%lTE-`=D}QinHQ!g@}!ScO9xJXQ;-o;xeLX#va zg1HF^5jdyGYKuz7O-$I6f+Qe2L12*E?|IgG>3q*$DC`Y;zx!REwbrwq^*r^_5<<^3 zYS{*RC?lerX*u(T7)2>gv2#sjDAqFsguwIn%obG^!(;=<@K8jNP&SN}B`lEyVi#4e zSW`B5%7VvvVBei^1%cOI-h0+kWbe|H{r8lecE0?>_=ddppNhRW6C$9-eggzux3~ zo*W?lxd^rD2L(f~y36tx39uJm!9>%Y4y;|4cT2Fc8!`CyVycVgIEeRzd$_bu_5P78 z6zP7Xrn{ERWRbMYw~blh&p8zu`Wegd4C{zYLy6gz@>e!EU@LrDp@OgJv5Yy?p)__! z*P1hEDyge?(ylY^k?`9MpN%eP^5HCQs<~TU#j4Pk3Vb-qjn0#D<)U?zn0}TO|-C6~$e?Wahtm^DJ{q3q8Z{#b0m}v-2#8xfzFcFd*II$CzmWpG?x;KN5=^zJ2L$-#M`Jq2nKz+p~RB2(~vH00%6Tlg2Ak5Dcw z;&Hk+EnA2Z*pc}?ul`A0WXr)5jICNa)~+UA1vETReKxX!vb|AhBDcKov;H7ImKm${ zHK=JZcItgT6|)VrwiPc}l^C(FPKXhZ+>r6_OX|>(e*JK~G-Fj~1x(+wZ>_9Hxslp{ zzN!2x;RECz`e#Vw^AOl`y}2b+I1q~_oRb5(RG_||qhI#qd-T%kZo3)(kVCt;0)^lv zbAoBlYj-{@WWAJPbU`>nxvGKdtFVN*M8vm#`3jl{ika6?p#`8t zg=kF7GLS)-u&a5+#SxjFOPkQ2ZAz~l>dMa(BBr2EU{sTn3*jSvCm7uxZ=lB^{wP?W zvXZ(E4+>l^#}MFnZj|PT)f&L+aiZWj)`6c&s#2PLH9?+gum_u^|oC!*MmN8g9;E9U zAT~04HQuh4Cy0CmiNX1ws>cR|U-9rn^Va(@l9Fs0p{JbMiT9;@t+*ku&(|MGHK!{Y z5js!0K1t;MQ3q^}E{o-ne!q+W`f+*g=DID*yf>lvTbAY+9eM*Lium`jkAczfosUPjQ5o> zF*IXAbN|H6kQ%+Xu0HIpXO}?j)7HBA-&S_T4ZK1Gpn?f-1{Fp_$B2S!LtBNH4?*Pk zekbUCgDIeb5aCVyBXca*MNPG6p&~n$IuFKz(CLU#9DgU*8edNwaP3`$nQuXVCN%-P!Ge z?W8exVhu*XtU58mxR}7+>(>CJa(O|EJk;4`MCA1KWY&Y^P#x^g}LR6C~)E8S-cg!Y6YU$#`i$jHQAW-C=;Y};3MgxD08UDE= z6qCSg?w0bA#JL6+=F$5MvyT5z&-bwN$2&D`8P)-C6;o(KZVvaiD+>l87-8jU^NpyY zBm_jMKIKM7Yuwqj!Unx)Z@ve5=wRNO!s z)q1+skBy^P(j=|s4-HxDobFHe!FYDpx#zb!bL$^eUIYPdGaQKuiYXGZ=6UIH40qGPjh7i4QXT60Du zP+Q4p0eKAD)^ZrqpxYB67A|1iPZ7FNc|qAUUI)k^%Grgiq!ls(;-u^&5=S*9U*!wb zr7O$%6Omu(oa#s6AF;l+jQ1w7h`t`E3DMbkDNuLrq1|}-f~!Z__OxdJ>6HB$sRcoY zr#mQ^1bDFe=|J!JP6yRJ`AF*pRy?IsRvVT8dCH90Rg%;JmtVz%!S z6r6y7Y*|lu*uP;+g<*oBPJvaMS5e=A*ioBn$cC-E=IvBs4 zgM?$F{@=(LJk39;;^1Ea(t|_u+u_W3US0JKAC3~erp-$?RLQBW^5o5EE~y*qk~A&T z{efhj!XH&%AnAyz|0|f$pjrIwVwGNp5ZVI1;*`ni@a*tAxh-vpKSIFYyZ=cX568t) z1C>D3#`+9I5#j@MDF|D4!+Zloh@7&f0F-{H$ky9Pv9LmN9cmYRAZY)xd?>{v6gdx` zRHxANjfw&SbCs!}`_Jx-9%L1nsJG1&3^DSgpU#J05BcgAe*p1kgO;@JR+zZRQ)~S) z3(97DF)DLuo9qI)TXLVO&kze!T@suR2_=lNUE-ZFrvT{Yd517NR zfcY)&9$1>V)UhNCym-2EsX#npl8P4W8V1K>-5 zPTy(X3xx;1xMJ7;sG(>Cx^}DrcF&CFH=$ZSlJ|~5nyw{XN6?HMZ~PpMQ_DU4e27oQ zM*VNfUuAzi?&}viSGSYMcMkWf+GkWhh_85tb)tOjzj07dagSJRW(xA&VCvvAu4lz& zR_LKUh4;d;$%^OsAID8Hr**$aEKD}AO4;GGuq38nQVW+8s zKR|oHTPNr}l-~RYyoJWYHsdD1*e_aH%z2LevA_|o7J~SNIEZF)Pi4Uwaf3$}xx2!H zIoJLnSnC)l8AD&qS0Xy95S*C4=&Qb)=S5%*#?TwRZ8dknO6+awhCSdsN%R|nUA-`;Nl0a8ZOCL zb5#3JOWqSoqU^jKU=pIA)n5M9pPbqk@y@bcX_8fp=iJ|3DQksDDr>M5h3@;vh780A zYCeDnvU}v4`pp;efhd{wgHQu$pYFd(65B`4+>M5~6Nvw;`%phZ$ZLdfW4&^b{-h)Q zCOd7Wlyj$acPfrL68LnY#ODTfN)k*!CmH$CD%rBZeUkK2n*%2Qg_o#R!3JArUHCWw zsALlnlla_;5?~_82U|j$U=rQf%IESfIn$PSp{~hxA&vo~mGwP$g!3B$6v15$G>nhupLwTUCwINwA^tuBJ78-ai3N3KVQANFkYq8x zEo_qFE&J&EI~X@T!*{9_lUMPT3jtMph&zO7bML>+@Tp6y7$s&~trK;D9j*z|JY6sI z6q4D5AeYi#AlY|!S8Q4q1!X|WOo+Y%jzyXFQb<+q zOOLri_(N4e201V!5HR?Ld;AihfYINrZlLdb@qO73E};7hs*?~0rXp@0YK%e00^cAx z_Fc(QMh6Aq3ZY5}CsqzxiBjyH$Tsff?*x;;TkTb6P_9GLG!)-?QapFNKxI<}7rMxk zU;*OW^DN`SiblOaS{{a2Ec;ZgPX<%zL6e2ohkaOwF$k(z#@c_o&!<8JoeFz#SR*D zz=1kViL;sBE9EpU_FfU$>-;jAut2q=7D`YZEQe4FuJ8FL1Tbi=u|uXdA%(ZOqMWKB zfgWyW2#Z5AWR#4$ta&F2LIh!Shuj45D!c(r9RAmAgF{;pdMb1n%7|qNh$B1kX~T3a zZBohYYVXBA$ez@#Pn7?IX=@dV<@jtYA{N)>{R7()Qm}R2J}V_0&ePjP!k9&ak&xET z^;}~i8U{N;OoH|WKZsyaIYL2sSYv+oyM~ofQ`~>=52xq*EX%jY6qKNI1cFPVPJ&D| zSq8*n-BSIqmjTwYmSSuESC@C_*%Au2{92otmqV-}6jIEnPTJ~DgeZl)KXs;1 zPt!deHn{LoW1nA0oL2A22bxqX@%mwi$WrR+Ob1c5Uok7VNioxOblSkMHAqN+3f9|% zqa;d4ShAs*C35N$mPTWK^}^4RpK=6*fQky2gAkl4_shfqYTmtQ_0BB{*%ke1F9QZ8 zEgYjc$AU-O;$jt!cq&I637>$> zB;JLngt5O!v=1$=t|Jz+NC2t8`*kmo<*R$0;z_6-%a>(I4Or$XV!K#~T3E5skAD`_ zq|$9ex$9aGdHA*In@$=vK4=zzYXc$Z?k}aCM=TzWnyCBltJ}W?j{=mz)HMnMy1)Gq z8Qs9BN1cLf3P=~we(|6?hRE*(f-sP(h_(XLl z45PIDo4KiO&Plmp!SjG>dLJ_!M-d!6PcxzRydDiQ+c` z-o?!uDnl25K-nm1c&I^ax=6{f&w+BnN8{{andpn{TKUsjq!+66z+<3TB@!# ztWaR#bRe(;?B@_mSWC^ehB-A$Cd$~wSiATRGO#dh*X=Wy&5-VI!GsEbZ>bBrE-K+j zvz5^#W8s`K;t*lESrkY&2 zZoAVA&NeS7Lm7#L&~jWqi$Ag!F9Zt*l9YlMJ`2MQz5}6; zb4Lb02B~w0$eji+{K6&&wUYg=UqO9P(hET)V?&M)x zKr?Kf1Qx3IxQD4v(a>w|y-kA2-nQUKf_Jq*ExbfQ3_Fq%Ohr!fFzm!w3O6QvCOcI= ztjFSlwhO-&_!GZY+Y%hKL#drfy-ysBkSu4N4#fDKb~j)TsQ7yYdw``2RV6mgpSpzJ z@M2_8q-N9Dlg#)M#Q-Kyw6__V^mvX^Dq!?db6aMIAf)g^!0?QNfx2n_$BW1VoNE!A zYH;Viz9vbF2+75IUL#QPFR&m)-uw5Tn8HD07}%@7D@O{yoZJEdyp5QosjXNKbryCqBU zs1v0e>SFS9*3MXZ)J;VvA|h*e&qXd=IiM&(^*$88f|`?2NtDluevp1P2oNeOBVanS z3yV1E1+MWBs7Q>ZB}1Xdf`cfd8$cjJP=#XK$BU6vxh@-GOIscjE^H-V@YbdB`?Qz6 z>a8uW*$AKGWbA!Uu9|)y+RuEHJ*L&HyoIb>9D{sDe3e-^kS zy=|n7m@UPdcOBc{&~wgt2H@~VCyi;E!h1BEV?YTz3@`B&$?4oK`F{eoDRidfy-5On zS(bc1Y;6sqmGl&reH1VtwhOBHqPJgIY9=QgE?Hi@k(&QrJ4IsZQs2L-tyE?yr#Lf% zlp}7y#D@S10;DVds*JICqwWB{QkGP!8;+|~Y7gx0YjVjiy-bOT9f0-lNSp(IvPM@4 zK5p!_;bja+L|I1^_O0j9WDJU018|Igu5h-DM@@>@bOaa zLg9@qmA&}(LcyZH$<8<|AbODftXwP>aVkD!^*Z?R!}3jgRBNS4h}$F~D`xpkyRiK?29TM6}OiN&am6&X#OZ=_#43@;8GxSgh2sr7OOt>ufW zP}vBd+~4-HBMAF?8wj&_LSp*WvRFAVl?mK56Yoht75y(bSc7&BhvI- zTWL(D4qXLE#ToIUf4u5kj1K-$e~yMu;zcDTEGwjOf;KR`phWLal6I~-LlFkYN8wbAgJ*Ny z_Oxmr6-w-RIC`7`P@v@Cy%>G1TEa%dr7J6vez}4htnLkP#K*%aRpDaPPi+joc0_{uO}y z&gY|SkQ9K6nCzyv$HK@#%D+J$3L6k6Lp&?tKE(SMrJCUq zu+m(i1r=^Z4H?Yg7(;k_#J#cGFJK+mLj10Xo8t@%{7AvulzdsI0`pOZO{V9nF9byL z#o9wH0rvZSSdn}xY3CZhs=6-!`VTl)x$@@T<;#W?`h+0^?VVOuF&m7yRzsT~9dtuC&zr7~A~L&@FBc*!!jQt*k8E`~;sL z{rZZmP<|fmE_F}u*^$1fZjHZ<)=-ebBF}0QG@v2FXIs=Km-C zDpwfUzQqwE+`&E(e^dF6zVmFN27vtZ_=lN!tn1#IZ7fnj*1KWTc*z^&(t@W6ww-hl zKGRvC6W2AXW2QiegpyyfI0i=mP zrcc^e!nspE^n!KS1H^p(JjG9_kNaOg zr-+2vJ^^k%-s@grA2V*zI6Gzd=J-xYWqOM>5A6O98`ETGd52ZW+o%l1m}6)28BIQ2 z+14ee`iQ{$#4SWcgYCG%cyMA}cVMHi9g;mZ1UE6jWS>Swti?A<@LVI66;Ax>Pm=Yp zHz90^cnmFaQ8>5-BSX;$Nn;h_u!bK4;AbI00G3yoW@poUVh&*Rce9L! zmju|bvIHVv1Wt&hO+bHpIlW1EEkg|_=_+!I)HND`@MnAkF`UT+GPQ`KHJIn(WD#lMR}rS$9G zf3dYpR;k)M%`EF~t%fIwB&;C&{qW0w;&QN9cHE_+39nLalZ$}w*cRmng*d7FL{8@7 z>QVJXZCLq&xRaK#kUWaC-LRVGebIKtQRb+y;XTBA4W95WKg+dj!mtWeVO8bx1N?IA zbg!5JTL7E|lQ8mQ1$dk}2Pt+j)(APyZ7ucr6bN=TZ@$*A?*MRzw}h=R4tf)4KfHpQ zZdc=Hxcl4HdQE=|%O)&qksu=!6mE^KTvr+z8u9nyk*N5(ltA4t$NhD|g9#f|L|OW4A7=QZu1 zK;~uH82_6>&P7;FRf>fhtoEMw-yjfPxx??IRj3s!!@AYga7vcw5zz?>PO}EA?k`$ zXSq@!ZWoC%YFJh95PmGGrUkhcZ~NbwwN4nu9RKR-=`mRa8#fKh3Yh#of&zgz!@eDF z*14ge!)TVoh6BONL^e`v-JGeEa%($M ztPa-GP3E~XTf5h1(cBPKW5zCt4<+`s#*_t-p^p1Yw|reg98`3oJ-t@G9VzJsO=c&L z1kcr*ONrXSYwte4#ycfe?%9WIJ$^jru z1zJFI5&n6@Y7}m$H*o%|IrW9fOjNV58atPnQC=N07YfP3-y*cAM{pSsxB%%(R3x*z z=G^`y;goBuGys~5IR`!_Itw7;Rgm6p!#aLL5G(KrxmI$ z%TznoPSzf_Ksd*R9^g331@+pnP*WE4vcH++)+rb5b98U{LP5V05I6OIgu}#{Et0lf zMEG29R(6a23fE``D;>7ir6f#6%mD}!El=+B*-vf~IL6=sJO%c|d%M@rCW9xggJi6L z$$yqD+{=qb$p(jg=3J)9@blv%Gj{T`?zo$6J@LpkiK4}W=Ltly_WiWTWlUa?^U5UG z1N__VCcmfh6Flq7`EwywF@>PTi3o;F_0oF?b<|}z#%Bq?6{%*T8IM5M9lAogCu{q? z6K)T9FC9;M?4|FZO@Zv=cx@-kKNeLYg-)eEewNfA?S}LO?nNaKc}_w-DSEPi2Y2%W z=;Dlt0~A5aJ2!0;=o-0)1a#;nQM#L{N?CB6A;Ik31~eT17=uBukQ~C#mpa^3!6G2B z-mLuwwmB{+Pw)8AFSikwC|7#9<<$N{kq?`iWLT&f#%aa9dtf)j%SZ-S6Sw6ldOTHU zz&%^#{HunwDI?}}kTR$k`V8zU-sUB*PZcqji%&WMu*Vx3Qw{nGYPiwP8@@v62CyNp zm~Lw*XDq&AD*)Yv)bVM$plEbV%H;Dc{SIR*nTIq@yTm%s{fO6lG3bzRQTF3CGTlJe z-#_833%|E58OyU0B-1Vd+XK2VWuj!9cjQij*?(zwuj_U+4<_bM zRR_C#wf*()S*2kn0K8(rLS*p_gc=zs?z{!Ch9!7~h&01vg7$29A4k7L1Yjd%`)xUb zi{L?xL!yC9a(vh49^Qf3%<_uOUkRoCpTBmMvqw8^Jp0?iV?9@X5fz~$WSyN z&Rjy6WvCVE$I6k&3y_~AIJS%C$tWxQ9$3=3MxF$MBW3-yOgEr{qrui%iR$|FuDvh> zter)%KCp5Gn#&6q@K%#z^xJ=&+>p_~z-Jyn^P+ygma~T>N0kmv=2z58XflNatBMtf zyMG96qAo3CS8EKmf^hzC=#>ecPwp#WJchX=Cu#Qrm*9c$ucm$r(vx-KWU0v3ysrdZ zpWLOPc*0C9YcLc$YCbhKp&(szNrFP5yKr!2)QJ$veg0xkDBbVF(4OFp=n6e3aaQZ6 zvOkhp=~odXcHpns|%GXYwOiVFFp_YB!A}FEi zv4+jYEK6oS7UYpn3UVu7#d}5aZ4x9Md|FlQ1G(a9P={H}^;HkWe5FX|XWbPcMi`^#% zz|&*UB;jbiTya7HqgbjV;ij$YBvOboX~%u;`r;)t?*zDj(ZBe;gHVn8QqS4cTa3%* z$t!V8$@rlZ;!xw^dpK<+FXqKw&%#dApqQp9~btan+9qjQ>WJE>r^C_sJ!BKkd z-QV*oWGdw4+`z_%9197iKD!$1M%GBuP4r9svQWZnQ7I~2tQ{_kKhQiL2e%5D+r#uq*>%7GE2bUaG(c(T)KAa%`ID8684U z?^hX@grgw`bi!1PIDAVD(-=$0BGq4aLW~a|&xp)ryL0(&M4b z$*p*?W3-&piU|ja161+sW%eF zy1CrK@AnblMyM{D0JZ~O3r2;N zcbg@~oYvYIKzo}uZs1n#+>3r()WOZV_Wc@O0qn!5^@i_MV16iqpmBM$7)QNxrSOQb ztFnMtUd&^Vrxs|ucol0OENt;e?wQ3sXm5dvU`~8MWs}yV6o2u0Z9?p*;U$sc%)3lK>7v?F} zi}+BW>}0srxRkOR@C=vV`{Gd8C;8|q02JOixA*v0tMW5EXy$`vK?Ku`<0}oH^M*4iaG4zvFdEe1YxW`r#L>`L(QYG*bJ0LAVN{_-V0E_S} zASQpT|00K63Lj8;AH*kx9aYFuAm*OTxt`hH^3kp-yHAGr2oR1TS1+mNBa;fTYCW6} zE?nic9j~8L1%AbRH3Xzyge0li3G$GI&w5Hb<)^=d1R4tWq0ax-8kk^`6FFYLT%sBU zrUXz27ffjRO>9%k1G4)tvZA{+jHYlsMju!6c3bs>GAzfd-u5E8r_uhUGrZYF9d~>T zda*>LOrYnJaKs76dU;(NHoU}}w-hT=7wY+-perTexv=oSOEOhG&)+O%zTZ=_@KLV7 zLTgt=MmJO@I1lhImb%WTut~<|qk=&5WuXVb|c&%Mhws_htuQX`YVLZrbBN^>9XRk(6$Uih z8)GozMRwF05IbxVQciObQd_yW${!MhkQ^!)9)Odxc9wtSqu2Z&c3GP&DhHzFD(Sz7`y!{v-lO?a{ta3Mp$Tc& zL~ugZ;Spmds8tCjs4KeZMUIkfrc6=`H6u`r%!Xn8Cj;!9Nc;VqTaCr?BEtHG^at;S zeTI5ci(0F}AH}7HFhuUc2kD%8tk+8q+aJ;Wyoa~yFFlNv|xX-l#E%hl?6o%nxu;ttsWkQ za4F-*0&<)HbZ)AS^mM8GujaM{NU1J4!8>h9lF}tQNtdvKWNR?a`*qI2D<3Pg_U`#5~nBvHktW$P8?STycz z?eBPn&PJ+ILVJ8`-;2Z*pibW1Ur$?PC%*x=jCB_qo$DYl5i~Zk{RtJ4fcW8DGHyM` z&WSAa_DCgM?y*Dh;#+Hy&u=-|-DiTwMaPrVw;cXlcm`dUlC~*@tmCOybyW*ZLzb7L(KN*PQhwqy=}xnVWwgco=XkO~7U1TA>5ez>BJa^MM*?U4U}~?> zEQGEHB1V_0T~tSI6+v0NAk~BbAw!fcK#uy1?JDZvk0V4x(?^-N7i+Gi_5ORmvod5C z!tNajrETK{C7s)2EK3@u;7{YDc&VlnuS~Ybpq$POR_-^ zWs~Kn2U(ecAO5;jTy8h9e1tS?SG$8#ZW12A?pzv|v= zsBz9y0d=TEZXnqqJs_}3D6clt8L9X7%m#Y`087nnMGakXP7RoF1V=(!0=pvt`(twB zZ#kgN8HT-b5vRzn%_XkkyZ^83m&&PvgzMO87LIv){4rf(8WZh)rElItXtr>Wx_-}J zhRzav2++BH+Go%Kw*ef?V>7k427_P-q`CQLMZU|MAq8jNIr^-DcyVLO(X{Goq`G^) zS5ouf1Rgq|zQBwF!?Zs0t9Wm)4sUUMhfjRHpe2CEKnxc7;{9srkEr!uYq#5H(7pt} ztDWjBBa^1jEgTIIOr+Q9+xGq)=^vO&-_geEg6sGE8~3S|iHqtcqpPk7t`k-DT$}_@ zTxgoAn>M{5rTj|QAFkWn|H~M7RF75jA& zn1`~EXJ@)XAlX78KOlk8zA`F@1)prNkE1rytk&rtls8Z}Vlye_bTGv2Mx|Q00t`5g zfm?LD1ZqZx51Y{UpxbbvUT}wT$~*_bhlWvT=E|`G$@VFE)+P0ThZiv~x)3Ao?rMf8 zkioJApP4&OxR+*k3+2_}j#PnF!gJ8MQ)0Th0mY|fLuy(04@XuMjz-*3NAoG3AOPn7 z^Z<{85iA=eT7qT8&IGTv1vFdPNW22Wp~!m%4+*52 zw$0QtuOGBZh(PVgt>G-(fOonrG1w8wCEiU3xuXXj!Xv!g1&-j3p!Rq0nY7c1_W^>; z%ohyhFnoK6^~oF|=nb}>dgKd=M0gG&$q&qniVJh8g+b*@ybdULo>-DZ@pWR2<`qOK zb~yQG^?k<(+V(ML0XLsZK3(#pzw#`cnoDns(XMiehJp0N$Ovq{Z-_Bh1*5`4tKi*L z*18L?*k0DYGXE7p`9QhMohX(Y1B}^49x&{{A^)Vm!Wol3?O9I>ERgC}e?*{R;c>s2gQ>b5W4I;-p+jc}`F;Yd(a4ZT;Z7KXh5r6$_yKxjBPmAY zWuZxl>x*%+OX*;h#8jh!){k-q$;y0xtO85Ah8xs?h!=eum7p@|HxwQ98Z;F zw0`v2z}0NqNUrSWy4S`XU_g;!WtiKHS-BP4yNFVN&MKY`&Kjet$IqN-$CDDx*O#eY zP0V6tr_VbH)iSwm2O+qJtj$jlWZ9M&URDgOB@em5_=d;OezD>17*Qw&*jPBYU4#lk z;0;x(t$Sf`^bou)=s#T{ZrfEjcvVGuEr}A~_yYgqzkJ;CvIT%@QRfcuK1FN-Y$a+T zAtKXk@3a%bg2FeB<$Ek@roPq${sZ9nxaS}m;faM%RJ2E=oJ)NOQS&n(2Y>-LZ!@^M z8m1gwJ#B`Vbad9G)fb4Y1p4~nykHYkMco=AMf`;TTtLu3VS^X_@>L$U&YhcjPKM6K zhzoygIFBQ-A;_@cA^jb|AM!{)=Z9INGQafm>OA;15?`hf0VS-yigW zg=OS9cJqtdkIUNteU?9I{O}8TguPg9Ab=`epYi=;w3cD4ObtB5qGFgS8U>P^P%FMu zEQQBg(0LLZ!batYt#v1=_!ane0Q zSu-Z;RtJV1ux;Z5JO44xCOf;3szD`4Kkrowgm;l4?tW`8bay_mF>vjGhd- zLKyDY6L+^s%{ToLR3)BDDqV*egZ#_FA0SFP_jsEQvBvU-_PVI+foC{R(v&-&gJZjB zmw0bjh^OPW-M$K6F+n)=$@M`V!+GEosbvRtMnqkr88lhjCOjz7T!OY-c(1|aD$Q#B z`h|v1b_(1c^3@I=t!LiAcBteRTC1U~@z}7DqSST zpw6Xf^8c=eD1cq~4c5PR=nx4t7q6SGapy7v7aw$brRJUF60s~CVs*Lo$xg-;dT_mf z4tEe!Xn89SZj@FGOOrg~Kycq{Y}5w5XE`Uf0&s;s#1bJl43b5dsQg1k$S6pvI&N6p z;cM>mf8M`HsT3@cwbq`t7HfR+{%;pyfQz-f_~rZH#Len1wseo~mq9Q{YPq}shE3O# z=zzHA4)X(L$im2m@L%lE6*a1pZVSmY_*Uw*fr|s60acnh8Dbs+4j|(NU9;gZFrXb) zl=U2!=FoM~qYY0XqJart%L_E`#f{E*LdK)VCk`7!hR`5cVl*m>mo{S=$^%P?s6C$%!LM?sbZ$C&uc4nQ=lH!RrFAksY>QY;|_!{on&8)(7g?C21}hU zgSX^EgTz%1U~e0&G9UNjk%0SfmI^syb6s3o_6jB|>?f9g1?tiQ{_uLlG$y0!dFEcf zaXgAw_wxr3xCS$bg`R83Hq1BpEtyH7SDJi4EaKo^dL7W6B)P>Y-`vxAHpW=%@mLDk z4+axL)Fd{g2B|V#t`fbs$Qw-yA zv-~8P%%b205(it;z+~m zEOR{qJ53njmyf`D;m7mQe@IJYOUP9701SuPhj|H@PL>}^Wa}&PzsMTKB9FoeMr5V)d zCEv@l#;oK4Nv+eA-MPbgClyve00M7W06f6fnfx|c2iakkA6D~N$vyO+*yMoPM5ykL zwTMAH<_#u{FnBB>I`BZa=Df;q@f`}YG^~?f6%skZ{a!O2YkZ>YoTT`;b-(>Q|E@Z8 zOwHxyA^M8++OT(nd7wm)$Vb+JP#tzQcNj$()F)oz3ln88q|w~JkXsHXyDI683~EH- z`Rt`GEc~b+Ly07$z6Za;ucAB*gTef0J1q&Zd7IIM&AH$6Tk)CR7gPc^FBghjZhTL1 zU;jZD<+lZk0@4s`&8OOR4FI!o=hqk$GRZ;U_nroWR+8{!OJAq=Cd&J^tsPjCN>P2i z``SH8)y)PD*$V~K%Kzjm#YENVu|@K(bf574hhH$az&sCX7gDuV=@9pN**U&Q;9SxH z4TGu?aE4C}!Mzak1YW{r`SPIPRLZ`R23{3Ti_h7Z->&7`E(M9TR-2=iN|`E6+P&?y zw1uDZYK>b|!+vT&%NTy}M0BH>HE zIpB&(0T_Zxa?<9?3+vK?TUv+t+bJhbrVbMsl40rrrOIJ-aS5*& z;t-yxY=xTt&U^u_zm>9=HOh(=5vc>+pe~E<0s_)9Zf#F>%FK=Mo*-l+vD-l1wr4iv zAW*ZunEMw_OIqWGU^XZ`h~P`4M&Oj+6KzYF2g)@zIg(8uaTRgQ%GWa%^sIk?OA?p= zi2M79D2Ua&DR6+HCBtAD94lllyCK8QjOBUdLfOY4N4zor!tLq;o@+pmz>NSJH};Br zh3~nA46IG&%W%fykZB+efPw_}PEY1p&wTE}_Y2ix+$3UhS0H^NmbXOz`)1r(VIUI2 z$&TO_PqPZ>7-E%;Do$y)r9p9}g}>9gd#1s|E^^9oQ6-72AQw|-jh1(gP>sNiRj$+A zy#*}@K?qG|{fHlruEs2x*G{#~r48UG+#U1C_7N9lxymu+m>NH5H-@Q zt8~;34)H;FBA%5oG;G?1eI*19Ff|ob*!rZ*6%)m!Ix#(38A;KP-e+E-m8=YJ=UI3b z|7(7wwAR2a(T*B$#yOf(6U+`W!p-0~pft#cn#c>!ast>L9c3+SRvMe{74jp30u4bC zzI+kAM9BU)V1#3HWufH71uTN3DU+3rP`o9(OT@`dpzc z2RxtbD)?X{?*ZW%HRL&1(+}beh}INpek6!@K}-3GFy@?O+m@57->u=#X*a-cVZ^gm zeP^siwF^x!{C$ug#^Fpn6;Bwqa(|VX(cx7*L1r~jdf?uzwHqsU9OR;i$b1^1hNO!U zMk@i-oAMpVdSpm-@$Cy$1xlrWSG2D0F8h{)5o>4N^2xL>V~iDdP`Pv_#@3c) zAN`!F?y1IGnv(#1HFr~@M~DZZ=yZM! z)Y)vbjS51D$Y5!dRWLmq=JVjA$)ec2v$9dukB-~Kc{e?^97haJ_ZNQVwZfKt|AHlb zCTW%6;kk@h1b!L~lU&{-8xeZUTzhlIO5mZv5yY+?0)6-xhxv=WAsU?82x3g7drs0U zVro*m`7;iWK#xZ50TM^&HM}?NmsY3`Tjz9VKQ-Omk`_LwKTy$IwvKpW#-w-o^5eH> zP}GL@HHVOq*%J5I7_erkkh1rQdX&f2q-=8nF9H zyfYFBCen;{;cw#l_skENe?bkPBF@@(5=yar3#_$QvA&hf=xU_gIq%XrrT<*v>>pnC z&6+_ltnRg3n0}%RFAU`b>GH?^*~Us?792dU6R+EAeC@eaS+OqnB{@g}HZr~}r+#i< zbXBmuvde49IfA3D2A2;y1>OabC$ZYitCo5!Bg7^#MrnkH*-QDr1&WWUxLu?|QL$B0 z4u}%-#`ztQ3-{*=5vBf#1AHb}T%u@%IY2}tfJi7|Eb5oI3WRL!VSZkn_MFrY1l7BP z%&dTK%P&ag*k2J!1r@vF|2w>~5Gej%`#fx3) zK10tu#04n~zGn_&fc(v}%&Yv3EJF3~{cBjgFfL@Mgkz+XYqof&(_`3P7F%DbUnIlY zy}uDaq5(=ZR1L$MYjY`S4aOQFBFM0PHyGGx8+Z=n^W@3$kmxj&UptTHW3wGx*Y^=X z6hu9x9$$ufd0%zr5(zd3YiHhG#v!^JHMnpGPgr6tZuTif?&1puT`LJzRxNQ2c!YodOVc`bqBHNI9XK?PsuOq{jpc%5eB*C#eX!2`a@;R zK_w8h0K$VY=%gkpm7^dz>ds$dKX_dkL2ya@m;B=1b7?*Hng`XQV_oR^>f7?q)>89l zKx^=E+l2cU2L68#1)Z@)wAlojA>W#Pl`EJT{MYoknNmR172Hh5hAd+Y$(rR~a>g8) zNZ|rnC(7Gzjrlh$kn$nel$hnYiSg1Dt}olXv-RAr_TCWaptlEt>@ZO986DepGbFuv zm_c1zV2ai1YmGH%!SBTV(#NuR(=Iu60c4LM5O; zd>5v0dShmUg7(Z$39^UG`s{vTb}QdWua!%C8slS<^O7kVfzJdXL0x6MnLV@l8DPmh z5F&dclY%3}9$wdwa8GrwKO7-GQSk9v!@6UXzcJJD@{kZ(!R0~n4S+M?DpQGFrnnio zXSwMCJiSREJyjC(fzvi4(KOz$1J4}Gyhk9z7FOWqaM(|f)`Mk+S@(zQT`r-LPYdc7yZQs0BKSz*JPJW6qW&u9FFG;o|QGDc%Zt) z229LSCQ}H*treg6<{=iZrWU!77wbj7#hiCC0)z+A&=RqTCo?d&AkCq5_C$NIs7w^Q zKadse-PKmT6{>cx=ZsBQ8A^f@CT8-CF)Piz?5K(%X?Y)uHar{Ab?wGILr@G&S=f3 zAiD$!uf{ru9B;<~Z+mRKWJd@zQS#rPqG;)}S^X&$gPW}EGSU&=nt*;*us=)6;(in? zy<(I^AAoqQV=!4H=vK%m|H|%}e3B*Mp^$x%s#2~V8tE8!@se0nEG$)*4|{9!=jN~c zPsR|Emyx1khk|f4HK{O2g^~=7_i>9*^DruaPjOTW6C`DG4A=igdLEIA52fGALfwOs zq7k#c3Gl0tMJcBMsS~CC#g?TYepQcv42x}^^@Y4+pv!0ywOF5CPnk`@xfhO}DE?;p z9+1KBGpjF&tc$TV1+~F`nmB@-RCj==#`X6s(GPcZ%_0>dY;I*~$VAzkr8)$nV1$>{ zD*jFJ8@99NLgUD9SR1CH#FU;O>MeL)$Vhmo0$9LGPD12aitH4bc^DVC-O{xHPFpJ1p1Q5vILN`VelDz_N`fJ)tcYHo-ml zvW%4wQ#|TII&wL|BtU=OhIjd_5jt7 zAo~;&W;_lzA5@hV^xp6XV2EKQim?|T>A6Cn2kWTaOt_jmDwC|QXz z3MjFwG4?vJjN)css|+`UdFh_Kcy9^CFyD1-p6lZCJS*I)86wW5N{j%&`g)D*>bNe+ zYT_piLkBhEc;Wz=6jqm7LvmvA;DpOJc^}jJf-iD~%J9KzJ*xFp5+QuLrUMQf;4%ym z{wUByxO+$|mNT|VG>e><7lwYj#TR zLCg5Y4yYQD@|LZp*1pi9W|?KIrbN>v0Qf{_^+6p;x`s^A<`Kh4n0y6!QA?MO5 zCIL98Aj;>Gl&2271A9>7ybBO^VAKZ!2a{1(L-|*h0-0*1>IPz%ubpLKM|LW~1~mEq zt0kc&zIk0%g|XN!X1mBa&UHMpVz$Z!7@0IAyMlTgsC){RDG5;UM6|!hoW{W~13mQ? zG^l-O+7SXB>n^_>L4OSl@!tNTvA2#?bW$ml4tgRtLZ}k>BO(1mNT`^*zb1m%Usx{0 zpoCa)Yukhikdd4TKU_>|Hxt?&&Lj0sTCB+#I6{24Q;CP{4H|2!?f1AvflQeA_z{Vl zRSV>6|GA12j^ldIwasi@J^+e)FTO*1-u;8TVX$4sPXxlTe|lPAKbXRbtwvP&OF&K_ zg!(ZydO(r@(te~$hq`KLjz1R6hY}Q$rCmVXzeOw{_GuFjBBE0X9d~iZ)rS#xh%YEB zG3+$zFV=%ufpOBDovl%pGNe`$tPl}izCujH3%c_sakVU-E5a6n6gugl~| zI9U4+`S(+eiHM@eFjF?HXcyu1o(_`?kpQx7h=~)sq4MzWG4q|{o6-`)(*%0~vUUQw zUJhoVDx&KDSme?b!#ddaDa#1@Hl|am^0#L;@jl6yZZK@D7Z|M8t8U327f1?_>RS1CVUTxy!dEznVz$?o1{2g z`9)a!x4b<->bfxPl891p)^n}8G~EXyD~e2qwEThcU@CBl%&`qybJUTrW*`0XB8OFb z>1>SxF!6Dz^1x4)WabnAs8+ZXL4nf-kRdgqWCg5M>p_gtZ^S%OwfJ-ts%R;XCizqV&!RF6kZd&@fhrZwth6=4P zqZZ;{xbfeM!Pl+k%%wi^lP4C>2w_^I`WTgEK^A^;Js)Yt{8^cPvL8WZo&!yy=#ajw zcM=OCtiDvOJBR@mHq0x?Kj#X?GjewjOGlT5<$u!p&`c?<4-}c|tTnji1{0<@r=_vw zB}p}w9|RJOQSL-Imw3Sp|8Taytn>QXRov28?)Aw1e3_29pasC^ZNP=$4Z%GJng1+a z$h-vj_yUTyk6HFJj`A@*sg`vq-@{+B6hwQR9qBNhZ?|Rh(iRn?nBM(14@QGKft^qh zc%iwWB(MJXW5!Zc?``R*0xKYYgvz(=39w2KJp?BiORjIryjT<{aHfSeVbzIP1~$?P z3~9YYFl2-TaeWYCgseIhiEWYxIVJUSY3N;y1S{)2Y~l(}Z%?#HP!!q!2cI=q z61dBjeMeda3W0^3euA;H8xtsry&5-W5!<0#t!45rAXgx^Npqzvzgv%WJ0(3x$_G+=68*SAnV-((>4`ty+}$B0R|aF_NvKlYfx4N{~B<(MtcF+h!*DAF#Q>sATGhR;ofn#&!=yu|l!q^5k`-duW7fORyGiyZ;5qX=bFoUEggmWBF zp&C-fX(@a|#mcqo7~xbBA$nxPuo=OafESGN^bW8$m}%YrfEN_0!HAlEOu4=op-<*349ZECAltjs?5N$}60@Yp*m z*jhpC&s%rim#f@Dcl0jEEy&3d0<3})R>@LDsmqK?)!>KfZ(heGmpg;1rh@jNb zd?v9-ERUn9z}Tzo98*)*KO5B;l)_`i3M1ADs6B?#2_;c%CYVaNr@p+&LYit>(`2Y7 zYGmjo&HgQCe`JV3$*V=CQ4V7^$fOCH-8!?Xb*yB02Sk;`b96;hj@B?tyB3zZgoI}&FuwWDY(q$g7m-W~$>O*HOimb`qpV|+DV)|C zxOxN)<_B9-u!@M%19x^WfOQn+wWpj(&qfdvd$m1$oS-AhtC|O)8NnK)E`J}-BLH{-%AgqSe0i|D)bhQ*=qCW5 zocdJ@L}TK{-b4o;F!!79;p-5eieQtRtk=)-zt(Nt-`?_mUdpq|v&<-1=>ZmddKU%) zix@^H{F8((fq9AHZJ!&~HHBBGzeq;uh#!Rw1-5{$p#LjU%AD~_b$w5UN@+7WaEF9n z8gG;A{f1O*cwtPH24a`^FU>+n#}8uON$gQ;EBr@D$n(y`LYX*##+dUHTGxZ-o__78 zGD&;|yOdn*4oYb3OE~3FgXllNhz4PakpO*35tP8alLNd%o|j#4Uv|tvVw8F=HGq5x zsC|ijfU(cv1E83B2e$@Fsg&|V$E7?T&$loe0xoMzqft1v(P z-f(nE2M>rZg?RP~6l$_lA!Zyn>Xez};b(a68q!zA!~Df*4S;2-7e73E-2u@%5~vY; zCH^Imw)}GbSF{mLEe-L%v)U$hWhI@|Av78x z{;00)X+4|z+;cxypyq_z&)`Vl#u?@%TnQ*96Dz;9%`ZFY+;*JG)sP$zUOBk3OPi)L z+@s545r&*Sfgy`YG8BG9p`!Sk#mY+Uj-NIirrn1a`&fd3y2v!%V_*V8pqe+ymfO|% z4xYL)j>K8YDqF5C{3U1VgCMwrh63+%WO7xV05w&Ln%;QKKqcp-)$C)#g+){}Mcfed zoLDCf69lhnm7hK>FI%qQtsQdsE%HMbuJ0{Xj6K& z02a{86&cu`4H*ZTG~8M~Wcv4ss;rT(hSv>>TYetp-HgpymaTi1OC^B(7xSg(?jVhGD)wjNtmY?wYtP9@s${Z^Z(0>1Vk_C!*Sr-gk5wQ- z2u6nBBeF~Z-7K>e`b7lFt^e$fgKSY#P=iX_K?q9w^4 z>~?s1@-9LB#Oe*wN!C}Gq^fHRL;VRt06?Ie7>*!Yn*1hgW8(mlj5Pu@a=;OR(0G>A zb7%v{NsdVq8dZ}V)I5K>2XOIR8mlHM&EhyJ7_UGynp^f!=EV)_YPe&`owT05A@dKy z+9Ia4cNAaQThsoh+na}{t+44t>db5X2WGWKzIq~{vu_9w!Mjhi@*f}=tPK{P3kG<_ zM~%BnB*d|ZcDW+D{~<(# z(jrd``jG!X-uk({P1L?i^gHNS?iRfuju!YEEZaTDo=^TQMc)$x$1ftu^dS#c&l$9M}JY}6*_f2`9hCq64c^|A05fbALNV+;K zb|L*+I816S4{`96is9VNVR#$X$YKY<1<^`%tgiSC;PM4y->x!pmNIL5T7sL@Xh58$ zC0TV9AHTqJbkgpVrDtQL#OUii7&cNi-+z7erfnZ{67!n0R2KWslG|a}4hUXIbV2r) z!GjLpRd(bu3JaBCi9XoM6=4{VpR zWwZx+Y~X)^f=qhKuRmf0 zJ+qo^6Ou!((EX~57nZ1Qxn?3DfgTNghVD(|eLk`S1r3z`sJe8Azo7-!VsEJL5X;11?ljr$MITKu5c!>Mf7O&V7|+P-GpNsBc)+`J77y?IuzW%dmST_LN)O?fYQ-lCKLoAip$eiT zI2NcU_skykJmem2tvdiS6NZ=^kh$pX`^hu^3-efdxtU64f#Q(89|{ViMi5(D3+*uU zTuzVm#cpNKWh5}S@O;oJAbJ69pqs;xYLYQHFU3?A?N5K--psKHjtga%f6-M#Ef&uJ z`Y^NgkUgNd;uSpCTvYs->N#WRB`gaVjXa@r)MASL(Sd@oYW-Jn#IAnlYUV6dTdx2g zSFVK+Bs@fBY#TKCiqbgz#k}(qA{2Fz8=~RI$otYzngIA`XjORl7Qo4#KZ^P~`E%h) zz|a;G8l-lEHH%|b)mvaT$a@&{G?{Dcc&x^u!Xm+;TLPc2Zl}lu+L&<%IEZqM>@g;5SVDD}Dm(5FW69+U17Rsw_(^Rs zsp35ZUDBh6_ldy(=t6Gzm4YvbWeq_fkd#1#2`rEF1l%|J_y+Dx0egl1t^A}GLo3v_ z$bXg}Bp};FhZZvT%1sziSUU&{@JN`3*)9h`EcVjHbHgYPS@BI!SvuH8Fo~rM=*J(= zI4lcOEpOoH!ued7Hr_U+UpzSAL)(=z{3#T9$8UX7oHpl&nZ9>c0XjzmlCdkzBXI2R z#|G|oxFaxm*foDP46T8a|AQSSYQ5q2()JAI^eAkEi9d4vI=YHLIYDBv^ewL2{v)=3 zfsSL3W6k>lqR0A0$TS$_4?)eb!;WfjYx*(BrQ;4d4NW-t{)GKHOYq8EcIU%OEM_j` z43g3fyYei-AWR}9&cBLFOSR4g<2o=-gv5GkH+>|;EX`9SjV}C;qQUI3U}Ypy46g$v zeE2a=+`c{`1S5C=?rT`Rf=Ky}@1mE_7sxzgX{}D84!-=c@FhgfZ?kzzcrGFlXJBFn zN?t@K&Xqr`r=mG`bep26FgqGb8Vn8?#Y|Kj3 zt~^Cos6{B89Hn6@MEXI|&2bfARyomH<9@$G4GinRCO-hHT?E>m=xNH2l3q_h^Y2!| zmP)Q_#VANQumXfg2O)4vLile1Lo=?{0I0oQCI$iEcM0%ks)A`Y9=-cyiL8V?n(NRZ znXXTf3b0`HO2{0Uv4i~>ipk=9Ds+)WsN<$i;1=TK+f^?;dC7fByGVW%Ds9tr(|)m- zLcK~Npi5FF+C5%RvCcqx+!na#z1xCHRDYWCuvojY>#1ln58+*KuEE;4BZKw5@8O@y z*gj)as2&|H@?b-~!zcIt5qZ?583|S6vDX{+)@*PlKs2G~B`AACe-QMdXz(gISVkj3 ziV6kf%xd|`*oG__cOZuCSG;YHe^DC34oLbhLIr?aesFeHlXMkTa^pl$32C!=sohE1 z3Gvu~>GQMAdMiWfKvXAYE%0mPcba#DA1QIS^n2mU*R8p6P*P7+pV`j8^w82p9x?qx z-iPQ>Hj5>~B}QVCr4I%^@6)Kj=?@ih4H3-}i@@VgWA6iZ4jncCFJ-PMoM>hby@CT? z#Kuw#oPx&AbGPN!O9yAas`wWcya-f47BU&FI>|_EeTlGk*X>Y5~FkA z2o!iYPRioCjkPJT0^~c39c5dp!|t4}K2bEI({d&@Ez6kLhv&+JpiYhpHhNgbzKlr| zWZWiuhsH{wm~iXvuSZjMtZMU34~Nn@P!#J#!6N8njnAm;^4DS#YAIX}yDEV&Fpv^U zsL{h9--9a7!MfiNU&c5vr;mDlKniNzUin|$|0ZrC;$-1&FD{ny{a`AmcvJGY4TX$} zps{nr7yy!S2hxNhaa<)OcKmFZfv!0%1);DNK=G7JRwKspD~7$uIvA32{8V6rYoSPN zVtRnO6+&_6zzTuJ?Qp~qC5cVuA%M;pO~t{HE4VS`;DhCE_7*$d@Nc1}tZ3CV8Xk^RLV?4{4&Q7($~YDhJ1nXn?@HQ>j}JG6W)pJt;~8LV~~`r{8t0_0T)t zUmW&kv-k7d_r2D-*0rvy0n?`Y7w5M0Wuc>AR|SX-tO$^HwqC|pn*H^=4-tP5moS__!cw7GWgZwj6-u-Nr{ zQRv$d-12kOz=&ONPQ(ZR3wYp+W8K`-p+g7}F~i2s5HO(S06MGZ%V%F4jB5#vF`$$K zQ_q^j^ ziSx}C*?Tw-LUqh)+|1{f@q)z6Q#ckmrFz0Exet?3C@^AVe!fE;qZ-)?=7FN^D-n}@ z#=fcw+mt0R@9sn1@zQ^28I zoN}gu({z*bdd&R#v#QdD$`< z^ol?DK4ab>lRizLMSi}SgCZ_ExM*l$LN*nk2;ZLdBv2a(RyjhWHJ6Cx)HC+Jf(l{) z+{6kS^ex9n_z6aR(=b zr}?UkbL6?tQc#RCOwoq}*&x8j<;(arBw-*6k_i&;Jbe!`04;6^LpF$|to8!I99M+5 zHn?$EM9{?bN|jDkPqxJ@Qt9U6p0Uh`NH4St<%I`Sz|+2we56r{!7@HY-Wv@B;6d3g zf-bPv%D#xEFPzLTOxPesQuV1h6{4h#sm-g-&iHTU`QdLInm8NZf>lG111(}hPXZ0= z{yCWQC?4tm#-Jm|PmCZ4P~mM@D1xUKHZfxT;^V2a+Td0@1H!-?-6n1wSA}bYvt1Q+ zgYMTmzXxIiPgusj|6K_!ZcKc&n2kxQy)J zR1?9y>4`UDkjQ~zJKHZ@3f3<2nA!q&e~OZyPjqP$wUwH7k9}fah_I>Lb?^|FKSK2w z^ino)A-u6%eBmwA?WT<78wkfVniKDgi$EBdfm^AKX?yQ!F1!cON9N_ziG*etsIVvi zc3C#*IUSt>t}0F6iX4FhD*JVYHbU8r@enb94r3f`3`iQYLUUzj#;=U(;4X!UPO7 zmR>;n3Gx@W+hjAv*R_Y#h|wXObBOOj)jfE~Y`my>J1!4n6uPRfL5~l|C4oG`n4<)m zV^iiC8Q^W$GgZh&DBxcOpHzZX$Ir65ft>DnMdvtWfL<&QG1(;BX}BH@9Vv65?#54m1CGCw=+Hv&(Egsz5?_aIu1UF`H# z`t=`kc&twG%HXUnYC$!eEQNqK zui_mxdQL`jw~;2gc(9xVJ|K!49?*DTwbpwO88kBdm?6i*;Wx$qiFTA|q6+i?`JPg+ z>xePjjt2PFxQo$-6V&=V{1=u}ov=R?`+SuUFSJ1Yq9t-lJbGj({p` zE%Tz{+J|8oKOA?aextj8Wj;D4Admw{x0dcJeQoyr<0Y@NYrGFxO z1|BxG2v*2lL%slW0@=)Oqq%h;#uoh^2o#9%#Ycnqr_?`zBQMv2yinuUcU>j@G4KT^ zkJH+MSari5)fST7cNYp}A1D_06_j*Pq9SG`xPK#OI0#2i@eD%OY#hJFd3tf7rewY# z@1sZ*W_P~Xp~{zg(-)$SL>Vu(ID(EXy2yiO!fxyG84pe!Du)G@|DT5R4_3ye2 z{I&iOWBWi^fhl|E?~zpq5n@tVsta#%Z_aDx07_IK?Q7ZS<>Y_oVZIe(MuZB(aO7a= z;qJu+vfLy;O0ygQgpJ3VIX{$dJtf*=&V3D@j{s7mPDZ?v)z9hZhx@MzW5tZk^5nAj zC|v={n$hiD?@~G0Qo#+K2OUkt@1Tw;3<|I7ZgTR1b|lR7kZ~PPNHBo4I!}3y{=A1J z>z)xy91^-8sk5GCt_mgiM5aM-km7PXQ+3Hd5zsYm(Pc56_FB?38=n%+R?=kpl~C3F zN~2#z2;%X$5e;l^8_FYc-X7V+*k5vYMAIks%in1l7jfM;n}{B(gBpWVbbL~W$S^~c zq)5~!;CVj%At6n)$K{KzEY#(c%ziElh%&G3pywX7m$~N>KqqCKI$rjntDrOKha^tL zKAOb;)|y%19==BhQk>qYt`*JM)51Ah_B&Yxd zCRSvIuuF@n2RrRFErA>03h7r^9H=ffdng@MaO0)?I;ShhuQs;Ds)zvaS=c{C3~Bl4@fuNv^AxsexJv1GS~WY~4oaVwtr9;XVPfU4&i zQLKwRUm9izT^!6@hoT4N%+NgMbH{HN)G;A!Mm6>x|Dv1VIYQv&f8fWV9^FT~B~ZOZ zPcew%HKPG?jZXlB6<>j$1ZAa;Y?MvfM29l*fO6B4>7)!sf#~i+l~)9n`^}%>e^?#^ zzzJ3bvbaFLtif>*M>RaD-;$b>zyZv@YomRZig{>W;NH-5D{%@OUyC7u!94=bpeK`N zup!gG#BYFtXSj3kMHae$VUUw%#a9K%r~}kHPZ$h9&b;dfm(yp_%V1J|xnQSnok*0S zHOIYFV`#ZtZ52`zYQ00S$K(^MLThVzx?NR43M^l71Vuk4F_{<8n0AN0q`(YlraU@Z z)@K$SrvQH)u9U)QuHb)UBAe-C{r!R}@sh_(hW&JL3`N3Pg{dt94)A(Kfx%63`#lsu zl6F7*PYF-r!-mc&t0m6wU(B>}3zC=7Xt4WqmNCs>>)9RDBK1yC+O@(*eVvUqzc$L-V;V5UOOd8Okg3x%gb~HuH4pi8%-Z0_?6= zeb$M~6b85r(~kFQ8KH)+SM@T8*-E5(r#fSzB!qW|cK5G9tqkwJE)!Sa{Mulvl?VS5 zxzh)VpM(rgrUu3E#6D!(>EW!FPB=Y8iHT<*ZxhX!;xW@o#RQydU3=uPN2aO}yUDXN zZ1@8aptv;ip^s79)%d8e!#7dmTjxtUf$p-kn7g_TEe_lf9OliFo-4sGRpd98WdrqHbwfRVZE=st)l@$1z=qDgpAxdU6`nEaOuPOu!>l6D}kS4Oc%WY@Vn z@rcPEHjEkI!A<>A&H!X$@angJ2>n6c!@cTo?c=pZf>y8Bv`m$qg;ukOe(cp9<&)|O z>Jkk{;Rq^aMGx3dI6!IxTqnSw4N2JjW~Ov-Y*W`mXh3Mo95Jp;A{c4o{v>}cO+9~F z^4tLdFW_!DNN(^A;AM zuG?zw?T{oAH!R{T!=;5iK3_rPNpT$pTHIcjZXlII@6U`MaJwZ;ddqxB4Kbm z{R38GX`)7fyguX$N=d+bI*mvJRPR%DN}}o6_OBnu`X6Drh9QIoEtL=g@N6Ir{>Sm` zIT&-x3fkS#UN#hDhwhf(92lqY#ny+tO|nM)o=c5czTxunD?TSQ!BLwH)e^C+4j02d ztuH?p1NjPgO7$Pvl=i;iv~c?|M|1q`f0HWFvl3b`NDFaiI_7H*xYuSv{0UlOM?y16 zM?2s5wm=LV*vRF7yi+7=n92Z9Jhr zz=bc>@wW_N*6uw}`ROX&j?Gc04+dZEUrA8IzTw_!Ml|HRN~{F9%-?IXj>972 zG0=lVqcnOP0v-}S?f3Rp6+i(>wnZ}9ya>M{r#UFrY#HUKRElK!-Aqq(EK%RKyL|_> zL2IA>0ry|XBBjs53YhkX_|&R$4yRUX0V37__^IPyFBEk-Rs~juy|Qn$y^WpB>=AV) z4t2qPl4a7p_h{%@N&L`GIHn#P#|x%KnejYH%UF7vNMhsEDLIj0$2~1(S3i^H7beY@fD$Q(=ePtI2T<$MEh;7FqWG;-R zz*RbmT{+c~I<}mJP}kfyJa?O+@5HaiBhR^T7B2)GX=fTN ziI6XZC2?eVo?iwY+nRO)%p)B1rhj&F9K@w{d|JST3T`@9DD0}Bnt*Kvb4D}++*1n$ zrH&phYYOm0Tun}RBTKsj#cVmt zvn|mY$^2s67jb46ywXKMm%`0FNC9@20J6Xtmy z?j0^y#AE_m<~x4R)eIRfo^F+|^nelw-d%D^?XPi-+KSx25SszWtlH|h6&&MVSA`^M z%K}5RTiqpzT*Ty+Ja8~-ZwxP{{8%AdvqfA7fY5E=Fr%fvKhSlgKn&=Sb|Ha79Q>1` znK%UC$G5oRF$^r_RiOCI8xoprFi-_mk0O4`w`q zD750_yHPK1bAO>6dZZEN*Z0pbqCFXG8i)7haqJ=I^Ab ztIesr8^&?nySA0Q2Cc_I>WU(j9nQKp8SpN>8%nxT4a6$p)PXHjeN~d9kvZT>Gao6P zu)<>}3zZY+p-O>u?>+5FSq!1IeJ)3+msFg)p1KeGOxF9WpxG%!{4wNi?WebbLUH{H zs-V^gQgTx~|9cEihPyx85~irBOApssD(BdFGE23!^N19@#qQ8^PF> zXV#W8@N3d%BRl5H?y~iF&&g#%7CHGRz~!6af~0s?1x8CD-Sz*4iBtQ}XofslDq8iws41aZAX;XR|wigmByft8Rl$B@Um zu1TOs(H>Y(^wbHqbFeBnl^ZlM;b=^+gc+X}4#g9VLGCv$5vsbB8H35$8(zN}%Lry) z{Ig*ig7AT=0r<1C zP>q_WL}MtBI}4&Y`Q*@IsCVRPA=Pq_Y0(00c)GyuLWbxud6i?3B5cp%X|hfkt8hSd zXLBdjY?=L;Oz^h9a#H{%YbvP;Li5ycmpehwR%Y%ngRrTpP%`%o}3LYfDug^ z6EGm&a#S{$h{Fze=k$rs@eo1<2(%-A#Zb}U2dz_Xu~pDUmM1roZ2@quet&AtK?2w2 z&#CbW>O_2XP@hU8g%vhQ0%%xW-_7r-xkuVWX~*h~%K4{)1~9& z?JZj$1HIs3b2FjfGzLNGLV%-e^#@GxR<-U-526+y0sMCs_!C%1lJrPVf9ijE5Uq(v zMB9SH&CB^S$pouE7uFw~JDD{S7vLUEvRe(ALb5+^+1sq;EiT^kmSH)-dAdJK;8I~s z*aso@aF?g@qT3IBz?%VaYVI{?FMOl=ODemO zq^Kxkc4sawuR_tmrooCJ1;cyG9>UNhHXl{R4#*X)R$?orU4t21AZ!I?P6zr;Wa$MogmJRm{(Qe7_1DUA5X7Up5u!eg_$noshtoI91>+zg9S3gv&zHi53k#Yv zGrrirbSTAQ@1md=@m3r#+)TSq)H^!XnSSU!uES&<_MNZ-h>p}@RGWBy_o^!pg^`pl zoWoM!uzDiAB~mqeXgt=B8*n1ViW-()H`5K~H+Ai^`~u*0+q>Mnoh z$cIxJGk7zU(JuW1auV2D&*m{uC4tx%`)K!_A>31~o@+5sBk{|+S=kj)E+RtFm{~xz z3W|>sx`5h^|HoVvanT^C@jH8yO(shz_&=dtKZegiUxeYy>6wzPogz?Sv^D^yB!3y~ z=^QAJ-V*t2l}hob4VD_aGc1cs1)H74`R(_zt}CIU#+ooC)T2K+=>Nl(?L}On={&$= zPKjW00|x<+2#^(80<`5^S$d0O^(vZ#D!#ZppA(4aOX`%N)5vw0>j3d6@7!!$oyel6 zCM)=d>ECiGu(~~cCwFbe-(X@P)Sr^CV5e6;b?bDFfE|OP&L)q2+O#T8|D>&AJvS99 zuo>ka2Fp$&)VGnoOmFs6c8xq_k;Y(Syy|)Vt+F)x?RXZBmY889C*d#U?QhTA)IEUF^rMX^fW8i1KhBZc?(6^H+8{q}Uxg&YRI1Auv>QU_w&57Wlh*W0dqwvYkb;8#f!p zsPHGOGwn$@6Jgr1Tzq*uLwT%1LO-%dX%{=Eao}fC;$7E2?PD4VeqLK>;V6J|K9%{k z{4AhS01V{5stIkzX##hT5tcd!gM#y@bb2w_BSlz5)~Df3YVwh{9`p&9iRlNf7NF9i zmCj#PPy%kmSUXtgeW$p=q7Hw$wn;b5!-3O|$J29>Ws$)0h<@UuhmPBs-dnar|)WC#E;7XvGU z=GW|5VLfr<05R?;nhYxp$$Qxg#@VU}dNY}#z5=0>D@#8C0$t}+0lP~gRsDo(&_Uz& z+S?Q?+2i@CE*Lj+B-At6mxNXee^G{(tgI&#q`a~{JC_z${=hp{6U^`w`8Arj_y!S|8qZ|E zZPRcB$7g?alP7TulYB=7ozde=<&J{o9}u8JvbpxW0*wh= z)8d%w6IJpnw_;=wkKlgSuf$vi2^SP|c^DtbAtK`s<#+^=E9*@nqqLz`{sb|C0K_WH zHyW&ohTGPZXCT6&d6;;3AU-<&Wel&G`#G^sgDwYku;z^?za zB%L?AXVn|3(sC)M#fvLrptS$FR+t|mV@(aot?S_J1?;v?$2AP48^VMz^blY|mfhnHBA%!>we*xoSZ9q{1U4gq)lv6kbhGIh@GJIMhp~&y| z82MoRcSz9%t~5gtW{75;9grD<-U%Vhxk_RrUdpibT+&XQe$IdM72e;SwOVF?2DvUYo_w z6n40;+|3_NfcjZ?Gm}>)NKRPwxs}IpBMIshisRowMOTIPTXS;bbqHUD%}Q5d@|2K7 zX6SUGPBibnmHhrS9_!tnPlJcZ8p*unrO*CFTu?@(kfEjv zf0wiY6j*M@y~e1<*<)SYRp64-W(xOq*8eaKh6f`GcS(r1KE`X&?Rom7GGV&`az=N` zb@3?N^NTv*)((7pi8?YxB{Xo>u4m(QtQVYAX)n=>04rnM3XG{RaR8H5Jx7ZMd`I7B z?nm1rq4|SqQb!9fzlIO584pADBOJ;4!TIEomE5Vh85)Jp#&kb5osAC<5EcPv;yih`;rJ zz%Xd7$eF$TMqtbq3?zYCD;P`VnEPD`ym#AXU2Yv%fcM=DD`rWx^4CaW55F7eliN`&1-fy(>)$3LSQL$^h5#T_HISz zXSx>1m{iA1Db5jxgn%*M_5GiPpn!3ayF;V&zpMyxmCyE4ZI?V&?DP}n0GI;AH~BZ6 zXNev3+)j(XHg%gidI=?NTLoqSGp_X%5uS&AE*ygVIexP8T?3W<$4%||C1#mdX zr`=%qPI;e>{;wYes4&!%-ol(5DXht8s`BNs4cJIJM^GAs{!JP&T3#Lhnr{B6)3++i_xl`sIN0cKS^A66=HqR zD?`0tH_D!B@p9+gAUv_H8 z|I*?mKSj0vAemW_9i$_`dA0qw|KD~3`o=s(!6#%!P{&sXZj9j_fYOn#`U)FjYv8{3 zl|A8FK(0lbbAlmfow5?rsN6@#AB_fG0>@4BeF(ypkEHEKGL$O{TH>i(iEMA@60{kQ zh*LoN^J3a@H6nKmU(Mwl&BKkN)UP?e3K462eQB_VEQCXz-!fF^ee=Zr)PZey*}dur zANLK0O3rY5mqF|oD_Z8taAuJpVOi-uWqp4j{}7HQ%8NGrn>nF**G0oQ9_HumuouWF#O#;zAenkbd|K4u{h>CeS@iU7_&0kU=pP+|LzHBF zA=&t_MU@Bn+mtKFgcy|aK<%3#7t70+kHFYAT-FH3JOmd)I~|X+D&6^<^#y|BA=wel zk;<}(I7p1aq1Z7NV?=_fgFr1(N*sRFk*WIqjA_@`H0HDbTL;?KD8n+g3kQ~o=|zp%;q;16$Sv^V=Ne{RW%@{Pmue)AGHUu16Y+Pd#Z zEkUu=n|$_uDlB{vBshx*`1CqtnE1`S(&*Jf0L0)S=^p?_u0qgaQle)wM}Q$wW2h~C zHAN;!a2%=-;EB&gr6L+`0o^%Eg?rRcksIJB1L6qC0MVhZb8cvNR1Eu4kbco#SA^mq zETy)bnE8+`S2#Efep9$!Y?s|?EC1|qVIy5=Px1L=9c;=9Vk?rlFT1ln0D1#za*h=~ z1#IF7!dtqYNW9f!Nga5?Vx3Vlp=3!S0CwaLT_H|a+=yR*Sro?q()GK?+2+Bi!l`x757XYL@nGK)^Mif7A2K{aTk4J*y7M1u}+dG=M7G~q^;F`@#w!r+oCAkygz-E z*nBLPF@4O^0zQ=vG2D@(#byd)rjU0=?!R@3m}$!tE1h$UBLeBWdDuFMGU}KUHauEh zA=~7yRZp<`WnX@|oH<@^mErePOXKS~ZPzM-PO4tT*o+i15)Qyg+UN zab9bOzK!O~WA0a8?bwvRq569t0$Req{OXL=C+>v_Qd?yvvAiG{5VHgRZ zm4&SGzkZa2F&E}JC}2MK+Sf`oKvb58QbZ8b$~i@2ECfVEN&}*KO2~a&Vh`9UGWcQX z7-wG+aF$tRqYND;rRP4dhxav(A&ONXCRQAfpcPZ`q>^Dv{}zS;Dm;+$2p+&XPt6I| z_K3n*fD>Hne0pq7Do?|=-J;5Ge^^n*J7xy%zaPP_tE1)P;Y3*d%e(FjgpglwK4 zo$x;3?D6IT5ct%|Cv0Ox>lUvsel{Y^7dKRP_I;1+OeVD3WPZYGs z*c8q^cA#qJW|5hMj9XQfig!Pz>6ebff%=ufMPn^?2gV@O_K`^C6(fzJtM3XS3cdg9F*;+hKeabgtOF|ut41>(LzAe*ZV0?32LRrEVJ z#qsnmsl4opxo;?j6!L}qLYxcc`iNl;n%nWpWN$&4^&d?@5U1qHq5sxBNH;g^K??mL zj2YhY?K&0WXz2VI)OB^1*J)Mmoa0P)rzd)9L#X|j4nz4Uie7I@TnV2jUd+6rA72yU z+qN^KBw@s59f$`lOM;P0shOL{7)yuB?-kM2z*$TJ2K60_C_zE@g1|vyZ6P8Rdf~w) z*=6NS{%}R<-v%bJ%s8re_QnPA?ZG8E6{yeVMjoSRAWHg%DslU`1mEoN=jh}#VFg6p z9>!CA4xN92`6JYf0%l213pygM-k@Dz3*=NJxW$m<&|1+kZ^@8(49o4Vn@^JgiA5N* z9Y9I>^=BEN8neNkfKos+Obb@>t>p@J0_tJTDT4Ju_c0;z1?GR~b%-1VRBXY#z`tqd zSDXN|E`N-dO@co6{{PFpFO&+9FEVBd))(h}hRUh{@?#tXu|ON(vp-7#C$;HiU=nM;Ip!{*$lUtoAag17pF6o}f>h&*$)bn+ zLOa8A+xkM)H6iJ(n{dbGb#HV6R*&Sw826ZMvtc1u>h+9{;(^H%6Y}b-H$q{EANa}L z5aH!FVy80NvPuNXpaEWyqvQXi5Ks$x&F;mt|3QcS+$V#&&J}Sg_`D>MF$)BhnP3(( zOI+xro&y*iTClfXn;o%W$X&=W3Fb^S*m z+_03s9ldL>=$9eww)qVzX%K!VOL_=>I6*zO?lT@Qy#ixTlcYgQvv5=~{=n2&H2#o+ z*wvA3fs=r8NZ3cccjGq`uh#+0y zm_jfMPkP1YiTe_A5{`!YWlRj^o^dk=tHn+|yr&I^sEGb(;$zXN3UhQj9%#ny3;_S#NxUz|TG0WdP zKjYz7%(;-A#DW9>MoTO31CGe}`S!Gpqve?&rG7fwkO2(aSKo}M!21X-Er~}^sOd_0 z6mXqi$x=#6BXLkC0yHQtvt`x$XmnnB9+MSu070wlMkiilgys08d2)R)zssb_tix5w zo7|8HFak(kP`ERcm)x{{=_p|9_hh=u=57NMgq;|y5WKk;EhaS(4$OQ|hwjtDiAy+% zh~u=bBVl04g8oPkV0V0PI&UCps)i`k^0)?bd`xf#K~Sd_AHQi1qw~js1t++dZa$OL z;UaC|(v>Qsfv7HNVal9AP`!f&RPUhuEG1i-TlaE;7wc``d`EI;V3sz zDU@>1F5|c0R$=eZRmV#>FIPkVxthDOU{cOf$GL2ESzY)9PZyFm;}s@`F{rW{FJ*MI8#PKAo0s>1-6y1=6q;AD^wn5j5e_{wqK+Z)KKxru@yZsWCJdmTnCf7w;g6^pU-?&4?4}?ER2yI#dW`ny z5LLv|mFvZ+-tt(mHUtq5D>km02#?VJ#TIc<_x3zciDz5{>=aP>+d}}fide>^L|Bis zD(f~o$c;AvDwjf{K*HZ2r-KAZN0!66L@F?C4%b}hSvO6(Q&^=WW-TfZkSIX1v$w%l$ikn1u#|)t z&!?L!$CeN0@-fW`Q2ONJP@86I!EAH*JChJpkvMsTbK!KiIrIYU-y`E5Di@v=1%9RU zL6b)%|FBaT1V&px%V&^T461H&KGn?jevI}mrj_;FxchG1$M`h%Gl|2@QvSvmynQxq}y z(3fc55Mv@bIJfPyKy;bb;jcp?-e{EpMJn<#Dz!uE*3hv#A#L zw`%TyS3#c+q#xK)y3*nAD6n789E53P?<)T%U<6eOh_4_AxAm8ceaK-qJa6Rr851RD z9cn4iuMVg16~$~OTwz!b#m5}PNGnUjq@3mAmxI{Z9&*y|cl`L+4%}-NT6bKG?D27%I!vaP+(9^6v4e&>}w*cr;}G`|FS7Jmc! z#J=z$geK7F_IV*iI%l7HHVR|3puY5erwLd6y6; zRBzp3%MT>d(wG=6D`{>UIqJ0^D@s0)VUf7=qj+JZxKIjv(o0+dIGJG`)cT4FAka^wfdC+O|x*3g;6&)=z4d zImLTLb{Hw)KEvQrPO@?^KsJy}raGVdsAUU=)c7+Ji=9)IZV`n_@I`=om?^OAJZ&()8dC_i73n1Qj_#JVN~!eDY;XbpUx=R%wJK>3NAd z8=j83o%+ulGZCICc(%T?iE^M3o4Vw-QOC)i!9$I%vYWDAZoQ?FWXKzk^MOB5^Auo4 z*I$ZFJ?+j!5*RY0%!U(8r(bZr{0P0)Y&C?aK9vO9EQXhBL91e2)3!~HV zy0E_H;Q1tm7czALZfO4Qx4h6TE1~nozqb?xY*U$;Q|_y77p9d-|=+;=v`asqC6e;Pt`XkF91Tjt63?43GUvogS8j9g{y1P zD&1V~*Yx4UL1XpXXU}A>0=7pS$gQOIj^synS1G3o+lg(Nm@DDS6a?j6=Q6~P%>OXN zb(TthO-UvAiq=JV2fYRa+X7^P9Dk(y$Ocu|!@M217&t>#*HT>*6iLB3{E~fN8_p|o z!s#dBN5D%|c@nd=mDMW2i@6IZMUw4Bj92oUb*6u|+E6`-LXPKD(THlp^b>s&*&81S zZx)JD2t??z8skw=KjWtXS58ddU}aRLihd#6gUfY#1hnWlnFTBDnAoIhf7cmSAmLRA zzVE>xd>@*3T~kA!Y~d?eK>g59Z3w;$8Wey&P|hI`oL2Au z%=3s!n(gRG5$H&GO~f;&CmzKHbi!lw9Uc&d1SKs)wAdG=$RScZA(Gjft041zc% zLg_1J!^b`D*pfs7TT??5#9;$t8+YH0R|LY}gyDLWg}*my%WX*|6wnxN57!d;kLa57 z`q>u58{<>7d&9@=n(40=AQqQXCv5%okGvMpal^(ksY;tu*B`+~!)ZG^rE#4VZy4_( z{R3q^9NgqqH!upbjBqd%VsZUwUrhQd6cK)jsUY5k$p%QqminE+tzL*pBRu@Sl{~Q2 z6en-L6Y*M4GoixcDE1b(VXvXCdFT-JE5;>u!6wb5(=xpf5*CcTCYZ5Ur$0b=N&mo0 zvV(uTS@doS*G8>)9<)KItR+8?$D>;S2;IyBU!3_m&~pGn5GzuiUn%HLKpfVptdd`e z$ZG;7Jzg%>q-Y_VL3RVU`{+6cidX z-Lsy~!3XC~+yEKFLF|xJ3{rq4`W0TQCeZ=slIM=+Kqz1}4b7HmeCKwJj&CRUKEV|a z3IoU%IMotf?>O2WdgeveQWxadLYUi?F8URHJ(w6w@a^V!BPc+^;AlAejdFz!Ib6Zk z!tj}h{iP+h{=gq2i??SRlB%|jN@oXmT%LGn;sZ)yz!~dqsyIM!{HqQRF{REBH2p!0 zOw@;!{qglN2#2vmGT;hC=pm-&4Uvd}ENZJ+-W3jS12DN5#=0x&YJtcH z_D-62Ur|f)QKyISpt^N@GuRc#en`x~jjDS=+_Ee#cSFb4Ylpvh(a;jL191)juq@d` zTHT9`Bd&{Cq#-&|u^m;_SZ@e!Ng`Y>YD*mC)_D?>262#w<5TeAaj+8;gc4e;xB_D& zFK2N6@lH{eQHezkh;%zTC)c>A&UDQ+n(RSK`4a2VB%u3N8{!rHqGsEhZ6UCPB_Ha+AcyXIGBlfo z4k)3E-onMWEp%XoYtYazU&co8Oe^SYp;p#mpT83f1|ygCr;f zq-E7=5zHB7?$r~+dg1#qX2G_;J?^0c6kfrk5#DT%KSB-W262J{%qz0X!0_UppJ@Z^ zd9L!3?7T&jr1g)cE)z!|uhRob!#)56aqg4`kle^wFTco(RA8z_BT+~Br$x=$N{ZE5c}9q09{=1=#0id~4ied&)Vt&rbcIdbsPgZyr>qgnUo z_e`GEZ@EA9Y3DeceW`X9pJYz`MIXwLu#GGeO~Dg6 zEiz+*0RkQ9fR>1v3Kpct%u08-!<3zsW|&1PpMn z6B>2cBrvLUM}384`%{1yh(QHb3{Oh9>b{G)3;s@N6^K>@1nCr|joP-)b}wm#TX0Ik zt8%Mh2&mx+EgIP85n4BI`JUDvxTPCx6d!~&Nc!_Mi9?b%L8O*2*ygLMN>5H}61QL= z>gU(-LyABGL~eN>GFA5_WWUW_h8rjX3OYfYsybhS732~WPG4JCB5Zq9^uNbiljQJY zk+2Z&MxasDh{ite^zM{|-Z^CnA*0MfWVZDN)%h* z8GfDKOOZ_+iD7SpdckomGbXP%m&c8lWao?e1-}3>$>xkapqfVU1r>q0P1&+(Qa07G zFb7-rfzpy}vs}h!;i0)=x`;>=Y;xsNgI1>0XolJaXV!37-oz-5X_`)ILEcxIPjE*v zxo|ffKN?y*u<7YLPbWDC)RNggx9uaFvoVq+K9*BwWPoGtN8XMkw&77Y19MnR+l_M= zGYmO(hwAv>LH9Sk{$bi?@Sk}t?Y$qrD;tK*ULsIx*3k)|Oo&PlyvkP1XyXr#6P`yc zX!D)U4!d6F#&U?|r?67el|2Yq(t!{E+?O;#Ugi!fIR@5RxYTAXi9i z4*mc+4|pgVW7!an(xS5Va6kPP`KdyLRLfCe97^{L4jq)P{i~E*h=F-|ra0^XJJ6Zn zs|_4dvaA#@z_NYf(6@Ee%+)6`J2t}*G)dLn@c$**-@vF-kjN^(71zeP2 z_pvo36VwR$e$YK_@MH?D3Ve1%t%HkM3wq?LPHoReOO006{4?;r#twW2nA9}hCdjk! zI4%4pGhFn_$}R#Dm*7N$Uuk{UvsJcLj3`x*o?wFzpq~ zOCPAre|-3mOAgZNo#u8c&$DcyskuUhrh*Xy{jp|(!uIu1^? z>)^Wt*a&N3!PP~i0<S3OL$Q z0W_Ptih!wM|0ZlegFTCJ1@UByIiF35T1=W3kgNpeA8{KnALVF1-WhgT0Em^-PPkrn zWJnAQa+z&Ovo^`p>s4#k%>4Ai&ZeKKdk*RmpPj46o-UM=QF(`%K zhxhm!qGtfp#WLA-hG_?0ll>X!whG@UY}WtapoSpokV2%ATwX!2l|IQUhVe&3xyZAz zkIZ0bX4q!6K#={OGJ3!Xyz@h&gD6&@e5xT8`FX=&XK%P9dq_umKNUjKCPX|-Uvo0HU9ncKQMdO@jWK2YoxK|kpF{2W9HHVHUS1rDn zV{zjJw~(y}m`ullF`wup3B$Oh`!uVuO2&q3RVGaF8pqYH@Bf97_bLB<4Vz2(u`uD< zoz3{=6u6BodX7u{H}~G$Lgl^}r7ZO)$6_u_G_e^mCFkxqzJHNG6O!*i=T2rv=9-J$ zzfdwePz@pw$ekc>hBdFo^=DRg9N#QE&3gf%4iEI0crSZ4Ws0dHGh4Idf2y_w-vjyS zRUi^z(b?QJC>PDvV9WlB)Py0iWO6V3D?syJL@B7UVtU#`o9>r-9pF``< z%IpcGCD`meN5Ang_>|_>mrz#;b0Vy|R_NxY%axqVdI!@q@uG5JZ#Sjy+?IJH<_4RD zOkp}5xvhjx7PERpjXlwRTs*1P>FJ01ng?FSb0KccRTLX`>yoHY!1BiNaFFnjXjBcQ z3fX9q#teGZbboAUmcgJ~M(w-fQT)Jm7rj!pnUY!$W)20r7TG=pGqE%S_N{FagAYtt z$yey|@R=&k#0eojty`rJP;Hp7J@BZX$8fE$u|`_I%B8V3Zj+xlDr%YAV`?D=e#isy=d)_(W2HYvaurH2}>;j?6%3%F3A{thw z%RK5Bvy?P@ZW7BR2k-0Btwm1N9-MRikpeDPH0_Y^a-TpAR}m%%;3+Dl1Ng_iuM4z? zkj)dzFZsdF2F7lnU0kKHAvx-#-O;}{^1WJ?K%J1ZhR_&;fFU6x^C0m52`;8cG^Sa! z0glmQd7E7?pnJ=a&-J_%dGA!F_i(QO;74mmZXlX6+Y7~WTaMbQHml%F(#h-u8vOT3WpLZ2)pz=K;RM8#ho$bStum(!=atKm! zK>}Hw*~K{ye+3D$$_ENaG^1ho#CJRfoU9|3+jC_v4j&@26P-JJg_neaKRrt7pGg3zZn6?=q29|jEf=5=UMYHHi`Nbj7REs4O$k=mSI8Kdy#dn&9(tqy^*3d&GY;(nHGhxKuQ3RmnPLgk~~O1NOP?#b*sdv`khNos@caVXs$D)bpLBm z3%yzzyUO7EW&Z>{4?6^DRE=i241`}y3NVG_n0|}xI=}*6k54NWw86Yc*4vn*M+S(@Zqdc*4WFknfxF5V3Yr*!*+xiGpfumt zp<g7+VFfw0Lzy{ZaEmdT?KuOmI%N`I4nO@3dV>-5f0?qhf?Rdf1anMtA*hOkxI zW!7aLLdVPUoh1y}ktd5?^&_%hjvDc&`si7%1p@0`a(r7v-!|lNjGodVQ$rEA@Dym` z5;Vto8BnF9hzQsi5>!CkLkf$0X~Tv3?wBPxFrp|e`T5Eg#FgrVe9M;ya6*3A^9o`XJ?bQ7GY~u2N$GA9;tj;q zGYyx4k8E!p6!qrPny)z_rRM-VLk*K_>Y}nRy7^4E;}Wr=qa>8o@HM=Zq6%hkE>j%4_bZC%xqlqT=l`af$7hlAEPwEAgSI37G}jJAqK9cn8@b(OF4jGtRH&Z)#Ar z>#%lfpc#Z!V&iQyD4O(QqU&q2zR*<$ItMh`Id=7tU-&-9y%rS6B=CyqR&EmtzL~~< zjF*IdouA>2Q(ck4k_J)YHFI+|AG9z3uu#5wKVGPj?b`@~3?`@df0q1SH3rbDIg+_v z0@4}uVnnOVDF9Ryc}}+C4=(B5p|{4OUFMUmGMfie`gF_^M>!(SD?X(`MG{(CH_qk( zK^Osr>ho&p(u{&j3Vg$y3jX8#+z4c=oqpZ7f%a%p$p}YQ_Py8lytc7_$x>$l&*by# z5V|4bmv~P08ZHH0vRw>d(13MXB1qemsr&k>Ejpix`fpMkB*@Z&ZmMIzp--?F%^E7MCf-NNqpI3eM zLWK-(*Lx3Gxze-kK})vbt=)`baP4ih`3_H`sCy8P%{4bVi9^7VidIovrZ%f&d+%1# zkhL7BpdjKqFzEoK9GMi6a`4fw{!{+IZmawjEOq`Ch-NJpz0=71lE;zxfIQB+$34A9 zy;34yIcYcVNiL5^?j7yslNYb2{&w#l;ag9;0T_mW0w~l# z7ms!Qi=oU?%(=FLPx#pNQz(j*Ofl(q$}EVlxGu*zXXq(WdAIT)Wn?XN0U0M{Tmz(h zR;hEL`%y(=p3KxvojCYbaE9|ypXBrRg*E1E8>>E$_w|Hb8dWq9|958vZ9-f^pQclg zyq1?biUhaoJyVHMBD=wb-bi2^!OAO%Y9ByER6uiWVN5jy&qYa=>~mn_fi^6QIeqW& zMz*wlIY}n_`d+dCh(SZCzD12w$>Zu^^YDu^}LCo3zQc|2(03MEurCqK5v zaf5IBMm#lkV#Y{+5GW9O1E8rthD-O5dkw#~y)vWvUTTQ@PLN7%uKT!DdWfq$cW^Hs*}VZ93z*vvwA zL5^dcA3R<*X{8oTER(jT9eB2eSqsX&eUAGR=>4%2eXs~VrF3K=fX-xix`k14x=|nIR$* z(CA^P0zY5~Z^KCv_8@3wP+-fH?GOj19h6n=MI65^?)56~@zO)`qX92;{+L+@0NxQS-597cUi18x8Y-L3!B8ketBz z(pr(Lrt+5ScMYdRENolxy{;(^Fs8@Hh%(Zutay4}6Ad2}fBAupBtTo?vQa+5$Ua!) zsE`cFBaZDNh=e;F31ajDC2ryO+%i|W!$*6Zq9-CY7*zS)H}kStyV4G*)+OBpiaFuL zhMQF_+ZJUg%e?b@{~`mZYg{E-NqbeILMO;@=h)(f5mMwl^k@$(IHBT~AVz&DJH!7D zLsa`UcV)iH)R9j5bir}nkaJO@5rVHoF%g=;+o#ePdeJ+Vw50ZS&lQF7vEI0~*fEK* z?%UHghs0R-R4!UDWbkYt=UIOly-k-ku-5(uRpVUeO zgvqvp0(PTHRRJV$%a=NS!W=K@3hD#lHXzM-U22=C@j%FdGt5d6^t*HO-iFN~GOs(x zDW7w6qo^C9>jU5$A|-6F;0xzAP?Jv=hLvkwSIA6?Mxbidv3HF~$Vyn;psn@C1G0T; z%*mO18JS%0w$4o=o)CHBywpME-m&>4L7*{}{A_oB>rO<~bLu4zdckO!bl=+$wp_Zj zV+-r3l}?gh9YtN(3xVM%P(GywS!1i=#V!fU>JJ1`0Ca^eATOaphv3XSF5PaAsJ7P; zz>q-v@g>1tR(DHnUyorPs} zO27dvs;WG>`JNLY+Ch^Z(D9sOWtY7rdPp^k&B*8ly3D?gWE)}2G>_ZOm$461?-#bUIjvONSS)%Nsi|WBvX*1C&d>bCRmh zrVBYxTayXfo3#bdvHckD*}h>IWX9M)gUX7kOSq)q57|~MjeIW{L%|%#isLv+xi$ z2v=#iq@3eQoW`^*Z=5K(aQrr*?Acy#;o|L8dr$|z9Wa=a==Az@s_P+0H_#gqQ%bMP z>~3?bJ8a6|;~}tdG&)g0UT)SFvX?f(a%9pxJ~d>2dJpnEfHT(PejITETom?p_Y>Ge z;}U^y8CG${mD+hyln+Bx`>^{E2^t^-)H86FiGO5qxL|y^+Uk~=be22`**UguvZJg- z&5>|>o}=xxjmHvJPplfx@j}2YlsRGD_v^E;@i13`**JkU4E2eA4~)JG2f`^8qr3AY zNh!U>`pVK7f2@&@p^01e4-gyR-0*93LtUhMe$QCI1`Tke?2Fid zTSHb+HyihNidfWxt2*Ub#4V1Pti$Vqr#pK#*PwxT%(~n$7RVjI8Sn96GO$m3b&8qK zh&*%5ly2gm)On)m3zUCY>J{0(=I@Z#A*Sl!x+%G>`95kXe#{;t{Cp{w1ja zJqd7o!Z|KK$9xhMI>!%jEkADH0)MJzr07Fx0}7WiT&CIw!8-xM=a}b*cck2AaDuS< zYEfiS%R?y8W_nNK?y`)`aijbMg6;-m0tEIu@+Tt;gIZ?`fCuwwWOdShmHX(m;{a2C z(6%N&Bus}{8}e5COMo)^xXFXbr%K)p6(G2~=+}eaM&524#>)UF5a506a^ejV@R;v*KM7!+?{(KIxGG2ntdqR;NM4X+D*$k(MmAx3E=J8z zjsa#rM$taNdgXgiKe1i1P5-S)n-A+9K@hm77Djos zGaIB)ci$f7a3a<$dtYgGN*gbjv|m|rh~vR3+Xq`>W&y|=*wl@Uw>22;S=YF3`&xoa z!qHJe{`K`o+!#;@GX|mj#l8cnahIa%30>J>r^x0}-y0gYbFN_y022`+L$;bXsYz)e zQ0(UCfzIdy+Fv81PVS|DFpxiB`k4RYqxl?&BGp*pS2f+|8H)?_Gu&R~fAV5JmZGIU zWs+fwbGNp@*SZJx8@PN?=KbMAWEu&h7CjbF8=yAw;{QMEO+*$91G5bn~Y1U=s4z<^3729!>_;}Hu z1BuSXGn$&>4pN#GMiDS4P**Ht`k)|OWnQj9Qe+B0>RT_Lv~ps1u6eubW%&)nvk-hX zhm?1n6^I9-05#B1;>#@te?hQzRUdcoUU%<_Iru#g94@jNaFc1yfpt?bspJD>btsPB zLve&9dp3aF1~hi=134L7|OUSq_`KAKC>aBWk6 z|C`Ww32jCO#LQ2WD$u`bh2j{-aJ(Rm!xPNhI?7OMUfd7{Mjxc5y&s3W2X;A<6Q+>> zWovTL0J?swL32H9=4FmKzENOYGGs9 z;wSD>NX1$oI&+S_Txd&-_0|pLlfstNxJ0bveL3t&ani1aoO4XpjP^OHc)6+dAbhmA z*#rpzCR0Hv3U7%iAy(3*g#zY5RccDz-chv%YkTP<9Okkr8j3h2J(8! z+w0J{)NGrT!c7QR)=*T68Dcr{c`NVPiB?PMlu=~b^=_d05cy6puS07D3E-_Z!Gdu` zB&aAj@y)X8d}!hK!hQ&g?p!UQux>rb z5t~E&Y<`pARXc~c-T4Y`?znHj>ERZ_K_=s@I2w!WnwYz92?2H=&u4eN9`C-mD0Lxk zc)u#SfFzg`{;fLCTBvRfjb%j468NW zk02x%3lD_YiXdLgYt&W&f&`Gq(p&hEfneXFmrVpC;upPKl2oFe9bVu6M=FS$in9ni zOPh((Pq}fBu!9GZBOdME;p-Lb>lk}+F+6T@1ow!S$CI9!3{6l!4peg;>)2fzY1)g~ zi81$$jIs0RhV&7<7$7L?sD`~=hZlZPzip{@qF|>b6B{T(+GT3cr>+1`;B3+Lj>cWw zX}z5IJ1ExOohhm2Zwr9 zo=iwdI5axFxgyaWP@#&LkbNc>;pw#sL604Ya#i8Ogfe9PC5Te;8Ap9>KwQ|5EUX&S zMFUzuL44MwS&}4%+OWk1;xF1z_>jxKCX#1Z$Vc7m6gOY+0`r7(4z`$5lJa8tY-iuS z2E)9VZ7!T$SNeHH0lYlID`RT?AtSu)VV(rbFmn#3gfPg%1Uj4Tai5R)WCghiroNo+ z9fT+0%!a?-0;MBfpSX`^ky-&Q2}`j{Q_Bmy*s~{i7gFa7 zCBl!fw{Vz!<3r$$Z zi=vY3kfq}}L}iSnc`jUf#?Wp2+9BZhNS4_30(QzTQKooFqQbzCxUddvO521nhr1#@ zfn2RRM0|f;Cc>ei0N!kXa-mX`F5h)l1=Mcr?|N%<{fzyF)x7z_O4ON*W{?El`!P0= z%fzRG42R$OnfcssAn7SSFDgn{QMGrtZEC6u?x4$$XoV$5FoN@~N`z&wkj@okADSOf zrG?~7C@c{vgyQvd-V~0Na|{t39I+oYpM6t?c5!63T6TjW{2!g(`lzS=4j~c=Y`+5V zhfvGMS-0+ILFQA&Y!E5`i(&=rmHUjq^=sMuu9qETNIN^45vPFY{mNCg@hOnqIfr;$ zBq_j$#(i!@9*f;`xbJV8ud^z1dS@F58n~IevDc(3YA@3HY8Gl-J=XM$d=<35`cJm7 zAW;t8Bl7y-eyXN8%VlO42IBj~j*52VKjHJvA10F>5PKV4EH#$a3#aXJxF(6pX1qRioP{IEHRK0sZm36v3 zett7GjoIYskeZg7mn8BwqUIpC@h(%!1S)aUlWrt0=?Ebr3TK)e^R9Rc>(;LJTTsGocskwFJS5Sx)ese)peO_t(z~rJp|R(AWuZWU^GBCehcQti+*|ZU!?aH5m>tT@~Zhz7;Udk4>*e?Id^io|^hAuwXGN=}e7!;i2x~t@?+hR_Wn^H6=!0`|zgu zzCj|LINzL+8w(OU&%K%pS*1{=u~2o+LOuZCPXio$kZdgm`s)RX+HTPHrc;Jlc9%$=L#KS7K2N+!pn#M zD2n`gpXte32fXt74M_o3L4xd9=ePVkD+DY8HN*eNWReM7V?}~hEK)ulg`QD;sl|+0 zFowfDwq`ztb|5P%r2CZ_`}VBU_cKi$*$;RmHh~eomzQoZ_UBVCi++n0A#wc*yvv}# z$!ZZJsf$K-qILTm(e+S|Bmg@ECkW=aYCG^Inb%x~1X|MU<#+ghun@81N5mctOJEtH z_1wjKeJ9Hn1n7;5yme4n3h2q)6T1sYW^&YjF$RMw??1v3vc(5PlXkYIICKo{2@9q! zOb;->?9jeP^C?jQ(a-Pt^L;1oD&^M(_c8m!zlu$uuJkepLQ}Y<06G%fpRzZ`%q|yu zBMYiN7I{C}fFWh4neu8m3-iny!DnX~iV}XWD4}=H-Ai^2qzVkA+L5Y;#KSRV6i?8Z zm->m0fn(Y-Xu|4Wo3GIrN(uT4=3A)nPi3l7;U<(c8AZAkL2#ZJ~E%+;DE}vqm%nBS~ zJi%t>r5~BU22R+C0K+b!YZ4^+`JGdxU?njkT>`IKYr2t5?zt-450zK|N1^2Uw1EVg zAc1#m;7*`u#p>PbSMsi!1{Ae|Cjo$d-^t8$&UAZCtzjqCJfNnytHmf-N!VrWEX*Ju z);>^xNr4u#yTlmA5A@-*&SZacHv3wuaQ*qmsD6z*M|InT5Tlvn?Fc}7QRKi7*?~j&`mI}UqaX~-q6kDS1Ubd zYj6seN6eL3Aj^GVbIR{pUl5dW*3mmR)kP)Z;dfx$fy1r;zeDh~LzjRw-?nPnH9wlDlw5|n^zZvNlVe&OvmR%MoH&!g%sD31!O zmzD{N3vaCxw&pJ#earwBP`=a1_)a+hL|~EA&3~`FkH~Qw7X5_LKw1B%2wBp5MZ7#& zRbX53MLh#0)f#!zbLB@P1#krzN6nG1nZ1^jq;bDhTG~;4wMGS?^P*E(f-^8FaORIH zpYCf+wBgFmE7<_{&asR@-na>~FKKZK)=NYrAm#;s4cxv#S=dq1^Exlup7#SAJcE$0+eT9aGadwwz7-1i6FdU58J01qlV$_|j z_A;KxhFSp#FaA?^o^hZxw5au(+csq#-%>$`96ltlgJNyIZJ)!hMaH;Q_W=0_kl>Q% z`9#CAD?bEwl0qrzi>v(F@ZI-Ag@^zfbQo&5f18{0mBD;v_l9-nqM$lB8ahIbF-Py_;|05=Wpqk(P@bxg_^#J<^s5*9u@xr(t`R5@K4_r^#U$| zmckNY#nromuy4mA4Tg)5xrX`#K+wD%Xn-~0GMJxF9Sb^)yD0^Y3aDL8*(H=~k%r$M zoz1Zq6iQ}e5|8GCQb$O`Q1)mbt_SX2;0ykB^y%$sKUP{|Xtqu9pAw=6&Fi&fv#i)a zKKt0++mf$*ZPrk*(GUvZ85MiwcQ_}*rWY{^;3ml+3yRXacGh3-BEax1@9Q4?3o!E& zIF1+6V<{uAeCDMWwLQPlHjwTKY<{j}o4&T8>Hr(1+I|zIc+N1@+$se@`VBhf8cHX@ z{K?u~fEbYxQ!j^J;ZO#X^VPFhvjW6%yoUB3X!1*+*rwyNWsm2GtD+2E7Rw?9_I{#R ztVxCR1g`VAERazR4`Wkt@s;7EttPcQb9x0y#gY~k`fzhQF=YVId#B>1F@|^HQ!m~{ zMBI=3YJU@UhQt85a;t9~>Ms-nJdBvqze>66;;paw$zi9@ds8$*-X~>Mk3rN6%18Pi0o!l*yS+2q{FWd!pk~%CHu$LN` zh_GD}IT+`aPzeL{2^_?m&$sM?E#^h{%EzDo#8?SGlBm#1bNo-Tl~UT@cE_u~Es_rb%|tYb@U&O(fod-E(=(MeZX4M7-6&rt0>#`I-ZRRW)CryKU8R zERu^AQF-ljLsXbPUq~lCd^=HTNzNUBq%Dp8jZlGgCfJ0T!X@RaiAIv*rn@(MhIvEJ z`zm($n}Z?p7f2%ywmx29A`sUz+j!8p?CdJq)WVV_+ z?m(!S`UbK$;<2jigq?#@%NjA~MeplLJX`nX^^nQ3A$2@w}AGpZIae3#`7%fErvd(Iz3kt8| zq$527W^fu;tvVQ(I|R#$YKDaghJPhbx8^rTnJKRgYm+7#uOF|e1H6Y*AmMgP-yzUa zg2hd;BN8>bm3C)+JplIlaUwD%_uvPKGW2LTHNRe#^Y&HyY?eqF@GSUc&m88+K#+bB zdJ2u{_!d*Hn>sFc(N&XgvLsM_Ur>vkamFPL*W>=3 zWl^u8aP8mBqOUF#J`>Nmf(}5Bdy%{~h&k=Y`YkhAJD0NoaZq%{*mQH^gge4v=#xx* zu%X71vPMyxpHgs@$x#Y~qDbx>a@M|1rF2;ZA9Z_+Z@rO8@~l~_HW_o8*31wF66y$x zGTYjY?Q+uytt7Fc>b-#g3HXg|Yg^oe5CEh+!|Nbe5I2F*7?UK15l}*~VRutwsEVf0 zA@B=W(BK;K(F!+lElwFd^gP5;SHX=gi6k6VYGL5(L-*@;>lIK@CQdDIXn zMl68?yikfk^^V32%9ivQgf~q!7|gyA!-}o*%*Ke9OU_Eh%FIg-qeWncK9|lJ5HtAo zYN!K(46t(KfF_!<;k0sG30;c&0B{iHd>@E%^JqBgX+dEgbENAiQlXB2RQywnPbXZ_(ZDS z1i44`o!fE9Y~!7Kl3dGSS+36hk2=^mf&6$DD(U!8ZD?u~lJeRH82`A2#t#qTuEXyI z#9{1iKKQ|>SmuXV>a5|_-Wgz|CqTCa^IN!t06wVf-ep4ba(M+)Vo*X1?+vSpQC;7* zr%EywWK4EMK89QGOO;n!p66EvGbNqO+7k$V{^HU^4LmUhq<;*{97 zYHVRWT{yViw33}p*twS+M@Wp~$b?^tSn<>ew~$5p(R-0J{6LAnuDQESuDFBmNQy3b zLl$Pe_Q;vO5RMRbHDbw#dir7G?ET`wiAYs`mE-G~k@Sv?*M&NtJf^SIm*7c{vuT_~ zVgmm|T#ShRSMXIuVeut^Ci%A5iO0H4JkW-PPQaSfLa1L{lcQmHO^jKzYz*Ebt5IN4 z4}jCm;cb?4W1h4$?SO#y<@FE~sc5Outq6a4veCKpsmuc?0dyT++Td5?CQbK5v`lGm zb$Gcj5|R@rjc=FUQJNJG{EBXnNiTsxF{XnhCz%c;C zz_>%3d%R23#YrpmVqajpQRR(s%EtJb`5A2Q*}R(*lGJTZ%`o!LpoDzHv8Wb$!GoP> zbS@>x48)U2A3D+F5pZhM7%Ll-D0C%O$5jVSLc_r5=Xqj}cDII<-(^LI7i74!_=EYp zD%~0`O~VQ;=`c$@J?YH?{@gJDmKs7PEb7vPq;BVql0oc^s3_+ACtgO$y9f|EVgL!z zdJvj8*OUxz3WBVPZ>85gZPpwzs9%MUPwX{J37?YF z4!m=7|6ctFe!*YJHuOuIpQGEXpF6-?8Lto3t&59P~nC#nQE%Qreq9JvPd8z_C4#> zF+I0v(K*s1yf{m+in`UiYvEJ_o}UH*`f0AiDRu|mM2`&**yi? zg?-{Q+C-0_I~#VML)b`xeYhj_26bgY5lgtO2^3`DaFlGqG!~O|*(S=U(L!3%2!<$Whl6S$Sva~ME`aRq~&FR%hiD&AbVFrag zpf86Ou_waj#h2jgKTYE#o77k!$gO`EvqRAW}~c6vSJ&9&32cCW2b-2<-u<9 zc1|5yrcJiK%fDK6s9VefTUsT9au_VP?wrL}N=D&m{*Kp#dt@1t3&e$3wAoef{8~gL zvKhoR#(>}+qr_Hcm<^C5D8EQoLkF+b4TZ&b_A(O!4l)ws@s8HQm^pjU1Nv;HZ#E|NFp`Ou-Sgc64OqRc#In%b`e_HqpvCsd@v+Xzlu4C*k$ zZMF)(N+*X~<&S_Kf!sRM4o?9%JZO{)U8_p7G2i~Sl77VLdVS5U!6k=snH*S=-3;d& z<4EOu2xFRV9h=fotOPLcRv{i*QNdRXw$crQM!JX3APwo>6k-@UF`@5ypI z%)4TGt$y$Gjzp+fu*E!fLDQinVD$EW@jZ*V&ZsLvM&D(9SLaESQ}Lby!Or7Hg;t-B z0#BBaWgW;k`^8`fC1@M&dMh-LEPQ8SU6mj?##IW3gi$rd;u;n}6?h~}craN+z+k7a zPRe6X`L36~Vc9-gJ)h#Hb$gS#t^O@&JaPYl3%U}WeT@+#geP-5{4NPZ&sze6kecv&}s^Y|LQ;_ z8b;FjPWu3UKyz1RHy4k4oYhgS*aujrMA_QW6*jALrPqYs55y{%mtj#4N-fGm-9vS= zSgGoax<(2^QJ;)&E!M=Mmws&9$=B53{+E*t(b#(jO5<_Fd@9ZJlk3{_TpX4ygLv1L z)II1358Vnp;OS5Y`e^`&;{?~iE6S`k)oW-A@2Ls52H-N|8S$htpi(K4>jun4R$8ws z?L%=Kw2lK#XLxRmXS_b!Q^N2TZ88cP1};cB_(HCzvXv8ZYoZMh(X9c|4(5+o64*to z$Yz|-A)X;3r>|?L8EEa6&~ei15g9f5GsgG^b4=~`Lb0E=P&n4w2tB~RbiOf$f)beb z{zz1GHwTv=a*J)yMK&fTxIceDd|TInU>@qksKI7#ODnk;X`CZYEzuUk-1C0WJ$J&` z+t@XBi;EQpW-#-j6~JP((cwI=3y42nWtIcOIb1TW{i1UU6>nkUGWk`iD&Dim>;gBh zN?#8BCKY>|pXvtG&pMvxz7#Ks&Y3EhYS1SeL6!^!F=QgkD?T?u6<>>nO}1h%BLo9Z zcSae@$K}@K2j~%&XtP=>6mn01u7r~vKANRYEhtPjw1Pd_7h`OK*)pv~w zl8?7*9uK0VQSui+e!fbNDSWg>8C^RsP#A){i|7v@&5Zc+3c>C8^bk-jI?`ZBt((Qvjwy`E# zauP`hIuf`pCcH3@_jF7jN~4c0M+XhyzSPjdheY^*3PibiwoGNZS@pWj&^AiS?_}@; zZ2W}8BitBZ(25ibTetZgcMh_mfGLPDhN8%~)Zb8y0uh%dwThAj8I-T^a`8{YgNPBZ zb+RqIe`4W3I8aA6F@}dH=gaUA1ZhY{3B-4wCfQHEz``JK1;qc$+O3}5*6ju+^TB`%3~yi(Kp9Zg0rNjo6;wJipT zz&JrgGfGB9lOEp7{+M3@RsN7Qrd(48#c;%@s?9@5MYPgP03gXPzE>AyS%ZdNI%@L0 zh5X5DZL%kRlYLvDrxA6*zI`75_vu!yhz+&n724Rvk#fSC* z4Cu?ovhCZZM?{`oalFPE*u?0#iG@^3#}PHO#)!`L8~chq+4E^o7}C}_w7b0}v*~qZ z_+U3xp*+B@JU>wjm%|mmoKdK*sUkaI_dce&OZ^T7y%5iTwV$|QzKK$*-cewDYhr}o ze)xt0-rloMDXS5b^>SYA4r~FKK{xe$un1hd;w>_RVmOEbDl%jQYd)WQ?`kR04W$+V z)S>=;v2c%H=ek}q{)Tr?CbtddN_})(2&s9O66=-7r~DzQ|z;RS#Dav5($#SYD&xVbwA| zu%3fqDZurCaf1!vx!RZ#ET-qkj9?iUHq>NdK+5AsMD&~3%36i}RDycgq!3AOL;j#6 zro)Z;0pe-MZ&15Rcl^&Mc@jWf+Gz90m=rK?#*H4Xd_(S#axuX(=g~y2eJ-f~Nf{cL zrYd~1M|G7dfCxk?%%V{$vuQo47a7#)1mRH^9i96MA_xjHbZO%V%@_yRJs-`dU=rR< z-%50>sOyaOa^o4Wz$EaE5j_BO99Q+t*|j41wf2>|AGeS9TGkr4^A?qa*RU-DUju_0 z##0CjLPXDBrG0sOOR#W2HK43yD)ZCjeu!2QJ)zMpgZ!k}UXufLgM0X3XT3*92%qrXfraG?Gg_XI#%GYQ3>mx zGyh;s3d{(gY;kXL(odw!8tjAO*KZ8zeAN@dWu?E&!Z9%JD18Oh9DQTtuTd5KtF*%3 zGk;z5E@NoaE*t1A&ciCl6;VH|!9TfRq?Hac`fq40$-4qC51Ra&noXAc%dKF#B5U0bRiF|bX zY3WcvWdO~;QJ*N;{9S$5^GMv8@B4$o;mTtR1Vns=7GuY27BC*QW?emBIhaBCidNiJ z%!zTQ)|&2rUm>B0*2KP0*~8(p8&eWoy-l_m5MWR|xy5GT8(VGey-)J8VnUcDvSr=x zx;aOkwn6v{_TfuLx2T$o&K{1}05LbVR(lOS4+Y8xR(NOV=`pEo;;sV)ZxHa}6w363 z`28Lrg=}6hq*TjA0t8N~)V5|-Y$8H{d%ta@k`z1$zLQ}u4g(?(gcUUOwN;L~JK;x! z+K_(IECaOcmV1|mqDWC`bkW6a>Am3$A-1G<)(?C-V3RF`-~A{ z*e4ZtM4hmp!0VQgA(bdlKR&2sITy4ViXhB8xrfSX#Od>?FFDP3(5>_Akc0N?XE5Dk zD0)Lg()6$;^J-=YlzaxzH=$Gkra$wMX*K7WDRVu+LvpCNjehrta%{`oZ1VjUq_ELT@}j z-Gso2{rR^#+J+K;L-e(V@CN{_o|gO{Yjm1k(v1?DkccG;$ItV!ZHGuzm`;NvIBE49#mR zA$f5|-@$~UU(=JxY@V)0A5=tndXwH~>wOeN>pR5=2oV#Gp=#4J8psyWfi4h$GivcY6Vp9D+ z-v>x0_zF_RP*k}D2C-VH5OaNlXGmE7v5T;O;*Yv6^#jAwoA0)Pt-2~j%V6O0BlY?}d#!E`ITiN$gDXNbKwnunnA>+RDJ|#!R~W-E#!; zLC@9HB}11$&jzv8jdj@j`Iny7XFIf%HXKL`sk``puZRT~a7S#i?R+h1_HWr1q}~3o z%k7ZpK;@61z(WuH$md6%0D}Sz73RIH3~=WUuhosTDq|l8#?HSc3X&0&v>g`@t7^ts zV|mRIc2Tz@f8Yi;Ye$Du0f8jUPQjAke@oN_Wtqb_M+SI6y$ncU)L`S#X8RcBql~g+ z!+2gdoLT6*K#>DObgkQ!czvX!7&|qFWFh|S@HnFx3*{v5M%-A~l)MulHQG&8+k*?F zy9DUMMbUH2gjIXdX-v}e9V}OLrBdrN@r&{N2Hh5q;d9?ymGeaYUS9d;$iF-2Mw;gA zV@6%@0CT@Az|ia)rYzthhIXC-Ku{z8@^ZcVS~8s(aXyA3(T$;7_Sfx)-#LZt2YxiS z_l)$5<<(7DK>jPnI)(^wH$Ye>G~`|MUBiY5T|G#H7d`Ej)#CTQ~u3@LWiHUBZ@ z9=}$8n13Vlw$xs#{z$DB+#Z2Rc>6*dT9$Khc578f_~`GAWh9z+rKQx*5PzU%tP8P> zt>GYmbdW(*zBRado#ZZ*8Ub{Fkwh?9m}TRJR#v;NHVf;+kj>}M zQap&aBi+3OB)xUwFh8a&+_Ik4siY8}V86CjRCU<;f!d2v9u{H1DA zDG(M>YmEqJmS>7!?~djN7Vws!-GZK%6VTgY{;+#YCgojo?lTKc?2K$ zCu${?US$=6aX)`6wrk$3&M+^R z$Bg`#oA>?}ncI|OU&%Vwjh9WPOt0ztsn31stD0-wn={6;|AYYS2D!e&I&c3fl3S-X zM0@&+XbzIo!H2LeL&;9}YxG0`>DuANJr-Ee zis#<96)=q0|B`7fVLDUsp$6Pg|&eG8x>fc;_D0a&4Wd{=uaR@6!z z3u^h99+~i?1|?{*k4(I*e@d|e_*`#){Fn2j(A-qbPQtQB`q?akg?ZLJC^F0@qYwKb zzkb8)htxYps+Xc^6kN1s-({A?ZPjjvOa)L2uVy0=ACE1kUy403^C(Ei(y)NCsJ;Sq zvotbei?|WN>;Pj`MhW=wwJARj$I>EM0Qn@QXY1}sY{&SV5&K2n@butFb@uD_8Ljbi z?}6JOor&u4^(^%Erj8W>qCI@gx?H+-fcxY7D4HZ3JnPH!;0~llBGdTJNISkWIxclV z2h_&~#@$3QMQ$Y)51eFVvv_cMs0zfrS&4(Kkk`yv-AxwsSNZ%~aQYr`oNB^==YlpCa$uTRdjIDFRLVO!hZfTSrmk(2}+H%eMC z-?dGUkao9%(zVmfLV%btj+&lTA`QPymXV>Cd+2}bh_X~)3|7%SqH6jrn;-<+@{3j@ zmYOJ3uFlZMDK?AG+A8ciH$WXq)QQCJeMyYKAC=H7W&#^c8R~{zV?M>B;A>ESi9=vR z7sAdw90pE!tb( zEcCL@k342~(8-#N9R_c;Lj#I?@XDJl)+6>mNJj#voP}jyx`ih=c)dnp3fw=}Ej~p} z9a=7+fH%Q{KxQ<+17HNtJ(FeBG5+usW;TezW&f&UJVPt#&89}d>p(Jpz8Uc#(BuX# zz>x~F{+^OaNqXAEl{-li=iH4bxib_W7!F0x_~Rsm$v2Q<-OdH!*)m1NQSK~OAGjLm zI|-$im(2NAH&4@pBTl15J%+T%zQ(n4gm~vhNJ=6^8na##hSt1IF`xHeh?<1I$5nHp z4GvySPm4hAIw$`R+Jb7t#Yz%?%5jc`cuN|%@YMo$$EKufWloRwv$S|WYq|=CIwzoGgI7Z$%pudBb za8+ff`(q$PAWYAH66r9K#ne%KMRBuzsj9i_;#XXiW*PT_Pw@TVbrSPlQae)R+fJJ` z!#yAu4{AJF+ZUGgO$fpZ{^p4o#=X&pS`LX}9+GYf%tFdJ-!L zP^akY_1eFNOt@gaK)eyrZb>Z*H>Y57cW|Wlbx2C!FNJ=7gSQ-BdF=eZ%b+{8<_H}^ z(fO@O?MnDz8{Wp=YHvNNc|e|9MGHX&AReN04G*-5F=d0+vhu3405bPLnV-i0-Tzi95jy7 zHaq0E`j|xqz;!Y7;qB#B9*pN;JDU!fxb3Hx@_83Iq-Co`_YK@8i%Zjoi80}QvXV~% z8;}>cZCQDtaW-_FgS8P&oC0!qO|hKA%!a-*hrek^+8p^uVdAgx6Zx)ZfqJjr{ai-E z?^Rj5+$`1ZphbPfA@g-7uX2Pz0?|YW^|Qr*WGcrBQb&7e)a}q^|WW53coq46=BEjpTt1Ct&4B7 zwCF z`Q6{EJGyxGg-rGY%E#%?64Q0V_HohW|IPXDo?bn@R$EpdTjs@8S6>|^iHO1X+7rfY zA%i0CN^$YX@eHnjz!t;NhuMP9}CgU%9S=->=XTh(6Ld zWsb_Z-wnQ}m)Q8pgdfRCDo1Kt06{Z~O8pO{6M6Ua^D`g`Y<7)l@7@z;4`lzJRWmvk z@K;ss2o@5dizoPRCYMnNQq%&1j?_phO+#MDIbD|zE#)K=j)KSpg3v~`!0*pGQjWDM zM3yxTMz7Rjnm*aw%*k#)xLh)>c3e0uWk2}E)z3i;0{{5M8ykLr z{sgK#0vGrct3L#4u{Gs=L(l7#xi49@U@H_VjVo_jYAAD4id7?OMHkHf`5H{ETHe&C z3J?=PmWt(zVDi|DY2X<5GLXJ_&(s}kVutAus_GV`iMM|Cr2kd?P{E=asPSlAm(*QX6jm}ZiWQcitjU-Zn$?a3a z&W!mS;zY_~{U6Bi1I}IgOXY!3vCZa*u0(^Mp~GpnfT{S(?|(GGggK#FYtX3c3;KHX zv^OCQzuDDUJ!{>pQ%94hOyZl(Bny_15dH_;q5T_vbq?)`g5)@jxlhrSx3=k;2$EQJ zVG@U-i(ZFIV5#dH?3KdKUkWQS4g~%kuiX#_(Dp?ivlv4uG??ESDT=Hw&3EnfZ&$)L zRZ)kj5Py&_02SpSxwpY+HzO{AuT+}paJ$1QiQGoZ9}yqWQq*jF z`z=UwGH}*+c|g2Lr5>7Q#f9UwG5jz9vm6MuFVh9dMBljG&7zAG+ zdmx_}BT*RhdP5i!B_b7&7+^Cc5dr0miUpZs``LS^dJMj&iqJlWkHKU>`D&ZGE$)*l z;f+`ak!am^1B#0WZ9y^AKIeMTktb|?U6?2He6%GW2NRs|xxFR12b;x|Yv1otZ)|ZQ zHe`>i+=!=Ojcse#COLs*eOc^&nXn~>TN~)k0g$?^qIlX809`fV2MzH3pcd>s3jQ3! z%AE?lBSvEks0g+U*TT9ztZ63U45CPuutqshDE&TlRG`of&ajLHLMuY}RR4b6G$UHb0aSu3+DQ&DMsq zU!mE~ty*ZQdq^7?16$%3wWR3+Fvjsv^zLPT2Ts{weDlq3m{v+VscyR^=il(_ytz6;-?kSd>Dk@CC@A zSyFwyj*%z^EKy5(X2WAs$_n{>^vl?nI`$TS`ee2d#k4%x=!fPIX_?xag$?`q*jEfq z5cLP7ELa4_A3t&3>}X_}lJU(&D`6Xt1TsO$dbvddLP;!IIa+d-e_n9f7CAyfeDjxY|G3y3<69DMJ>M%)1DjIkXYu$U6=v; zC1T$VA9TE?V-RB0#QClo6PyN^3}7jMs6_LC{3LqC;{!IVKCojs4hBX)KumJfSzp&A ze#)3>O-x*HgBglw&JW7^Rh zbT)W53*UR^hQ|CCRvyzJ04DFM_A@(;+C=dl?DOE(_$#Zw*Joh|*t5%t^CqK}4Mv`R zJ`R7ZzoyB;`>+N%ujs;EA{em8k*Nn#-`PPg5zas?u!6rcr zNA4miBQ2ce&3Ak7FE@0~O3axP;*pUKhWh|P#@r#=9W3^~PJfRXS=S@V)J_#8*Ya;% zwd9*4Ex&=+k=njVpbMiYv88tHmP3|}#b=hfOcXp^dSjW0xV z1>mXT5lhXvJ?0Zd+{FPNOmndMMt8Kyw#vWWNP@Y+9osyB>l~;V%JPO4a*2>f1KK7; z7Kh`w@GC4yXK2ylRXg%_bss$gvtu2<3#4p0H?$)E%cpVnh3CMo5OXMSj&m|CB9hxO zGxc5LGITkM4FhsFL_dyI?veTv;6C~D?c=zxU<)=yD-e(q%)VeTOFmdhg~5{-C;%O& zGJ7e%_5&@G&4<*jQjk z9KrkY#O9m;o^U9pjGi;*doBuCxeqja@a0f8fOIrk(6O3Vt6>G!bqR@u5DCX7qOoMN zmUf|&ru5|41Z3i8z0+_`b5Spn@&`0%_6~lAovlen>(+`o!FpI94w$mC^Qgbl^yOVc zD7_2n)?(DKd3qA8qGk#loYn@e1cv>cd+!EcD0Usgc4*xdxBmD$Ah4j#1v!E|)urQ^ z@`^C9F}y1r3Tm-Ve>jns0iI6b2QmMJCWZ4@KAFrnqdWL*@M7Pd?=X}R)!=r)@bNp(w~)F46B^^#0VBl~+BA%>EPuqv00zNp=#25$ z@EnnaQ;6pRcJ|Qq{5+BxF~`WgbH^XY2PXEW(v#vPhsE)_M|!`_Odjrj29gg932*L8 zfb5oW)~vV#PbR#u<41sHEC4NnJt#}6;ntNQ9pQqeeP{-~@vrm4-LU52Vb03-OjCV! zmJFg^_O$t&(QbvBK5`UatY*&#z#&R>$@1R#te3bM?szaX$+ z5$+f(w%DA^n^$ukn)zJ*D+>=ZhRb;lT@6Lo8P?d0)$u!Wh6;9{lq< z?%4{;2{Bq>N!Vjv$Dqoa?(S|#eRs~XA&PGU>3I2926%WhpYQ8$rR4d^VlaP54KF8P zte2iGMoG8T22cv0?v#es?HD&d(LD!4Fpg8P^2O+9i;H_eaCsfcMcEPhZpX7hew>G) zFr52UKU`3IC!{5```|Dt-9OIMbko{YoISju?e<&#k@z-hvi#*@=QU+|Cg;!UKWodC z4U&;qGofuU>bp%g2+|rMbwG9a$+xriw@s0FgrUHuSe`c%zdk#!aczvu46}CbmtfeY zs_>&_MAurKc{N{q?u03ElS+*#qffDkzXIcSch`3~#!XuK))Sut!YMTE2HXRF zuSThI&bPqL7#z+2Swp=_`DD3Mt8(P9Hp?p5{$(!pfz&IlZXC}s5!5uSh6-kMMu5);z=h^7A7@16-LxZW@Xt*t>(a9& zUUa$!|J`Z>jkfr?g#guTNv~t6kPHb3DK8?lU~H^0e+4U~KixW@QUAC2f18;6ae^rSeYko{R&+cD5N0o^@^w%&Z)^MV zHghRjwkOG^FxHSl^!6?n&X-#W3njO5UKDV`8_cY>Ti?jaMAdD9{7^~nmkKkDMsZhd zo1fGpZxZfAxN_A01abuS9OpC&m$0TtB|Vnb@0|V(HZ`fF2fXeSuoxD3ZeLp*bNIrj zATTKjxAb!=ezXBcwihk!JmFLADbGWEqKW)f*Lxic%55*`dI^@vB)=pXSy*b4 zD;XvBU`>naGvG7*U&#q2OC^F2zxe?wT=WN(v%O!)Rk@S_kQ?`DKYykf_`+_Oa*cUs z#wFv_VY^X!ms}%ga@@FSGw^Wj(5!tG$Z0qe(>C&Ht<~sbjIspBdePaSI-uS)3$`AC zSLM`(JUN+wxqUs(Nz@hiqg>p@2~lkpcOeUII=%8xWHO-##_R(|U%{m9QGgZ|9f0LK z>numW`!6Z?^@4h9|N^$;DfS>hN}Da`+n@`7Zm# z_Y{kOe6(@@qOF|NBeKWXPy|8)U^Zc5rB*OAzt$|KXsf%#!-8ZAW#- zoekKZSGR_j%UXNAoberRqHeZqbk6Hp@?jlcOYi)I z+%xQioUz?#>0RZ_w@7^+i_1tv2exF2xXj;mJ6ssm7PrCkZNK3!w-lB=mwyGR4u%vE z5c&rokOd%}=hp|lhnsibCfWN&W^_TKhBPbSMUr{L!M8a<#Z z*`3W#7+g^bN$ilhD57e)MJ~U+V>b?Y@+DxTiYx}qm0Pt~j`BA%i{!B<)E(@2-xwl7 z=V5+6_t`N8GN4?3jSQngNh=`o4bQ0nR({AEatoN|bB`Sr%0U2{_+~iah^Yt7eT{3s&CGK(H@?aF$s8@e7e4peIm3}k=&lnw-gK4O^ z5s7`n_!Q6}{c?xY;9DpY<{yi5BixWS=Xz(nBSpIA?pf!}A`iN=KK1bPh0zT}_%q3? zKtJX^%T@deD#v~{s|bs@T#B+cJpf@$Kk$nk&j^Cx)c!xK-hL}1rZ49s+(8rS7_?i7 ztYba^2xMcxiI*hd;urpv;Fg_mO%^zR`+fo&^do5>iY z6ZG;Il^UVGMq^}(9FPmIcwXE8a#vs(Jr-MZawe984B(S4vuecBM2ND0ck-<{AuiI^ z2Javcz-IW<3Z!qP1= z%)J!1Yyf@MUX7dU24NCT4$N0Z&tFIezYBy62#wgQ^W+wg)>Eb58kf+2FiRA;(AHN! zcQ+t8R7!Z?+&}x0w&&3Kd{32nk?8XoW60Bni+BAv%58xQV!IlLnSpUbvpl#Dg;)#T zW4VS0z6X#Lt*V(SL2?F^t}bgkYIpz&*7LVaF%;1HQLg3^1R;e~_zthL5DOW2;r6tBEbPx((Va8D zKktL}m2ur37pY*f8qDw9dxQ_>`c77!Dxw}sFMIX-0C&8)8@kG;55V6;rPa^ObMY0s z$urA;#a;?nKLLlUc~=-WfT&*QHEVr3OJ};1jcCQKIa1oYgvqKI%LA&GNSIN)CK~*Q zmCVN&IWgEVvlg7-@QrA$uCA872&Oa92bp-_@`bi(vn>Hjf_&r`(T`jEX;UxL8XG7^ zkrELcgv@qE)zA-`ygm{r;TH>>L2OzYh2kna+QEblyH7D?lQZFX!E_69)qd9hWM3IR z`3B$#w8^+@K!?iQ0Fe%z7nhwFCa58k6&#YJq((8n%kXd&4GE{ou~HSGM&z>L?Mf!2 zryWx&e$AU@6)N2yGtGcL$+5mT=S52JZSdalV)L?!t^-pW4!lzLjQ`PoYjyr-yBfQU zaci0m8`}v;l_fdKE(7a&#yK^uXt*A2KU*2%A9EVPu+`==_6PA5&ihA727=1Dee$jt zuZVkdI^DQ1ipt;mm5Ffz-&~r*=1X4<_ooDEcW1FYsAA zQUs?;Re2E^J)yJgHf(AJ=@DQGM*8m$fub^K3qJKCS2TZ%l7KfE%1^x}*n1LJ^GRqA zlf* z?yvgY7bVm3C8#i5@9p=T`)y=hq< zUPjsj$n``p|L`eRFRNhP28xPz{;8FGzXczZAxfEZw}c6D<@44r@tC1j+t;^0_Dn)O-+f*WxZ9Hg`3Sn;`ONfHdRdw z{m{(0VCKV&);b}{+TG=J7x;@M{dKP=%*+&X9;6)M^4LBmG`=_Q3Ii-FKk%TD0C2hL zAIU@&?>fVhO*Qsj?L{Ro<<+QGtoIsAH9ee86A(I_JNe9^+63yed$meH<&Ov>H@F}Y zOEFt7AQZnsn7t*~_6x(QxrsD@XT^EEwn;(FqZ{yZkcuQfM4cisG&GOo{Vi4n4U`Z{HSO*1^lME+({BYd>+A;pH z=o2$v4s11y*BT(B$(hVdPL<0j%aW1+j#V{fHY)C9@KzudHXrB&?iORc8xPAOaU?MM zf+GwjPk1j*7ykpFbi;e$+1Gf*E=CCbZk@dvw>V3<=E$0W0wNJYETZ&N@Kn%JZL)ph zAE%*Te#tVUc}9&ItBd<$G%kuT!e@(>eb^=8#fth0?CA&4hbb-z)PNj_^Zh*DVz7XZ zr-y|a4L(addC@ttEVh{H&Lv>O=~aPzK|z%t9fegY%CFEx(~y#Q-&5|=f&(SLV?)9P-9-D*by-OY|dVvbX~u@{;#R+{3nJh1E)e-RImg? z1&3yt4eKQAyTZ@gEw>wNU*Qo${f{wmq4_hY9jxeN3Z=CsYtP-)!&c(IIT@h3=!I% zAAsLBFfKS@*M62i;QSeM9G*2UXBnt2l@r)Fu@sbeAv$h5%M5(gHF{x+k-yP5mc8`mlZ@9)nZ-L%at~_-S>I`73%u>Sw!axwG=N~b z%*9rB^Q7#XNcltqMa1hMhQ5OCM%8r4+&YbnJ#fc;>UqMz=fs4Z)^n>S4oy#b)03A2 z4y_trLCN(8vv}c`fE!5AgEw}ekZ&&EbTY_#*eCIY_|6a)4(Fyg@3g#_<3Gj9BcBlT z^`Ri6K*0osP#oBP0*bQyGwuQDp2EtV%ObA_0}uLHCp_}YD|p4qN7Q!GL=OaXEF)4w zK959qxaAcwuhID3>_46r5a{Lpn6lB- zdn6;lHX(1pUtnQFU@aT7LO#pK`=lo?pkZoY?e4Su1fu{uVg?L8HKR@5C2U+?8mR80 zax&4d5w?gq6Nl}Jue(b|CsrF5EzFC?&mW)1YFBsdof|{VBS8vC^uteGG*t*B> zm`s(uPWV%gr55plHA7D?{~CN7K8NFG`PalOIrBVs65W!L2NZK?b$u*k{`K=s-}e0m z4G||@l0ZWCxLP|x%ZjzETMHp5CMmj(K?- z!wh3Md_`y&s%&||y*pOEI(H$(QrKma67fe?b=hZZ3iRTJG^FIsBLS3ierEGm|P z%50z&EI5{-9y6l9FIq4_JHuDtYhSPXhAn|T=M6lED;sU;-ZB1sgtfv!5$K==yqx~V z-^MIiU^YPeCv0tZN+A(>(-73dW^k6#*Zx{TNdWKeLq$d}0u1yrkt6DUhjl+zfVHsv zGX_yHo^#C(uh)iBv{}GcDjrcXt2T_;^}DRD0Q3&IBf1TPL7W9teBoOZ=f=M@Fi*1P zc-L9qaEf(1vcGRD#r=?bjzScZxyON%U+g=Jy>waz1dybUpb{WR$mvN}qjmb60d1-2 zOZ8FqKLC&z;xy;_Sw3Xja4@#<7f3?a6x`ZA{Ff-EwVgQP0Q$7)S)*yFfmOSafPIb7 zgx_+)cu+Ka26FX&;(y1^xt>`UJd-!Z)C6P7QGWj1dy<$~mA`y`J+&$*$Bl%ST)aS2 zu1$U!7y2_+EWwr|ElNlQ7sfI&DX@~hN&;v zS-7nZ{@HRIfVx3?;}ltS%o%%c1)eY0gh`0324uhn+FnDS;Y#)bV;7pmlmzLvU*!jx zZ!Ejr@C-gX2>6C7_KsTS@g5g}L4Rl#C>`vxjBQ7I^F|DQkcv^<1mP56Zzg6@N81X^ zj2G4_Io#X8kT{h`M+&TgdDp}azjnY_-LwHqMfPTRe^M=%UaUo9X2EQf@bz6+Sc3w{ zXpyi{{LI9@f-ta^G9N4?l5SrSQx+O$st5-hy3r#~ck!8B{Du(+(b5$EEZ}~TE6frh ztcqic>N|{f+X07PqsPJo)GYep*YMJc8n`)S5}`tT#PxRRoLD)R@nK8)5mZj@b^tdZ zMGu=PNiRh4B|sd?tn7w=*@ycFI3|XQd!84-m_lT%YgYE-$`#M()Dr5#;?KMh3J)UjH(khJz()Ljv!F;JB=3tMy>)X7%=L%Pc6qW`%_wQ<}JpR$= zEonI?`|b4jU!j4PD_m=;Z$K)A}RirZu>~tCJe_SZg^TZo-N2^^s8ChOiju zZdZzyaFOTyowMLp43GPPEDYR{eM3Jl((yatM{zE)^6+0k>eoNP?TGF5b!GkdZp(vl z58;^fowfByp4pMIVHvzB6J(2^VN=Z-!i`1yeS@G$0UrY_8P~3zfw|X_&CQ;vzMKMg z9i4p4sB)ctVm$x!@{ELWzg7B6iZC{B81~cb5cx0&tF8DL2xV~MJ0XNrrcG6IYt|A6 zf$|y!YjHFik6!PVj;I-v0sq6a2?ZxUjZMPmlLJPGh>rQHr$i~T>R;D(B7$NG=2w)> zLa)`=KA4w%blhDrk}g&Y?^yq*AH=wTjvr!gQ`LWl{__&+gG&zS(2dZ2N#`5Dx;P<> z!XNt3LPRA>8z-Z1qew)AAh^5-Gh91YuKkEyYnGLI40vwxj6wcbp~> z5Yxm_hKy$$v|D>1_rX@VZ*CqRv(gACKu}%fp7ha>Q9_b439IN^3xJ2Ar^o7O!5Y)r zaJjO zS!^=^UweP=t>_~0=gq_?N!kT%tnuKFkFW9Jsp;-J*YSapEYFVrM@u zXO8Q~K_eC0B4SRXezc&WB5wv#XcWjLwvR=lfGY?lXwaIVE5}gK!kMb}rVoql)C;*B8O$^Yq=%9fsNPDB z6cR58`^2pCwz!uUI(!pzb~;PPsE)#rnl%s}kp7gDVGc!2PaZ2+T*r~e$> zBi2qcNP1WH@wX>AVugv2296YLmSA8EU6bWlw`S#t(@PPuuHX-b%mz?>63^ejK)-s* zQLrPr7U=&Kt3pi+aa(NhJ;puZzIK0kZGcMZx-Lz3E+{lW})NwivKR1-@Os>7+?RX7YiI?OaI6mQ8}Ra z#p?Tzr9enSK~2CB3<)hWk0Vb=u;LW+4j37j$a*)jU$-Ughi*sms@J$&h^;lvK#$oO zFSSGW3;YHKB<@8)U4RM{;4Z2F(}zK-f^>CqO@Z~q&dxrylDu15Df&m6=lJ%7aR!6T z$>>l3a>ASjcM##j(Q&YS7sH~q-p{EX`3c9*1(4-=Zlx{ps^e*W1Riij8z;jm`zV2X z&LcKm%tlp-i`V#D)X3WA2;Oj7poZmH9rha-9^$$7KjiUqFBtSvWCh_wkbLCrl9l{N zeek`})uYY?mf}+n>IP~;uzKhwiuSaevTNv`WaN??$M7eH|m$xaIMSjc-TSX3w+ zKDhxPUEoAay3GSjS&PRdQV|;20qUwK2)AcpXIJdw_R3MM?K9F(W= zw_1i$Q167);akj!kqc%SO2t%n(hE^#8l(hcY(Am6Pdd_Pc|*h+aF$q5-on6>y3H-XdCc3x}OmMV9n-WG~I)5=Bb*skeomRlD1Jc z9F_5(f+?bZ#?sDp_3ld;=Ba1ml!WZD@r+x6hTTqwE0!) z8D(raNIeS1Wu;~yM<|D~Uo^O3R+`gj)pU+C6nGvLkqWMUa`$kGOkKsv@3#4(t79s( z;DV4o6mMtov` zS^dE2*-^#pV@QN>GpPyT1E`MnKg4iRvzf&YFrLi-!Ii!@kWpSC6nm<3&oIDRN244>A%6-& zfUG2{Rsgh~ExZNRhF*&xk$m4U>)Qol5vf3By@_BA?(XQPy+Enx7i=FcU0?}sGM3yrzoK0=79q^Fk=S)TygRlKDaz^ zh537KUk7Y}_W-)#=8Y$fHGJ6l%z7Dfz!$6Y?O$@wWYXY!9ge$CLO@aMS;$h$mBW-G zHY}$#bm&Ac=t?3f0WHox!p!GlcFy?0-qQNM-zeV&YeO+6uZ|xY0ym)4 z$kB3^Lf0vF(CV*H@VTe4r^1lG%E&Fxxtq_KQ~A`RK?8Zs{f*{?rNZ16v#-I)#>|7h z90Ei_0xRxlhBf=q-nxUbir-f!fmqun{feN~0a3G6txo#jANY!1CpB?4KffMhzmDN2 zI;i}p;|vBCO0%V~c|$P>c(kDqOvz`x%mrP)tZ4!2JygWuQYCY)tcQlHKMxA5B2mE) z;w`3jL(w1r8(1#Ij0Vu0U zaE+{O!pb+<{9VfuY;L6nW24C{}32QztCf>(j=w7qC7*d6&Y9J8f-UGg0-_zS|-E(0#^z zcSESCY0VTXqMld^sq1gix;A&*Zsq- zco1lcu1Oz+^urcaFMDI>!LArj{$@hCRike#+rZyQqPG|bmsXVp>kzq~(c59D2U+6u zVS|nfA;Z#8hxK{^L1VY(2N<%((Ma=(VMfZ_8)C*&bxm z@?Qu=G=WyrMF-&TDZ{kJbO4x~f)nmc3=Rsfs^By8U(8MGb$~(e2p9#5$JAZGZdq5u zT~IOSVE-NF|K`nlKaV~NGmuf|v|ZdVMw4eB6EF6s(ie~-h@K|vo34)3UF{Wd5F{B{ zZy4+z!r*PmaI5EAgcE?UU8oT8zjvkd0;akol19f3#frINWx3gxqP`zyi8rrSOL(?w z=qnZ+S@iC`5V`u=-Ime#wEWLzn4MB9_JkVZ)D*~bACY!YtpEp(2$>LgD# zh+QNB-xDBV{Nc^sJn~2FE!v49;LTGTc3+-vHt%4za*l)+gf?|%aoG>I>PT|P!Tvje z^h>aOS2JcW|Hy+2R&K+hYvt#GLjKS)xr&~x_qsp2iqET@P=3zpE5d(eOw&?E8lIn# zM8`^m2o{w+mWCKbiow!Dg+dD2B?w`!oP%8`;qXNyUG(*e5E0(*6(g^dusbC3E!OEl zV>JWBYWgVXka&hppkk&UmJDb07#l|p%?1Kq=s_}cI=9!s^sNaW*6>kwb$`e!YM?{5M4jI%mm9iR&zdehNTW;R91XV zTaFU^lQA!H`=YY#@R>(`?y*!8p%o@Zg)b1M*wf&f81{~5q zaLi;}I8sW@3f#1rd3eaCs`kq2YNMLl*7kPBjb$CLel-R7x5tIBWAk~xgiYVM?8$Yi z_SVLgd~iE?f{fPcYacXRF}}R#AYBWJ6CyJ|z%%+cyMWt4{c{Cxg#t5ozxzb8{<~^` zckYb|fX=5o8>u#DzinY>m*ZaHC>abQ*uor`JK{iWk@r--AK&4+zITnHeqb%{14Ar(ZKY3(j&*awz`NBBI!y2)kNi}+`CRfYz*q!vRqZ8b)%#pd~RHj1fjEBTr%m<}t{Mw%>Z=5lOindU0E z=AE;uu}H5#Hr=woVl7!+Smygp#vif0^uq(D!Sbu&1MmB#2>CMWG@cqE$*fz>c^dJEt?t4*BQ8s`uHcd*p!8aE6n}@}x zm#_yK`OEG6=w}`!x8f$(M(X$t5FijZQ=QM-$AE)D$w|pxV-L#Vly-y$ZK4s=q12f>;sx+5a_Z>b2z%^aXB^{|o=??-hnR$#Oq+*5(%pBQd zrN~JKpIgFh55AT9rCEqs{;T+Si~-%#KQ?;Wz+(}ZL;YztieDq9`&1u;bLS2LUNLeQ z)Lu39^{p}S7GK$Fa5Y$*eCcbpioK+xzIN8jQL2k>mEU0)K?+)eVvs&Cls_r_p%^t} zC7*iMir8>VT~L3Tug>TVK;?%|24nPF{`7CRKcwQAZP3>>`_xy#9d>gw`R(!bPQwb{ z5p!wXZ?HV_{w$CDEIxRv?N}(O?^tjAwwnvz0!>Xz^qIxuF>3>HHn*tcLxnq&HQ&~% zBi=E&JYZU?Lp$7}tO_C8=u*@Zr|4^!8!Q^DLz`uj0Zpj9xfV$Rm6CW9p04X*3<~lE z+=%Z=t72fBZ$h0bmxjwBj~V(1ZYiJHuFtY~nQQ$wF@1tSz{Jl67)Ey(br8%xeg6ki zqu9CP0RED}%0&t@c6xyGdbsx2uY9y4%J&`Qo%R&?cMi+fwgpn+bG*^{~g!<#DAyuo)#oz1=E=X7Q}3szvQ<8VjfJ_-tJ>BrUKvc$&63DDY_JM!o@S z2ipxtf3qN0*)a$3JC6a>%--?OFf&2dPE-oZ`DWY_or*Nw-P<-r`CPkLT*rj=E#FCf zl=EH7ExXbzaAC4El6TWt3+}U66mANi+VqeL?cqBw?usA#Pjlg*m})A*J=1_>KzAJK z<}{uH1VLwc2&0~RZ{!Q8_dx9X%K}T?51ph+zKge54+1b^?T6USt78M^?00Eo<@i;2 z9VjfJ?8?t)K;CGr499mx-uQ;K5%Mo(l7PvA^4Fe(2lezaXhy@!V-2kdg8!1HXkQcj zE5RGyM?WzYuRDMl9uSxd%sz$km-xlq69hD-Lh_ry-c@TkapIppJwRRsf;lx5k`$Y& zwg5QdN^sSi{s-WVh=*GHBt?<=d=dFH{NOgLrk!awyv2!FW0`kg>3)LJ&EZ7j?eWDp z{am_Bdzqlr3_ z4Mie%amlKKI-C8-Rf1eZ>Z_M{>>oZ@KS_69nKsWsZ<<)+I*|9zN;m6BXRX$+E4t{P z96+Lxj%-RbWc7K5D0pE_hhe&B8D_+~I5Y2nm+^*Hh#|BwmX#Z{qTR;Wp5!fR6vbM>36aS) z5e|NK(O+!GUO3<3McTo<$MGC&Zi24RD>Arh?7(@@HQ?I zxAd4N?77qKj}y&OsUqZkc z8F$sIP(66TXCtr0VXM*a&lUS0yfv;&TE&2o^#(0~4o^HmW z+ZS(x4S}4(I~_mqXRiDgDG@IAAN*arNk3%`$4jA-FIYp@ENE7Iy`TkJYj_&=)j5AW zZD;+Lbz7(MY#N(iSa6H*AHH&D*0FyxL#5Me2lM%{I2oh@AhSr)d?7}FG5VU=c0jI8 z`Ka!fy?wcu4BYK#w-~--o)ro-yzBW1X=i5ePzsgy6S`5$B;3T8FUJvx=7c)*mX`{j z0>&tt@Nr1kk1v6-NAEKjlgIg_EvD9)DY?c>uX13?r&ED3v@jXM5hi%lxL*nL;h6-6 z33rw#h0#WEo5xG`5;Y<-!2pr-!1`%cM)gSpK9VF3e=AHh`oim`K`xu$th;urfjPrs zTc8NA!fD424&McF^F}y#UUzYvA4jVZ1Cy8 zqJ#s+=q)%Ou}SuYa3zifxm99|SHHaoCk|}R&fWxYnm^ZcL4XIhKTCA!pY<4g zn44<-Y zBT5t#R)Z&pZG`EZ3LzYMPrf+J6XXX#fHO>Z3fSEct4LWyg2j#02f^*Hq5(T)qt(W2 zR>Bz8=}mikJ`%lt8f|BW46=_G}08zWx$+pxvcf5pRNYQkYuIw-1U zUT5rqgfa>sdS+FrnYG1inSpEImE8Etlu~M)kK^QH zWylmkG>geY%pma@ggv;5eh~oMNNgH~4T3J5s8ipzW;piZ2v#dk7z_unDh^A_Gp0QH z(sLhbEbN&5IbsZY%LhxanZ|g+GQ6)3#G(zTcUAJRTrfL2R-c8MmK0jxT6CKo`Q70y zB|qC7x2i-Qw~ORO*R&AQ))Ea4r8$e?q{IHog)4tdqs&uDQWa(WOZ7WzzkaYQO}3Kp zIK8bW@9VC6w5kt=p*YEE%bya5>Pz6rsj0K2wDjS)tK3H2p}!qa3mjPNU?hZ}vB7cc zfyZQm-iYc+Avt=41og&m>XqCaF5xQ{$FRY)&F{>!x2Fkh59Cm=R*5$xt*F#a69GEL zqnm~kwkCI+qXF-Ul@lNV&CBh2T>$k5|RUYL*i2$?sy8-wb_2F^G zu!|*OFta!-^AU)h3xJ#DtZa(pS}U&R2e$R{CU87|6L=n)_ACGWC*ax$N^i9Wm>!Mm z+k(VkmV3q<&IBF^xqH|qNq7o;+?6F1jBNM_2I8flLLqX^IUM_sJ3uH51qPLfE9~oN z*<;pE3m2oZ!A+Rz7=hqba?uFR|`2eq18p7oXmI$_IJA03U`-I$cZ2+VtKnH_4 zENde(nqi%HQ5KDlo7AW>e{d56)LF*c9J@O907rkL#;M#vHAbJQq=ebJ8r7#f8JwVW zi7??9bw=mGqJ{e>-fkZBleu-aVTe$o^}qk5z+RB3&H}sQ1qI(;I??cbOV1UuNIMEQ z(DL|_aIe%PRz4U(*Yd`f5e&SWMK30@17a9lXHdltS}_|F0LK`RIU4!2l3G8m$^52- zIyaT!gB=Y+Qb2h?ifkURNMTqjSh%6YY_jC>e8|uC$Viv@#oCpXWcN9YLu)%gG%+#6 zS-{T+kY8x$RyhnaU;=9@pd7;<>jb6) zyN){vCxS{!~jP*f>tR+RKH6 zW1a>a_(nMuhVe$sDeESs$vW>pc=2F~aE#I^63e4RC3G`;EZgs^DjA9?5-AY1+G4vN z>s^dyEV%`x*Y_xthRrDW_Uts(_@nNr49d-3=#eK8O(?Tefo6}1j{E~*DjJLI);L2%}rPA35`v*FZ-3> z>P&a?WC|lWG4wU=ZSXk}mu=(n1#Unoq85k6(BPT|JPzx!H>d5Ee&^WA`$!DArXZp$ z2keF|JaH1;GNT9*jEp<(k3Y=Wn^l#;RH~TmZu;abA86eZASs|0Q!iLDCn_f1@~$wF zNN5v;FBnSbB+hP6zQzd+4gxNnYGPaxMh_+x@lnN_*N|@ka>rn0bv!(m($p~szAMl(>I-9PwE{8_9+3%k&O(I!J#ObN4?K7AcE_MKstySu z6Ks{PKk%JS$F}x7LIg7R)BoW=w)*X`f5bA1U!gD7V%32Uq;8j zpE}IbGM(QYBV{B?KwZ;$N%Q#4#Wq|OwFDY~;-@_?tpDOduXJxID_lZ`mvt@T357Y# zqhIM;6m=b4cdC-V2D-fp1XS5SuJML4hgG{$Dm>#jCyXTRjC7seWt5X|7hbz~K_`9X zu_5A1AXF|iiQWV?){Lj+kH~IyU-{*Pg@O}M5u@c4IqI*}k}gXx(^C{$LZ`zvPr>1n zSuG|6bqqI%Jr$%Mks<<%%-Pn!?SgL*nJIHv*i<%3t?JAmZAHg%Qa12m zAd9IH>wO|O@QMLZ3FB)!6oZkGDntbxo(hj$lsgmiiO6m^C69gaK2irk87_yVSBcd% zm`!)zslAVF^t<7FtIkW6TPIKA`I)$;n5#XLrUS@xs^BgqyVaLgy<==bt=4m$CSjzk zEJV7u&3m~AcU3kH@f1W{w4SkARnW}n`J(4CTNTpLQ{(e-=tT*nNBK@;l0(85Y$Iii z!06&a)Q}siyo)p?_>)m12FJ>zMxQY01P9mxN~dyM0q?+&tfKJ@X|2(VD%Elqe5oqs z8Bu`@<(Zn!V+u6iTLmlzW%%YQoz|FT6l%vh*!d&2Pu)ffJ>s~(=lA04QJ0iNXN_CvBu#>a2SS!#*3v9zAxh-W_*aU+5e}o% z@e>`-v|kthCm9KCE{Aq2Q+7_*s1fK()$f{Em4BAmVD~@fdNghJBE;AJ90@{47MyIk z##;pgPT^fxH*RkiMg+!FAvi91pxdPGYV%gaSwr_6y3=Qw<+X%$c@+&5Xe05ts}ihO zVkyhG1Wq6Lw%>l7bRv{v_=4Y|I6j|d3^(P6-SZdV9J|2F)Y8>!9|x?3rlQbjI0Bv& z%dR`&M}&z5**);gA*1;ZwLhBBv3mEUuM2sA z6~Z@kAT*#^jrd$-g9+cGVNXhw`drII;Ya(J_O1-97%nq5J{FztOC6;@=B5 z^wJZdp>+R~=FzqEY6Gbl2@v3}Jhq>((>YI1C3Fc{>?NWBa(M@gpsPxTuGlf5}p^Lkt`(q)LW*i~WW!egm_ISoR0D6AU+Z z9C)T8;RE~=3@pcL){kIJ3QX=PHoa5eX|3o_a#UF&qtpr41tr{4iTB&i8u<$vR!u&F zo4W;uq`xJLr8fbLD3;s7Uv5EL7X_QjQFMOiu18s4l-nWoyN;ME4O_UoGLsiE^ridnpPi4kUr)mWkEVb8UO%fGHerm{{P)&oEYockp=`+zhXM$(XYfO-`g6 z4QAjc4A>YNW5@~0f&1tIY^5LR70%Dk|~5O*@M zbBi5BJwGfKPQM$9tVojDz;OU^=?Lh`j8B};GhpzcETWB6Zy`oUc-T{L%NV-tDM*^A z5)|&GDm06(4tx%{S1^DnSYObw9w9+Q^ef8?fdWk&V{#HiTPEUNyW)Woa~r;JlC%bx zGc;zK(TBsIS|}r-@k}f=?AS&!M(n(?M)^E*%E|MOs{oxsZ2N;;RUdeo}p>Or~g- z-8btGkL2S+QkF)l*hF1YDP~p&;jQY!di~%UCG~Viw8))<|D&oV&@y&XCPe$Q7@qyb zu4f3~bf^e+FY)Abd>uq`5JzxQ?^zmJ1wI5uq#j%-^?RCe=b4QOk&HF9s@y%>7{aXk z)8$ESf4;YyCsV&g$U`6SV(~6Je6GYb%l=9dk3CCi5@>W1N^ay~0}TRIu;(rjVsu49 znfM`ovPDPR49Rj9&1~Wdbg-O~E;ov~Pd~#0FsKg*1Xa&^cf+WC&GP!VHJAp3(^R^L zSPY2&sUlyO(nFO-1Z^jyijkjWbV%*g;&YBH3WDL}CZR$MsrmzdUJ1J=N00Tj4(=cW zRChn>T+N4Dum4nrs9`8O3%q?G%NZdGwnqL273ubzE+<~bf0|Tqw&fRQZD!}d$APNy zn|V)aimZ}NmS4s%mYMVJ?S4&(j`}aFl!gSKQ-(5V@h}SA$-iaOK;;)Uvj9WTrcxk0 zLn2ACIl!N3$}mFQAXdSzD+JguI*S4_So9T_XLvn6Ig*h||)*YCHYO0M)1y2F%*nK#>S1N|Tfp0aqP2E05RX;1B zGDhhqc5E%E8v=8~X(z&_@{T2#9EUT8rl&uzk>}4Nq)s}P4OMcc1a@bY;u3n=Dip84 zOay|bHoqJ966&}#B!6jOQ;ciM9F$qT;bq>wGManox);nTrO2q|6#VQUxM60hR^Ppf zho1l3gRIoy{gS)YV%0Ho9-QboOPK5+uR^9Uo{Wn6-PchHiL&4AN58>bpvIFa0lUyT zqp%pJT|g}dh%?dr`=c27TnaTy2}78@(y5M$Uz_m+WQmT2A~r!S@VCT#08O9nDU!=U z&Vx4zhJr;1Qov?Y$6;%s!|>P$?z|HnimxZ+I>^&G2#z@D`lIZa{RhYMxwZGP6{uEo z*SNoi>CICQiL|%C`a+Jh4e4V2ijWcnkPprJV^1=ls4EP_9GRqMu8;Zf`vK;pit`Si zFxf84Y*F{nQ>>{?pGF`J8Xl|F>ZZzvwpaZWoqw(qxl8mhxO2*b>1c@K;IfDCUNKrZ z^^1plgyXs`p&Xb6G#oO8&j;lWn76BU;<*!U9W}fxz2lm)L!^WX1T`FiEjt8A!x&V{ zw6MV^z4)!{l=bfj;T${!;^YXcPbD#E9VojZDWKc?`s`3qs=c!v!ZbyE`-YsR_!5ai zN!HBkTEC=hrtnf&jEjt&C?nm0X_WsP2uYacYAq2gqQUHOO6+l9P-E1_)yo4BM~Cr) zO$+!d?^i2kF(sZ{Jo=R+85W!RLxJB*cLcL@`qZ~*>@hd9HuXWvd;2zFY&xZsS&OMs zP3>FHViZ92HZxEWzTE*QWXR9;hz5<;=&Nt8uDq6Kt z_EuHV8_}Q@P)UyK#;;`!pns7Wvm`V>7$F!#aCTbVI#SFQ!`?|>v??>{d-^J;ugYY)pV74am51`(H z=>zv)*scicX6~u>Q=c7f;}{1*X3I?Mp;?)hXA9+yH!f9m%OYzy)^Q)f+a5G0A-ZlU zk`z$qrw*h{CdyF1dy>w5(3cG>UC4A}Y)m2#270pS$fCfFFOmHqbMBb48%-P<;C>-s zbR(m+sk*-&Uw{(hwL>!{I+S5o;MiCWmMS7$%*+ylkyq@dIpB1_>ygI9mGD*g7&F#HvjA;|#LR8%k zsAVO!#OmKsK%(`(w~O-sr3WzwVJ1)d0)%5kIRck$ehE<5u*;CI3rEjPFe?}U2qgkj z4W{#FR2w}eLui@kvTV#g>6IfMALFl`A^>=i|HYlmFdw=-4doERWde5w+Ny#U*!@H2 zB~R8oDl9u-MW9MR9$Y39k*k|#T_mo6ay0u{rJuT+hGq`u?cRIj!L)d;L(BRtGD#f& z{&7R~Jl85zc94TPQ=H_XAnAk1B3^8<$|Fy_YM8*#c?uU41{V{5mZMz-ATq-yJ4S}! zdZ2e+!8^IPxwo^sTT1u(w&&4ulHX!Z=HhuYpO7GJlg?86Gliz-B=(>T<2FKj8{7!! zhjkZx(Cub0Bxr+UX#C(0Q2PPd&`k>RgG?EqqUYimH3kUk#7ATw3V{#0xxwknL{qGO z$4HT6#&&npS=yAZd8#Cf*E`;?{1v2m$V@E5^jCsMEa%2Mab>M9f95VnS%%aP$`%sc z+|E2y{OuBy%Ak-TL8yTM-M0F-X~Y2_#^sm*;EeHy7lDcJ=hzkx*oDRZza9OFa%S7-3VkyQRE`|ZYZ3XKN zSlyG9%((z9QHGqZHJ)y}Ugt{K=oXBDb1BtxyhPLwA0qxHrkw>32bEr@n?_Ybn1UG zeXxX=(tcHW*%fX#m_k0u?UDFI9-j!gpiy#qC>sac@I#bKz{|=P!?6%cL&uY#72ea= zXEQD~KRJQj-0^X(auiF~2|Yl!RoA$aI@7}h%#&C<$Xu52@*@mfP5|Fdjbn6jFQpTCH4z3S8G^s+XpUa3ADrgtB-s9O3)x(vWJW4nQ?e<(2v$zmaUPv{MtUrhWPhrd$A z{VeOKbdHa#2J+PxBA2Zf>NF<;HHxsl3QhTb$J< z>*dj|ur@>_9LC?^2zvQV)>?k^0*yI$V205mykgTKzHIefZCT)G$*tv(6c!hT)iTvn zE%J$B#$IYxM>b@6?iOZp{Z5-5PyTSxNdKn)x4?QF;lR`WB+aMBHl~Q+^R8JD<7QIo zL*XlG)PdoRicwU8EW5k>ruXH-{nm5C2$aD)2Rc=?x2cNDfByITJa2N z#vVXWLhATd>r}4ob#Nh^!OE!9(TLLKqE+BC?$&88)E|i-lyR-ZJ`urG5VpOAUO>vR zqFE*(gkQ;hO@*yhzDj{k1aB20qa7)0mV|bh-uN^W#?EZR)p1Du{`~M!H8KH z{&`2^hTR4vkHU|LavMFKk~Wg zW*yc5mev$w`#5vfxZ#mm`=&Gg4cFHV%^&*yszlz8m@G`;o zrhyiY%vJhIRxa7G!j`2aF^8~-9Bc;lhA%L*no-Cy_Mk!*Z);$3)sKd7L>HTqLYL#? z)^QL6o0qf*>}h&>&};Z?3-?j+f4p6)M(`NpDv)*(4oN}oFdLhaYa&eoJ7t5=AN~PH z3*L3;5>g!uM}=LK>@2MM_^$#c4J&JWZ);N8v430nlcN?}V>wAHgeUqALb zYAzPV>(di+qa>#XO*>=QZ}IoHpnr3z8ngi+IcMG%NeOU~Hf#}I#fbc+yBqMfq$0)% z%5deQ!rk~{@ypZF^A&cuQN&+>Mt2g^QB%2E)oPXIJ?X>1tw4qXYmiI%^RXT7L^&_f z;uPSNczY~w23N5(fX!y;J@Bgcu60l$EQbOSHc}~`iH~q&l|KgB8n&uO?2`Nz72qG? zWQ>ZO5xUeMu=3}raQy-C;h38V1{ct{x$Zu4rcLX*1%(JGdRJbY@_j!lVYuv?oobi) zS5q?#$ikw(<5jcWD?$B32}}=bte|WU(T`xPh{=^qPJ6el%RlthNQ~o=+bXK6{t~o_ z1vmEzW9&My+O|$&TbI0*d(m1VmuRR+9hE;9Wvn*sg9`(k%+~S zN6rL-*Y;=lwD<;}IB<6*|GU>SEo2y{U3O3*7LU#RhbU|bN{1I-hJ_SlM2-klj#Ioc z|B%jNtJm%R|F{+so$(ARb5MWq;p3=d*j`KzJkpbb8~&GnqZHANij z>B15Qs*F)~@hybOz>^NF6_Q`Zpdd~o4;Sh;Z1)RO4O%UO7r#;bVDy5#1v?a!g(6jtOk@nsZz&32gZ3H>Xr4-P#eotP0e=Ff zcz3!Et=AM1#HY%@@3C`8#O5#*(iN0Upnv?@Y}syw$ttKj6i(pPEP5+bm+=m}sMelS zfB9in(77+iKT!3Gt~R?8H5vd*UsS|!X_6!>%p3#~8^c`KK+z8DetZC&Id~oVosK4j z3ja^?c{w)h4s$O7aPJgwAdiL|+mc^@Uv)8p&r!K_0Jk zDxrOQyWkitn?Y8S(e%hO9xL`U>+39f3FETt58mE>U8yYrenNRL+O~BY{#~5|F)k^E zRukP*5z~+dEw{!;vW?4v>=@StI7`BPP%jQB8e;cA_+|H6Uc)ZjegX{!2LhwbJ4JmK z8*9Kv3}xYUtL7O&uxk%AB<;B(JMIpLVhqs;(Y_T};>vjHs$!d{RCD`Pcj-sgr8Iq6 z`U_v>Quy!|ukqN`si$1}d*j`%f1Z>NrrwbSXPZH`HQqpQyp&Hoiq$hJw-%aw7%!;4 z%WYX*m*Xe#3o=k@gWccltscczaOa|#siD7#5X8v$eJK+`zP0O%-S(a%*i*8QD}hRN z$@fXonN&D;8rwe{Y)bR6&EWxiGCRcC?Nin`;dvYJ4lE0<5xEFIc5q`hhz-aW%Z|uT zFt)vW?gyV^+qmM=?WDpczUTZyV1EcJB01^_VbIES)ExIqx8UA^g{Nlnch-BF26WlE z{X16in%hM9@WBSGb$;xR_@td1HhX9f%KXCKlS*)oK0mSHvmCA-%uCOZqRX_#)5#47 zJl$b8=%Uzo!{gpBoiw5c*m%n0EnI?n2&;7Hz`IWvdrR@Oy`O=XWjdP;qR~b3n=tXf z8aH}QkqNTvOjEAnd9z#R$Uq~eI8*3|3lz3eFhJBOprccy1=qf38tQ7fg4^5Q<8f~o zOL~_j!}G0HiRi8JCrJGj7>WQ zjxe+NgeUd_Sb6c>S8>Ew3&dA|uc7uZo1g5iMeCR*yN4^F_>hRu?ou(oz z-fpMT0Ffh-rhLtSRX_b+8URRyp2Y(tu({2=&KJUMM#nEw{b5tXU^Vlsu$z^NhBml* zbQnSs$q<}maRIh2Z^FoI6>CA&=Sq|VysLZ7yl5dh|L0hTy@T6Ogc)|^5Ptc!>`(z{ zqjv{8Hr6sDM?VsaS_?==7N7DU0C?mW?QXr@$G8*V4UotM)zdRmS6M}e9}`5dnR{hCiu~(I{BK%^VmpHo#H60UfYb1Or;Rd&)u#h+8?C#j`Ieg^Fgi?*zX zziZ^*?NKV8$Q5BV3oeM%@TGX}E>K>P6K+o~xW(ihAfI3qt&0R%Z&bt38z(VTYt-4@ z!J!eI^zOiorAHgi-yxz68*l9;_SQg{O8XZ6S@Jee+9Ioz^;f-oq9OM6_XjDEPsZy~ zK9PbkAf-Vhjc$W9pV|NZ(adX`h7ySnxgJz4+BW7iy4JYWk z^#s{!V4Lz&J)}P`O%FpvyU3-`DwMk+ED=p3PSXURNiuKZ3@0vyAQW@mqvbf0*$IC( zH|7fh2m!$~^i7!Wfp#x$Hz=Pd{EA-m_@_5;r{Jt&gBQ7z=t!Ww%ZzQ9de|7x5Q!tmty5|lHJJHLpq<=IkeQdK zPDe!`n~w762>C%p!bO*v4ftFSRSJY9D;w%_%=oM4Pt>#nV%qE@!^MSA^NrJ9*`gCM zYYM!|Fwr!U#|8A{8pAVLI?%X+>3`H7jJx462oATb6(xwy&sO)4UNkb!209YE@h2el z{VO}P2kZ`mDuF8zzT%*v*Rc0zs7BTD$?J*&H}cPS>P$N#m?+NzMNBJareXuO0S~Y} zN8Zth18pjl4_J=oHw|z3#Gb4A%z09I!2%;vuEeJhC6V&i_@iJpvkc;i)Umh1n4oS` zFv98wArJlINY7Q)OK7f3f@)qf30lHz5HQeqWb6CQU>9h`B4#&Nh|nx zt}!;sdGN~!TXZN%i0N%Tf z4x3u8vjQ+oKH6jZ@VszVxxc50zX-LYe#iDLCf7GN;1UcqGmvJaWcV{Gn_ z{EDF>J`u@YtrA^_!&DUH6bOW3!U>0$@{It?F(AQw_S}XoLZFyL5n&wmBb)@cu%7T9 zzs`ntWHKH>?N~I_9T}z}c_UW5+hFNgGSkF@sk_WaXLBKuMvww;vwy`?*el4`Z5bkv zUPd^d5pa-mxbpedZ;}45F~nY~eW;En0k=vMx9}{y1rTIijxqRz-#XjsBYj?j$=CT^ zW7s=a>+plcduaL|HH3+QcBZ5HPX}82l2Zm!)X3 z_guRPU#T9gIm>v_lmIVP8=_P|;_jZXn>(_3W~k%2&pB3gj?4!f!X20)1h{s*Dpfc{ za8~*`R5X8^Ep-+B+F{Q2NCswu!dyojeP{9!&Pjp!SpLmq{+zJ7P$gMF&R~$Zgb5o@ z%2Kko8Lv{E00j68cT-t+%v^Zv^(D3}Zy6ZRQ{EWGwj>lhP>KaCJ17y`&%9QlB5>Ky zOIG(lj5s=nYH4GTt;V|E7|G(wzT6}rVD}eWj8Q1SoQM#7WlZ4X9FWJr$I&95|FP|E zhd4%vj6KPkSsMlzBQ8ty>OU9me0G?FQDC~4=MRv`6b2X_*U=J}CXVvp&}p&Nb_NxV z6i`mbs;vIVHkEsedhm|)wB!Nx17x`Rf4*0fFmXnca4oPbZUFfI_Uejg=LP#}rq>~| z`dy-5Q((MJpoYv6vADq({q&9?5CJKq^3~_8`D~(a`!JC&s)a~4R9{TK3y!fdC3b&P z4#eW`a2LU;k6lzjgA5#7LT6Zy$Vkx&5LY*BNeUQDZXWx(Tbu+1Y_Lg=FeP%)JlQER zEUNUyLRIty(ZYHE8$y40>f!%3dBMe=SIPgH9ae5DEww++>d6^icY}Z6vRP-{tl0b5 zKxTdQ9T~@c?Y`%kEppEYet!prn@5c@VIh(}RRU00O;14~lTO$RNLs}16C%(KTgnz; zU9R7Pg$1HwzM;O1nXG`mL4ZVB_tp5uP_CR9j;AkX*>Pf~Z%I>6C@RyG3xx`LWS`=( zF~d@QRf(lA+oQMwsUecubN+b!w)&Xqj>g2*xeO)DLW?Q%ss7>@*$v|OEO&Z=A`({V zo>8ni=*u3Jy{uB@6@@_iof5x`X;12-WD)A-L;hhv`A}Xj>vsaEfHt79H5QIb@)h`x z4Xq?z=Gv7N%1+9>@Uq(t6E9Z$Ob`7^nU>cjy%Q3LH@``)0u#Z7D?U#H!5oRLF?k3U zA@jcm^!^$!z`;?tVhV~Om+KYgY$LcwI6+mXi2W=#q3RhfyqT$nA5;~=Jirkt&{@WG zB@S#>pdslfjevYXw?+~o#ilTrk?*TFC2G#y99DHqdb#XLSbH8M+mGZ29DQZ)JC zRkZe!zy)z%nJEmu@TEoND2vx(#Dd<^`vIg15^hXFaTXVu`H1DhWVG90|5gwg7WI3U zo}!2z7UQx&UIf$z+%FhMN5q-cI^k#0Mntau>1IQ{=!&?A-pQHi_?F~uqgrvNB~);@ z5$po_wx#Ny=J|b}#d&M}tA}xv>`3EPfh)w)1~;|HNOhKwB=`gF@7-&VfWr45Q|(o# zme^gQSSul|?U#$nA@_0R%!L)Cq5;hiJ+C(|4j{Brqkd8~f4K1qs^CwBqe14LUoK7& z?kiATmRm71qb>%bhlDSkkuL?2IMK{9Kv=290J&Kk2wh9 z3+}EV!VE!tu6h`eW-6IHa1!>eq+@h|!r$P)5?(kWz4mB~oUs&R#h{C_T-{wCbL0ju zr1dT!1r>b&KaAT49Yr){q3#0|T9doZUY|m`@j~zHkZz&?D{nm0@7qg|v*F%h2D=LV zV`I*+Xl7OGBDKB9cn)QCQq4i`j-s1q;P>?u|Hh0?e;k;z-D~n&xJmE=B({76c_8L0 zMsQuZorIYx*9Y{W>>VTd{Gc7@vlyP@7cXqA;qr$PyhW;^fZv+=vz&uW`2K30alqI_ zC$@@CMBl=hhJJu`1eB_$^SDa9(3Um)SzrzEmgSj6T>qfY+KCTDbw^~fy0jxHW_9G3 z(XYh=u=N1VY48UPo5QxAw{ssC^nOyPeQ;;M6aNogE64=ClCS%3$@ew9sBMmJASV)x z1ICx}kW(1Qt@Cf6P!FgnaMIi(4csooYi2u0+6<=ajXZgWVfnUNI+@;p8VeW_xPP2t z!eWngf(#rJJbN&LWMdg&saWWHF5#gMSzPuHQE1yLr~nLJiA;Xy(;MU;^y<8se-Bx(oZp35fw9%V%s2<6;g`J$(XApX9Z%nHL?n!zodCxo>Vu79qP zphLPiXdiI2yv;Tf)+idLugTa(I}&1sCYC~K3w1K(sDOEfUM!Fe(Q}arYIK@}#FwR+ znn4bS&ha9Vv79XB4+h1YXqACmcPND2r$OfHUVxMYwR08|je2Zknn8gRe#NzphZB_A z1Y&q6zlAjrb1)hDBR>|}RG2;Dj*33FR>7@HLypIArVv_UsrnBW$neR&>cXBxbrpOw zRAkAYjVs-F293YZ(V6N^9P&_v&M=06u?*Zq_L#8N(Yw;R5-^tJH-?+*vRG_6REKIJ z&=Z~4zV6C@4d8OURu(1PyYDinzzT5aMqcZl2C0@@8yy*_qN=<>m%?v^10kgX?V|Hr zpc%oIxJO2u1J}!efEr;?#K5#3o^ivu<1V}mqu)yyx5Wt6w4Y^n1)Rac72%ZuYw-&J zCt)KHEQBf1<8laZLT+gu@t()SrUoTvLO1 zP~O>wHD8^A(~QL|MP`?wxwQflZCG~@@*5h&=U3MiTPc!Yiv|SVx}yrSO|CT5xZ+Vu zegrmgdE^ZIiA*D3+vx|ci%=>G&t`Db+&+_4#vK_%@mO>Uw?9Kb;=~tQgjfqQat;*5 z<&+HX;~%@xD>={k$7OQ!sXq_jm%h-SboeZS*RNC$Z7FhO(vT=aqDAa2|%TeBl$ zCH2I9TmbuYA>^&BQYn8ZFd4@PH(KYAU-iThb5NAwoeEAYqn|Znj{tGt$MH4IkPzk; zFySyVgjF8>hVXdvWJqT49|snDvytSE&A}xLK{`Gk!u1Ak4}Ewa%SVV;z(8Z*Yv?cz zLK>3CZSz~!K&8CxemfsVO4#z(38;t)6kQWeecYWxE4XOtY%_ol$yg9}hVL9%{2 zP=MjDe-hfy+XzDTS-@-$EcH!w50%yzTaNwW){AElSVnUo(57e(6*o$8;@a5@( zoz%XrBIQ|*$Jeh-@7+|0+u>qA*ie&>7hL@GKfAwi-5W%8aln<^OM? z(Y5PkL)_@l2lIG-scp4;@j!H(o;Sn4{kohvxM2^)r442}lDp0K^Jx;k%q@(B#khKq zNC9mOi}Uv64}oRnDxEmv2vkR%uP_E6+Wmy=~j zYGa?ojEfc!&4qK(TH;MA?o5YZqwFCy_zZsSv<_c%ee7E&GwQJOapLW@#8{*35=dO| zKYUS_Up275j++!Fc-1iBOmtq3gNQj`I*hEy1~EG=FSi;iZQ0TjTW_muL_xa0wrbtO zu-}jjSoo5;?h9qe!;HNWE$g;f+##{g29c=U$_<5McU~$dorI%wFiD4TB)+RSQn`?7A#-<4w^`}Ns>R-bDWEm`p>Kw_6L%G6oY#J>*MNqe8nWQK|_2Uwn4q2mY=}_Rt29I1(vVO*lx(t`HBO;xDDcZ|Dk}?P7vLRtJ^biQ#~^v=$)w)C z+jMM*zVM9n+Lu0F>&z0L3Bl$k!uvkKq6t5i7~0-Dnn7dX%qHjj2^ixj&{>?5UR8C@ zvwq;@rpNvWx<$+lT56d^Ed(s72D1?LoP2gZ$mj`pa|Un$O#aY%1LTXjmD1F2zY2_5 zoAZ_Tg>X^-<&~sn5jgLOvs6yQ!$KLanZ{3S5B3Ga(~d$fm2+6NEHZ(A7YYD*tGPiy zX-ND)geYJyFa8x%+yX01cZ9PMb+|rbnICxTv)mVXG=f47q|*b2FY-|!X~E5cmKAY} zz>8O^Iw&UaO z1)OYEC3%oxF6y`k^bC2|kUXFql*?rik)$BULbkZDtoyyNb8Ji0r57qQriuAYg7j^F zh6!S)?0o}F$C}NMW|bflqEZQwP16&3Y!A^06)@0y=`pmFLfUijwge;`R_9Stq0tpC zfvdRqi}i~8rEFm;Nc%OOSYbOO4#J%mPcJ(nV4vF=rhiBfp~WQIj*LE;3x%eTKzt@I zG!_k|FocJ~rVLsFNaIg>5Hy3rS9pxO1c$h&N5BKB!!I{Qp!ec`T`dVzH_u^CGS#8; zflyZT@)YNIK&WY@w9&xkA1oLKF`+!P$_501BwV*);SAK`YwVWwQ`7I$Q93ACC3eT66zK z&{Gkdh5`&i^$5-u_GDfBgBb&fckZjDMqT0B+{X*$L1SnoVzbDKxSQl{<^}<7=G(cu zzPMfz7qq9|;>TD(U!<#y)dcDC*g5!V}A2HID*6Wkr zo-Vmx_f7%4>T|jZFcZr}&lqwqnxIS=Fm+zz)s6+9Jw^O7{06D%2_SLCE5OGDG!PXu z$^jh23^6EHAZ1oxD{<3dT-CblJlYL%qeSHK*QzFvj5NFTjED4|Xi-(&I7QmJV8q&G z62WM_p1QP{SUQ#n?RU|Rtr4-5)|y}DfXN{BZz1*+(#IC;6ny+U;`0gCxPa!+&?K8z z>V9U4C}H_#7}b>Qu)&;{k9`82Woqil@Zb}9{0Bswi0$EZcZ;O?WU`Lrxgj%ASeD

?emz9`7;WPL^nq)=r-9xY9dE%_^I9H2^Q}mNxBBr@Av<7(?~?oQR)gbIhLB2q zLZ9oZleJR?RtJo>u6;i4J7z=fgYjU;k4?2qnct}2|4LOpe5Y|@?eQ@E1o3fPjzwo) zU1l2~93;Cat*C&-cZ~)XU!Qca1Yt(t_3+o;0Avp6&yw&vGrf<-&xjLM9KMB%?F9^W zO2CT^{2pGN3{UMSd5vI!)iO}@5QS)APCQ*Q%#**v`Rc(!EGXw?w?AK&^K>6`FBI?y zQN--fF+LyxiSrt07{9CKJj*UB)uzMcfcFA4Mu}I6#TDfhCR%YD5MVg zWdi&xI#bHM(Y+c*uz!_GsYQCaa8k``0Z|>R>>6@JKn!*JQ*320D&7(^pI#8S9e0a_ zU}1?20_TkhU3l%iz}#3Uqv3TaO@;&1i7Nx8L)9H>bs%M-1Sj3>(BeBo2;Ido7-PD~ za@dJNnz`|sCsHoHNpXk+oSzw&?!28p0BIiz;V&(+&4K`47&UoAzAZ12ga1lyI%yCf zU##%r)rLgAVQP5W{3`cdZrvic&bzXcK=H$XFdK7`SSZqrTIN@%N)eG~#)~hv-^OT@0*f8`NJek#3|)v%(I7TN7#PWUx+ z;JVGyl|t(`mLv&grXuhV+rDn8)z}EJaJS6DPv8XRH{p|kzZF4238Vsx$@fJ~mNib~ zBw`*j21d8Y!$!jiQf9FnaPN2v(njlr#<~Pj7yQtD3*Wb4g@wCzvpn0DD;x;?;j%1n zAZW<0wgAq$g|-RN|=&f%MlK=1Y#K-HmUV{x6&cpekbSluAq`w|8p-8tf?u}BKZGHp;FY8M0( ze%%xZW%3GyFSnBP94sb2uc61!Q)UW-F2LZsS#ZK8M_0*K2AQo=V+@)84nKnv8$9>a zGL@;E2)}kL+}ZcWzstyj_J9PfA|dx3u*<}Lo739!wxI#0Ljez#d@sG&v8xZTSdt!q z_qbES>xKcA?iFJkP^_L@9v#4T;>;B7nkkFNQ2YMBjb%x z5`XShG-A$Is0@u$zrasf=3;D*^pzBJY=Duzoep3iQKtIkQly3W;5N(rb_?ph_>K9; z?{|-lW47OXIxFEXuS-x;&)jg3Ne@A<9LZPDe&|84(<_KJ=nB^c%;YmWkCGZp2#-T| z1Tw0os9d_Gjzeq2Hjp!2;>TmAvPr<=x^F1{6#}kJ2U9pB63> zz(L}DxO>;CnIpG62U;ydQ}b1?12dzTlewF`US}Ky2Z&i7FerFvV8=J_mBcPdjld3a~jnwn)mkORY(!>ed zXL)m;#2-UK!NEaZS`xQ?Ip8Bc2g(zUa*^H?eqY;J4`ED*aD}>a`o(lWgdv}w0Ll-U(vd~U zi!FrXiT={~pbb=u6|SJHENwp&oYuXQxwm6Kj$1}6pxmn&5|7{b$&R<#rsThPVp3(_ z%Q8c5e1>@`q`sZ+!JdJ7m)<|@i!1*e- z-cU+_;6R%XG#|kJ{6bBt5>1Y5Mop>8`L|^3Az&^Li4<-#!jNjKa&b!g9YLl-?m%q_ zuLAHZ8qeTFajRN-7)FD5_pEUZ@*9g;L=ym&)TRd-CQXF=01&SJt05NN?HXfdN{>?@ zz4yy};hJZVOeoNSQ5^jbkZgWI;PK9>mvlA|XFOvt4R7Ae6DtD zN5*B%g`NxZw#?Y&e=AuUM_N&C?I38x>Pg%7xGP~isDps}LZBRsV`t>AGK1M4 zl?8L)Af&MF+K4lVz=g2vK<1s;%|T5`)7Kcb+J#Bs*pQU5cGES(^1lz=VJ&8KiD{fZ74upa~&T3K$y2^+OPtsg;Mw zYQ4{H<_LW0dI&41(lr23c5rc|@Zo~=3hUw*_{FDpQF05P_2O+la3eh%>x$28hi7w; zHMS=EI1U)wXF%z`^JKY_i0~nQk!nG^Ea1!ncbMIp8e*Uf`7QiEe9DYtE+rqmf0RhG zr_`r#%B`RMfT3;YhFvmtbS+P^bf1=AAQyqlW;GVdI)EBG{4fzH;t(e*?+Iff#fd0);WO{ub80;fzqrtCD5~JI6u_+_Y{(&2Mq#uo2uKRPM8@@9t0J7 z(~bG2@H`7#IiL*qyfL;1$!g^uC0HJ2wLWDA_E%YXPc~4Sb!>YE=L%sliUbES+-LqN zk@{+67m*JS;cpxYQXU@)TBWUmNDfJQn4zkt=T zXt?&GY>t(9=1&cC5P)+WL_C>c#Si>dJb8|Lp|Fy`yaz;pp3k;p1@!MkiZnP@GEO;&L6}R#w;yBk#THjw-((X9H*Dh!IpU-`=#ub#_WUe*y<@83 z?;25c=T7quz)QfS!Yn;+$V~1nIzixcThEUS@jj8k5N=(rS0#Nq zk?1W(rcn^iB>p(jzb{Jmg0A@Z;No`mqeoK&aYP}hD6mXhd+hXZ)`5_;YjMnrl#w}KUy z%JVq|VcWPcJh4+XlkJOOivVOca z<0t0+hW&Q;@ddpxTzwE21M2}W-@HnW2AWS*X1CfT3l#B+SsrOn+BbqvaOYHfwt7gu zRk0(LXt7C73t?{43j_7h z76P!wF9^^({GaZV@(sZH%Ag(?NSw=mHAMC*+`aLM5yVvZYhyFe8sMFnzc>fAX6Dy^ zrfnLB0i9a<-oX6snKa>(Xf$1P| z^A@4KkG6P#+%4m0Bt-e?x43!1KOzkH;%*UarWu2xVlaIrjip?Oy&MoIL^?(IXyobL zr{CaWVMHK}EmHr^+GD=C1>E>IHELlz+7LIgXiLd=zjVfuX!nHlGBtz^rj&!F_%-q} zCzhmACBaf5>^DX$wNC@6A0~qujga-?Vp@g_0-T2309%07p+eJO1xXxY;TUOP(bSEG zMV-`5^@|nD@iCx0rg>1j5lq>pfs6*6Hi500g2aCXN*;L+E+OKvF~-JqK|c2qPP}}F z&fD^=n*3Uj0sq7siGQLPc4TwdZgLz}jsA7c7Cs~bW z+;qvwM!3l~8W}ff0}RuoTu2NJ74WOK8_mgHhxZtSCF&8!6$W^&^6`*uli}Lkn}9ip z>wLS2QzH8nI_=!;P=1;SC!q^9{8zqw{Ylvz^<$yk&0Uh* zy9x0g*EQ~E{xm;B-X{_fPBr0hU;Mp{bndK&5Qw08J3KXZxrm(xn1g#NL_-fz)^$>W z;0FON5^o%`f8o#ioKIggWLju5jcwo-kNgkFcO87)5xwHS0_{QmqSTRwqqL~IqEq_m zU!?|G@;z%Hg-p1Y;D1B6j3Wz9s_q_!rzY`_t>CG^Tn+%9$oLd<7vPv+#}$8M7-o~m zCQRp`Ei~C>NQr5*Cq$YhbJJ0EoBmF%a>${*rY4q7g0lH-Jl*slLy!cEk3bEUiW$6m zBahe8JWvZ}caY8aVL*E1*#JM8mvk&NsIWQMABqu8EP5>+%moxu{G!CvF9kX?qBC0O~u7*5|Zl+gu6#qY$f#zpZBZjv2 zPG|O3eDteyq#7}IYM3dVYpefF1tLN$5A_LPGJCib_5F#T_cY8gHd5BAGVt-*zQ=i4 zr`KgCT(FC&^`&F|WQT1PJ8xR%2)IJUiOm3b1;7Tso^z z9#)NR=(Xanwy9ml!qOPM470AuWzZ2HoSicmr7(OHk}e0DAr;G#3YW$)@>^cBi{T&% zDPd_hd^=ILmK}jEn#Ah5pObaiK!#^<3{wG?1&}d=Y(Sg`vIRPa+Y6>vr0^Lswj>ax z5q5zh3Q_C8V3T%b4%)K#YjwWqkP~HkhcpC*uD}jmhHC6Crh#UMbw$FJfNZ))DTijU z=Fw{aZ2^1CFmIv&AHZ1nlW;@D1U4ylW1>U1x(*Mp~WVf?2zN(TO34Z2t`hV zTcvff6is_5l}WHkl=leTixN5Tm1wCiS5^eluM;XE|4MOEww~xA{cfaJr?Qn{WQsEz zSeBuJupJK7##~kiXy%A@*t7g(u!&LdTc(o^Y)s$|gUEIC&367CGHhaX(OB@wv+})< zFYU;1uHlq8x)17g(QoRM9Pz|j0VJ}w9ZA>}Jev zSQMNclq+-y@1A|UZ5E#ea5^mQc{=X#xJgZ<5;2>YnYw?4myy_T)tLZUasZ0V$~+Yy z4@fN?xK@_kgoyA;A8;nnm0$D;*XpznzCwF;XMOtfqDC3hunzMnwXs7%{c#%8qz$GM zt{v}jBiVGUlUDAHp+89A;-+FAm<2m|KHwJ_yGZ7}FO~Hh!~il8ii7ri5Yry6k~%c0 z6xK14zjJg0`_W*S8*?47pvSfvYJOgfHLx9(?b`WXDj^&!Kao-POM)O6rO8o<4#`sK zR~N?|*)gEyO0ifY;R%xclJl+hJD5LUJU6~uDc}MTitSW6%AiA_EnxpE)d7xFvd4^8 zQmQi}k-OVWRz;MCHq@W2x7#Xgy9@r%_!Ps^#lmS2x+qd9W$5OzlI{;~$Sy;#T(Cx@ zKYy;_hbX#LZUBB)q~GKMnI@@?+BBQff(J=bL+Rb0)fKsUbmf!C;$+Bz`v~F{T_Fx14vE)vHK4#9uNCzW$(1Hf3pKw}&hicc}>zz+GChOD8)A3{@bFw>Gq z22W0Sh`>W;abV_r8!ng|lOWW9;sFgKPouvyBX$b&pA(@q{<7&b{zUBwVYM{WoYui+ zk{C&=qs+;&=qv(GRrz;e67SD2w7NM$AB#!KkOUX#Uia^I(=k9MKv!Da;NJ;vA>L$P!!DuW2$`rL2i{y&chj!{v({2>p_HPLYNBO>NERiHhc@^Bwetq)KWlQydnC5-bf`a3TT_^EmzIioQ=8_CE*<=h3TBRgh$SAHuY%`!{FjcchBm$MO0VgkNF zM~m=WW=>k5XzH!L@p9cd&i*l2#VXPMlDOvf+eo3%Gr!6Pul=gZ{RdfBO>s+;xP^ua zNEJT&g}DF{9+UOtgb!<_;Z$}1S7T&-pi8w+q^yPPiFI!6#PLo#UUs$~`XUlLus06* ze+H(Qc7rc=><6czcL<3#bbRV{gxk@&9dA+B!QA2C>DmZC46Nrn`Rxo)-R)bKq%4%d z^-DtvZ#Y*!kB{fx|0D}|i@0WGxcLCh3j^J_)Hm1B?3mdL#0<6QY3=k!zT8y`rtXRH z6CQ9Kos1rhjtbDa#5*Sb>8Gvk2?-)>XV6GDybnFu8DzrC)!G(~x#GSLNm&@1RiIXb z9gg9d@=r7r(`$c{u?y(A2ufj38(fz9n9acVnk}yEr0JzEqdM|#zX^iKVS@L0$giuk zmuhTyTedN^fiTIFeJN7f}mk9AtUrx^4H3m93$o4ZvbTt=L(? z+D#GbrQ+-mV;&Hf6D1OBR;?JiTYRK0+v^vN$-MiYh<4XN^9#@`AVHxW_`IP?1jh(w zblt8uY>mP7;$fa>LCWR|)OgjBcO93nA4LB&ZHY+jS-~5_aCI>Hwb#E@)qV%JOk)hA zchcrbI@ok)M?8YB0OxM1`KmGz$=3Lfzg5Vu@iT44KH2DtZV#f-lci(tsXxOcTj>E-oGVLLwc%*`;$v)@-%zO z@q_yh#%RNM_QJ*y58}vt6w*3@{^z;c~ zzwP330>785>(Y-QRL8^Z}9EE}u@|#$8f3))pHuu!6YZc$udbL_7bPDcqud zXQ;3=%EBRsW2cI{USf~RrhqF7hy`mzeFe9@-g&!c2h;#B(k2730JoVOF$Cs8rT|U`C(LAPKIsCkXGAguZ(vuHz2RGgq zi7b>@SY}Ei3Vub8oP|`MEayWCQW%oRLSsNgdF{WM^kuvbF-!v7L=!w2h1lwwDTe{; zBNToV+C#E=-K&t&5Qrw~`_w&t!oeD7dkKNBW;25lfY<`O(%iej=ePkq0NXOAmnN)s zoP3iQU%qPUpU)o;Qj9Xg4JxmSNEA6IArD6uhjL-^NF3AGd zU8^p=21BPp@SZyDE(~Rn7T|J)aSC2UQ@-bI!xTEmt>Bf95j}|H#C_A4pMg8YiCUa@ zmG>c6!hbCRXF;$k-*lvOD5nZOd1xc^3%M%|;sd(>V6P$yFVH#X#M=(0q}ZJ>qiLR0 zVi&cV+1uG8I7zE{AJ^?D0Th9v=3mOoQ!fp)C_$0HC$xN#qM2*!lS%S?a8QE&Bvhha z!yC9M1M=~_FqcLsVh12OSZ*mb^RSb34i^qe%%!2Y`g^jjLa=p7&LKJpZ+}9^j&x!| zI(WdSpTt3c)Jcn=QPpYI*SJ~g!}2zY(#jlnF`A~kN*2`2xh--Hb@A9y8E9iNNrTF4 zM%Xu~b3jlfj)w)PQvR^EHm2Jnbn9SVf(N{v%xf7YA`omi*ha)0?edH20f;O_;BRme zC^Ce^)0!e*a5veu%(W2}(IE42L-s2o&R?c&afuZW_^3(ee~QQ z*x)@b%>HT&c?y~H(TSbFPqgS#?{D^HPn%=kU_K6rL3j}vuhS+I5ggU#(#M7MfT;$& z0875A25bd%2UlVQc^QkJ)n@?t2!jDdj@N~(;XIJhlH-Rt|Y`mTZFnrJE%QXPcd!Q+w-)!_!B1NMcc9eAq=hhtrB{{kSSDA0YX z(2m-7L6ebYxa;(WKB|;@! z$eO{gU>S&X&dU5XCM7Dk<9sf8`!Y#9QSxs*-Hv`;$b$x!E9ms_XbcgFevG_8X-JA< zZld;$FS;2)IQVfsKhx5&)Ib^gZjwYW8e@=L{IMo$*sW+hN|CSvU8Y!y`RCL}CcXfT zGgmCI`T^=5h~_uuiFmRkE?Ou&aI>W>`>^OkV6_h>JqroJ_~ryI>5kVu-5Py(+csYL ziwN^$IbRd0R!qMe+&mi6PdLo}lDj1jSOey19dh|qFtQ$~zSAunBg75~Rj3?W*^tUx zEcA}$%`fZVZL$O)_U=!)6PCns{#3@1bbg)GLhh)xZ_|zEtG{?mn@>~CUFjv2Ch-fH zN3K*UFEnPuk@zEDqOU`*j(hkS;8hXzy~LAyk7jLC_AP4&tFX2?xS$krwwv%r&U~OL&oyQf_FMPsMDp!I?T2h)#V`d3zrRqv>z{7eDz-&3 zWlK*-V+KPJ46_OCbscsu-rjy&drDBL5PEn@Aj1^!Vgbf2)&hB>J7)cKi}YyqX#uHj z@jtRjTipPQKJ761wLEuwS6U6X09N=eYxWWtfIwLu?v^!&TBqQ+yiD%;R$rdc)f&3!CY9gz7m}!Us8C z5%8O2eGWQeQ9Q)+!JIA@RVrLQyVYl3h$48ZW_>^~i`kjmj)Y=#-RQLF*b|+81`6w1P z#}O^0J;0|jy(>41(@`zYh1??9fx;D2@yDna!)L&oA`7J={yDp~CqM5#9s`_q2wi_Z z3r)UiB*1LLxQB@Js-;nTrEsq~Pqhof(t~A9MBH9I2u$;m$2bo3gqUuHEtnF;;(f$CW zE^e-7yVdp|{IZE(#}@J`JS=72$(V=47%mdI?a2>KFH9ZDI-B9BwY2)~?KbH?wP%D< zWfFCT4h$|%R||Gb^>)R6at4YF@E~QGe`!s+@f6_JOcT(`Sz1R_9On9|Ol)8W%6uOe@Vjk2_5G*72Yah15LgOCI48%~ldKBwS6 z%lu(4a~4B#s1jSL`YZ{9WyAtmq6!-IxIl1$m^lYEc%g1`V4b9nQjeHCVB|c{@c^<3 z3gZbd#O&{c>ZquU6TXX$)&KsMkX;NxTl=~}OD)zS=P;`JJy3$4Cq(Rt^pZKDH?2Dh zxNQ#Pm&LZ5$r)Hyf;|Pi3e2%(B4!19qSWD~@|$ZR>hVx{A8ZFKHB1dCodMDAI`2a( zd9N6a>N3lJ0SBu~XK}cLG6@ef`NYy()4((9U`6dZsXm~0g`vf7!@p15)qc}!O>w)cpXoK`rAs!d# zbsWNkofJzsUJ{))_{3I@@5@0d3Og4s(OyC2MfNL@>I+Zn6m-}kUlk<1ZGTxSCne;; z1p+|_%0bd$f>4nNJCIuNj!xG$xAnXt6v!#X_Kzrf9|SKL9j=X@v!j9Ov>&wCm9W~~ zvKfXHvME|N&pQyPiu^d@YLyoR)oq5Sg8Ny0A`*&q_CTEny|A~WGXrRa>^&rKlyb$a zU!j`F6xZApo)Gqk`k>iVRRWyY>bG%n6NnvHe>QGxFrh<`W&5*C+KSI2%Duy$ObG!m zs%Z?A8yR9Q{rJH2e~Tj+U?I{VA8W!eAlCTWg+eB3*pnj6{)_fMOP*LK_UwK;rAoqa zTYb!Mx@A$-Z2|xR8M6&5V6`Ago_Nuq;aE!FXb?yYadOQNE4>O;L%Gkv4P0P&H_%<|UAwMDN1n&37AF+%YefP!c;uf#ke6*FeC$B^INx1k!Q~_eP zd%;?hb5bD*dqWB_$rj5>s_Ug<#?&59vN|!2*6@@M0zJERY7kp6faV6W^LAby{Q%5O(x_ha7 z;syzA-53AjsSX(TK04|=X%LVRWMyZC6yZHCOTcjiH4HmbU>sX}`sM%A!L3J=0pxaE zQ6X-i#`m*PQp?H>H61?;boqlz&s||S#I{d8oP_Zvd=6!_V(_QV4Zt0aMaD4Pzqe3% zL9{>cL56a<+(sYgoGng2SXRo23WU+6aRk3p;gD1V6HpvH5;BHNd?8RHkW4QW;`Rtg z6Z{&BEYA71;OozUn*dLcm|2^2j03ulYCqF@)fUnv>e0!x&CA z9efPU3fW7XJ^?#oWL$NO>y?KaaiA#lBWDBS8TLE3Qf}Qce<(umMvh%wNVy`EQYW@D z5LLIqcOki!s73Qhu~kDp!Ut3V7q`u^i?8$7ZP)Y5$-qpY5&#PVzfYZM0yP48+1OP6 zsY4l}xqK(LYMHTsLA*#z>|Uxm+FI#Z6|!kH#=V(=kDs-VG7j-DAoa-xv@Fk7-ZclBq} zcrg-Q5~>9BN*gNWO%AAf9_EIY^CA>azx<#Bk_LFs`4du(W-~|ukA@@!3brSc4)H`9 z@zxDE7$aIf`VIb1HnUf!l4Ie4%a`6Z-KpgM&Eo*!@0_$wYW(9b*w}*3$pQ;WrnIeK zc&eCOTT_O=y3*J?<`d1R?^-{}ZPI$e>2Q^hQ2zg^diS`h>#TeH^?jykrmW1tbhOeD z4I(d1kQ#E!dz8wEigD8}M@djg5D*Y}x;SQe#akdkqZACa^V#pc_u6Z(B@UkB5=Z16ba@r66071637Ja5fl}-X z_~U*y(1XziZ0uz)2YHR#fwF>9fiA}fCVhMdLTE)15%k+BfjNV0@Rj)Sx64XZ-=T;x zg$EvK=44^#os{Mod}q$bcaV>dgn@!sL@W*26cj^_(n*O%Ge4Uykch9V`h$Xe6Mb0W z;j+Q*lJdW(KAPbBgx6>Pm47xlzA8Af2tQ$I5#ZbL6vOECviRcJD6J9G!W zqTNpwF4@k2e3-6abUY^uh2V&(lCopZmuU0(Jg^E#O)Zg9)(hr?2#O-nBg(A)h-D#U ze>d|gg>95wjpnE+ully3v*I6HmaEN`M`&`JUI1*lieHf9wZYF+B*X*&RP1M_B`&q5 zF^6IET-u~k{tqb50v`eVfVKyP81&aDu>xKH8Mrx&wWNjaG9Jf7)=;By+tyF(jChM( z!?ymD56&i$$oY2w&DjObhU(q&DE;02o_6zY@ z=S>3Q4%af2NkZJk-J5SanX@FAY#0p|w%LKAWBT9zD-d#e^qels{78WH#qr|k31X}~ zqOpCC^jROI|D3Z6FC4td!82W?IkkqMTL6~v;GK%Gun#RUjc?s8pLTF{AWK5_?GiTQ z5@(t~xbNN@H~w?+`CVAUT*@l18i|En}rGKd1jU;;?S&%{iI{qy;0J5>eEz`>1jk~*I($q<3!}HIx+`~J5 z_Y`xJWtw5h4$cOmOjs7Y#$&SQ>n1769SNzMqN5iyjINFM5Y=VY$IFe*N% zIJyA5Xaq=%K%S8S6!^i@Wnm2R4a@$>h(0UUYZP87^OKoYYrm!EUpvyV$*DUTZ8r{z z3ME_&69f2aU9%4cebFrak{VJM*@9ma&hRr}%B0V}=96QrLnM(5;f-vGVs}EUa6JRL z0Rw&6S}S&)uz^1^=p{?I9yrXio`RLIPdHiJCZSkAN4L@&-$}3`4m-zO@u4p?9Ob#y zo5^XT&JsG~wiICNoPibzVQgNjvvBMr;{Ji@eI>UV_yiQWSWWhi_0p>xK{S0hh^*lE z@nEF^RP_S^c)kDWo|F#ZV7~z0MIZ0$cE^7B9Nqm2G) z?5{$C83zmztO*QduFJ*^9V++$`V#l%;3r5z(dI(k3dvk@wL`*avKe9$aO0d#35ijO zX5~Pf+2ju*HF&gdz-o=y@bUjR47D#}A9sxutyiG(SW<2xEF#vqqo0esT1PWN_ z{IjS?fON*kUHz=hfFq)0j7&JeeuB&wW z1JKE1`4aAS#!PxSK2|NLu0g+MS%7qoL02;RZZj+K>IpUzn~xvN_da=!=(wk|MjHfK z;D~`LX5-s@kg+(NviKyI4aGf^(GWq35K9VXX+E zQF=<21t%2235Zi*ef<`uwmbIm1I%ZUg2v9_@8FAhuM$FKx>Y%1h` zgh3Jd+C1m4*YI)P+21nJ8p%W#(>$meq2LJ5}3ABt@TG+~buyjZwM-HY1c=icNf`Z+*GX3bhtPvJ^i&<~1S89`Vjs21yXZ`h}iEwPO|ccqoGkDGj@uj`Y0I6bbg z`?K7IvB?Xi#@l(4%vv-kvH}GQZ4f5#-;v+*+Bv^|IQ%;yjyLpVL8qB@UeW9C3FMca z^89lQBzP5sa$gWruV;mIu^d_!?6`99D@59{LX(0OTAUaAW1!rMi6tn^B#l$rC+}dC zi0eTa8$A*a)s>JOLqjEeEqtmAJ@%I!mK2IJ5T|iS`FU9!HZ`GhQ;1 z8qZKph#$w<@lQ1dqqP*2@~ijw_S<{o51Wa~dn4r#|`oF_P zAh66d*?_bfyvliUMuIEEvJS$_auY!5U%r2SMWPgUZoptyw5{b`5o@ZV!~^&jk8CFE zHNEzzJBqzIj*VTfp$NdYs2k*wmZt3B$DV2PtL7zumLX^wQvHRNgn)d_N177u=4=#r z10y7f!OV|9yFe>cbB`2jFk+|LIMK3moyzqymNqIZPwVl+Dq2R z!;$Ki{2*CB$mW8SeTafU8JdCx$8TJboFv-T0N43pFK%ZQr;=wPTnXacG>jW0+K3b) zB<;H~6vivH3Ps79+zzo>Pu#`J6=;MCagA;a?vYhMV3~|691BIVuK|qiDDjFKe>qsT zn?e?|C|fiMtG|<}U<6UMpAGHI)NfA7jcu6Yx~Mvy$hpq-X7`ErSaXBOU(%-Iu7>yE z`=F*Hzt;mh#3!1MB{_nGy0KxG=#_iMPT}v*0XZ0WW15bmVy<^eW7QGEzX}!$XS@C7 z>o%fh5Pn}5*QVxn9@)2odK8G?8j;YW95HWJSkv$&;Wjpi5&&u?Vko4HE7!xK4u_l$ ztQsv(!wsFWw)}D)L(u&7Zm{Yg4S|OLRRE14|!5oajx*N)Y}9E7X$_Sv=anG!kit-khQ#1_d1# zz$`)~`QRPWEU;|&3Qz~M;Pn2$qtWUZ3h8bDw5PumCYXdeZ$LroegeShDgK2_R1Pgw z*$jc!nHO2m?EFya6d6b5yY6Q%$&rck=~y)h(s)P)8$ z1{y9L{aqt5VP^gI*gmke`45z>2oY>%h*Gj=#80>Vm?mmk!;`C-#VGyDst2Ed;Q&>6 zddTS08R}syZB?lVvBk?}ZHTDW^y8f)Hq-EB7wS>pd%m(Ng_-Q)f;$IxNKolQeCIcY z=eq&ScW;F-gb=4b%~dyzaN+TVeY;WWK)jfwGGi|np1fhao?&XMe=Zc`tpR=>lH zkT|RK6Ga9}v7m~FGfcptImlZLD4>T+5$#5yUP>=`WR8LTE)2Kes5`(DY`EA6SOO#1 zqO0=l+8TiwU`k@F99Bk8fL#XCrQxI@9-doXfcPIV5Bv?A?;{y<@B{DV z3(iH~b%HgVzs|bw>Stfcsu~#zGG4c@wetHZm(;@=SC0S@hRxM#F^G0A7F&8u(7@bz ztZaQ#bqAVvFNHbT(5W2xboK~6f&F)ywQ0d?H4<7CIs5Q8l`wven+oZUi^pa#-Rzsac<`PinVPG$~|JBy>Hdgz*N z+BXXAsiMm55xyARN=>M;a4A#sru>fKC5}XD{%N#b?PIdtUBs(K^kE^iP+aA77|90j zmO>E-G}|^_8kiD9D>Z4!59CNt=tPTP_ch?mx=xy#O?sh2F34}Ku?!#D99D7%0(`V`!VVJu z0c|A=W=Jt-=K)>b^S+57!{U{=rV4kW@%@t+OY=LJs2i*9&AXp)m(iFtI_y1>6>REp zOTwz^5|mBtyi0LaKr$J;`$O^JrWeEP%Bq&V{ZvC&+ROh=N${zNh~4jEig>Ye?LF@t z?%i?p^@0cfW`q-gjK{3Aab=}|j(HFW1=^y=)rxL73$oD`z*F*5s2L-!@QELJg_j*W z+&y=(Zh|F8f#}x@)k$}vK%*(hk*nAcdG};$Uoq=jb43AgBWbvLp7qib!nsy&c zfF=(v3qa^ZNlWc%CTO+tL+S#kVhn^Mql14$kO_=F#Q-s?gcArEeFh9Fi~aDZhB9f8cb_*!qSZipW7 z?HE=g j9QNz?Bqy1L=;@%rl_n|IgQ}s~yv?gBWu7Ro7m4Z>~lKQwFA!ci2tL!QQ zA@UXN3CylGy6tJ$Kp4)TUth zeW|qrU+;2v2@^0JuJ|y06Pfs;{U8q*Dk2YoQ6lot1yWig4Ke`deG|^+h#Mu0mNC}t zezMJHfNN+Xn0mO;kFUcmUGa+*sJz>Lda^Z8I0PMAHp0-ALc=@33X_Ef$RKAA;}xEF z3iK%6BTZ3%_tHF>@r;;s4EnyaDzcjQ-0lLrP@x_)cKj-yTMiXV?nbw4BZdK9ZLez* zKXxpq>^zy~;!ldZkeJXme%S(G3|Mtm_p<+HqRQwt22v1j$_)wR;ZwQfha>#;1f}e@ zzAc#FWV35VFH66sjcPzRF0o|Sr)y31Bqkl`+Om;J>9*R$;`oKl&o$|m)e@k6amtV3 zg0*+h43r_QnE8+4^0i#Ze4ayY8IFIms*O#k*&C+nC>v}nZ!FCT02n*WGw?h73L$X$ zpG=6+&-9oa1Z4?4+7Zhf1$k@J3J6ob`sqTIf6oI}0$uZ7SS+hDD)-8jAK39gNOJwYwfK zQ26=CjK+EgWtGv_U7=0{gkX}D1DJOcb^-i``{#E8u;`&1hiE?YL9#$bj}cL^otV() zh-;xI{D%1Oh8aU*6<$*T0S>cVor*ml?W0t}(p3#>u1Dj$=m4jp=a7jncqego!lNDh zvWF*E=Xll3L#4%#iVwf{eoN(8equM5Bq+?1Jv!vxs*0ee>6w%gQ8;+^vBa|h_*ai| z94@xLd&=M~e4_W@?aSwduz-TP!ONZJpDFd+N$TCftju$i&BRiFe{2QE&b!isu)4K( zWLKbaFOL4$h3A}>2een$J;rQ8H>i8E!>Gme1!{9)3|KGE@6e-6TvM~I%XDZ~lbC~u+nGe1(z3cCej#$hTF7Yt}ijB!_PMcMNVC&0CQ z_--uDQ>}t(##m|8ZI1T9Y!MB^U4r14kPC+&rBoB_EkUw#ckAf>T?)~D+1pGP+ z>dHD!Glh0ex+LGXxykS2k}yYETt`UvOeVk~8tmee^G5`s3ltz?j$`P1dH2bTxA*20 zR8^AQGaxG)Fewy#2PUIEvwE-dtK_Y@opty5D-kMCX2a%%M`T#VEjB}8Ib$!IrT&Iz z4;y+ezqSi!R=cMm?eYOQ=kSTZI6qDEG&ri5R~3mK1=Ln0w93!DFH<9zS4mi3_m|$m zL#+d*XL(st_C2x-r&aH8nZb3|c^Gt(fj%cGI>{#-hfHQW=GX&#Gb;n{l{LmvdzG^( zseGSNf=z9Ykh%0&F=dEdi%9=1~Wwf{;E6Q~$tq>j94s*y<+ z6e4*adS5*EK4WS@T{h_OQbyLDGuTPTy2BxM$xA6^R;v&a@7Io0(=Ii`K^X5m}t$zS7sc9ousgNYcxXV^N zs_#F`c{x0HtA{hsqlMW_Q&>kRe?Cs!2K#qgvvntt@|GW%a;;)J?FgJp+TftypyvW4 z36AkO5++nhkpMaZodv%CojoB2#TS2rZ|O8XDO}Q>CxsIMoCp;5OQ;6F&fF%(-#T)UAs}cf4t1~tdCC4%MG_Cu| z!6YF;Qw@~jgT5u?Dgpn;Se#%xfO@I2dV_L(R3+1#Fr-p;e*e7TxSC<59>Sevbr6X^ zBk12jKov}5#QYGdE>xJn40)B;Ux=xDx;^qMFF!_-Y`K!`=N$Ndd^vFmFGee_w=6agUc)pzCZc%B*0y)f`KM@)eJWU0gn9QCGWA2@wllS<{OK8P#T0tcv zVeoHMn}8(JwVl|?yHHTp0WXVdw!XR7EYcBWdBj)!jVn&Ye($8R`b)BgW}bf@xVI=F z2YZQXM0)b`AM%ILlEnoOq_|hdW;#KLJprf%U7Z8S% zM(Vg@kcMQAi{c>zM;WeN|fV#{3EHtU2q>NmV~fVn51Sw9LsJe{vIW~Wg zj=I)ESN6x?c$nw?2q1FYOe*U#1~n<$Kq$U>ELV|iph;d+FUVsup26@6g5#6DdM+}U zsGBNlxl~L4P*YI#fikMa%NKiOb{K5dTER2MRH&PS42Yv(* zQ3^cO=Hp7Ek%6*B_Xw-446+)mN(h!ATy-R*%39w0B{bgvumXt@-6^g;k*0Edk%9nT zX(2Th@Y8S3p*O6^!bZ4Y3-FK*uhi5K4fU}cWiQ#+lDz~jfuYhj4fgEUY?EL{B2%aL zK5daL5BLuI857=;k`R%Xhv>x{C1(`H=$)pnYbuV3-E%OPgeg5%^!wU39Bv8jW;$BG z0c{T03izOpG{e*a9euP}*bEPceT0NRAoQBoPw7Q>UnFrnV#=LGE{0P|meEH%%OCwxoSu`!_XcDC-Z-KO5XM9)yG zRUq6M+%$_pQ%9L9&S>P>XOV@}H1j3LV9AEXQ5VCG#X(epNN;(J7x=Rvp36K6r!2hd zHF@3qB~t`Thf2YL*!zv`d5mENBb^E0O!7_Et|@A-VM^NoEJY|?m)5#{y+#gB4yA7G zy)X)TUi=5BjPlzkAXJZ{^{blz-$71>(CJievCJGm$y}_??wu8> z&C+z%EIgt-O!#~7a7j-s0z^S7Oc)h!RiZ#Bnn0?lXL*Bo4u`jeTT^y;W= zuNMS)z>r${tc|2dBHa39*?wy<;a*(QYTE4rA*2`>05O8(2y)tNAfx11E@3B)-{~*6 z`_!-zz^GW^d3SSPkY7_H!id2DYc@WETkVO8UuyVCBz!K=>g}kdRpC;qL7kMqxaIOO z@qfo(Q=l(=;T2j}<6cp=dS->)bG%nTkLA@~=pVtcFSztJw^>R#y=Y;=xq1@^6H^ZG zi?bYF39tu^ba@~B@|cw1`Ec9?5Hg+V2vNUilSciWNtgH>>Yrg*SWx!3yHFV_YfoR| z*GT~l0f7QB2G%mUa4?ygnf3;*j7SU1Z;W_GW7+dy_ zVmS*)SIs^0OLu>fRU$BK&)6{{GRxPElXAz&yg<@cv@3_b;OI~t6Qy3N8*drsIVaV( zrd_w{@rHOguvo@h4F$={X%bKq%Y0Vt=d1~gOnxIh<^EZ-cnl>>KHR4Lz$*W0Uf{U- zhgyPnF@3?H)5rwDd8^(5ssRGaDJLE+2n?HvJWjkjei&9m5- z6a?1k)Lh3}+4lhE&S=A3p%&Nu4co3%%ZO=6_Okq!F*3~+C=XM-)D+$XuDCX0xbYt+ z4#SZ!l|83jOvK+k$o~k#=q) zq;6UIwe(E~4l8Bb{uq{Roo7}r)6{gKYh%oL@+77uzN8ww13qnX9*uFASZ)!0F`|x! z$CEY7JVh2hYHD{X3kh5KDCkgn*ZVY3G!}^tEnT^!AWOZEA(7N7HfnRtnFeXSVR$7o zp(S7+sD?l?Y%DWcbs!8E#}g-g5-zZig8?aPZK>@Viy`^&*E zPSqlEifi~jundZ?Agdm10^?!0<<_LEZ@QeK?&EI6xkxSw_p&3p-ecmO5L;@fQgD0% z+YP>V>;+2^m@yx;`XEOv>#pW*2#@8RQHzCw0oYJj8vyD&H_QF#F8djPPSRNF^v5qQ zvK5WxqJ}g$%gAlgP>e5_$z-h5Z#Xt74io2f+rajqwZIMopdlC-D4z`Nu%rKkb{L@Z zW#=kR{P$VOk_(SOG2D80B)pr>TV9kO_wBS(JUx233MIL#p4o&X7u%u54+^{VFNA?0 zVx?jhh?@^dNWbHb@8I0oJA+>x8ymevrW{irl5n@cYscR(ny$u zMp51z!=TRtWxF0Jib52iuE1oEJQ4`4uMlm2(6ZP<;c83=#59d@p>k6#XKne{amN^H z9<}hjO%*Jn-+VAuOTka6FpD;(4j7{R?jJX)3WX|JrccoYsg_vt*87ohhC}jbEYcf` zK9N*{n|O@drFiMEiu^dxDk8lE*CS7sgdH(Yd~w{^iiG)$)Lyr32aGh-x<_Wg75gFU zi_8P+Ug=ZV$d+%KmKpNb_lkMc8amw!2m6f2-G#_DS=pr*5TXQ7d1o0*z~+s|Q%`Q@ z$#|N39ow6NGu(TxOE#{#A*}P6#~QLC_<*}_iX_y|g4X@>q@TK&?Jr$VM_>Z$w!SA? z`u>51nGB|ahY_D3A4bwX!#xq2&GdbM&!jn^NZ>}GEO^}gQ6|mojrnN+xYI>>Q~8YZo+a`G!FQX; z0tD;OU-l7EVAz(qK!$nKzV%Z@*f4$C@0h51WkO_Q83j@KN?t(l0hM+k^b7PvWHA9| z?n_>8=$yf)7o3c-eD{Ww^4%Y?U5-e0dQ}Diw`UxKrjZoZi++#>5=-7t{t#AeOBKcl zqA11(v<*0PKQ56`=f;L5f4hcz1u%o#;@;1Ec|C=yr>f#!HMh004k50St6T8w~+)Tek@+jD@Of}e5$IrS^-eLO#$F=!Q6xSnUXIAF2NMY|^XTP(vuRf) z{ug(MB`AB`#Z@YgQN*1_*GWf`tpfbH_~jy3RwT?5v2r!>1UVRkBociSrv&NF?@Yf; z?)dWAvsTD92j+NB%Y2Gz2E}dvKz=x{_Ny|66Qj^v)Nwc`(sGu1DDwyk@}hA*@-PcH z*K9+Qve0&NnX$pq_al;(k*sms%YfARN;_yuC>l2$?3)wGDlOh+>-ne&;4dgp>lrcF zw2^PP$88Mt6nMGf3a0b~-;^}=dpczdDORD!H_$f!l>7Sfq8>-Ks)5{FrE98CF+#mV zi){h$F^#lgyM3mx;(KwyIeX4Oq5X4egs`5UFQbCvxlxT_;AW5yxI zRHh*rvW0D`4P)7w@Df}tq?=S%wXN~Tb0Qd8->;I6XsYTwYBp*rge>vXa?+$VjuH;0 z<&y@nur`%r-bH5t6qlFciS11sT=eFEfcRm5e{N+6bJg8vQ=xaq$V%-A!H2~h)LXJH zNBW$9R1qiNZdxg^HWyxNE!+4F4KWA3HfiT(f!gi2E;}nv$~3!&RmM-eMMkV_4zV(bcvBWU?%`}q*DXUL$Q5T^d%^7 zbAORsr?=p919HWq-2^7^*^R)KH2(q81b^ZB^6xJ5P&!d8^o9#yXUOQ{5l!qk%JAc9 zi8Hfyi~!AH3Jf%#+r?X<;$1i9HPR0p-yF-!aacgetqCvYzu0*Xzj5{-{X;&Pc$9oU3l0BCXB^V*39YbOQprYXiE}? z7*Z&$D@=JKVHSTBxE~t4l96&OPY7||t@bJVZgG{qvgRWmnA@Lwx@j}xeSCJcxZ8kJ z>*68|_2UOywxtmL+lg%;h5JCk%x!!YafA%4t{Kou0Xj~DCsm#RkB9Mvj^#NS_{FNVw*HmI7C=JAI3s~i%AA~6k)m2G$J8lQb=jVA0UtK5lKDgIuyZ}ky`UQ2>ylTPdg8hsQKQDWcSI^VC#n`WsDda%;%5(Q`#tz zKVlmt{MpbWhJuT+i3q~n8hRLFnGB~SZ|-`Mkmh^0|KiO5VXs_s)48XN#lx$H6w(!O zeMfI`1UvP(#huI;O>uS0O+rC)TKK|wIF@!>~X}E81fJDgIO% z6Mx)}ab|&$(U6{!VRJ(bOyr}><**+W4-8m{Oi!)jXNbK~`P%s1^%JTffv3*^XAoS8 z`Q~ZA^C(`xJYxMa0;@6VpE3P+;8P%h)Z8OqV;@}_QH|#ql?e4?;Y8$YU)dmISg0^u zPf>4={3HK^16XDWTT^ip;pi6h4_JNWc&UU0p#c7WRJATJDK7Ym4H2TW60UuKjC4~< z9CL;J0c3CUF3i;h37+}LDH`5jqax2E{*zkPvDj>7w-WC)0c@Kzw`i_dZc*WAnAvvU zQTIZ$*J~ws&_g-c6ta{<6yq!2;=H~(o|+l+>(weM^=Lfiy}&*aK{$t6ZN{Rj45=#| z(akk~uz#Las6|l|Bx=~-Z#ONo&SC2DPW?QN3A=N_713}8>GsrFd6*TfC&;C!9OlHn znhCuez=1~+{A%=K$+=xwUOgoWM0gE0CfsjXJx%*a{uJb$>@(tJV+oCkB%8&U6NeQ{ zRUSk_L-4`aWOP_q@Or{GQoFK-;7TEq!w~BTXpmCfsbEsm%J0h5%-x|{$bC(*_hzm% z0mP%|103LSh9~_~LXDD`;8pO^h|uC}Kd7|er_tc{;j}QEo>C)`dgK^YQ#Giq2zhUg zW6*mC%lOC|tWyXm=<+HjyI~$@6sN@WU&cpSH)9SpNmlX_9t8CJ5p8qKWz$t_!s`X; zq44%$p)Spa2lNOyI{;)Ow}8v5WjjU?!6@R;&8LmAb#_w_Ci^nC?SfA(J|eG8NJuvH zq2UYeJ$iK23}wZNVKqq2+oo_*!<~F+%dsvHi8xqgY;LR{i&u^tW2?Xzsn$@3^ZznD zGJfv6TQ;G$_=)g>r5DGjPlk$ZViY8#i1b>6Lh+hc*P!n|c4P7saOp&T1PB9N=Qu*Oa&6v28PNU7u>@Jgf_9ywnOz;)L5prB*cZ=K#_;{iV^R z@+ZuD3R?XD`k}wSxK0~gDx?UtvMH+brHC2-j=!7mR{A@p(aUDbA3`!bp8U?BWI?P5 z458$q`8;Gq*>1Qc!;bJMSpBI=t|)4#HtBVTZ;?Ds#(_zVe=)kiT5&#p)Wb(>g4$_Xw4Qv*{bzA%0Y!3l%O`j>9l z@MnnC9M6u~UcnG69bWAtD(K*eeXOhn?C=X(81w}~3W;@J1a?AHI$T6xXq|)r1o>zL z&^88CI~MGTir+5CTd(>qErx%TrnSZgy(ah2YXHxt-Zx-9F}JM`p80S7BGeyX(f9?sBMOZYJSw z;#Va?F#@kBE&5N4ql7IY4O*Ju0a6W{4ueT3Yl8qc%-*c%%)>u>x+%yyizJI`3~=$0 zquIz$aFpYtxoW9b2$tr=WT~{c+$guA@|pJdQv;Qy6`4)sPeB>QM}o zPzQ~}x(7OVm()PD-LPCTr#hkD10Sh&xA=#C&;%c8U=-V%c73YS2tm?oj;$i3&b=OpPicMU>*eFe0@_BVz9{@!RQlOT1vI@jA+})H_tSc% zp;tpM79UA{{vGaL9?(>7EK08dO3-4(!+nA_~A z%5R6?Tf6!RDFYV?3j)yQexz2`d9LVCd*CXC^3;AZ+9eJpA2XuH!#YRm=%$_!{aTn& zWJv!p%X9RNy)gBgZXdCryHzqv-cfJt|ZR=n>>c2*n89%)zFLnzM{S=a(F%xUpN%f3?Q;#$P}HKFI9UZv8TU zMOUBz>c=mlNvb&S0+L>?GVFHrJnt)tT-S^I#_&^oLXD&Y{Ch5UvJOqCdf-4BCQM-9 z`b4RltZ!lx?~MdKtIS;YB|0ki6|{qmo#ukT@@rT-L^TLh<`F>!U9;VHIk2iqnrQA& zq4R158aDVg%0JSR{d)$jyVKuk>Z${Y9&iNkRb4@3CkEhQL#0qFz!@8@YPIadp#_6+ zwpc*H73PyEKl4hUmSUJS^#$R*VE~Mqw)Rg_D}=uGRa%cv!eNGrG+bC$qw*{WZ))7p zN~$_ampY5NYB8_77yZ_L#GwFxzVNXI#4x-Oj5vU%I?{@rN4a|JeqS47EQXx9 zStV5<$R|4F^;q*!fD=YiM9Y$l!4;OjMOzjka=NcuqE1q%10+)kjlxh!2N3hjPwd9J zj-&>5EWvzO=Drd$&ccP&l|vM_y2KH6O13;WcypU5l_Fe5nbj^FU(!<|zg{40^|ST& z=y0><$d=T0Nl?|)ir54T>18j7CTywy(~aa*00D?{75#@kd*AFRG1urV{X>~eA?Os) zg$EJODfQ?LKxORv?#Yl=qTtcWDHdT<$5{n! zc9tK2L!CU0QstNz6yzoU^gWYX?{HP!3j9t|UEcga(d7d=vT?voZUtXuK&XH$tfo;3 zV(Dvy@6n^sr9i3M{80}B=tS2XpoFG0S7G9WssQ2O`vR60Q+qgtx{25I#o!=q$l&X!L;dI)C@E4bXMurGo^1%76N# z6Vz+QClr*?!)ngRP3Ar4U1TpDV*lU>Ds=@{CyQU<-~7krZW*W)_N7zroBlluf} z@DfA<0<6qP-sJ%ZkKr(HMfHfn&vs09HQ_7IqP4F`D`omv$bfEs0LmqRxQT%>{&JqM zU)ArKxbhXYQU-LGE8EWbC$A52=ijW7NEoFM*T;>;7_b zV=!}1db~$8kMc{(l0XKV`EU9}H5i)c%{a6S)s}(2=^9~>bb@fbEL&`h#;w)6=LM8+ zC@EEMHYNr>YAD-E{Kzsdsyksn8D=*hr2jzcPq0*!H#?Z*FWxt(SRYE0#>;4|yk_UZ zdV`SwukVK9!IXEY#CO1c;~pLH@+&%f`}AG%|ByD&AU?rCcdK@$ma@r|tCB}~`I*nF3`dsv}m6-5a3p{UlKjDZWD ztge!W{r!im>FN%0?u4;JT$@d@IR-v|OXTg+z5#Wg+&>~8v1rl?B}SZK$Hc`r+#}dM zT@d~E3f6e0)%%7(z-4w|Q~+%NI8m{RGD`6_f0Pjv(qzp7a1%D;37XbHIFGqTsY#-T z-$4QNVV0cc2X;!7iH-|N@7f!-I)2AYRaE<_{Q^7%+->T77q>pn91`tkGGx%PWy{NO zV*l!Igbos(kFWRyjWyq{K#`>YP$|HlnCotDT0em!WjaSErtlT?j5LE@AUSmAJhC3J z@}B`HUM5lF>{nthgy$>}23_5Shyjlh)IiK*#%4rKffmxOhUs+GtzXRofv3;1iJT($ zE2mO8|4YU{l{N=KWo#i*l0$o|)d15;$&>*)JV$)_;uM+#Z2Q-;z$mg`ksO0;*2@#7 z%IIm_2G}%7nkvM+=m&mIIk#2$1@kOkLl7KU+jWc@l60;H4uLdn)eYW0+690fH0HYL zVRSaoI{jUZ>aW zr$t5B{8I+lAArbXPa;?U^dC( z&;x?(WY z6t&o=xMQu1pA1rk5pW0z{tjECqP4L01QNh*TaIHIz2jZ=#d#gtC8M^O4YKd#^}&;m zN^gXVj|7HIqX;qVO?PeE$YvbI?l1Af#;hokFE!ao6~$8ho&$)7k_~Rx#@+@6X|bZ` zFwa<+ym}h7o?YSy^v~R$ADQvYMd8u5$+`z{AV^mTUqp>Q>A{F*frkR*j$i>R9aKWX zA)7z8@Sn{)fnPnwRN?m4n|u^DT%k(=(n3kddn(|$>K*C6FwBrTO>|grciVN;2bd;m&Vby)v&cT%0M)vOv<<~ax6Oy!uZjC2wo!rNfa_x zqOUk#lueEn3eHk$z|0qqsQ~jCM@S0qc~=Q*_A#q{o5=QPWis4Sxi&5bL4%hb(gp)Z z##1}mU65{g!+?}v+~v+#8_eXHgaOfq!Y^a}e zk8pd_AAuP#nlSAaMud^PK$lJ`V*u66&8C|u);1y`g)4>PW5SR%CKQ+$jOG0`Beha4 z>_cfjFY!@8#33JnC@u7Z+Vc1y1`6Dn4N}0q`i2)JBP2inIes1V8b5xW=S^`F6ixeA zLspN^w`YBIwcK1lCxt=5l8yCO71o|4z2n1lu!*>KAKpxQ?R~|}PY#XGN=cXrm>wxC zV3OBV#KbU5e^oJ-Cp1e^Hw>qQu^FEsEtg18;6k%8kc;J9i*b6cly%XrLRqplrYVsC zF3A{H1t8fUj44q$dO=~*@WOq@!c_lukyK1~wTYzoB>*i+ zQEqiP;#xT0oO-2i@MScD0Fb0jyrl{axEV4=9iki(P5^6pJ~Z|pJKBZ&Tg)@-erA0Y+VI(%{zDG1pZWz8&?IHsc} z3Ng7h!aL-;Jq%wipSgu`(O=h6{XZMy%8)(vL42mq!xPOeU z648r9@>SYm0r1>xzfv7Q0V$9_U7h`$8^Lu5kPeueF<1^E+5pCwFZGUaim}RLe=BW0 zro|FtBI=A7yL?7cy}x>eQZm<>ckU=$7NA@^(w}0mz*BT8W5tv3=cawGxX3bm3d$g@ zoD%!Y2qim~4Fh^XzXolhz7tS9fjHfE(;goYyu=X}^gp~1s^6!px(2BTm!4SWO% zXxXzHckv#Wd#3VE(wzThbvW4!ngv{LKKQixBg|c(YggYLl>7ex1%?$&fm{O1nbM%W zr>j6Z!nCELL5KmvbCaF^$=j)BVH+T@$9Zi7Hhf_G!Ef{Uquz6C!5e8gX*cP?NfTRR z(d6Jw%s|(%h_?*DYNF(bt@5-f+ATqUG>uuxHZxdKQ`+lA31K{_JK>hsv80iA66Dd& z6HUi?RGY0EyExO3|8xP@FfpW7MJ+#42yS)Yk9bC(w&E)^u128SVzozPTrfSLW;X3o z{}o?mLJKQgah8TT=npfaFx5qIt{fxyI*;R1=>je!6E0$Vn|TR(A$Yr}pDPCc%)enx zX*PaxKB){=l9<`Xro*+EuSZl_upsuI`mScB3x=3?`@AOfmOT;sXapIt=&Zk#S+Caz z;EVGRiO+b(_%KtM%gEl&F&5*+kpN|?Z5u_+$0ZilKUpDCeQ@eQS%Fkoxi-BXaJ@BdjnC)h`G>H2tfwae3)ICBwGO~ZSj2OtzUSt^q_Z) zs)GPKi@!*CSnw#$FhbXmcR3d+2@7XQ3D!4o!U2&AQ4G%?AULAPej|!I9C8~fvnC;) z=6k0Y@pQeDzmQYquRF*PYQPU;CRO6epUOVWns)>=8fn%@Ew)jNeHY0y-q2vYqT`TNhZHKqDl#IcYK#QKi;|>}$C=Y`I3h*k zH8PPo!l0N0)j$bk?Lm`qaBoyrrE zk)|vzIkxA>iab^WeLNeKQ~q5jL5b#v?bGY|jjKORx+HW77`(Ss$+wo(pAFJ z&G!)6BHYljm5Utr#&`_yCA{(`g)*ICJA8sZZ)Y@&t8pXFL(@x^z%y)4&$v6}V8)MpGNymY4IZ zxyFI3d-=l-c>^lShq-57JZ*4r*L8o(A1v^Xn4QMI3qxGDMm2?3{@;Zz_gC6vQgTHy z3>Jtprb(3arrKxdtRd`u;C|?%Ms(jOjpC^afpWtW(HPh|*L_++XExl%RrSnX@!u{I zvdBF1t|xPaM}B}i>!1XtboejX2?zoY9ebHcC+FNy0Dkoz1CTc^R(DZ(QGv|HbNU}- zk@vd6_zTemC@UA6hFkv8%tZ|_wIWd3;gUEsAu1Sn*XQ*KP?46SNChv~^j2k_19C+D zNA0$ast$X_OGG0uU9$?457Q=HIZ2~86!qXy=HX)g*N@~EHDeQ69S;KcFU@5KHQJMQ z4NxY_E`$vOQ`dqcmD76%gIG`|B)oXbRsY>o_@epcZ@UI6kO|(ezo$j%660%fd=k#w zP11Yc_u}k?yH5P8*-!-MKS+t9gbVKZ$|a-nJNnN!P&yr418V{=;QP_88RVOLgF-7r zf#8e(Dttw3EYy5Hf9?PDZ@o>w4#4$tGc7y!#@Azi?Q*Joh%wf&cRx4}c)%w?9!Ou?AduRF9{@Lu z1_ud9s)5vIr=I*(Uf!Pe1<|biD5L~`z^J+JroWS({8Oj=5cGEj7EHpV1M5+q#d-(M z$FTC?u{?)@6r(K<>n{Al8CjP}xT;O^b{%^nqEHgy!VT3!@1ur^zDjhd>r z$mlWsYSwh#FY3Z;8kbL$!*$Zv>lW!-S@Nei#je{n8b>M-n*L9Y9t;(Jk-@xpq=ub6 zPC2dXp*qEA9E69a@@GwV&p+QgP586W9pWc=Wm$mYdWM#f8eFIO#m%W@*p$!6GxYj0 zKLYW{7Tg*}UVL!}yikRE)R^>7Feb@jIx~VOC{8Ua;tS6#I69lbjkr=Mi3|C8o)B?n zSR7o=4;c7NU=wR&4_zQ7`t470*P0RQ+o%kASiD_MPFkTXd#ql+P_s-&* z$(=3hzTF@PDAsBK`Dn%z*`?k zlg#F7-JQWnFcP_YU{Lg+iM1pl2W2`ySMzI0dyn`M?aZteO{4UNql4l!T9iKP=}A&6 zDrd3dpdV|-FY~?k+?ZuXph&^?IEYKQm$e$ql$BkK7|LSq>YR7F zSYj7dGHbC=ns6+5?h+`-waY6N* zSD7j>=hldVM7u#mmr*Z*5=PF#2EORS&e2)GS9p?()xf$l4*o< zA*2mC3^tCB_%haHE^QDey*sFrIoz^-sn_pIB%9iCngx2!Pn})cMMeTD4}wh8O)XY~ z>8n`>cQFC6?4PfE|_m=U3prJ z0zc8k`O=Uu7;iNsjPS(9NC5=Q@f1UZ2-)Lpns{VsgQ(7eatROup$v!zmIk~AVH*OC zs_=+_7?5ls3jpATU3*_H*dduALLA0u`TsUd7pwz9*u#zBWY#Ki3C6utz~6-a3e^R6 z;*k9>AJTfP5srzfTttk7DtxbF3*Z;Q@z5&s{eekDn8M-_LPKK} z|Ki}rElL*LdiP|!E)$L*$;*~r6kf6ck1_`2ccC2hs=x#(^JEdDCneVifpO&vNop!F zi=*)>pF8)X>NfJv=Q=_aO;V1Icrgs%WEa6|^F2Wew8!YF0GY2vPb76r)P(L*%%qSt zA;h8@nDC{+{EZ2!>HYrju&UKh0D=(Zb9b@Z;8V*>F(hWOF)Ken{0^FGgh&g0!rQ9i zVhXXQW`L-NoM!VqYeq(WT1PCO01A*`IBbJ6WhG(>K7x;`O9{FTNJ$`a?wY`%fYore zYg>*yG9|U1k3_dQnDP^qUY6T$yt~|&qtkKGf(-znXDhx7qR^@;OHDKTjw`YJK($mXAcLeO{ey|J2i7@7 z5uiSxxLTEcn(MsSxA^@{7BuaP+tb?lkN!_S$BsG}6HxK5!^M_sq3W7Z-#VH5H{Zl- z@GNpEiIQUyc6rQDTZ7@N2hN1+q;L1b=47+X(l1|SFhPT{hGQrkpb;xc4#Obh!{>h+ zeEKc8$QW9)&OciZ{ZT~hh&XiayBV_@)83c4f7r(}B|f~3s`EIJ&aDj;!DchI!b^lv z*--%Ko7drh;1wXdi^iYjYH<0#_~LOCREFQn_Zn7`u@@92L%x}*I$n05(xFiyVIOse zh1l9U(LU$l6WlH3I%HGD5D^B7&SIdb1i3Y@5JW$&ZB|mFe>Qf`q`|6(sZtOdtnmS& zHGwvUYJGT#VvEgr)xdrUz6u5#xD|I^=fkmPC!?puH1cB12#r$bn zP3>sv^9NQh^mUhG4JzkM&Ny;J-F^YQ0ezNkOFY=biVq8ukYGoWLBifFIS07SRgR~? zl3s%)eBM8eg{Bh2^J76(afhU@BLhoIRhnX2i%ho+MNDAsj(;7?8}i^=X+Ko4S9&bOlolGQb=>-%I97Bgz*2_NUfSeQ z=(sYiwC|vz!Jhww351Svsz96on}}L&>i`)$e}bK!AVp8FQLK`Q3F z1bWH$GVhhm5-L^9H&+W3IH%LU!_P46g`Y-0cRd=XW(G|I&mJq0$2&ms>sq~!7iaKG z*$Rdpc@kgY)lM`$$YPH;aK8yBJ&odrTYBR^?cW^8dtdDa%t3w5&TUpHF&55f%e#+V z7wPwEca2qHe1jDfeg|qNSHAG+NCS+BD{5mDP9Ls zm|+UDf`DW_|Gexc;s`Y53XX|(&5-25ctxg7q98U0sMF@D)q-8HP1PwdJZB|)<+83D zjsah;Xkc30>vD9AC`=i2v2t`T-l`jR2EtR*L;jzM1i$dV?(ELM;j0FZS*0aZxb*n#C%x(-WRu z5T5`9^8OTSV2}XqSiuc_k(5=W86jY_s)BJ#6*#t=_MO4f6uylt^wmw&GMK%bEXGc| zbPQ+q;yd_AH{f^!sQ`E6k{{$dp|=-CSJhWI-Om{p$S|thauy1e$#2O6dTxx3hyrLh zv|Y8lYYVi-l$FrEP{0S2($v93F6hn?*2ln*0$hJ2@L(+)-&fN3s&`WPHX#9k;~ti) zMJ{pp2UIP;S*8(%b{cCN9fJVQ^9^7of0Z z+Xa3Nu>P)qojppF5FP0D)vW9h0mAeYm7xgB3WUQkP6e-?qtag)_H7W#k?YrQapu>! z0^eNGnYCalAa-EeAf5}PU5|B4I1gghzZbT`(Bra&Qb^Y zaBg+X^S&vFJ={-K}z=~l%7eJPGU)I?wk%}>*swnMNG=O&3)NC{Po zlruzau&~HkZc#0?a6%d2g-1T>Hx@hEV0{CX=nL1SiOLHr+N##070)QN;?Sr(r2v26 z8K|%yg#8q5{}u}}J#YW!9iIx0PT&KBU?@PpM&Ay+?8@;(9k9)!pAknVyvML1FBYh5 zG^DJCgsIky6q)Zy>pek2D+fix^Cn2B(S|Cpqxl(Vo@sF zAI3H%I)|q$AWmQf?;?;P%1nUJ8bpx^dIVwRe=KjA>}$PA=cL99vN=iYJq1o}H1NWf zfy29a2bs;qIRetRZSf%+u&VengeJC5%6c2p5!WX4!pdUgpZMhY_r?(hG#8FA{I2cO zL8QRNvQJP|AVFm3n4>QHjPGU=Na~%(^eneKh+rQE)M4%ChD^+-w{dVe9ER_)G6rXQ$qrBy)aLxWCeMe zZ4^jKKbdFo{KnlQU#&|3Lf`^@y+#sH;Z0Q>FPL#WMNM!(e`%9VcN9B4-?Ep|mENX_ z#U=e}#_Q}B%QL}rrp2NHv2>;%2{A&HsbS@!;CCYLvItY4_@h}OX0J2<&Dss4QG?&t zc^U9HZW9Qa4U@7CqYG({f=r9~!Z8KoC7F5*Vjl7hgfEWXDnJd~=1sEc&}gmsXHNi;iA9t?;2G-W1(u>cPsC zviBg=$5skmod~#qS6;sZ&RJ68aTN;XNDqMTefe_5A6UOF{WbxtO zwTTQmsT{x+{&x|+SNwW8@=`pV($_*b!e6eHCHX7MW=qa0rz8?^gX+LQ+Ko~q`>Icl z1fVAjx|oE#D^I8SzVX7?7v9g<3vCIEFE0%KAYr5<@cS(oAQsm zg1%~d(+EiRl|MTV;Q8&I8@}aCMuZ^n)tGWR%%_3+VBd7jE16B;e)G_okg3ZWdK|!7wHivNCAx=Qu|B zU(ehoiW3;P3Y0grEQMKGb0=98sDh+(0#huFin1(ybWNszyYlc{zYIbgB4Cvg7`(B5 z9P`30HzXO3P#H)~yB;Rsf?_2$^`vMvfEdZQob!UR?$_g}(yV59d##-Y^A6m}kB|5m zgDC<2fkDeMN7IJlRSX?yvt`md6dV_5yE~xO#7%YIyErQi+@?ct^2_DONOK+iF?-`b zNH9VufsRlI3MHeZJ?o+&)d!{%mvjvfzB5|4= zrgyldr=pdl2|Nu~4sqY;tZE1@A>j{)Y86_lJwqT;pKQ{b_w=~I-=K(Zvt%V2DxQor3gy`)CLxK=WPn{{Vgvv_xT%=bXy8@)!Vf*)}uyO3!6Mt z?RGsVnPDnpG-rX2IDjCp%{B!8Vg29>#by*XD@{kGIX_q?w29!==kDW$6YT+XA}46lNv$T z#2xY#sF@@BkB!6gbvZh^Vc82hNx&@pr{u?Yg!Nd14p{Bk@<2fFQNCEzUSR0)P<8(5 zMyZf!WA{K?oB@Lk0HGCcJ>2x^$^>bg45f;un8Ry~RYG0H=nUMuv|4lmAmx)M3g{S% zJS3^AG{l^G@>ymfs0HvaF`yj)&RWY{UzPoTz;r`q0&Se69~5eY&Vi9(T10ts#7)bs zDY*C3U3aZxqN+pFV!r_2!QCDiQ$;y;B&%>y#e`tW$^>~ZEOO)}=(^zfXw~b}{)yYv zLYyE&nCVL7CE!AlYTqURI$KZCX9k3+V`4*`e9w%w5)dPBUe{g-+W>K9m=WYp03C)mSh%OFOQU!C={df zee&;IyB7~L?}(cq^A-rW@Qel_3_*mceX*;}Z9D)hh3B7CIRtFG$RkvMVmUM%+SEQRKYIs5M5(o!UIx(5&rrk`tZm=39v8VlLW#6KBqXP;L>@@PVUdr}HGeN% z7J)rvot4v`=En)2#WQNahNUZJD3N03oKS7FIPYvDzYo9C6TCrg%{R{S{ls6*!=*`*{ED*EMb_{Lmm!IZ=kU46p1;ATp-9W zL%YHvs_yX0AFtp7Qy}kdPH;H;VNejzVQ)Xo`=#zpk-1b`A>k3RAMRNNdygc?! zkwD`ab$J_5a8>e z=j2HH9Tc0u25Ra_Lz*JqLp7GMT--OflH`jXgR?-x8xU2YsNWFm#@|ZcdOzX`Ip~PE zEz~rQhJU8e)zJir=?6^Tre7LV>O+wqTx>^M8lQF`qNlqr^j?E3#RF;j=!W1G#T{fQ z%=K=sq!$IUu}Wmd6)rn0wrnTD^M%3yQKX%n6!rrXA$o$mYvd+n*8MJS>*4gxR80?i)@v!khaQcxDhG?YTZpToh}kP!l~nI~%B zM*ph<(#yOFujzP)OrpZ_eMgJub;^e7d|^`*gM7i8;9C9n2P6*X!c#-az!r|=AWmmc14;#B93Gc$ z-makKwe!B>B@k-n%7q;7meR<-*q0*?2(%6=hQwUc8-k;1fk!=b!M$2qTv z_5Cj44&@y>$J4D_PyzWR=Vxq%GF0}w^pZ+3N?YxgDSL>$siVun^Hp0SxcvEhc-Xjk ztu0zPSt`c<1!8UNnV;dTXhyQ!0=);{lKJfXuQn|`>L{#X129Jg`SlaC@WRcxqtx2bAtOj=N+8}HlNoW%JyO}ZHRb|ZW}X%!o1xr; z;=SWn<}WA+zB2@DgZ28_cf`IZ%8Ul_a^CB_es|m7of;y#VvuQBMO$4Qmq#dQj^pL{ z4t~BnOpkV-%3~oH;AI9p$PX9Nce$ka5qm-C01BO7rNI^EZXu9_Fcy&&5H{X8E4O9L zZ6==7+j(<436yhnw^GN|;t2&eorjIh#p0>ps$ic_$xG%vRp60Jwi1VH?4Qfn$?`Yp zoNS%ZkTIlS*S((i?i4z8*k(rB;D4w9j4)G=n|_~Z)h$N;(&hI%n;WHs>j;ch;%{GY z2{z<}68lMz>~X;8OYXFhBG=MxUFTT#c0At_5XLBBQUH5Nf!MVqxNzY=c&8eU@I5AE zR6*^I{!_?DQD1;#B#CW6a0=|7$|6X?r=r{f<U$}%!Ptql{@N@UmxHLz3XE-*tnm!4Ebp4Ck1ptVskYm zuBE<)qQQ2<^w2fKnFoE%S5R~*X`^UGbmALQ_uX6~D#VcN7rqpzy&b=@rSWku=Rdds#h`c!-*{7T)FUJy;de=yl~!H+ z<86`s_AyWC;g4}nN;62lEd4C&{nbnXKqZ<6=_x%+_{&ALCo{Migy{D=dUwCp?mb_VkT+hOJ@iscqL?tHN_D(2?O4D3^-inQqjGszkV(1v40- zON;>^0{#SgJgiM=PuV;2d$v-Y5WpZ>3AYaaYR;5IAF?d!HMItt~BiLSR-*oE=7gqXnHf;V=!H2*^Xi`IdJQc7#t6p!4o{ zSEvz2c=Vq023!DN#vQli;Ycp>-x5DzRD?yvc;T0u-h-NC-|C6LwT1JF0^+@}s5^Zg ze*_#Kc6*^v^cqr4TH~f%LfInW;typ@0(s;msd>!Z6h~;sGi`y&z&%9-46$yYL?XQh z!ARsP4EI!1LMTG&3veJ~)NSdqYW_~55e_Bas5pMceTBO-6kchL-(m>9jPSJ%OX43% z1^?-!comKrvYNy^4;Mo8_H}C!+rpB-9K_2|gf)4e8XgP>!oTCgkutR~6ZqUz5qbBA ze%bjonj&lo=ktYw7(yusoPXcEEeffgdSa|>!`MRD65(Ru>lpPYREcDUD=FrK^p|yr zq>H_nHit4K4U`<0Ut(z{c6(7euQb)ur){43=#wHvhL&tDDqBfSrS|?sEA{4Aee(+g z*)|Kw6JckI;IYnpAYEZl( z4|O%6Rs-^|GUQp@Z6+H-KZ2G8CF=%#t9LR-8Oy71F&c~ecnHUAbx~65bkZK+QF#ok z3aQm@uA4@hIF73$OK;l!nYT`VWX9pXISY8xG-kA&!-ozu1T+Vpyld;!jRh*Jcq${&1X!T0|R5#23e-RO+AyzmXCi1e=kZQ_V>+8^p333Un4(p+XI z_(Yu75_#eP>vjxWgir47?_Z?8HkgAjFfRb!M`oqO%WXl#S;*{B#jSN&gQ;im+CA$< zPt02^I{<9j+!ky-<*k1PXGh16{j09%nLWY)7?_GNINkd>$Q zJw{O%aX2&|MIG1^MG+qM{X)rGrScTf4S!g_;$Erv&MzAV-UJ}Q%JCjkhl9Z5(BlO* zD*zrZ3?{)Z3bMg5EZ>3eCkV0Q68+<)CIDS0&FMjs${XkPVGtuVk%tO7#M)#{3gQJ+ z1_hI|3~p zg~Hw&Z8<3nJ&d( zEiqoYJN1aKgkIFCm<3gQlJeVm=d}_;)%+u?G(x;oSVs4vm|v&=x67g=$S&mGI#4gW zP_d_0SeIz?3!17>Glg9w^2elmS5dvowx`U8s+k&VR=XArzQ2spNQnduSG!o9P3;GzEv5?xquwA_Ygf-3) zTt+TkIZk4ejwz_-NUK2OnRb&!67#^a09m1ju}2m$2Nd2$4nubcTbq)Q>6twPpFxGT zm10JC4-?Nj|11hlkfN7RGBm+T(;nkBbn{&&1$-@jh+=B2*MQ19+1wPhv(~^U>F&f! z1D6&0tr7x;In>}_lJ1FgoThCkAu12SDu!!jcC3abr}`SkZ@E6bNIr(k)7$b@cdv)* z>^z@?VEQB#O3e-QLU>zae`O7)P?_v2_TlIF{>Z};Gxq8DMON_jxPL7zKNx1!fPbBR z*GY^F%0ARqlZ!D*aH81qJV!GIyZn_mlotWUQhY`#U2vc~Q-8}?)Luo65VIn&LJC>6I*VILDA z1=$G#A-C`Ito714zrWDU2KIjUyFP2JXFcnAxCZZH2VFf{TO*yoJfU7n2bEtgtpo{~ zO%XTxG}9zCLO?e{rDHE+qY1UTVQZZpB*k6l@bLodS^_^+iDGQkY$*&8hk-Q_wK6O6 z_Gn?%R~ioeC&*7=p1yinZGk-Y!mykTF7nt(CT8SFTNL&300dsap8(i(ai7R?!P3*? zJ<=r7X0Lf0w?4|EUBBKDyZ~D_5(`cKYD6vbOyu<`_LJ^07Lu{TRBKdzq(PYL<^y4#%`mP4-TZU9 zsD(>`c2hvXub$pvL1!h!GhQJXf(_8X7`9GXeq8EV!OWbPawnmSH zjoE1YF#>P=u%UId+`hP(0oUy6Td1q?z#l~FXlL3dKRNOhV(8l64?~;0H&5Fdn^2rK zff{CWz8UOS)N}Tqm*qM1PGjM^@E=0g+ z1N}_;hcM`1m*pu&9net}u0&k%scV^G_+>z>xVnT&B^~pedd!ms5QqBBg~`)JDFQQ< zRN)+TKPXj^Z2+XO+W2Ilo7{}35S<m@O zKA9)khB0NF&p-=E_~Kw8DP|lFPsbIIRacm8=-mMFh z!!P**9vUx9fwz*hhz-Z93tA*+)fUu|SA?s5ImP16O!srGAH+LQC4rdauF@HSZ;bFN` z#J`N2s+x8h+pr;wPF})7bj=}JK?F}UV}x2nN|yeBA<-xH)v}q$qZfN$OkXZ&p>%+2 zcDr^)MU3ia`DENsqZ-h1c{M$|p~9|{WKaVHeh?KGs1VB9i~6>P&aZW@Sr$Y+c+Y-}S5x zIBN^&g3|i(y{>y+tpK#Kx*fIRR=%XGBKs}UCi&8P177KKiY2N#xAr7Y6fcna+`s@t32t%Y;=y36xBBJ#S6z?_(e=tKUO|IzCW^~gfBdphnxys0CBksiM{)~k%I!A zN`<`PCbJT42S;-Dv>*CLlEkZ|+@oc?lHWmT;p45BWK{_^x}U+1m46q*GYPGkKTAj{ z8v-@>dh|NdFXq@G5p$ZC0`Sskl7XE^Vb z-5-4F+YAS%;fYWk!;Ahu;?Q})2O>RJG7L_r^ zB*4!_Y&E=+-N_&k0=tRA%-aJ43!P;T@#H3QTQLZUa9LVB1fev{jFBYB7 z8um(wv1zzt`d^hB5#g~zx*gq^BF}voL$+Yz&SevMO)=7I%3h@ezzbUHMdh7^3z@{2 z!_)zLLso;g#`&a%sfdQ9vwr(PQDnmck$NKPbB)D)U>0&~Uv|C1pXze+Yodr4ZejR6 ztrD@=22jjm$e9qq6~#HrU!*`0%;WPpU zYRF;NF0e`3T4*LvJ5djDEK`p)tDUsyId6)xMWu%ne-oZjLDQBzhsm4P81vjqt zTxl-jri|h{Klp)Q<}-u-DF3~e3KwhK7=b=5l@FycGH==D=@cp}hS*QrE|kCAmY^T( zm;}IgV<2m=m$OTg+&=J*{T1AD?@7hnoR@Fan4tB)HtevKGc698c;T#)>y*Z3CqZd3pIIrApMsm1pO9Z$M4u1u+q_5y zy>Qz#HsWveAKlx#Qi%>~xm(f5{xH5vi>;eLh}59~?k@Q)y_<{)1=Eo-Zsqe`^*cBA z=}^2b-0Andjl{hQv56!JeEy@$K(ZFT#L8$(;-`|g3oA<)J1<~J zk;3nxWf}E=6bW-*Vby6X9FK4_H-w5c^hxg z*)3OafCpl*c`gT!a|sxfHOQb}9D6!EU2tZkY{T}ZuO$F-o=N; zRsOvgOX9dF7QyD4%BNCOV@=pdnU4xV240>EMN^`$>%A!}xy(V06~V|Lr@?dDsZ#d{ zJDxeg94Rf;rj^_G8)zf37ZuXHVzT#lA0d=W70vX7&?bL_8FsA zr@p-Ec7AFj&3fe#IEDFtJ(k7yM`6-fan@5 zd=Q{89D5bZR=5q-M=e90HVmg#m1L0?{m>c&4qyne;6NQz_xFXe)r%6yuX}e^F?ehLRJUez5uc%OCk9fFD-XIUu*Fhmd{ zRkz_SQPoBUhf^odDZ_0e=%`Q8`Xb(6TW*IyZ&FfFBR4tlHw^;c4xuWc0d`C&Ev5|%~z3IFN8jCpPUxqV0=1V-S~BzI42{38AQ zi%0kX+Qaq`2dx3}y&av?C5TOCQ9};IUP5*l&+xH(m)54--}p{}(wPqdkaDpggHui<1oi2UZxIW%c9xLt5K_ z7$*g?CcCuRO)X0&tf$ua!#dUEJMsZb;N}TmrPc8t0Bg;!poJp4HghU;UL-Ml^3t4p zT3(<>q!Cgq0GZ}Ok3gME%QuoBehZnh&&5lM(KOoASp0zPoo2TB&Nv8D7w}F-Qw^-NGDQXO$ zmbZBi6kta92)!ccC}t8Y6f3sm0*(OpL)9)j?_Ai*1Je=0PAdaP;|>knE1TRBd7U-B zmVz;Su>bt8${TeG1s4#VR(8Svu#_mV0+*dPZ*`rOMIAVUj7L3)F;mjz_CEGo`_CBg zx-mZulawX0(U^pI$5$AuEh8TA$8JCRukYp7Khdisit!{-&_-%BLxmS=W(uBx3e^2vh(jH!l)$MZrA)hA8iLroAMp z5!ZAI##&~Wi>mTM|9K3b9~0-35;jY^mP?zL+dM8y z@s{FLhw|lRE(&j zbJR|13V>U|4Ko=^wOw3LB2v#X1_xyLmR*16)GlTej^1bEqxC9Y4{RQEg1TV`gTMPd z-UF1eC%Y4zKw<3s{QpD#(|vhO&Qx*Zc%G)7jRw${FG}ZVG?}EXuEX`0II&8}=xCX~(AZf-dB#uzSAls4D@H;; z`kPK3bsy6SNm^EXxo73fkH9rgNqza$n1$B_4c_sZE6-E9838`{BV?foKC6`p8zJT! z*#5f1t7S0#n#)eF((E+vn94PqbT>*@80Q~tPpe~9_`;@x@W3F_*Kx>aCIuxRAJhir zq#yzJa@-*-AQXyFC@T$!L?mucUExL%mn~<#b*#LxwVd7t`D50i@sw&W`&EoU-5-10 zY(UY4TE>DbwAk_z=?HHW5x#$Slgcf1#Vd+f?^$pr0ZD3WeRY7Lo8*5#?vyAC zHGrBfQ+80(FNnx23L(LB^RItqKzU2Mxt9yY2*;v zq=7v3yL{v|j(O?pbUn0&1s~uc#mSd@RY9PN^Y+hurFNCo-*YVrv#~o7bfvV$ximnza zTMag7g;Xw%0s`rX#}r6qmx=!TW;xXqA9P&1gqjFH2=@wIG43-p33a-j%~`Lk9dj*i zTj0)hIHCB+p7VDHXcfpotO&arho;Kk?rJ<@^k-RDL^6OTvsiblQPUJja!{)jx4F!+VEAFQlu8bR}XA0%QkZXNrY0Cva zsRTFF3a?^4h`+v4#98ejKtav?lrvSw&+?-!aeSJmOV_DLaw;4xh3nz#onAr-!Dwnl z@N{m<4@2*TrtL&8e;*dBb^ECyCgC<&6ew~)A#wA>sY?&|38-V(EOhi1y$p>{{B$BZVQtTBHC>o}#EQBGiv31$H{0%@zLjKl9_kl_;%hAP76xeD=q_VIu)ihyq(8fvZ$3r^{Z3t_89YvoK_b+Q z{=F1u4pBik->nlgcwWeYCfV&3P>!Rm@W?({NbHL&_g^JAJC^1+vqI|3o+|bokr)vQ z(w^y#q7ajUO$4cM!pLw7<@7lsg&wlQ>tYAW$0VG^V-hBr4kvIAw`*$2Lx*}vRVph% z7g1F~DnH@Yb(=>~clR>PC)`9w0+MIkQ$t_9lEe!% z`3w9MW=?2o^c&7>7;XaziBG89iYd(0gv_Zw6Xu6>C~IXxiB)CJufXf$`Uq_vW`6KD z+-OkBu>CdZ_*Uo)miD_&Q=g@-#onhSP5=SQ(k{% zC*f7bxh@rQFEq{kS-#=fS;$8KJJV5sdZDT|0q+Bbnn93>#n$iC`WY;h3MK|&s|5e# z*3D$Y!=sQS3y$?biA@*4)F1($_DPWcBb%VhB@@FOvG`E%cb?2;zX?;M`R2!9lW@GP zt&Xbd^FA;EOZ&@0s1kxdfodqLYA|otBWK=|HO17qTs+LMC`r(u{}t|rlQ*AMpSSk- zY=bg!NVaq1>^DQ>P1=Hc7cwo(PiSh-8gq3F&Y8jMBljkXCcHuyaNT|uKLVt0d=7@w z{1*uyjE0_5A0DD6GY;EZy}In^pc@|)7nscx;8<7$zRoqpFPkHvd|*oF9bd`=-~)SE z{YEqG0Ik371^;EqP0~FL_`zp{3%-}uEIpIcSFd+V=)jtd=maEE>-ztmhV#$rn@u4x z^nwXL+w$tJ%M{muF$vxAnWgT|Dxq5#sRwM{Y z;9iwqFg3wYsA$GIWTVWw@ZN+^9yw-Jm#T{a`{#!ZgM@3)37fWm=)3AJ3eKVog_R)4 z^;>BiW~W>rKZ78n>ndfPh$Ax~N5L-3qg==L>5D`b;u zZy;oZq>7EVe~Txb6u1n)zAq)>y*!=KD3QLv3}lFkK@eH!=ve$~5`OF(Ayr1mcrqm+ z%C>1%Npn*VQm{-}@?i;Y5%2~aY@*2mG=>7_&nA>@0aQGdf56jQq>>6B-eSnyiuF7c zEXqd&K%SS95OjCefw)^F-QmE!Rs(H_A?7$=*J(eYBihCgK>C?y!a;eV5;Dt+giZZe(AD^_npGW^ueXgrSA#mp%;} z4Y+57J%PXcN4*q0DvRU>yy@AHvjXJKV1DL)*cg zTFjqa4FbR8K0fQ|#tpsA*_26gGwC{c>>GgJZpeUc))l5M+V9*L#+~=d7akntn|7g) zo~TaHk3LMK>l4)wK50Q=rHGAL>vEHO-*v{jbL`K1UR5AH3Jo>qU1zTI69{?vXHlRi zy8f&<31B=WEU0!hpGtg8@P)B8o_o95lAuxh<3)tpc-wZRrzdigm&XVV;*ot`1T+X9 zlO=U2asrnjCmR#3^-Halqwsb)U6pBgWv4h;Q*!{0LPo_WR#BwGdcvxZzo4zQph}Z; zK0OFfmX=~C@!kE&1~ENR1d=#dSdGi#0gonWr6Pef@Kwz~tn92m9VCh@^&6Z;eWN~g zv>TXmCy@BU-<1t2cwRCyA+nznUVA)L#@Q-KvXa72HO$r31{xC93)khnuw}xNUhn3U znipT``;iaFiWl>Zdlh0FzYQml3}=hkL;=!^ej-D9XY_~CM5$3cHZcvK0BHQP&LNn- z<1$;(`32a-l&3E9CrSuRy^M=dfSU}z!fs(t82%ke)+3Om6oSQXCW>uh=m9M-gc%Cf zrWU5X{)_9g<&{80rL*`?KgZd48T2&z5 z2D)AlHl-!N(l1G*YUoA&&|Jt<_4bYA3(HOKif^-{9q&&=Q#*?VTL}K{O<^N7M3x;e z95^sD+F~~yt9?{ZpydV{y{x{^G&c?Gv5M6@rV{c91)~V3lk>iI#U~QmEWPI{sm#`_C?i^HhE#2RgU|VW!eh~5`pma2Df3}&x zGzQrw-TQ)~)_Xhw#%8Uw-yA=-h?A-fFq@G$YN#^gHx%)U+HqMXR=T(t@9l9&5Oo|V zQ$e_u0YA0&R#rFPXGI0b{}^ zaK-ksM#GO_u<)3m^FyLicqHvP&~MjLL1JD=@y=`oyGjOs#CS`}yeAtWxK!nz@EOJ` zqq!S;W4e1U>_T!>V1~wY)6oXw!$C|@*cAy%-@?zPh%mWG+kgQjMPan)rB~z%;{qB2 z0=u_yF%dFwLnzpSgF!8MD}>iakKQZ+p*mn-$F+G>9<1ZB{K!!i@XHjWm4sN15v8rp9ytkjVrMov<5AAzY9$V*n5 zu>tWei|Y@tn}U@iMQ)*x1vrEOgqw_CcU?)4&FHOZQQd0m&*Vy8-Ao4ZOox8Dw=0tuPE)+kju>&2zCnpYi`RbZ&l95~3Z0|y5f z2J~LX`weWe4_;4Dyar>jR)=~erqpMX*@satT!sT87)?%=RNXc=_Y~*MWt9x{aeB~k zQA>i>>lBqyLPN_i1B)y#JB*=F14u4gvJjR_VpX;ewppqyQNMD~U5d0A@=fSAsnLvs z1#76hJI!N&nh8NyNrLFKy1M2uJS=H-r|K@J3fX1kf6z-^A8o2 zqmdYs&>3(*E!nR0{TkW06^p~oi_h4q%lc?H`FX4#@|my-1WzL)&qNPl|hz3~p})98g+~mudnXjiRRWNV89IC1&GN&r2{a52`IZ z>WA|JlB8CZm=FFhQ`t>9)pMEw&9bpCuMFHI(WN8byfv~U|3>TlSaqccWk!Q* zy$+iSmB2T4d8UqXp?>iVd$9&%3&G@DVrDlZ)XQv{#hdgLNM z8`s}MC=4?@rI!O=mgH9i1`^sUn+fQq@G)wc=cLxX;Q125gRT-*skWs>jTChVuPX`7 z#I+-R6Dq?unO3j;g{rSBH#l&P=ezKwa^kEiEcFlrTO-6*?At+77oi-5bzUUYHPAhZ zu`^`D%T%1gW!PDR{0bjMCY83{De%r=Uct^pE!?cp{p5b}J*MePZWog94w2gF$dAa;(B^?|P%;$!7htVdByM#2eWHr=cWlGRx} ztwl<7#MO&pSUxQSAY6Jm2?!S9lmPJJ`sisplWchE-uCuqz!^`xT=aD~U=tg`!Qq z`)7_~s2Qdi4iPrOip8GYeAZ+w>!7|UTnJL|MIGM*(v?Bq453(3SxN)-FArce= zwi8#VFL56=6(tt1t@&}{n&ZqLYe;H^M}tqw7p}AUnk?i)0A&n+75~MW;!<(t*cuq; z{MXC6$6+lG`CJAu=XFXho7L z_FDx4)O{-W!YW459haaT(XLmGiZ1fnX^Uu2pxfhQw(GyK5v=4P4+Iqo<*qdx7-1{m z)5%i1VYKmXG#*0U>|Cg8_}d^Ez{nGOKpZCp0`_^O?0Xd@9jS${$=4Q{)M8U2kkQo$ z!RbcMnsoh(%ZaFr!(}B3PVXRE@L9#*lrJJ^8hB1jJP>eePQ;|#;B|n;^>5^Ipc*AA zcQidlDzwWxLwWO@TZ0xqQtgqvpQMeMU3VbRgrQ3J_`I5X>)FfGDeUQKq_SYn8 z+{WGGWo-M$YbqlarpP(Q6(qx=aDP$>6}mnJBE4=h!QToYAg@WXXOe*9PJZ(T_T#w$ zh2u{&BgSRu-Pm+T9Qtg`)OF^MK{E)P2T)Y{%r*RD`PZkqtC0rolT5MQz`5MeDnwW4 zPpS6_NCL=M<9*IX9AoBkntg(&iTQBkL+FD9!Y`iUg0qw~FWgX4U=O-|j)yL)l- zAg#~}6op-Uh*UH#7j<{V>FcyQe&6ig=;_^UFQp9>M}nKy$~9Vuj|uh??*X~Pr?q0F zknl=;ArlM9UVNUXoOp{oR0{yC`6S7o(p9P0^HU6+z-Hs|sdClX_&YnuwLMcg5e*JRv3s#R@%nhCvAjqUa}EV7Bq&;v!6IquF9wO7d|U&P^hRxH!f|&xcYRTQ={O2Lp6U>V z4Kn#99}4J=m0Gx95}t!#U@>nYaRz7M=J?y%=K$+rn1G)sN2@}`&W1EXnYKY2li4ey zswzviF)RmiFk2YK>y-hWKxjw+-#ZhLE1>>I@bgzz7PH;--lceLjv&A1zHj!`{ZAo%^%F}LsqddQ>TDkPE1^lvFzvK z94qZn>LzpxEE8iq$4zKbmhP}=Z-(HT$c!VUY=tc|4*{?rh2{YLTV*#?6k(4S^_F@S zsv0Qrgf5t3CkwC~T7DHhoV?y*Z(-i>Y%nHj?zzhQ+ zsplm2=9*^Bq~|AAGT#(3!2UDjFkp1+T_h?-U{epo5k&GBTMsvW$okHZTro1!ZSG*; zMF-VR;gOc2I($Hmbkjn1mApKuIF7b#o>))QyvqgvwA6W_dBF}$y)Gh%_l@}Z|7XE^Z zP`@K~1`6T|i!ki!H#+M)aBNm}s7#5V|H}>s$K`M}A{vtwr?s9FsCIhaquiAi>A*p0 zk#umAYC+zCT?R6e)6ee2f7=tR(T7O-kqp{(-Y7oC-nz`my&2HRk4h2mh zGuW6PS4SBTOXZh9Dm^=)$LZ$pF=ayP6Cf>X&RN|Rbh`F3xLJaP7nEOSO^14C!%ff2 z^8hgPCHbwy^gyOfjT;~t@t1{Aj2FJchJ*D_@kJzCA5OpZ7=AMicvl_ZU0d1>dqcH% z>iGCG3n{~*HpkUlbVf<}{;E6I9Tl}kLaKg`_f>yzsxeY3&x!9C#iB9#`7rUarmM6J zZ7RDCh^+?wiDkJRr#BwJC&K?CPt5#TG7#AHP);K0!2|bZ1YRSwDUSj1A+A^2861U0 zM*N}lx0>~VJQ)_@n5Z0G#y99@>P|nSGz(1i1;ut#bG6w=|9u$WGK>YKfU$3u%jUkk zMQr0e;BbTQ@2tnlEW!%UBcTB;#I+g$=9p~I9#pI0#`-sj#QEE8DWp&(0<88?bbh>i z&Ctr>g=(1frq0U677agX|1@hiD4Gc}5JW9Q{s~*LCteTTKnwooQ05N&04J427Z_`ysd73u2*`7ROBs3LJ7_qkdN2)RJB> zj8H7}d`0sOl&{f8*inUGeO&?hE+|1rGNN-EXbhMQDM!3LQg;DmkVLYwG(O*(T0n8Y zM_56*%lrG_F5JH4N5)=hn|ehcC6)XqND9yv2szP9wt`3QE2KRD?|Q%T+J*9U29>ma z^90ewLA$f~Z=>*{9Q|IO`N1#pOh}K402EH;EQnw5yIzIs<2uhnv%KTgyUG=Z&m=XL zaUx#F*=)n=?ZzgEk1gTh0g{L7*SFCnp?gdWZNg)3*-Ze$QluofhJx35>X`a);8C7j z)XxkTf*pLjWs|KS3ndyWD(dqH3l?SZK|pvYSN3?jOSpL*%M4?~lr$Zvjr`Otk1Fw4 zFmKru5bzzahIy@iy-TxI_chy1AZy2KFm8R1zque{(*T(vntOtCO_|#}(pe8y27ZdY zcPwvxbWZl)qatLUgsmaLO=zKu+bI{Sd$-2dfTSYmfg0IIUi=jGJ7QsI?#9}(A0rdS7@ z+PD$XRf?Pv2DA#(1(P8JGD4W(YiKtZOmwzX5_$6`6u5W>c(}hf@%HMprbb0bzPjrq zUhYo6@n2lQ;uixo6v=dxrq$P7k3!-J8R;L&S|AipcyVEL@F_nyWfeJ%Z(1RwJy^^?U_1LUtx&pMWNrR7nP)+ppFfDyL8 z=D$s8yxykl6}IBRku>j&f6;E90e!tmm9}nq(sp6`&x7^!aSQK~)13!I{Q={)CwUtH z!{;ql^VO&apfwDEhH@kvu?91SH)k6D8lZ6WF&#nkUu_9Pb4M%lXUV%=H3z8mj2f1w ztRT_|w~iSW9=2e^@eA@*brq{O)2uL=!IyUAoZ!mcfS=yIN-2MqFPpPPl8y?!!Ue95 z*#ye35vu%p9hxgeGMA0`mlS_7PY{yuP@i?AAwxdGpts`37$`~5PGKvAq(;i!wEe?F zzq3ljoNscyBB2yS0~+j1!iAIncd&0S>VGZ)IIOt3CRw1;Bb5Be>)EMft^)5>Z%z=w zVxEAa$?$|?f3Fs{OgP+2Kp#iybP|_ph-4a;Peg3EZHk4Yip?MLasjjk84-Fbmwe^g z->Xt`LiG7s2nsSGtmHs<1vXJ@npD->#8m2b9$y;mrP<6ith&t0ks|& zyDthXP6)Au^aad(D$?qtsw)_bm;pfVoMIxRu&o|jZC%4nyd5puuj`aUNb=_PO@CJ6 zf7BbsM9E`XkTNBxGmcL?C*I_*fL!aIMjp?yasf!=UNbK@N)g$@1(fRDGVwKDo6%V^ zg2mrbHW?DR8nLBJJVo^{8o4S_9D_?{A9($7>W??xMiom;!!uV{3J_%2lp(QbYQhQG z55!xJl$tbeg{ZNRwKx0pvLVIAwjRHS%=-%K+*b9ifWJhFy~V?n_I z3)=KJmh!#ca_%mf4BAE1lK{;T1J}fb8nt>Elp}7@OGP@Mi?Xtgall(B-K6Flrcmaz z5f8e1-uPjTqvs6WpHH?{Ll(bi^dai*RgT3k8ShX>@C{47r}Kn}*f1;T_!(v^DDpw} zTw?6%#P?k6Z$CgOTKW%+RXz29uAVyFae zDX1G4lI+%ePhQ4d;FV!-3@D>UTVSkV~ z1lvoOwf0>fd#kK+Y1_YDmR)$P`YnE%dpNjr45@OG_E`+^BXvF;B2;^t8OCIAgPO(Nd3~? z!eK4^x@^`>6?x2SM=zsVoxtQNMLlyxmvkPQ)yLkbd z3c#mF{Z>9Ja3ItW1(>lEyI-yu3#1)jBh~*=z+K!27y`WWpE{@=@y8^%A%hsnGQ@AjnYbfAG(^Cv0)! zcytX1YJns4cQ+X>*;OEHBhe1<63ZK( z@Wo2A1k}Uhgdi3Hz+U;VtAY@>zxr5HJAr&~9cX|Dhp0`}BKn_X$=C`-7ip8;cdrmo zvOcwt7_A6Es4_DJlO_v84+?ket?)-FC4uXm-gKr>3PXUoHDOvZ_A(!#e6nw53^nZT z|F*8yZspYpK6xn#U*31`-#+3w)0tJj?m5*F+;l#B#Xr1KTc@SWoBF$_KK~MGez;S) zK!{3JN=b-U{>9Kc>!htAMD$3Xz_Yqlx5aj`e!f_LpgPNPWfgg~hTW}B>%Sc_$e+2}{l@!TaaVav~ zHTbW(!)l7h9uOy2WQ<1?a?&d=kd4y4#HqeTdeaE$jEo5` zhF<}*gu_}AI}WJ~yn_$^O-8$(?>tP(g4t7`>$LHL8~X(V8kbNkcE?uxTRg6I7+L`H zvSa}fY#yTN=I0?BLa?uA`Ue6V!=u)_a1S~v+{``^HN1x`1y6&_gbLd84_71#KL_=L zy%&ZXfHPz#*U(j$4GjF!oP$lWnQLkU9e{RFOp-H(@pVAxS!ccA(lZtVx3prmhk{pU zA9s7qK;A*Na$AMdQ+`klb>s3}Pzf;??U`6-OnJYZF7=9QAvE@gRdg3jL^C2VU>iFu zjIG~OU8aiNvf~SNefjQVv8?MoKUS4Xr?=PCka7fe&s|W7V7izHjS>O=It@cZnNESP zD>Y%%wHh)bt{%eeFA+LW5Mqm&cDh&@cltR?ko=bC1jxY2GGN<8-ATnZNJXYwJdn8R z`W-@fad=*qJUX87DN}B2{-9o?KZk)cLj-vXG$5&h9MTT%mXdMX?037RHupFogI;@a zHcU&di01Q_)#=j4%d+Zw@$)9t=u6LqJf^f(KIo{7O=Ic7ZowT7LMiy*+i>WyVsu=& zYcj>nsk5|Hhq{oqYvm6zpO6hBlaB zNrl@#a>HNob)9jiEO~yGuj091seu5)2-3K zl)0KMn#{M)<)rYl2g1aG6@E>m=eRY3wCn5_B=jkwHj8FSxDur!a$MdmknUsgyLTil zJuisg)gJ-1unRl?m)qu7L?5hjGRYIH*`dconw~BW0PD#2*L_;Zh4#MkLE2XOfe1J_ z>uY%Z$%S3wE$Anx;4Ey89#NQY(vnFQ;(s)#KtBp4Au`YcrS?&0j#8VftOGkBlt*imUTx%D!DrKgkd({n_vmp4zKTmi;6!`|S7ia|Xvpwf& z3yDhe$M!o#QYtFu2eRKYif7bs=n8tGO6!^FCf#qUc^hj|l8HnFX@r*LT(a6j&b>Zn35yPWIafi zOpolJDf6R<+TJt4^GQK1|75-qFH)4%ypR6^^Gf~z{~c>qHL&cTp)(9Uh^Yy)GQ(W( z4|K`CxJUY2PW}-qwcRlH{xnbm1U9T#{+%JGuXWq$SyxU9%yI34Fe|XHv5v1he^CgY zLYs%(%xccwm_71m(jeonU)Q`XzaxGhf(IqsF!P^{Ra<)LvZdJ|1E;JxV+Gm{{SHMT zWY0%Q1t^UgGU!i^0tiluD;lg*(2H4XW{!C8J-#~2yLW@JV1c9hoOjU>02PS(iOnnz z&2C?1Y=u(9WV>Z8Trfly`bnk3fSS?A2GA^}`EbTKU6JP%$Bu2!F5p2_A-p_DGOG0_ zJTVmx?CP11CAR1Ivpym*;b=(@&QBK!=g{+z=aw4MzzStF1|64?a)>yAu<$Ch|ACSR zgjM(Vt{m4@@FIL<_()=Ucai%um?{H@=jT4QW{T&+=b5=d8l5869;ALFrK9Z+5*s$G zYcR3IR0r-f=@M>t89+3KMUrLnhlc2W&R`_1xsGLDioK$+wUP~|sF#0cIeAwRkVx2n z8pU>MR?J8ND#R0tUC%#xEIH4|0tqixm};&7lhO3GZa|kgSSDGmU;RNlTD(q)I%Cno zemlYl$~&sJ3dpl)r3i8X2_NDlpM^1`Vtt4q)w=^$INcwg@k|1&!9C?lL;G2lHSqD# zmDU3F$S~hmJ@`vScfC+>g$Q&e-(B{1zm) z|FFXL6Mp{9Q+(lFDW%$^`QFyKQq*Aw`5EEM0nv}$i@pP^+gI2VwEP92ur1z%+4|AB zy8)M^?tA$Wz{E&S1?ss0D>Ul1pm`fp1`>wQ96-D@lw>d+ui>*V9G9_JzwXYX51{l1 zBn(wSA#Ud&2$m?%4K^O{_SLg?3)~sd@=Fp+Fn@@5OU;9Rh2N3PB)*-u#zz7z z!}q2pETehZWV_VW^J7Jm5=%euO2dS#R+$e1U*Ia(=1s+93ILfeysu~D5_wrKG~orI z-Xsj~)xy0f8CDS}tlyP7YoB2Q>YP#wsa&B%gl2mWv{34<5%*ZH=Y)O{WT2btVUcm} zfl;%#5jj*BAY}mBf{fdll`pI(Ye2h&dFhr6*CkLRk6xyN1DoE~muwTR%X8#j@~m>Pl*F#IMy5Mh)bJHOl_s5*Ku zAJ8qNd6Mxjx|;4~B`p^iXZGH)RcaV@U^Ows)`0znQcF+Hp7hB=PR|hSX0}-MQq*xL z5U_RDg(e~MIqPI<*&J4b1L37pdEk3b?QSd{&v(4*6Qg0Xc%oy%jp8oYUN0IPf5kB@ zHIKrYN+LvYLC0;gx$Ta`L|t^2D9aw_-#DwHVfW)qx&hFzebp+%AR{v5@b1R+|HD-~ zXw_r%N{t4r!{;)6C0TqT;2?#EFu1cf{H;2_JX3Watcdp)$^p7P?X}#$J;tTfRa6x* ze{Y)l2(yiY#WR~_Un1-+__g9O6m)2HXF9359+GTsHm`s2*wezto@efkOxLCaw#jdFGZA z#tEl-f=8-;}--+yz1v4V!)vLb=&}%aO~$$V>pU5K9*+6fwj+ zRqa3_CjcP`RQc=Qy4}2&)m}5Y>2$H(z#f)7pvvAgZ&z7$LB6Q+8V^#mfG?djkLDY~ zt}#Ej8g%hDF5nT)vi_ZoK*+ZcVU)4U>GXYks@c6ZSE0AquU_sX4*~-@`?M9A?%~n~ z%#^f)Tmkk4cq!IvcfTp;)kc#oCLUG?yUVk#xJN%pB1e4PjB2G%jaw22+6L<3RC~J7pTHFPPd64AYu zb?q5j*-6oVPK9N=aqz1q%=)+RYyzu%YMPq|SPKw;$(K1n$FJ>6mkhS&&BG`x(&;eqj;>UtLom@a(_YUA-V$3 zP9%e^^{JlK(7}_#<@^9+B3nG9(qLaKd8)Qxhn!X!6R)N6_A3cJA`>iZK{~zJDtD)# zcJULh9UhoMX}}o@dS|N+h7MZ7g0Lvq5e!6OPedde4jpjGWqxuftDLQ0m$=K<#;Rjv zTWy?nIq7nc9HKkUi1rFa^iXVC)``RC5OZRt9nLC<1snn+%{bXk57E+Yw3`5WMhyrt z+Mf50@nl+XNHq1()VCkI4iQ*%ixIQHGq4$QZp5@&vxiy;O)wfJ~%D9t|6O zQd**k_#t=&lBlVW8inHSJ|rSxfGt_r z)%Z2zTxLev1!Tx=E7BBVxD3>Ikyj~`(J(cknS+W~Bna~lDN}Zu!2G*jP390#v6Q*K zU2y;hQL?nn#d@d!4oc^{JHkh3M{WUIT=xReLj^|rTvR7->l7sLTTD!+g4hmFRh|vbC!ofz_6{WX@T`|EMV5h+4zJHHn-f;}tNV*J)z=D>| zBpnuQ+jpUAChrZZAtBD39?{?MdN-ym!_3rP*7e)dNtcD=-GiIKJ~C{`V(9*@k^SVt?}pdi0aQ6(#x=gBRmdX=CmfmRwM5-V^?s@v^QJP z7;O!7w^mnhENhPVa@mq|#TqWhE&Q>>-GZbS9I1$wma2$Z2_1$ac^G<^I7>b%E%yx= z#Oq`#p5zEc8(j18kt9sL;Zl11Da`%)9kN_NC6#j;mNh>$+BGOSG78H)%MqgIpzCuHu@pYs8_~OVpF z5G2;gZ|%}S56cF-L-GHg=->wEBd5cWVU^(62S3#Rx zjJjMUS6LJM2UvY7a)ezqDf(n994F@LW#Q1`!>H_wrO7Qyaj^et$&Y;Kg=r^aRMPyO zM-@d`*hCsw_fH?4v;O=$yU}(6{M`rtG69$z^Oy@bS`gypv!S-os@$e->|LvZmlDxY z*LR9pZ(!W?!&v?QAI|^~k%djsmwyYQ<$ay2A%k|P**P4}o07Z_sQY*Q`lZKwf0YX@ zNI6ojXY~rtT`c@SkK3#AscxnKwH968IHR-k=1!PL^kEGZ z$9ea}{>DR8XHk6`yeFG`z#Lu&V*z|wc$7qBHOjvN&!^hVQE!-@Z<3_$!_N#?wU!_o zdGanF@#}z(ueY^wHb448`nlnGR9q%O?IV;fgf5Wv3@opj<`uOpsOmmJS{0=70rkKx}=c@G!7Oh9uhD^JA9n z-m{`56;Hz{)J`>RSuxa_4qWyVj}2fcJq`CEim@A4Ey+GA|KUldARcKKZ>^UP(zOO% zR2BoQ@@T`@(b%ejDpKVrzR+xFS_o2e_0So3rD@4eJj|*7Kj@db7CT?| zl6+);1dEi&a_Rdn6fnq4x!&0kn;0!GOcU~K>(Sotr2q)5s{$dC*J{8KJWggO$Q*on zOuvuQ2mb`x00*83=oJk!Ka&YINnbRMo5V&a);MFXZ9UQqz(cm#3tDC~rmtiL3mka8 zY)6$+b1=Qi;w=J@3t#c}2Cv`53m3-7VQW_Sq+Ip;%e?jA#SrPbU^_x|fCO*`p0F+O zo9vYaJFw>Rhi*(EU2%(RWJ2s%o%bKd6Owryw+!+YZA#n}=W?VIbdJGwfb~VKw?Gf* z=~aQb&E+DbTqVn~&qDIZvVBPvXG=nJuTi7#{KIlt)9pqmO;)a~awG!#pURmeJPL^L z(g_54YP^HH2<=*i9|i`(*>1_Pe=_E3-mWzAnJAHsNpPJ%$z6n0yxds|k^@}CbCfhe z)zn7gzZlu|hJY0yZjMfObB5j}-qcg6jKB)nsp2vdmJZRuW6{%7Kyepv6bw%6l3lMc zvf;%e6x#t8GwhuZJqIU)XgkPStQs+wDO@C`qZp?bT|ACPkTR8S~)qPB`5F^(ALcJjdq=mT4 z99S-TK@>su9Nz{OC%A>@Ef6sHbe+}IV{0Z|n(h9#%BH5mhJzpQp&WNaA32c3EWDQd zQB)-2GXMBBwm(c_^q`iBmVoMRXgr>0N>D>a*L(P#tP_yk5zr&2gnzs)g zI9L}y3A^?%u)E!fLJx+&2cklagOdjw{Cx-UWvV{&fL%^yO!W(l`|rPcSzvmwS>%Th zz)~X?X5Ojc+jp=E5k2IaZ21Su^puSPVxWpqBj2uLgMo_+{3%joc+sN-eJ`~pZr8rI z12-<)Ekq476>iDATsTO}1N4x`JF4zJ4YRY~o`OH@ne-Rl;(~La-zmM#;;|=Zkg)*i zEPIAP=meNJ`o6*$3+{1mS47cxECE|MB=Rgcodtv!x+T>pQpD)`FAGl@xeGP*)aRAW3!-*Rqv6wtf+=-B{qhU=r$DOC0p-W-8^>$`ZJdzp|pe{cw zmq~6L+B}m65eCklkbuP@oSKcQgB4>%sG1taLSmX5S*nC5ph)zVag_K*N&WU$!Xca< zL%xOAp%2FJPv#FQL@zxrx%K;(C@&Rz-mXPWk&y*|9G9j z8Ty^yA?Zq<_0xF}1xH)Ua|K3Uw(wBnR+XukbXZ6eacAvwG@p3{G>yV*Fl87)1=EFKPOpb@9$JGcw0FzlWMJSmUC*4WOj2YVNo)s<7@tRt3~% zGZ|j3(Fc-oO!4Gy=scCLniC=`Mv)dCqlLjfaB{;uCM#$Y7Wny0HSP*6WM^?51AWMG-(fz*#jJ!kLy`XM}Q!fZxHT%5C+7KOZj# z6{PZ{B3kqBf&!LElvFJ9=iHZoIJJ&&gJrt&fdQNr`nN0;`;vY#B!>XUD@ttOgQX)1 zfqd|`GxV4H6F*p`I`KykPxyrgqJ&Gk4W7<&PN_-~s&9Ve&?zj3%A|B)cWMxL0QI&a zAn5n!#M>5JV>V=QnRZR1-2@IiQT>?kn85Xw6tzuuN7Gj2$4EapU)X>h?Je(@6vASF zCtqb5ZKSXb@&zNNKGpHhZaT=)h(Z z!ukZ@#GQfg%mVVqtzTGO^Swk@*xXydk2Cb!m&?|VVRSX0tW1A)4?p}!-6fce9lY34D<(;nD&1cUqeHba!y*7Y%)J_8HZrPH@#O@=bYN#k) zKoLy+KWg3y${iFtgO@Gl3Dl8@k-b6kU*$?}ux|zn14V!|Tp?eX&paz~u*uf)+{up! zT#^2vNP=-LzxFs|r8r{T?OkVtOWUd-JESz_zZ*

;^72Gtw8tEn$O+uX{{U*rcG zuK_XnzR9zi7pOwx1LEujs@mrerQ@E09y2yt}D_BV<|`V8RF=DjQ;LH)#Lji4^4 zWMWIy#W#=loruZey%zd{FeY$?XPlt#*qN4i%a;X`8sF~FBekl=7C-T6>Q$3+uoJHm zZ^E{qi0e{#F)Z_t$`~s=ci@z*?AQvC4uklsWU28G+LcYmW8E(idYPy z27knO>OxBzDK%Yn_mRmeglV$=*zSU4g$qMu9q-TBQPjvnVU{U^m^a_gR9!K2$QauTcbZ77 z!18=B#xC-N@L%=ECmjwFz<$q);vu*e?1u;freUFd;y}AuTH~eUWtO0fV)cY zK(<}3n9I?Be~RzSUx&kRB9}=Q@Qq3&vRXk(DdvQZ9C5{awjg!tGWh)vtL z#mH6)9%eMkt_bvV1Z0PvBnMhC!xE|TiINePb%ccA7`o)0Ykav=8)?UZ9p2gqNkPg- z?*SSa0RRIPq(PGq1u`6yv2Gqbtz`KbFs(tujrYl%b3qA*JX1_eq zpsXs$W@b6vM)M4mFg1@tA`io<)t3aiUhP$w>dzqS0e&I)LPm=5OqUeMNC2K66bpLA zJjb76mw6}bW)M~17l{kpthRtFD&-*Dqw}h37w}&k$ITDqVT#rd8nye|`6ig*;VUrW z9uvNjc0%OCBY$Nu2!3;+C%}rSC^+Px%>*UW=KD%L7r%CDwV+hyE$s{P>txa!`BmgN z`KIXR zs9);*S2kH#)dPcR023pghf`8e!o|%RQ*bRb^}HwWg(B`ys7H8!(~D69K=Du=S6t7z zaoOn+9z`fc49(c9@ogNy?f>w)_sVx76)EgC-3$R;AmJkK4WO#ZBaUmFvGTE4>5>+g^(9@}1rTm| zkkTjM+l2Qp8FE}T3?6D`2(TWD7U}ML7P&icR?4~R`)ym>)gjmx$?*1QVJsGTmVtL5 z^W9x%K%*FwU|SH!3XTkm*_X>d31`abcqX%m@E~q1*>Fh=9;Ai)ZS05 z#VF z#;t+6k$ZO_fs-h8$ALxfWv8D-paN{+TK@2c9hbq07-s%BNw^Z|BIj5HsIYR`es1ga z|5^r(1wJx>CKG&TNmE+{BL#`GLqO5-N3o~>0)JKKOwhY{x#~aOh^~zn750C+zQ{Ub z|4gw!1e<|-0blE2xO9Xc=ijYk6D$6~ywa4t8t>$Y^ z6!F7vItowu`}>+VLg{0 zix2;TzjaS~Ggdxsjki^C=ZrGS#n_mi{Yl{UB+%Ee8;4y_&acK83emLbMNt!&6H`vF3`~`OaboF_82TMd-thw4>Y;lu zSUz=llDf(FCLI<7_!YPCRIFq)1;|rz1cG;gTP|V+K?!KMcFZ`eJ5s26{S2p%z^CBl zIfEH>Smpbp_^G9g4AHM&#NsQYjdY9fymv~Ee1izL!x!`#$EpL#=%(ehmBjY4xz z0N>{$v0sDXBnBN&Qx;A|U(HAZ2nJgeR-Nzh^6W?*)2a1V3)UBrQ@~hJ4oJk#)}g6d z8{2&^Urkkj%)xbj&ZmWzYhb>N?(@>+(ZVL4KPA-nL;Yi{PvNlCL`6f;tSWiN3XxjI zGuSKz_J{xLkd@w0b6c@&3Uyk>6z6@~VHtgsjd@+Wp;M;2Ql96cJlmMpK|E)Bh3XqN z0sO-fO7(dwGiB`;6;wZ4|^vY#!wj|&5XAY_nA^hA^O4e)OGCG)D#&o zS{(wbACtZbE|bPKVTIM1{$b#!i@9%DP%d~y z{KcJ20d^8|8^g6&6@?}M^Ms2t#VEh4YvbYcNd>-+mk z*TD&t84U7+`bba523RmK&~gVsQ5JEl;9lu<%+Z9Xj=onQ2X*2AJJHBTIVpd~l0A*W zs|s3J0EW0LyBj|gDu&N@Tcj08 z{(!s-bEP3cZjM(D2+lI^l=2vX+S9)P4+B+W=K=5|(BL|ckrbZ@aR}xnKnKF7C~1*> zDxmwq1%;~d^9-_M_0MwVC+_TxQ4%jF5)Soeq3{FIZ(g*Nb?D}JlvSvA0E$~V$sl+^9xa%ca#YxF z7y4uKh9b~wkxA0`tvIWx2@4~2iV8rQ6b8v+3xLe*_~E*OFOR zZZ*cy;4|_B#;bi6;xjN>{kH!yw{f~ZUXgVvs(%ZIC@5nr{m=78I01)_S$u22w>tX= z@m5QnmZK2dPxXi8A?%qies z*li%mdm3$Zd(lK8plnS$(tBUzEa{}7Yre%FuDX}{0vF)LB#XWQ=qk#PknknboqJSH znFn51ibd>&9lGHk<-WXV{SqD;o0pyyJu*X|8>2X?GQGcWN!YjLyt+U6XWeg`vIE6s zDTc0*R@t$CyP*~Q1$^?awYSN*s?TX0HiLeUAK3UI-k|*P*?7Om88Fgg(P#Sh>f0~> zEWC$?J>J&T7vyBN`(Bm~(Zg~+g*R`L$JdJ0+Xt{oBPP^=(_2N(2mCoG8NkE3uv;isWJ$;Y+h^cp9^iIZcg=e z9q}H&kJ=jX%WrzMO#BO<+N%aqV8mA6pUen?IGB)Ba019+iVGrmQI?f!)FrKb)B}mc zf)*5zSPlN8zZ_V}n=#lB#geccy*9on16+>b+(KR2haTFTO8S3+zz<1xmY+fLImhvd+bKSTO&-$#f@v<_ z^<5)-1=Z#<#Fsa2kgpgbOQ>Y~r_31hycq)|9u7riRZ&;O#us~p08fr{zlhzwohY^$U7!1MSH8UT*}VN&HZzVSAg2JY+?z+2aQIA z%Y@6c4!~!=_}tlQ!hfK9s~iPn8=4|8FeGD8d8 zxxy+DV%~DiMuom17&$F_zJBU+6=f+K3Odnj>&;jM$1{*ZlMTEJbnAx;LI8{D(Y>2wLItyeT zo{sSgAVQav)X@3+XY7M5Zsmp4Wyu5sfaqZ@QP$qdULLCirGt#FB)`%KVCkRlOSE}h zR}UR*08W4oaes!ye7qxSLMQ+SqfZsoVv^=G2&c$&c|wP0UvecQG10zQLajgM!Xe%1 z=ym;jp!!X1L<*qm8vYKr4m@8fm;A`)1og^(-iZQfVR(8J(^hr_jl8z7}An5K{%GF!FBA@kbAIQyzkBbs z*IoU5*`L^5T91J0`SNl9MuZITNZp;yLqNa7zEijdjdNkhiIw<#`10rD{rMR`ff^B# zDi?naQX${yycKef@8$Si0ED>tA;^G0q$+VSAPuGktVnGoG62aRG%rYhHy987;RN0|ttyI_A68?OtArIMzo5S{Ua7M=)iVS$} zA)=^x2zLQgdDRi+q6-2Dr@S+)wm^7Q!A)HzU~V&uW%uFm$sKyS3$hSDx`@60Hpo$mLxrX;h zd)4iiO1HyV19*R6E#EsBTSvDKDGD6I=Y z@hp@puu%96zXC^3&mY$#xYagujayxLGY{ZX7MZ>3E}!8}He^xzcTZR^2{I5zRKbV! z79Lc&#LFB;P?t8y6bn6Kt4w^o=qQO44Y=fMRmXZXJqf*x5Iw( zDrplvc7|{bgjxj`mchb>A)rLbp5{xA+Fy1qmskgI{XM+zl40*h1sSw%(##xuFk7Q1 zuQd$`EdnEMtXms(Z1qBZAc1nR#;Cekqu;?%2e`+GerHix0~KO01vHlO*7*RMl*LpE z{nFqP=~$yjSNX_p1)PDH#^92I(hG?D3)| zn8$?^##vhOYw^5oEs4M%+W#Q|B}m~sWH#?H&;-8%1fVp60r>tN%b#G{+J?l_%6O43 z>ba^T)P)@*8)9Qfd}_8R>Nu2o-=_B;j%C(?9fhuvwLm5MMvYL7m^`&;M%rvg`qLsj zK-4C9yHCIf6h{6)f(Z#1+w5D|VI8uTl!ohpM_w~yK92r*zJj?AeF{L)14@6-_jj>R zQ`cNnYvCuK3<&vvvGRCR65j|0U(*Dq_kTfxMn&77JN@#u1FR;xDJ_CqFA~h$%Y)iB zz>FKjT*z_ed-cQB3-fk>Lg6>=Zy-Z?7mcU`&5+O6oDH@Fs#?!8U<&hmE*kw}VHpV% zA@9#lnfVt*#@a9OV7$tN7EvoG1R~4OjOOMdE%pB_E^kWo@foCjMK-s|36#O1Mt-N<9y5<$eRiP6{Uh;hUP&Nf&` zM1>4?Lm)kC;W><>+Bqv&UZdH$8<|QxJ6#(@SRMm6@tt8T(m8?*zUn>-3 zZaKDv=P~zSoLb3~MLq<0MRv7OjU#u7GL2)Eag$%&OBfw+{HGgy3k)z3GFe*@}5$eyyYU&HscAi3a&VMPodJh(8`H1cX>^@>>%%=$tvncdD&!($w_4E?;Oi*uN zmcn-T1H4XJeL$6P>6kuRXiwN4DCPsr9wZ&7p*R2y#rYjflSm!PY?HPfj4G#C-!AwYt}v- zg^NHq(QK5CCcN<{_qfDQi~*-&W5|1s=biEQ;Y7g1Th42%FD=NoQC>Kw+~5vskOAx4aIkWa|-d`%}||Vw~p5U$2WbB2?&~ zjbGce%xHrOX754=$#+Ew?-UK49)G&wt*)Z(r?9pY@(p~1)IpVq+J|jcVSrFVHH&b? z@EJ9pIc?eB&NZN3Xk$}{LwO8Cr#PAr4A_=C1x9fiEnX`O7aT|u1;7TUX}e0~+v{lc z>T9tppy3anPd?J}6iKyKv`XL)j}~sV1B)lbAizU64)KXt{4o1f&373ag9-G>VT%g& zK25f$)lR(%%f<)Qxgk&;HX<(F>-~1E&O?R*wd8IUe?1Hu$^YTiN-26u*e2~1qQqar z_ZH^bsNU*jxKZKeUwqy*0U$VrBMW-OXie-uUx$U9q{wi8S(Y9Lu*b$vKi=%j)c5xm z9tcf5zds)b0R66!o=Ece3;@I6%AhFYv2n2c;~d3C>>v>*N^=$%W<}e1$OFr6(8!xZ zBscHQV&WT36Gu&bxj{}HMKKk3k`ON#a9&Hq^jz$Sf}+;hZEN`Dk06*%-ltKcev z_g!k(N5h88p}O^j8!g$V0Y~6noay^oS#*Vdzq5U;`6tG>C$5wF0LHBzR5#;7(>ZV? zu=Hhvb~jT|<$0s289r@z>vD-wq`ZP|M`30}DIhw?Pb$7-nY%F+#f~#(Q(PRIi`U0Q zw7zwMV7_(JmyWR<7;deb%f{+PWFS7u@%;FqBBT}=Qb6f2*g+?4|M6kX%g{y0Ij8&l zG26#1-)d98nZp4vKWRgkC{21(+;*eshkE62f8|+5Zst6&0Z~xohUFU$YTL|Gkj?p1 zVA?#-o@u7HM|W>l?eIUrTt)Ipr=RE{n)mjA5}?kS-fvj?=^mogV!Otac!yESDUAit zsR~jLr=Ek(7aOR!b0yelR+-n;nJMt%S4IQOK6Pe@H`4~=3B7-%sf{2>280OO7!W~+ zdEAf2Bi=3jEfjiayY5qf03gxq*Ce2*#blWQw20Se7j+tjQ2cqEYtec6eDomtP+lRz z09U#LDW+q>sJg~Xg0by(gTk2{NBdy|N}P)LR48vVS*`C*G$8;Pmeirqdg(8S*_kjzHc}Q8(oBf$P^Oj1t!hdWE{);xJ zVHJv#DYlf(!+iL`V-5DBP&ff;r`>bkAHg2440u8C_v44_Ihw)^vp%1EDqM()V#LP3OG{~S|ettqRHX|on+H#42uUkFz*uFIl?%0K!(ALd;LqfV}(Jg z8z_+b_+eoCkB@|+PA*6{Xk?a!@~v0SD?Y|%3)q-nBBBd3`%1IcfkQQxSi^vA!DlG* zf|<_5(`Fel%rMR&7BBKqnUzQgc9 zO=Fhnn?71nu=ig?%3yW8Km%ctC!_enJi&M~Qt2HV$(KX&LW!u#e&bFQ&T7C!XiA3Ln6!YTvhWEjctO7DIE*@h0;1V5 zlMP&!vSZOxKnlcFxuxDB`&sqy<44uLfmck;IFY^%wPgb`hwkU*Ow6rFKnuZC8evm! z9Nl6H7lLrhTQ*7F0>f-~bpPW4-Xy6tUAYf7<%m!LvB{VuU<-hw%m&B9!A3R2d3?)k#>a)AiX`CpeOq9f-fP+vyQ!BBVV`(evimLK3e z0NyZQUWWGv$Vyak`c!;e#E>T98{u=qOSs&4aByZD-6xD$CGjvMfXR^XZI<)|;Lc$g zkz`Y#Yaax`yyGlfF0DD`u^u`W0HQ1J64L^9$G9I7La?nReG8Z(YSVDwCM`{Q4IQ!+ z3}7@(+&(ZpR6qlJC5% zxvTu{iENI3+9D!M?1{`3!3rFhGg`>RLTQs(ThZ_!(+H=fXP-+WEg%{b&!Dr%m8huV zsxOjkHmkRq{vU4R4e0&FT$9Zy2#t4pt?(#6>X^o+O#2VNmR(VRyA4%pS*U)F1w!#8 zM2f(Dd95;%7Yk#6s6qG_gHYEPn-b&)%N)X6(LWWW3XpI%&vt?* zf-B@+0dE1Eg_xw+&moq8(@_Vh4|mM)fL@N?N2`BC1Yvb-*$ml9KqJ$D5~}ukIkL+I z2G%aGJZX*aL?QemX0P0%)Me*6!)K};I+(k5NdY* zju9eDtLA=`&@bY6)CI7#$9lt$3T63XY!arKl#L_r0c`fF7*?dd8?i|UA^vAs zD1yx)a)IsWE7m&wu#2TydwLrh9BKj9R!vk5ezC_2qcbz5U5pXi`?)vheSztyB+vn| zD4)P%MVW50fARb{y#79pd65F}9%*;2EA?H+rLQ#-eAqL+JiI|q(KK&ffb=mzZ*qQ> zCpJ24LucJG^oTp}Hk;+k{u$L|RiNgv%-}dq!io3ETSgG$HDl8Y=nlv$52pERf+}e= zFuCcu42e%{Oxu&vDbC1{=pNaLtD6$hL=C84VOJgVzj#0XnMsZ>-zz|^(eZ7@rTE~piZ3mkZV}%Xc z8i&5c&c_nCgn(zgu}(z<`0zctk5>q&BN>oD`;??8I!em!cucrvVDbLxai;wHM?WkU zqWE|ejiZ2 z0Gw=E;Fje7A+tOUrxg7jM^wm-{2!W*8|#92g2pt4?T>{*1qAMTW%8wP~FskBXK6z~cUThM>Ds&ES=nN4<7Pe#Q8e{IE+=C55V z9Lelb^I&RN^EACp<17$94GpqL5;vBuVx2a>7yAgLitwa)oFshx$nIS{@V%2>nMdbe z-@JefK7Mr3@4g9}0`vh7b#wO#eUYY$7?N5-6@fVh(|g=nFZ3?Ttl$(1)2ENh#xq~c zTCeWQWxL-^7O7%EEW}TX<`FH zSVSp}axPDQ^uS@RCvs2$13eaBlwrd%caA0G;&>hr!TrW`BZirpU9X8o_OAG~Bh5;n zn>ITz@G?SWVP+A;J1_c5lgd6*CXNm$zH{Qbpt~MK9!HE(Lhtfy`{#Ue0+EY%8B0Y| z3u3c&B`@BBC-RwXXXi+2$v5^HQ3A33%n?-Ba~WbvYzP^UM@kaw;cSYQp*!7Up?xb4 z!^MD+SKNBklGJRz4REce_S@gdhnHX)s+|ic#lfOYkzsU_+P+;?z_tkyWJpMs{tH&jZqvHGRFy?)oydz%=(;(k<2m+~s}u z6Gf+|OliD;XenL#=7mWH8roCS=C00D31hQ&CEx!xP=w9vN(cRupI8$@-5JdK&Gz+E z73H0oPfF0Ta-jGS%Lj1Nmt!aFCgM{rbY0zP{5>+3z;arXs^j-tLKG#hnb`*nl6_JO zm`vlUbKIUVhwa~Z4F*7{oeAI=cK82voP%^oO5U^*r!>@x#fZ7nKjU zDYA0dSOH7e_S#wm3sz4UXO*xn0ImUc;zUmvexk%4>~=#L3h>)winb#vr17b$@H4Wr zZFR@DE`RDZphl2f8=D2|5V2@OUC@}lGMy=fcT;nJBd$T_R3bBPu<^uB8(%d}kz{El z6+#!lq%MeAv)_bLgL9bRiH9??u`7a-Y)Hdcxy2<8pS5erfAhN66ksRTaMQt(cS?iy zkrf>AP68>HYMl(CSup!nFd2ntQcSP4v0yM*9m=<(cMhmU1zYkW($Lc%t@O#=PQfGQ zybedxIU_)890(F!=XQD_q+l%NCXXC@a;B_)KWx-;zZG!^$DN|+dVdkUP%yce_FJ_|V{+C;V@L$heOq4)67|cWjZO4o96i;*5R==6K#;cbGe#Xl^Slv>V?z64_46_94XS0YqJ1)s3gsD>eI{cfu9F1 z$U474Fu(!e#6@%nH{qZ-`R#IxQBisU<})PZR<5{m}1WL z)N6bYM%y(EE&uDYoSfrERr~Ftx+a=gr?mr;QHBQ)r>NxD(WUm-kt>2(6c%pW99(gX z^Qs=ER;XXN8T4RVHHG)4p`iPA8ju`K7Gnx?@{s(ne0K6pSZ6cLHK4BL$220Icm|Cb zKp1WOxcS9h;|C4+7BMmzkl=Mj#X*SxZmfOSV|NTwS(oP$_`hqE1(vN@8L-6vu!uwW z596@F20ld?nEDX;<^t=GZpz1yO z>Tqc4cQ-Lrhr8=kjtfI(y)4H9|9vxmUX`%*_EwNua9BW@!LQ3%bf2SUWLw8tK>dsz zc(9f!xN@FN`LNi zpswQ=!NMSE3NOCu8Y{)hmy9ue?5tY^H(JixkZkx&6QVn>0X|1N6k~RvXdjPHvHWAq zuLVAQ^JyK`)U|kPKGF8b({l`o_906_$wvVZ@PaiHa82va`)R@3acZ>8_6NAf+6V!& z)>`$*Lfd0n)v|D|O#MeTgZ&k^EaOY&-C_JswN>{C)2nteyUn;YwHU>x2fT;vbgg+kig|+%=niq(qoUOW|lwDas1XK z%yTiQ74^ttxA+rr89o4M4xBY*F#__`38+gTQG7+s2^`Upy{T{}_{8E@6y5vIl~Y$z z9e-w;s$R6y!!}ibh+hS5;`%)gToIfdmVOT$9u>kvz<=h1KPgsLC-Fqa^rvju4ACu? zm;8^qG{c@q*aR^fgNg7 zxM2zlQ96BZ`TkD1eMsa5xFZ%hb{bkBT73o$3KN``8?TQU+VI7JxN5f8MwlD_P9Pr?DV%=K zeLtxQOkHO9xxfaZgJEAg0JV1+@Fcvt>du2~dh_Tg9Mgb^1XhfQKbbTUf{B z11VHl=-Iv)E%zkq>uD|d{g6IJ3B4U~vG%jEyYxEBy!UINwcn2qjrA+RK$|!qM$%@r zD64#BkA%1_^|gq;3^0e+VZ@!7wMJRemV?_e=}nO=xZSZBI-v{aNaGpP&$$^v&hd<$ zgI8~M6mKVY3EIAFCRPUK%*R+}W9`a%wb+$49@F6woBk)ZPZj4f%z8Bi$R@lRKkT(Z z;0^=WIpS~A2mZdBy9g5y)>Dq^n}q|b(~ovjN%xIM6^I?%VU%?tbAiPlB>QTlZV`eC zLWF80scV%M$)M(@E+c&8c?T@_cRtQjD|%!!?>eO~JN=Uxf(ivLywX%3j_M0G;nw8^5>%NkYOa3S^za z8WXwx1K}MTqm4Jg%rj)M_wude!UrE($#+}#ic^ceP_)dESDT~tk!>JOGDIF9sZk%Q zBDh&&xRsE0ai3%YA&bUwa6Knq$R9yaWf!KmIan*czI+VjJv+U~KT4kdcl2UFvb~#YTpC9QH&tHCc!o)NWYTdqen`Z)LMKvwC^)DM5HM2>FQ_RTFxNxK4E>MuYLWw z49vl*AdM9Xl?OvfO&wpm=0#VpM`65%?^yVqIMFpYflSmO5ZH*~Ic(FM`(g8emt?%F(00TcJH&}W z!%I`=^AKsv`u3?vs$OGvXH5*hhP)qG${i07TD$^_XgOtO7G`8c`wswC6pjW$mX0~8 z@C*B;nV<2~LZ8Dj4|4}mlIWp`|AqWh@nsni{15?LE|{N86loK9Hd-vAUQQ1k#n<7* zbq-|dgoojH`U%I*G@wF7R5KklHN^{Qi82FOf;!?AqgBM^l)|&@~m}*YK__ z4z24&{0<2hK>YAxt`#y`qPj1Lm55!DSw4KG{`Lh<^hrQr{~%>D%Jy3E1wJf)&t9%HwIDikO&?DxvGgBwO`T^bo%6qYHY1<3u5 z>ir2u1Icm#!EoK8KU4)}SP#bt)y~hPeG@9>4|vs2|LuU9{{E$_@Fc#TXX7B0g!XXG zvkiuuCFwElXD#LrsEdTJ_zZTL5jIx^jP6+V7X?927$xz^u^jnQQh{G=@$cUwC!4gS zkMelec`g?33zfLB2W}nfDSu#mc4q)_cnzI)I+_(4@V|tIMU)UEB% znqjPLFcnhB7#3~*;)(|4v8IFfY6c+-MFh@F^*?dxk79esM8jL|y4>keqwuOjre*As(hK26j#e?>><3f5jE+?(&@6(i<9V?tj38h27qqBSB}GIK zLf2>O%>ycn20ld#?JuT(LjDlAfxRj|HJ+;&(vfm;s!dDq`HNu0DKJ6N1Hxrj?lHTT zJH+H8*X-Ev0ArQt-Ef1+$1+`UGTZ>ahG``+v=It1{J^RHX~6IY%Orv1pJFDxQS&ka zIQmV1-jQhO)-OQgig7mIFK5KRg+!QyL>jz=C@L(v#KbcqM$~VqWB+6OdPM4^SxTQL z@mBBU6<&XzBr7`R#sllc&(NZcbv1%E*i@0k(p+CGX`tj@@-buQ%3WWc<416Zis|@o zmqY(8_}V?xJ(Z-Rj;rN6&CX%WX;iEDDe~BP{*wE)U^7qx49{|v5|Cz+3ynSrP+&}q z`vU&WeU!&mR|I3x8l4k(Mf1IxgBlO#8li-%(WTqv4ES8rk2{U{%dLs;T8KjG!d;Ny zNJpc+DR7(i42wM0LphQ8CV$>-tW@%h#r6ep2P8g55NR(iN4Q*-PLmRxpfvj6&Un)1 z=fikmF!J@_JEwQclK_KoRkCSoh!$uu6!C>v9Likv$F$-D;oUrZR z$}3$v-_!~es~M@t`xZ2;;&llJ|Ct8`G+>3`xfEC)IdD;$g%7RgsKiI1!2-CmTzO&= z**B6wj*SV!gH4{RBS^*v2z+1N$ z4g<_E$p%4KgFrz1Uoal)s2`Lv(m@nuW+EHG$~krN0YQOHhyw2)4}_P4yHLLHlivbb zWibaSfjK$UWC3+5uttGXi}`vc3pwyPQ^exKG#_Y%)eqrrdmvRgCq}f|o_Pc~vPSP5 zmGmXEu~8e{YcLKpLVFM98DptiCk{E~(rY_S*=In|S>M*1DYC7odxn|BLdlyyN`z%u zj;IyJ;KRkS3%JuC7%7)#{vGGy5cRzqLINNa!H*S{T5313^-7Pm`FsC`F6p@@cveap zxv4bdpJv`y8QN&GSF;+q`g#`Z<%rYSu(ZCKfB}*wyPe`&KOeH3KLQ%VY~O<~(Z}I# z)aJ%5wIFlXBy)4ddDu=Ew$RK!A7>_O8=Ha>OmMiH3-}H=BR|IwWMjp;7FNzH0Zr&jYGoO69vcYZF3?}r@Ei}7%!RXGWcF${~S zNylk73S!m7cBUKbcYr6wMhU}~=qq7NUvV|t4Vqu_g|*MyBNK?!z<@s*mFIYP07%E{b+w`mP&cc$tE;Y;=D%ZYPMBWEEz)Hr-<+|ca;sz9z#IH#_&pbm@+Ky8%vx~5 zC1~fb?TlaP;9Z^p8P>igQFO!^rvdy&+x4$65lLKZQb`34j0qUe@QJQg>lZv}gS9Xd`IpSyq0 zK~y^E!Cea0uHUJO_75j3vqdH;4GPE^zuC{1p-J<)PE@cgN0(NLm>MI z2Uq%5#!e$t3;yKCZyd7J75w*oVMqZrBWFsZZ_?zC#Q%~(sbC0QT4NOPFez7&=NDIrd)U#HJ>3GmLw`}!#HFxf z!l)E*IrM677ru*1Mca2W z&PEqK{Pq&+pQ93{CJX~d!jeDEN zygwn%AbGXGdxN+W5DA~P`8D-X?UUk;{TB?W{IPVQZ-n1tNECaoLSq7Zz}E_ap6H)o zZly-G0^s(=N>^RuLM$9jF{`;_Yx1~FSUbd?MgQx|gA(C;>9qm||DS2&>p%eF7zs?;XD?4xBf|CArk5pBT! za}bykIK0m`RNci8*>QIPKhN&QS)d;S7{?X<5nT`J1;dH0De^&usa%vk<2g>9)_j!d zdhsahzL#9n^7!?Vb4$9Wml(-2N}-?dI(d=+c^=@a*c2!qz zd{b*mO#MJskZB=M`k392B-V9~F3YC0*rfnQ8~pI(+y)$>K}%04PXw2np>9*@=1iM$ zb_1-39dI%Mw=)RtB{v4Wvu!Xbe-O^yT^3$R13rF*o zKSpg9RF#zjQOIDh%ZaZ0wgbt3+rtv6qh-T+`@Os_c=PJMBBYAM%6mQKW6_mT*&J<7 zTDAG_5%6RYnN@c!A>+u!F&(QX2p_-1Zxuds<^wvPKNOoqLd=~$E=&Foo*u%DAlWI~ znEM|HEZThDY_Vf*PW0sGY4~WBixMWVh%plsq|P5Jv2mvZsm|{ziI$tjQ3q+T+gy%_ z5^BF1o@2H)+PZdy6*LI^zOoH4IM0oiZ^1B6CThN>-yC#FT9tZAk1t+5Wp3vqlW-N* z#~nyX!SEVQ+<7CMw0@jduJ_*mW^+^&J7~6a1aHAT$B3^4oQBFXe`d*uoy#Q{PJKG< z>W1u2e;N&!4#D-r!qNIK59S08l6$MoUT&&!<``}(G;};nzYSdZVUo2WRbdRT$DL7t zZ5(fu`vz8K%bC==Jm|Yjmca%EdrYjoGKh*>M+h(U}S#sr4^vjfBn! zbvKW9Kf4vyQm#dkVQIWs=xTv=G?jK;sRUKJoe30O!etl&jK>=r9XY~m)k8^(|3}1f}D3>)zmOYoFkt;r`V86K+d3yzaC!`_8#8^%76`LK|E1|Z1p5-TRafnWR1IQ2A zE*ZA-2u)!NnAepJTp~jbhcgQjdyJB%38T_>+i#f4?r4w+`{Rbdhc=_;w-!No#|M}x zNk#4dAL=%bjnW@79P;93oY>@`hp6_refq= zRQ`A2eT`0k%}at3?LWR8lN5hQiWD_=Ol&u%SpgQ+?;HlQy&RR6ei3#P$0z~5?cEQr zylFyu{%-WQ==hJW!VeJB=i9?uj_USe{Teb@>Za$o`gClbjW5raUgY4{OZ z_xx9-$EqC`|L9B?R3=cMw9-|60~B+Itat;jwiaqnJ{xp2z4V=U5#v^20`nN}xwu7G zyBFrIk`FoGIUXgz6nNY%VhvCmb6+Lw0eK)Df4br2Z3Z5f3~g=Xu$9A4ixP{4Pb_Be zh=pAZb`hAA4tczaRtmjCP^(#lPaEH`Y6h>WLG*h3+M0U|I#fqOdiEJh!I-X|@H~BC z^r<<+6ei=KBEjaQ%H*BX01OmQWwt_biC|4ze*;U1coky}3MhzPIgCqp1O_mEf_K8y zxdlg735_t4dvY65m~f5o(c2^(VRfGMHOiALV=IGs903c7l_SI0z9%l8{cltHSJuSC z)R148$W+~W|sTwr3V%X7gtZK2Tq`@-Ixc9)) z!%g4U-bHi*?}bbUWU7Q81Vgm{S{Z&AZEOVK6ix8&)M(HpFwZg^*i(H2*-m->6!_>i zd_*9S8>xuGKS|3Ocj3Ve`skWYTxSUnLEhJP@@9AB*fEme^_maEh>PcN#G*d!W%g>< zRfJ$A9(M{=r4dRX^u2}@T4~a3+V1f0LRr$?cqMB+bC{M8L%Vyi^YY#>Mj>SMC%7*6 z0dM1JA)Ze!Qc#X}Sx;d4t>BVNkAH{KiF|IG~eJei^}eG{K0kfLB*Q2>e%E z{KhQJZRrmwn?%QaF8e5`BENi)mt}l>n7$_e1jaFLGm>EOko<8Q>_m^JQV5n;$KluEZ;%tpgw30S^bpEwP{){?+8Om>sB?D;jl5tRbv` zqbL^iU<0`K`}rgGWA4HB1t zKa(svj%V%6^}#N$sMQmH7b~vbDoOQu?p=4E0CNy)b?O>y&(gsJ4-gU(5ENd#D@Nle z|5fBwG%{yZ8v!1m&;NI=#16%rEnpH(leL*PC9pKXfNPDvEPe>o@VnMb*% zF`b2^Vqd)s0_ZsXxwebExl`hmZUG^$^UZ_DsY{or=rRi$n8jm>1y!H5Ag@)F6LWr# zS7AFXZ6ve-jn@1%)j}d>Ld6o?@6~yj@+kW#I^Y#7UmJ;6!cYQlWa67Y^B%3U zKZ%7(ey_6Vmr63m*6Kj^EqN7`sowhG@ z=Vij{VeobTY}B(#j%AoS3g-_FK+p~H{*GFHsx5i#5hb28T~E3`B$JJ}39n|5oD<(BjyqV$ukJE?GNyp zu`_yI$oj}#mY~ehAU@#WC!tP$f5)wLafMTeonSCCi?6kC7#oF3NK7el-iib6&TE7O zW0S%jMMTY{;Znp=KK5YAJCKjyU;v$$=edUfSRQO8;`f~YjJvB;*G&t7S~v&{8|dh3 ziCl*M(ntUTdu&o=$E5JOio;MBegD$gul4Ce5ldRL-Zj0@#3=p#E*8xwA>H0Fc}m-7 zBGH!Oh45u1glvYMznBnaD;^6%jFq_5>o*kO11J)^_lZ^B;YnazI!c(0V1c$$|X>ihyICjp{)G#=a~X6Mlu z1}J1E`TNLAAY`{ld9X;6${rq&gE z9US~v1AyMQ6%JTytT+1S;9fAoApKiy2E4BYrqVe|UN9*rr~rLwy%@dd5aVtzvH}SN!?4iUXH^-N6**e2?C~!KK z7c|_#^b%&j(#c^n<=)G$JX#0z)1Y2Phg6{=qq^^ z5WeFWPXa`Tm-AZeXR&Ar|Jtqem>z@LC*s}GzB=rj<8@cjl2Egm>QiPm8`cc57VsX+ zQLu%o1c{cwEFlB(Ca_)7h%nJ(K+-Zq;{I9ch%jet$HE{!(Z^+?yFO%|na^wa9rr5P zM%0W|$1&Kgpg+!kHP&@nO(*?mErxer zkJE;M=T^E5>${5w9jxO<6r_l;kv|V_RF%I}E(b@!2Zj&f4jUF585?3lAo>W}p)H}C zjncwO-f$@?1*)w|$_`061Rx6|uqCpgpLTq4cfdJaK;h&j5Ui-Sho;Nvic$>(T=~E| zETPBj$8bsbV~Xt}M5{<2#ob_xGZ|mm2 zAdN9EL7algND&R97uCf88CdKmi~$W1qMpl6NC>HuKyE>&^j!O&da$Rk0-t|EXM(8- z$C2DOQiA}1;cNxXH5EbiGlfnmaSYHU0hO9B)C_RR9Oof0edQ3^ZCsP?8RaM|@$K z=f_({-d`S3_}wfOXmBIJX-vnVbSuF%XR`#ll(lnvv?vhPTB@7hbUVW%e=*+*Gfm^j zxv{hf7`82|%sB}`^ql_!1TO9Z5{NMG!wIT_gKOu&(s4`~(@+=!9kN*Cu=T;TRT_A1}$G z_V+SV3C)4w${xJNXP4 z2Qe6WTnRmqceF5wAwS1W5adg+_^V1slSeNOtqc}M{%IS>yx`hIW^t#7Lc?|_8}J5P z{a|z(*PGBv9aw@2h1V=X=(ulGHkfv(`wU*^Ov1*tc5<(7vV5Sy!^k)4eDLJu1{$Kx z;`zT#u%T^69AGs?|HA(Q?V@e73vOP3e5 ztpo#wPgY*fjRHeAu8$_}xwD$2QNq?TQ`;v7ptr#>4!hF8GZrhB9$zS!jFc_d|(7 z%$OUkXB}J9tVh=X(AfJg{1ZC2b`UG1+r6pknt2U3$}R$_!_7QcJ&!Xg4774eRPb`~ zb}Duatd{MK|GuDK47Eab7@&ak4b&Sxxb_#aD?okq*vcE5Xz$L`s`NrbtYiaWq-X<3 zWItMV;yn>R8n1UIne+PUGM|`Wqbrtt7;l#tMCE>Wvx%EKW$e)#c=<>l*>={5k1wY> zB~RRwbF}G>{S;PpctySi=8_TN7WS~zo$m#nL-b-}-S$1)MR=0gKKwW83x00U6N2L! z4p%wBK zfa`ZkW(~V7AV}qh=G-4S0{^Uyxog?%!uwI&;U;|{j1|M=0-CLXI({;js%RtYxkiX{ z{C=Soptf2xz@DAnFMh)E7=E5Q=^38LHQX4)wKq!Um~1KVs2j`q1(aZTJ41>{ISTc= zVP)JpVQ5Vg&&DyuaaNEcJ8EbESR~g4AEC6vsdVBxV+~|GMqkx;fR+4*100aP{^-t3U<W$R z7T4%Ge;~O0G*|yL<`et`R)(D~@-RSpIeXgiJVYdrNKd?H*cNj~8nE~BL_Y}03(ybn z?V1rneT$ZVRMTl=^61NTo|mNrV#3eNP?2^cfB9S$;?W@9QrMjfOzCK%A0x;gH-e{` zYI*eed09+c9Ue7ciZa{|-j50q*N*ENJV8}2f|#ICJkZ(rqZYi)mYl}?_>~I#TE3_3 z*1rZ;8@r8_oH&8c4!P^RXW7V|g~YCcsYU{`At|q6VE?+#SGzo;iktfn3(}Ak+7lKN z9d^`wM1%K_V9v`fi8@RH|Dhc2sv+b)YKxRARRea?f2?gGJ)ECfBP zTSGkG{JS1yVd|l-D}^_pRk}zCBcLKB-v;r6opz+5P9-87&ecm5ZC_|aPSR4&OYCTf z4Z)&#H?AD@E=QT%kUmL1!a@S$iatyvj-yq~yYVY+?`pymqEUnK1PL3&7!;g(YzFGM z$N}fx2Bkh6F2dSll9-p8CM?;n4%*lAr~&Yd>mlg*3@J1%1xx?M+Y^E@%2+jK=QlQKN+MW9a4vT-dx z8=Y^rYR;VqVUM1`J0<&B_y!vc@nT22%1fL>M2Hj9=S;KJjbg_&N}+q)MQ| zJKP}D963?~61^NQizvK@RX}T>)^e>jsu7^Dn$boj^t!_tbyluAZ%m%J9+;}S&Mm+f zg(0P76jI_55sIFmJyy_W2fZ#h;?q3(0}RfuKlrgar;Zc#M*rNn(O_XYL-Zkt#=TL+ zLi9>Px= zHPB?EZfqaN>oXD*vh|xk^U`5$)sm#JIJ0$_z2=GVU{5-Ej^|`z zedFx`e4^-GA%LcE+XQ6j!?(CbVOPV1SRI{U2`L`WGMt&TOZ&8?gZ27mMoA-1$2^*3 zpl(19l=97ai~mq(Z_&?@fbzo3&xe520#{uGK(@V_$!Yo2xpdh;HRghk8{dLTX-ND!$OgLs;RB9K;yUi2U;Z%fk1N=$w2)kOF(Xf50dPMA06I7j2odGTAZpN{^E z&V{8~>A<}(A*Cs++=@G%TL0!`9WraAC6?y>!7_eg?W@j0 z>Y0I*n44gMilkG`hLD9%HyF4Zu8fQs=|qrxQ6yjjB2snA+p+<0^v?o-z*`6Yg=K~v z(R}YRr7Vz8EyE`-c#Raf{y(wesGejLLNE2oxwH>9pwRklK1oy}`vUeTnTn3jobfs8 zJr~EGEVJyC4|FtERs)UoV?F`j0o=H+mjJj!rTz-qdVB|}XEBylcI$Sq5+xC7`rUq| zr7K@&!nw51+w4l+gs2~;pA*mc*oTwEmz5W`zq5VbN1sy{eD?MG`+mUdU2|}QVS~(n zI``ttzo5&)lw<2N_zLw<3k_`knTvkf5*&jBA2QspwZDGZH7Ni2FnkL?8gxB0*n=!S zd1>JOO4UgNH@C-lV0T~$X)FYwNO5AUV81!@#iEFLUlj$pnS3NH|K!*~Q1aq2b~`J7 zTKSfdd>BwF$p=F;n-)@FsC+}XMZ(V*?~l9e)JA!Orw$Gi-mMgR#|ujT;sL(+Sg|9Z zH+i7CkFxr>8>v&f=+aAOuC!p;3(CmG?-D55YP3*^Ro%s-t*w^<-91CKDh2U3Rca$) zwk<_4^bxiPIw-w@9e|1lX7FiWGJ1SrpAs0yQ$TA0JXZV&Dzs3o3xj9C{Gnyqtr?BX z5iGDg?2tkCI&Mu2e8#XN1d>}!2Y8a5oRhR5E=hPpNwt->uiA3O+zO;p(Vc4aGD&c9KGu~|T=R$yeprtPDS z(Xee!o5i$gHv@g-%b&5quO!5wEjr)ZjZ#(l zal$bBh!wbK+vu_LcflAym54hYy?>o)tRwSUz2SKf!jRz8RbGd zoGGg9OqN9~g1vPjUl%}q-EMDZoR*{*L<#VHyr|x9@EMxsJyAIvd?PIAI9*RW^)FBq z48LKy->vr4-}s`uMw1E1qjiNk223I69st6Ot72u$_duT?p%Ix>*4MNk2?%KFn>OWN580qC>Km4n- z3o1mhABf-~3hU10Sqtvq`{)P#sdEWS8%q#u-#0tJ;QS9SX=Ck+F|@8h1X2fG_+`DJ zw_-z)$*2~4+cK=d@iGzTo%$csP5GCKMBy7d>7tN8WifEbBd~0UH-Dvx;n?!+v6}!8 z5<=d`Ladr0x=|DBKVU7gv)g|Zc)3~`7K5<(o!Fy3;xF-7OcRV>JoK3if-v3ASDQT3 z2>*Ne+2-f3=U;8b3{MRaq+~v=F`#-J^_3B}NyXwAp$Ly0q-ROrM=f(MCSxBQ83y9z z4`d36ke>#9u6M&-g117=<=Qj#)e%_dnCxGWI$t-zitj{-2smvk&(d$bRme0|XSuj5dUuCCXROBjHu%c}DUQx2pD*8v`4& zNr_78H$T$iDeX9hxcJDS)ZS+Hxt%qsFa^h51Hal#jNnrHjv_-gd#y_*3%j_w{`oWA9%@1@tPt%91b<@mHcI)E_|XTPz3KKGt*Sj@OkrcaibAFkqP zd>5~0RaHGhb?o((|ofD?JK72D5a9&B#SV72wi70p*h|Pk_F5Yf+=q z*9>!MEEBlzhW$~Vv^Xjy;ExfU;>oyNgAjNGsG=AVq$J^Ghq^PnSlfX4)u~{Vt;9E= z^KGRYv)E+TMFwme4%9#iifPwq9VP$m3ts7lYdcrS)f&1jp143gWgYw!6+M1~G-C>l zN4$XoWvpt3h!lX5(8+QjWGYp!^{_gkeOLwYd1-;byTn`Af~pWJ0^3`t9N9McuG#kQ zfA-ipMDG%wp0Y8ZkgDgs9PKyOsfwxv7i0IE%4|}afLMvgi;vzWd~N z^WVm8V-6Hr&>J^v7ZEH>uq7~_v&=WZnvf?0_5PTwyM`E=tJ)!|>?k44`Xl%f$iXIP zPR+R=J{*YV57Nah?BjqwFJr9hdg_P@IpL(GPyz{2e*R1JhdeGH*&fY)()QuEcg{(S z2s1st;gyHAkjry=A3NgA8oycmERgxBKL}>Si9c%eA2#v0ICY?B>?f;~5Z=wL>;Dck zDwx0c@Rb|zrC#&VOL}zE-k&JA!?OfZZtS6&6J>V;rpX9Ae;S%5zQE{3Czx&uS2Qhc zqdFzNV;rJDQzw-VZfMXQRfvI!QeeK z#3C?s6Gl#a0TU@8JWz6xe<1;R1{FH03N3T|I_wT;v~*bThZt6#Nf)@%f!B1gk{k40 zfzF_8IoCL{=Y}BNB+w5`V0ndDeYL4?!3_-A3}8e!?5gpu?Q-LVTMWA|TYS+cmyFRN z`ayPCFEtpxL_4Zwoj_$Tedoe^)bn?pYORU}ZnAdKJf?cKU&i+!|EVIPeap+)krUZJ z7xzxTL~aG{^y{F62lFxp&%Ar$2fP9}2*5mF`Yv62V`6+>%If>enyK6`hsg1q>!jU5 z`yKOKG}Pl$giyd9`W27A_v2qW6R9Ujobc`!Qu_en#V3Vv2$!CkCIn<8pXb`;;I`YR zxFbkc7R#}1?JvnX#^csuS3{pl;o{w)RZ^*~c;V`}57XDD@bC^9_z^lW9If)UL38(G z_;@_G|8C=5-PDT2 z`YK4nSo34gvt4)_C>Ot~)7t@N<>oG;?Cux9LpK!Kd`ZE&DYiC*lw7cg+W|9-K<<#0oWq<9MhcNI+WmjV!j6Y&ZmB<_$8d{YG@tt& zJ}%c!Q(YMcXG3r=-YVIYmcVrA-BN~1OH16RQi=sQSZnqUZ}D+h8#-3wEH|drlR=4BGV(KJUI4uXHiq;%{1pi4n3rAz68Q8-J9y{-s!D;| z9`5~p(u|<0L{m%?KXahEBSPaZ@q;aCoeUeOKk@J79U*06GNg>?OwCB>aNq|xNZn6` zJ$UC2`dF9z#K(+YgiT1@J+mg*nNvJZ*th<`SC19ZvZoyQCr;ZP3Hnx3uyzUy2X0ca zxUn`Sxfs+!C`)kd>BrJdzoFu+4;ACRhQi;>XynDif%UBv9bZQpxC4(KrgOVAC8CL( zLm$^T!qZ9H7D59eyqC?v6tICGw#J0DiP~td;iQ*W8A7yKEZ_YY^4{P zSG~pbgL6{jC~=a&g$j;bLY;F|A@PVX!26C-5}$yGsm@hwX1N?)tvr_Su=ShB>{G%* zghXJ8a6(FSq&TiuU;l|J=eTREt@qY!#LwV^pAg=_80)Xin=)B(dmTn^qMC8Q0uc%u zCnt|yqcGUpIlEfaS7Zt4>U}t(Vc%z}kWmCw(a~^Q&Yn7 zti4@tv)5bX^1a#p=~>9FTx1lLs)^zOCy22f*I?H-kOnuCY&q~5kn~451wc2QX0lt` z!?rlLM~^wZ{W*Xflfym->Ikd?6sWTRK~>k44ii}dfuUo;uR^dn?gT?wD()R@lzn=Z z-?0R0tUWB7-PVcu024Wef;Ki$2{d$(D*OGN;+PlhL;1!;w9*o!MQ4Lij%HCFHv9&I zN<0!*Q|QyPa*gv&S2vRkB=Fk1**+Y~i>M17J7U{C+s?w3sjmCV){ZON7WMD>X-q|H zw;yi1M&~?h%js}@Q)f3%nzDPB)rAiB*w%eFAw237^j@7;{fCF~5!){Lv{hXMzL988 z$v2_}Fu9hI#i7c9A~saUU|X_d#&?r2v%QL`L2_#1Kc@GaeoCgHZiZSwesdJoK>29bwpFoYNMsxT=RtRF$qHDgpd6?JDW$WO*i=eUVb{YIP` zY*YNz_qO@H^6fiky}>&!-fvL+sd^(10;m*J3Y<{Jjjgb{&#Q3Nw%)lRXGDI<0dHEh zk!cPixQ>R5^&$@b)-k;{lUSVtLYg<@U}T!9aopzZ=vGNa|h%QLO5>ep@<+@VU0^S zysEHI9nbRz5EvGBmkbpCKWg4-+sC8$CdOM%9q^c$n9t*}6Z`>+B1t@01PY)DW~vKL ziR+9mjm_G|4loA|*2&b64gak%W8G3}snLo8nX zQBc=N3*|t)+Rs|sV#MhXZ9zScPva6B(=CnN`+LI-E4g_Ks?hBlhEdL+~enht( zQy|FN!6lcM@ws+{Hv};r;4CNak50%v<}pQu7LQQi;5DzYbM}j=m$Bd&425>y^PUIH z+3dAszrc(kdtu~YWk0o3Iod4*#Kgo73;^n^D*cR8z(LoC(6OREefof*wqo|FO3*v_ zfD~qtH)=wWZ%`+IhA&Xi7~NpR7Z~|EKjroccf`;oI>SIKmTMdlL6Factr=!FEHGs=7SqBJ z>lY4-MPAM&kp;ZLfEw_BWhk4B9S1DQN-sXdVbt-IigGeIqYx{ly}sP3o;WU*!n_%{{Xw|ac5X$V$94wYjjaT z&=sTI2O6-YvJOf(v2YCkyW_f|ez>U#5#2s`R&)|YWaXE`lLa``RL`de>!cpApi^Tk zbfhc^DQcmRFrFL!`_6TAMNa#EQrI`{(5qnJ#pTTsNhN}e3Y5l$t_@VU#LXWynwS@Q z*xaF5V4N}ZK$+hl1qf`f1AG}fwL^ZeP(MzB=^+{#acnhy0=nPZ-W+?-Sq50&od>Jj z4OQE?v~^X^eDN8{B<+ARht7pgEyP2bxQ-e}8$wcAawj*Dng!*F?78}u54s@J2tZ%g zxr_&iH7@|Re*;iG)WaseeZE~JUH^R%t?oe()&XacE!3sM^+AX=bPp+`v7yL~39*rc zQ3wL0izq%5d4@tO(Kus={pQF0l+36c$WZb8sqVS0M*A%M(=vPuy>i6^D;3Em)>G&6umfRO!IF2Kw6wB!F`F`olKjo$Mw?*5wvMS#S3{aYU4EfCSs_RB(mJV3%BZgRoUl;{&HJj)|_XBgi!!y0RN`v!J`~Fr4>e^Z-V9(NL*q~ zIOn03!?*B=um8zMge<&y>%=eS_P^NYdjNmnf0r%ihi_3cLP}o4s5bT9AVTP9fAhbp zi~VTuGK{FYo^-Ag_@F?ex1{Z37Db?xupA8uu%C1f-dtH-N(NG>3e%551#cE##R({ zILLf#vD{oQw(|LQq_>MDvA&}AkmbCFSs|yAS(4rzl34cmZ3GiO6N^_d+ex7@Kzfv@ zKpn)JfRn3Ilj|X2l^OYq(o5kmzT2kYuz70qJ=O-_gXU^-)Bh#_VeTOywu&MPJIRNQ zJQ7TcVufw>!>$PKR5Fv0#h?|P;ltt$<}Vk`7|KvAyw6((ZFN*{0f=vM8lvgEsvDSGA0X=(i&^DW88e1d1!Bt?iSw*xxoW?{NK`Z2P{8a7uts?mjWoa}V{Z07 z`DBG;kw#-MZz(#@e_MqepvrSzY{6(RfFc{QReS~fcA)aJZb|81J03_`JzG%rq|7nn zaZ}n3R^7$8pns)%8b5D@A_Jtj^XF4$^b5gY5isDK#IKHr156lTUXC;yQr+IDQZ)nal{?=;&gW+##_$Ps%b^1V2BtQ+P+RWsrg0wJi05jRM>qPdZ+8Tp(BSvUst4y&@>InuFf{w9btu#2|$TTDtXyJmVlP{I`a@V zJP|~|)2J(fRx{3~B`Ek^QGyRYU!BJw8yzsiNdI_2t4AV8sN>%G{6EitddTc{7$ky@ z6zlJ+r-Q4jfEl+nUEc&`zP4fe!e{V@!@e$E-)K@8h(wIH{CVfiu2fh-kGugu z#X0jI`m$eP+2(ym#xI$hxas`+?Ug?c z>hoVbl$wwh32o&6bUqI}TdHn<>cdvrw~b6e5|58P0CX@R&0+>x63<@7H5O%( zZ6e&=k1Cmvi@3SNSoT()wEs_es-Oh>Fl^)v zd!%*pPFZl&+K+9)Ryog2->C8`)OXbGzHxt|hQu*>OZk#Ri!;m1&#+ZkdI__6Jc#t{ zjF;|?Wh3EVlC@XC8vb4U-0DHg{tT0Zj?XdGDhLTkZ!ck0xFcg|s8|=vLdCi$dTT)0 zjoYcYNl+^S2vllSj^rb=jl~=WmE=Q#;=HE$iHiKAtQLA-ktOP;81`j$yM#CJX0V z5zHdcyJ+^f6jC!Tt--CXtQ7iX3BO{;F~ePn%`+vWGm-p9L;^=NQ)C5GbY+Hij5w1X&^8*W2 z-#PxVl#^l5rrs@8<%_La9x6qF+nST!spL$xX4t za+B|i=6P82d2YR1h>(Y@Z5{aoY58<0p+%w+=tjy7xuRutWOTDvn@1{l&td+p)Rgqt zVdRTgPhu)lrfsb2wq4CD;#0 zn=)IH$7ezgjsDn`c2$9=d<(FYg;mI7^rl}&@mX!|7L9%}<=k;(Ev%Csqj{mDd6mZN z7>_@Z&+kLx82v1Z}97j){ir#v2*Z3bM}_bXsj;T}JN9dpnyK$JpuH;P&puQ<>K2+4__*5WW16Owk>)$8PQ4O#{B79&?d*c zf*Z^*l_yLQ4qc&fXILIqw!^JbnqyF>pnFs=$H8*40R`akJ;WaUl%r+3<5bAG#nBS? zq!6x<6X5+LFu5cKs;y#xv4K8C@nd|~Y+oa(GK2ZY%l8PBa-!|P^cYsOr2;3CI0rU3 z6lfL^P(m-g90L}yI>eK@xf@D_l%COk0Pq8bFc{nNh}dWpjzG0WVg*j~-eL)$az=3=|p=5D;K+`p^9HC1$3!?CcDfU!i%T zLgi^mDQ%&3aTV{7qh+~6LBrTH{hQZ~C_G{uzlKKaU-gCw4^`Vp!4(jBeG1ZfYC_xD zrD-Hwy>N~SMgR@}{>ejqPWvQgFaN@2>`xs&6V?k3gCD8!%DL4!7R-Lu3CGto(EbTY z)evl3kDV+&q#Wi6Q(<4-dw<&j7zBaf&)pVlq=k;v_gk+|`Q1-nf z2%=Q$i4N?bf+x$6c{j-zbifW722?^TDE)_5;QEkz&#?#k2!GOoq$^WhEHp+`@vW?G zo(5-9W+Xejhj{rqJD*}Yc@E=5l#J+SCSdf71}iSoE7$Df5E?|XvY^5DR$eKC4KHr) z&vQjz(omSAc0>;ja|`{(*qW*nh$G+qDgqb1PZ4a)KKh{cLpj{Q;KS3VP79WLC%marAdQ z@0|4)RdO}$O@X^CXu0T~)TkG3bby+w=FGJynF;Rv2O7b7tgSLmyER3yKSt)75>`Rl z5}v>R$kX&=!}?_@)StV!DYgT&k6G@tYJ-$eU=G_g5c)0&cqvP{$}9_;=mY#*V+w2j zFI8U}5apR={mpceZezMl2A5!>wo4$!tp%4xLE2pskrtzb%}gtbN}??U5D=IwHgSmy zt|)@7w1@~Kg@}N_Orp^egrpS+Mqr`{5s-?Y5M=w#x%VYsf4|VGqNsY`=XvhE=bn4c zKf0(!3&><<_Xo_UoWAYB&uvK)MXgN6ZxQBTAzR6+7rY`Fz?yJDdn}OR703epN-2{N z9%rk=9#XNYXMk8qy7#l^pq~iTAd~ZG)QoOsH-4hOwKM@W(+I$E_a79XgC$3;WYNi! zk84`%)xpU`8N!!i)R^X#(+B+SAHY6F1zq_JEF}P@W3bWs2*0AR)xjXQ6Kli1-667m zl8IJ`Lm@aDLb`?Re3cH;e?6FCZ-BYlG|S}!QY~5|ZNzgA#GyECX}JW4Je4Bt)5ljfp#)>Wtq&21x-)Y=i*S zN7}^MMAU+GB{YH%_E-aqpm5J`dKu^iL@g8{JoNKoesfW*fS}0I1a2z`h~+2mCGN+z z;L4R`B0;bt@tgpzHFt8A*5yw^lZ*!QI)<|@GA3;iIi*40HP}xZD)HW-yvt)tDK)Wq z-yP4io{PL!#lzm(Dxo*}`k%_S2tBkc{=>xV!%VHLz-$!O1ss%^f?z3v?*zgTZH8ZR;y4hYn(C0E1QC z#x%4TWK;QESZqzO0r=RrLU=iy`Ms*Wcbaw@(o< z-@aLbUqWl9*-LdSV=moengDxbn#IGW8UFFSk*(fWmrX;Bgz7Hhs8A&^udE{I5Ru%- zDyqk`d#R456ToE83B%H4tj2v4SlNQqD2oYwHbLg_^+(!G(DHN)Ik zX(7Dse-|W7yi&Z@M&UT*l}*BE1Cs=mqQ#j9uCs7ulvrEfoCtilx6^H8N6~8_yh?0a zHrtj9jWy2j*i-*dC#Hr}Dp$4~v;VV2b^Uq};%-YnYipu;X`xQV!@NtFNIL|cz}S#v zcP_Ydj5ctc@h$aP$>rfy$%cS%2YPe^b|o4%;iUolc*aaOZR{g;-({Vs2E?+g4CJ~!jRq3 zQUB_81KrT(E_AXHUL~1p-m*6Txo%mMH&YG8j}L03e|0lcHez$4BhM5Rn=As`UeY8F zC>U{aY|v_*1tBR(&>-1rm(Ti>wDil~a=NzwYSMCs5zW92#q{)MOSlNd4oE-x@*_!t z#;PU8wxovi_THkA9eJ-6PQQ`%j(58}+xrtBPMkOpjF85Dv8N?hOef4}o#7E=Ar_C6 z4B=IQD^(Ev2@C-`TQdhW`YM4SEsZZn@IT}?^~0aQU;qjlu6mhY?5>ypUs>{9CpBCw zhX;1fNnj!0k{VpLGELqd5mt5HF+Z_&@&Guu-?21!O_MSxp?#63=A)7Gu$EP<; z^(~hS1$8hB-uI1^s=N+rvnsvRl<4Ql5^AoUH$I<=U(ASZ5u!%kogx2Z{o+|z#+t|s z#GOo})uSp+6Pe15f@LqHV2}bI0uM^$DlpUE*zXe_dMWiUFmUm}A*BS>%DTHEhthF; z?~28xQMjfMQ<}Vr*R$rgZXhTkPE(Ku86FnMn^Hre)xxNKE7O@z2IlvFFH)102vRcU zy2tnH#HB)=v!jTg16)|P1w-B2)0ZIMei_Y9xYS4hcNV@&O*uL)pb-t^mWi6+Iy zT|<`yTtcr<1}czZnLT3us(Zf_VeRxN6#6?i{?jqv3T_vbH;elRfo`abR$-;1Q_Aq= z*SjM6z_QugcBnEh@dH{Ge`g{&7hM8&g>n23bL-C~760wpC z-}f!PGVj7#j1!|LJU4KDDlOwdEQv|)B9RtXcznBZC?Lu(f!B)_b)Q-~s1_*ZfApa>veyZ3(OtB zNR(!wiu)ifgHiyl+!jE*kx^+4#c0tur<6MSJd0;SUYhZ1mg#$) zwn!=vI=tA%0t)liMEd?SUy}4buzth)_&aF73m3`ZRGAES%n!tlWpa>MlP;87aH*W% zE)iSaKFy`Nun|Ru8cI!C0Mw}X`k?H@W$#jM2N%Yl7Q0BqVH<_2x2LOEq+Q^PQg0|w z5E%z9(_TARzz8A^Qn5H*QlTce8H#O{%#U)0c!2qpOlZe&9pKKPYsg^M^OZk3p`*qw z`45yU)9*8ZM&gnN2g5qDhEG7*7^@e7!s;4@(F zltf!*jVpn1wE$~3Js-j&@A~5B^CPAlSj?O2Q;rSkJ@UaoYMt&tFg+@b)AeIsB-Yp%5`d$v14IJQ8Y!aD>mNU<6b$h)tAWv_bRA znvOoa0=PaSTFR348`80&XD^4pCf=tJ6?0t^REPN`9o`lM!t5OCNJIjVtxSU?MGY}= z{i28l6enFOuX3FQUZZNzm`~$P=)Ke*wEKT%MfHksDpm{{auzrU{3(Ap8?r-On9> zD*^#y~-zWcRwiCk(%ajQ!7 zEP|Xb8O$<9B%Jj*UQYauXz-Ul_CQG=8g_l2o9yDNHwlXwVqSqUZu00#c81YzQ}GJS zx}hB`S((mwboCr&vuwE$_^lX!a;hIT?>67Ab$zH+Ke(+wyNs!bfL@}|v23=*sl`_nQL)$u|%|gAfMA#K^+QSA^ zK6z_{1Q;g6BS1QJVmQTv(~Fc%t|X1k)2#uR1o{tH#0)>&T{99iVw`ls-3!~u7)hbUy(M^&z+b7^5#YntFdHPR$O%R_>F?qll_AQU$ zy!j46VN>viewQU*kEul&rDO}Wxp%)Yd&FmqY94;wmtsg9yq$kT6?xJ^977B| zHnd$u4+>yDmmy`~=zZadb$L54hE&eD~lVltMzn_F~bN}qc*;;e*}DDc4Vbi6~Rv&0zAEpFcTo+y~4)N`5< zv#2p1#t4@0Hv@P@O4MTqa*gUl)dGOZk|;G%XmFad5CG8$%{NM^SEZEN8k!D7l3V9h z40kZmyY4AO^b~&@Vxw8d_%$)sNPp*^>p3EU2yB28=(fzt-!QCH5<2nhz)`<}916;a zqs9gY)F2V!kd)Q38p`)r8*qWHTA@>o9B#TfkNyP>FC>ES0HM$PM2Oc2>ZkS{fY|T2 z{y`pin0t%?3Q~BjzW$QYHEt0-3xHf{ny^#9ZjzsXhiDgZt$^EVI}TQE$p65{;6A%h z4&y1{$HK{F@`J@LIZ&u&1{|XVSr(tSdC&{Mq0NQ`!4sG6(-`GgJZ3dU%4ab68|;Zs zJqzj5i|6S(iu!14MWA}~UT>YVmMI|AK*5NeT*D#aBSG+ z@#gkdc(MR?Ci#f%KXfIe9turH6hhcjIVIID0-HR6PeQ#VJfF_lq=&6YXr3wY?63k3 z>MB)vOI4mh>%sMyV+tM1bEEh>=(q?NX!X#R3uwt^xduKK4+oj)r|u4)XqjEE`5Hvv zg1A4hVq{j)BGTwB-=_|eGykLV|>03+5|C{m`1* zbhL;LzpF=9&dpcE&M64i@A#wz$k!(C+qE=95Os6Li3-*(eZguCwP~q5ZSN`6G-IqP zMLNZ#g|t0oS~oE^RZM$<=RCc~I0X4FP9eU%=70G=q_LW0K_OR4rE7+NThys=h+a#V z*sYya7HwewAMiZ)l5NzU4yJ@yon!@u@tKy=@QQ}&C;p+Yv8uwXpw`EMH~t&m9+J|^ z2d!ClZG7@a=_~@GtbUWJ^9s%DvZPqTu{4x_q$JJzGcJ%>u^z(c#_T#2ET+W*0~RHV zh3C>bJu7yxbyUAi2^ALZJ4#UGs?km&M3{oJ$%9}&y(3D>o$meTR!q#0c(ts*=uebw z6kiT#8c5%2+~3HKK$eu+IKDL3TN+k>&lUDfXV~IC)*G5vbV33){+7++fWMsb1)*Jx`oNskkIrn`O6c zV_#9#?I`;V5v)Q7K};Ekf_1pcY4R#A8gTCMHh5nuz~Dfns6;{=^M{4)_y> z729#O^Jww)ZB!jUNQ+OD)jwJ6RB+&^;e1aIUNV@Up)k=$|7dPwDSA;`7S-(^FSH0A zPSwnRJZatrSA^eVccBGbu4;doV;TkRcTe1VfuDZW$!=j!7)~IRBEum?k0@lW87ecI zxm~90M_EHe-jRQ`;GQ9(ZijY>)_nFC^xn6D$fyt`x)|~%_2oe9zy`h>C;^brId&5L z;O{Xbl5SLjvWh5^jjD<>2Q7XbD;k*()Fiq+k|v1C@^cT}<{pZloEe^AAa-#6lft;b z*%|t6Gr1NZB=kG}*`s$&I8#SCd?IV_VQxh4a6s6vuBxcQ)7z5H1v_^}gN+xTcKdh% z_W-A-W%qzHUZF}8ZwEAjDFNgJdJy^RtMTlpu_0Bk??A#j`MUYdJVBDCBq)Iv7z9JX zhK*Hj2WcJ!qJh9W7NO`>0>_t*j%O^&2S9i84kb_SF5WNg+fqL(MF>42vO@UXK}UB^ zUdD-j@Ex$o2AErWjy@IS_iaOdB4M4UxJj{wV_*TlPx8L@iUaL3S=bTv`gN}gek4L- zk0^jap8a=3vvEk=xb}wIf4~d`ziEid_J&v?|6sIsH>x9u>=D~zH=09GF@Dj7Z>qnL zh{KE@GorHeb}bX-#bdmclZAdVLm4KtBobde>*0bu6H}XAowaocP7~r$O!Q!Y zVHDMy`T2Y-vr!sK*7x_*GM8A3%7fp%>LdaoFWT7Ti{oVjGT01{OkNzf)}+-0-v|)ysWWo1;LR6}?hetU0N#K=D&VAsvKg{_!TUTXh5<0`n(Gazjv zK^6{cGvpUD$0&Bbcl2YORAIsqRnfbf$!b|t`XryP_@Te2?Dsx zb+dzD>7wCGM;OsuLRC$^I~?lAT5P&o<~!he@8K1|xc=4IX!9~>$>?EKSD&P)E9bH(MP3>ktU!-J=3v?*Ow zDRR8pKw?&4PgzT$Ut&)hToEh_e-S}PZ(iyi7#g?Bz()!~ITk%VIAq&s64({Paa;06 zg!Ol|b1|H?5Sf6rzqP&aegFm0$Us$ZBr$gIX6!6s8nI!i;tU z7@p4|cNVLeyopD9nuQ=$c8&AGD^i(&b^(zGUi+v8jl_~j!I;1o6bZc?UoeyH%vjPAM3PDBr4jj zA;05+T04DoYWVXV>%$05XH`GJ;R|ZQ`T~h_`_@>4b%8?di|gB85Lg+aWL_8cq&K<6+ZidSxK3$OsqIj!&ku9Y!mlgL-LEfkI=W~ zIzj`J+lqZy*(UFjiFdp!kc$oWLrd}vbAsK;Pc#l4P|A|6wB@l!E#{>bnW=*IAnJd> z*=v1C>7%|=F82ry?>l6eAJ9C{-LuVPlchFgq#Wv#DgL>qDZz%9Qs*O!#Sk1^EdpOUk<)! z&2xr!O8oL@=g^Y?S;yF!P z`pu2xS27q>Po60U#CKqB%cuM2Ss3lxx`)rZKFE`Bo;AvCN&QrR0;WBJOQ2KteLrtS z`D@-?hB|?ta93hbKQ&`GV1F1qbWmP#R-u~w`kErKY^>!qN&;toQ`)s+;Y$K$C0D4? zW|I2m-pI<;=S>_^jHv~`?@1pxi3boWU;ctO3(BR$=F*crgo=bsZ@3(;#B3so{ow*R zq!V%=pmbI8q^B(gb_)Ve>K=u z*9-I)C9-XQjU;=vLn0|>MofFTC#AE-kGXQJuCu!reca+@Xxa+d;BH#Euoaq?VkSpO z4UJ#73Jc(hd6EzzT1x#}ESruxa}WAOhCG#*cVSpJ@?fp}8_w%=)c~HCKkmKPD!Cx; zp377DiG{_h+*;*T$~_;Ivvc|lzU^F7erkZHkjzahuktVAAZkT9hOZXz91^c}jI&S`xKBI`Q0NxJlg`+2Q)^1j6X}HtG6D#+Iz_k?)dAJGc2J2n~q_M$cDrr?6Uc z4ceDZCW65&71m9!ZYP1wv$IzH`w#e#(qxrA75eiByxW+=)OBOjxgEl0##Kn?MPpp+ zHv_i@1fbvZno~k?u>Y6Umfq(9>PL)A_Ky~}!y`KAIF13#G)rpfcy;oy&i>=1TKS~2 zOumat^*dNfOT+0?g@i_#>%Bio5?{A|r&VYNP zDjR8gh;W(chpd>}aI9ODCm9@%98r%U4s-4vN35mpyrBW`L|;~q{UY;`_?1$d@A$d@ zY@yu;rY>a!5%QoaPKFYGtd%$r@Pgi)uA<2Jck<--c{m&n5@iXy&~rv!7}3|*l}6O? z*oQ9K#P&PCLQxAB7$|c$A~hi1p0gU$0)z#reiKl|I<4)=@NVegvU>|3MTk|%fmCpH zFA`4v6L}Y(>+p|p3%eS`wO!-)51`-NU=8+v;eG`D9Fs_7UvTh~wE^N9^F$P?mg`_#lI1JB$;9Uqr&-Vp4PY zB9mxQz--X!0^SM9!{2ns?@&Jfc;KC)$Ie@(g8vd^FuOaS0=jRaE z+DlMG474{@M-Oqa;xcY8rsTHWBk4rIsihp)ddNB z{l1u~W4lT|f~#FNW74a=-cIcXDR*R=E4y1eA%wvTGOO?$On z#L&+sIB_5dVkE%15-Hw1yKs?n#`6SxsQ-rwleMIX zxIS2Pio}ad(w0SSFGd_CQejsSddF8%B}k3o0ZwVd3P+fN+uMZfM^4M|Sd4;kMZhgXNVUV?y(?YiG{!VZQkYjs$3cH#E8u@XJQ&02V5_sz}WX+Tp7kp#B^nR z=}$Tw+@Z(_4%}4d8CIpHnCK>-|2#1EPd40m^Xqn6-aGql{?; zUX#+MHhWZR!pkxo+#20ks(IMkwj1mDWsvImOZo@OyJUFfUIqoQ#cA8E08ZC)yTnPT zuX-d?F37SdYl|#P-RqlUJkuGtkY(-m7ks@85IJ^LKIS*5dg~EA9$(wf8B0wkrV4hZ zs+nF-N5Xjaevxp)_b($&3p5YmG{KQts7j&W`ytO5ZhIXNf=;85ok5tVXmZud(KzzYotgv!Fg({KDre!s@$B*;m_aVWY3r_;X!hojRw%{D)B zqm1P+)oMyvK9o;14`G%&pxwXl%+q7yP;E#?B z0s0Fw`vJ6e;`YiVO^-X~0ni?9u%?cv!$PM3Aa#aDRQE<%(~X2LhC-A4MEqmVs(Ia$ zR{4i9?ExuQ*?#R#ap(j^as@z5l7|zZo&1$o1=`5ZS|NT@%))-nn{0?uIHr|7tG{AQ z`l^{oT+QWAr94Ag3#0?Eo-D8ftkle#{{1gue{)OZwab)P6hzbII5|sAGoSl=91%lM?0yg7n%G33sup zrp4l85Wm=UB+ZBT2P##2Ps=BQB1S7$2Np#_r)Ve5kCmJ3s5@z8o783J>vWytg$A2} zrw{vOy;pMP8_YhCCt~u8%m3n{w(ts=l47j*j9Tv` zRo?`jsCqdT7HF`Ttmq@wmKN|E=FE(IhBq|L`ekF*XYrU*aGJ$~P-m;k^15m-^Sm3X zltPi?NV+WujF{$zTE&({)0RHBKtdGI>=JmZ9~yUfp=<%G-!vwR4I;SV!k4JqwY2#Y zSb8A+IsMUYZmp(!k!~we$KdsNpXJ{W!Z5}@LDjw@6ekCW0?UBn*63M{swx&5{1EyW zq!+muW4?njn?>dMz_Z5ocufxI)TvDK;CX-zHqU*kbgwW-%AyDu_yX>R^xmrD6tBas zZ#TTBj{PV11L#MS5on+Sv3HvMbbWT1;^>4ba0}GC3c`TGQo^DCGEdvdZv`^G*N)Ch zlL89DaPTCdECu$HU!KwH1x-dRe3jd1O6+yr*_Fp!)ZOxvwQQ2`5YP_<_@do48bG}^ z7yCZwwyv6JIDbqEYs?btFYM$R2--45x$kg8c2vx5Wl(@?1*{!8>lH*t9w~KETZ~D{ z!y-8gBQk@VbgNz^oiEA0p;xdlEYM-$fjeOZB+w;%8B(>IPvJnu+ABAd<7qMlcz-~~ zdk9R1t7_ZU!nbk^n>KC~MDJxjsIDMtLGmoN*ECrA;z(+b8(;L_{~Df7mRUVy2H>7> zF_M2Cm-m-#fJ|m?v0*<-_7RQ=_4X}(GU1iirB`4n9U&wdWXHr1A;N+QOU4K20W%8W zSUL$Yc5YZG!^;!>K9Tg#_KAe2ctfO2arlN-8DunBLT?(TR11sYuZ0^OljLQYAqjo; z6d}T#ji})LbLnt&;KO07V3-=+9785Vd-og5Lyrrr-ns?+Slq3Gy?8P6aup}H17|Nx zAwTGtzBgs~E9R#WV$=rJWDMxYAe5JPyNbGi^wO+Tzqi195Cw{${CEGM5}5_gy}!sQ zZFL6RftdN*r;B+p@^Z^J9hK8G4YR&q3Uy-m=3EP&q%<+~@tiAGB1Qp?o5q)oE&VBu zr?3xXQB^0#+#67A(7tAy)f>$ws&^FT7;(VkO1%uOa1ZjCcBePfv|(z2v`JIWqO303}U@<+h? zFsGz#D%=T43DI&JcbJj7WoB|tHCt5^W6wh88PNxdZ-I3Ez;^id@~bkcvGuP{#%pzC z!0g)8@+$~{`;x?(=<-JpUO;69F=e-EVcGW4x52-vu*|_zG-0ckybc;{eN%~HI_c3) zjuCWTm!`8JjATUZ=DnvF6SGzirOLW`Rz^a8(a|gk4W3XTuA+X2lLrIjhiFydn;ni} zTq4rRf|yIOD5zt*s0Kw%!yaP*_U|Q>vh8nanS+;selrO zWPGai42u_MdVio*ye^s>4_8c(z}?H$*TB5OjS*>x%NIJ}!8TQB6v>>#?f4VGxE-GOu<q&yru-u>zqB30RS^k3sA$UgeQJh2UWu+JjR%9Xhw|7;2yV}GBSN9_ zNmx4e7y`x&(vr=djB6b?W}~&nXi`}{V0q`9s3l*{{pW0VM@|SslINU4fjM|s-nlU(quH1WUiC|%V0&DncZ=Z%8Xj(t zbyN$QED^k7R7#s9cW@wJ?2_g+?$eGURz@^WJUsuOvs*^!NY9||(lH;h%_~j506AFc zJ|Z>-|Shg5@mT3ixA~oio~oyVZJ!@{$Ncf!ASR5vE{&alX_AC zM%p7*2Dl#0@ z97!^fBFXO7K#@}~9+sMLSEhS`r&18+>#m@NMr}u18fHKZ^l)D=1)3!dtcat?;{b%o zafa*On@)#%+#03!so4v1c#4)G;LhAS5#Olg>-Wl9sX~EP&?LWK;|(yVy8LXgBibui z=<655_F-6`F?~y99Tgw=4A4aWl3yu|Bo2N!jGJ#Hxl44qO6bn){WWk4h&hV+sOf6S z26xLw9uGd7z53!*bjRHzZ~<6=J8Ep{`f3C7mJOyGu65oJZRt6r zgi4>u=VMev5?`u#^5>k|E&<4tk?f)C-+C5YKODzgg&GDmDUdg9Ogk8$YA%s(OH?ug zeo3%_!G@gMGB4Fx$bBf2d2y9mfp6VXG@ReMt&Nj=6znJmI#8C7{Oj zL>+aiv7w$-K9f;{v~%+aqY9=k#`CU5`6nOllsMyF((?{)lf&dy%=+NTD~|NI(8mjg zaJh}(#DH|1q~7Yb==?3D`)TopO)xBV78w;%voX^vMp2^iHb9t4c1U5QnnKRqo(lOh zb%mV+E~TQ1m8TMvz<1&7G%0~@z96p>Uga=X7uadhTErR07Blmo^I3E{;a z4LM(FZg~el7MFXe*y=5c{bO(4K`(jpzW9GoNdjH5W#ze6yn0yVcmUvDYA|c2oKpvF zw?mx*0n9TNqzP=N{6_vpQh0)+z!sL>$?9};cKb`hzn~qqW5P{|#fD1Kp<2==T&9k0 ztiKFN+kBkmpYgPE-F2quPb{<`G?~ns0*qdM(cT0%kD>hT22Ty60=YbJF@iGlFWOq0*3&E z@8u-Jz++DQ)AU6r#I@)2@%=uK+4SXeIY?L*gnMbZF_ z)!%Y`*GxcnU%+(R1uyUcDtTJKL&CY3*UU@Ol2WFjCg_R)MmQd;+86T+<^6!V(T)FV zk1{o-GZBUz)9V4;P=2M~c{a-|y-Qf^k2shxikA)V0EsVDMaLb7XNWX{Rc=-W+0lC8 z>^#=byiURJ^7=9Tb9b9bfX`ge6FTY|kAAw7y~iskAOphC?MjBB^dqR$SY#d0)D8lj z2t&hT9~EZK>C1V&*=5CV7$f-Yn4-t$26;$##gpRu#L2tL^9}{D*+h1*1NU`bBiKBN zFay$<3`GE2sve!kwPo#v32B{5af@RL=u#yL6z)qX?$Y1?VryrZ?D zvFkw3eNIj!RZh!_R%;NLd?BOci;^6V@fCCJY6&7ES^E7Y=4LDaAX*``3E@=;Bn9y_ zC6yohU;Y#{HgOJi515GS3qIq;=vt6-p}F2Kn0xPzbJ#2Vn68_1_@0iIf|g|yZ~l{enK zXKvy2q0g92a0?0{LgnQRmY>19*KE$ACIg_qY&AY0!4G5$riDKo6d`CVtzFz^=FrBW z_kWj24C2;)iub`T0A&F2a%4Bgo-wqZ^FpgIuEUoWKly{&QBz3oX_$Z!SMwgI8UcGr z%Dwf?*PVcwTOO|%U_DFIU!2|RU8Mr6`N}WV(B`w4G5jse$eP=^BE^TG_B$Cz+}-=| z>!$*NwCr0`rIW&LsxbCzk3teAK+JO+d0#>F&UJSwkc3Z9MlDG7qcCfZ9`Gn&xBSAp zdo!sb2BcZaZ~}V#UAPm~!|*ZlQ@3dBLE%BP1=5>?2Z5!uuzs)T_+#7^*dN|A0aym! zGro?S5Y8!6Wf-_JbmvpoR)IIvS33thGD+LrXkuj-@yhmPv#d<}H{UZ+#P zM|Y7gH1aTrMNV1_aXpL;tlQlrJnGBsszr6P5A-pQ(x(g$qnXvaoJf7Sjh{0w)S9=9Lk+5`*TGM$k>Wb%hUb)UwUWXR@=V6D+*Zkv?_{4y%J6U0jHqZAJp zY&~=$_X*!`mAIB*9O;M{5IovD(En#jQp6-3S=TFH)^tL$O9h#OXOH0ZukwO-A}#?& z;>yq7jI1f-M{nqsY_Vg|W1tH>?~vy-Vqan~g<{W$i^u+xiALwHb23qdP+%_+F!BoQ zr1@q$R6hfmX0Dtp!%~$7=+Ha8>K;W^5Nxi9Wk{37NGTu?;Sp*f*T7vRcG&O|19E~7`8iC zLx98Bi7Hb_2}lgj2%erOZoe(w-WA4tIGGMqhuP-0BukBUDk2h8qQQJ>sG{g92%Qqj zS$wdSk$s^bfWe;%!_`-<<|{plWZ)OTWbg5{Mpr%IV?gfY;VO-zHf#%E0>(Qw$Fyms zTsyzRGOGwgJuN`MzL|gV=5{HZT`%(_qry=5&O=< zpkdok?#oUvVAT{4j@5VxTU?PNUc)%-2Dc!Z{F?6!$j}7`Mks_PhQ}_9?`jch1;ppG zrNRn$F3QGrRL8r*O$RxQ@tK8JxVPkwL0Mj`yTCq$@W+9CsP$&sonkDC(PzoEni*vZ|Y!0?({ zsnh31?0s!h?h7B_*J=}6ZHw5nT0OEdM&cYYWs^GuH?7R?U|>iZTr-!zl98!nOXR4= zR1qSBhW;Ds{h=nNNPsnNrKhh+dxh5~S{0CXzk?Q~L_e7gA*Nr>DA?!UI&N*GLJ;

D!dRC8rvAg+DoGd=UX3BPnf|g!)B@gz5V5J;mDeG?eX7C9f@Flh z^m$?7gZvPwKz!W=MgiaEPkg!4l?Mzd$|}qQ;{>aZwttS<9LAeoS)=07kWT9JFSW{+ zYAn(wLeL9jdNHu8$AKc|Q^=aZqLT`?)oI&f4L&3gqN9BTeo#yvZ|S5Imy7`CeC*Z9 zjUNe*0%p82ATkpM+Srn!TZSY;azrx-ZevSnoI~4eSq-@FJ}AI2Q28n-SlpH!$L*Fj z=g#D7250sx2Hv+b>0++Hen!XF5Oc<=_2G^}YXZ)IEWbLgZJeZnfu!N6ivhSrk`;=m z<`DDJ^x3@~^O-$8jI^OJ#n@-Kw8C;X;ckN0kfyj+winM(^RQHChRF_W^p!%w0n|$! z5r%jA;>Ot!wJo^@O%^T%Xl5a9&2gU&Qac7e9QDpIIxHeej>Lcn#t8uMN$c2hx9*25s$~PUG(7@Q*Dz@ z#u3_^wEn;aQArAMB*jgdmRu3^%_Lr9!+9}v4E{HV^A3s%`TcHcwkX-w-1Kw!wm@L3 zg0cqLFl$cWzwnR9YKSoLU+$QTXM+XRK9njhE^0HqpmrabW0a}me*EiO@`BR!&0He~ zg1)Kwl`VuREW49cZO}2=G@bX?$GZYM3e&r1YPL<`15v70HdTy5wnMgrnJ@s8eBMle z_|B0u#Y(Yx5tInb`p(=c>J;EooQ!07j$e=*4E9l2H(XIzMmzm>f+2#ZxA}o`j!8rQ z6N9#KoJcjy7RV2$g3CROw)guUkwxJBo?p-Y1rc|FvJ(pSM35Ti1m@W)h`By!WaPyfzv$z_6Q`s8ib0`}+m;Zm|7So*2Dip21$j>fJ zq3U~+=d z$*R!0Z2H4&bSNRnJ2yqv{CQz z%7ZN`X`?DnP+Ee{Hz*>|qQ>x~z=2_kql*bJ_7j>BAz6N6!6Kk?z;y<1>ltIDm4)~Mnc)JQ#9kN%_1#o-k zW&dP^q{0#MsZ&|6*HD#~suMH#YwjhX!x&25wl^&hw0+=VFe#w1_$H>QSn$e^mMN$4 zsQ5{c*T?uU1#%P!9lKW~blI`i>KL%*zj)LseTXx;GkCsa!0;rY4x9+eI>g_#!-Mlp zs9x|But|e(C=B(oz(chH0(KZaAPIGx9C`1HUt&-m#|-h8{40EAunKqMev6R$G)iIU zT(I7&rEj!q098JLHv9uIGS7=v3X#u5F{uq7ju#W^A!m-*IuJ$i&zT!o zP^w-)+U1?=enp`6_z2 zZBNS0WfM4Q{ExCfJoq340y|G(R+pbCJ4HyNEso^p3rG<};t~{+3{!17&?9s@-2jjW zvpG)2D+JXp;i(c1e+IC=l5RQwPw)!r2*X!;iuFKXzQqjNU4raQscSH}z_+Xm{hQls zU{N6K*xnXlZ)ljyfr^oA2{M*BN}~Yts4;7A*CZ4#j$@V^z8wLLcoG$e!wu)q3sQk} zu9l1~H25=E$AMjecG7NtJcJ{CyxW9$0(wvc%j+l76H)sQLYEc)L@-*`-`@b*!LmRl zS8v9c8{Pmq%Yk8h11JlvVm1e+7&TGTeTD;IM| z-Wg)Y8Y(VWi>Cn{&;mt*t}OkqmDfO{Kpw+EdX!|BlJ~Y$XYrpP#kD&>i`a_ER~hY? zYg!27O1_#3E(oyvBMvna+#+Sb67+6vzLt4InEDH^LGh_lEPpwZ=5;#ABMrA3=$DNh zH*0_PO1e`GkIkQt+KB&vQh-qm)oanU%m;}x0GuxZbl=mHj|m;*EfNLX=yN zFqI}k5l*tZD8?u>52I9^tolAxEawJh|0t?pE!}zHHw4~*vLH8KiWmj}fiWC^b-1ef z2xv}_tiWly{d@U|d2<#|DPlfFeSiXfpYR2uT-LnKQwFX;2p2bB_fk$z4KQ1K6Kut> zEdfA8f+vqfB?)I>n$psB^cR#m$?T4QE=}SD+rv5vA7xuOuOki=*dgW>hK^Xc8(=&n zqyEX5*CUV~xA2DW{al<<+NE(651F#Q!fJjC`PfEl*k?k7gU%cJcg0_pxi}84iMn5O z^_0U^B8*g+ymy7p`k*YvZrOK^+7$Tg7|Y+9aEYH~oHpv5{H57NIjX8WQIx0)KLUZw z(vJ1q3mq#0NQE$6ofw*&A!LA=35mfsV7j}*?sMr~jad1_F-NF=DCrJ;D7*_x+6tYl zpyUkzBTj)73quJDbSG!S5XWi{3wZ+CJ!;-&6*85p1IYl+c^-XmlQ?^@D3$k5W~uP; z+xLxcrwIruuP-huYM<~!rgWTaZuiE|{*cu4PFtu_P^D#47N2og1;26GFgjIFwc5dt zd2*laNQa#VeF|@_fes)()1|$zw;E!8>ApWu33TU=fBuA49f-32UxHWxy>(m$DTC|} zd$5g#v%6fzfsquteva*f()vdo@rSgFz=2@g2Hrq#_y|ebh7-=&oM2Ka&~w;q%_UYf zNAdiY7ZPOzQM4z({oZ4Uy_ED9FVeV~UWZbRnfxV)T0_iFn;xTafF;b1@5Mzssvao* za6i!4gWKC=f7GkLC?s(Ncc|GcR}TaRPT>nWzzb4+F>Na&3y3NexdMyV$RxdCC4e-T z7&@fCq!~QN4pJGcmGmSsCxEpejyf>z(6kZkAnq7gFvyl@_FZ@=uOAXNv=t7i%R)Rr zF$EqL1fZSZ`U{8oYls)!;nC~Mt}s5RE7<2`XDu{!EtWv!le3k_jY)=jlK^l_qNydpdp56G%3dK; z$UHh~-Wb1Y>Zj3pXnbBH9@v)Dl#Z7}6#lyL7H)SCQA^JcM_0d8Awl0PfiW)?*q)j_1JnlWZ4=HHZv* zu*!QbguA7UZj5WC2#WEd%q-MsUVSXo3J34C#xU0MSm{k+=7vU7h%pJajmvEC`?#ns z`9ilSEE|E~#w4(5WV7(h!GFsWtPgH7@CIWq*I|^mC$Vy2KS-H6FS4h0IKP=NW)UEZ z?>q6vfHW#cTV`_`f3^+FFHkKPpUDl{1@5ZLL!RAZao$!t>F};2!LyZ>t^)JuWT802 zzKW?`D)MAR!r~Z{5t}7LtP)asTus0Ag4D^_D{%ynY+*$rka&j-8cO>YMi&W85z&nz z#g>-IOc+g9)5%@OcFJt#{{zpc=n)p2Y%)#v`bJFD*u&Nf-8QK?xwm*~N-}tFWZa{m zH%pa1w5O>}Dfm#HGIW?jtpxW5-w$)Uj7P1P&x%<+`U~r#dCA<;5^hECAK<gX4EZm1sntV)+abA*hZ*y#j!Ue&Spu~;9B#>bhog5u@05Y|hb7U5P9wKk$H5Ox z;y+(IJw-axV8yI8L}5gt9Jkmab~@Onw&RFn_X=y5t+gUT(BCkz1N!%5Hz_(s6*eK%GB;zn}}N5W_R zyiXqS;kHfOLJ%pzw!?+k|1P%@C@o+$!8!r!LLSbC0=ph@hLhXOt<&&=@nl2^p(Ol*`YSsv_xDy2D&t(St;3-M!cxB zKLnv4g1EZ?f9T!+SR{e4b*$Q?lFgr)KvV^+#YQ!6D_mc|Z_plBoGYF=ewGdQWy>_! zlh}uT-o*?043gv^a&lZ3ugJx(5crU*OC%lGPHTF4S$?BHO;CTp`8nSXLnRQLKx++u z5z7P?8Bz_HCpnL^u;U848D5X}OudbHWN}r?IN?D|s2poZrhC z@`iNe<_3Zb&-iqpbIwL7qaIXd%dSeO!|M?9R~=4u#xq!faF-}dWI;vSZyonQ^Fh`@ zdO8{xY;nQ{xC1p7Zr1sy5t7|t3jL`ztl<8qOuxL_u78zZOS`r~{S}qn$}1S}d7MuI zVZZ$LH{Ub5JFPx!n0%Cgl4dl0YurNceaHjiPe0&ov*Z!K5&NKOD8FZk2)s4(FN6$D zsrXVnh@lC#0o4^)PYIC-O6;>1=gQtmO5h;9P3+w%rxE`#4^sESfxKqi0B|@v_KRxT z98(bq?AUM<=42R&2UW_jkkvc{22HFOYeSa=864TxV-~NKzP*fG8HWc(PyvuAZ4kE1 zJa^Zba=|czz;6k)Eq|NK2B*}%QYMpu()D@4_d zd7$vII)CV6Fahz`*>BN}!8w)Re?2sHNCzD6?-EsLkGg!rM&F)Gt@Ma$yVek97hTdb zAOGHA?b%^4EDqIn%A8W<)w4?$MHKRwbHCI-O#iX?_gndoIiLcV6W<`W6_Zb?UhPr9 zfuYTnXg*nfzjGmo3fFaey{%Rtu$wqeaRyK>@*9P3fKg9E8advnHZO{#Lv2chVdCL+ zwbfu}g!B||A!xb7H`KC_tHr?^!a&k~ny|v~pAVcbejcsI*1ap}x`CuKhv!}ctDHZ; zy*Q%+C5B=KmNQeo6so~Muk#4Rq-vhfWZYM%wa~${ z@U*-0g&w~-t9Ecf&3A9I0Z@n|Eg2FOST2`8m4J_Sp7c8F^X_3*6BqafYB%@M3iFD@#18X`UIj% z>3)j`rWXh}vpRu_L(_^hpN*2nESn9=1u}&FfF84XjzY%|5@nDE1bQ#ri&?7==QTsO z!e!ar@ zk$a$8j4ms1=ZY-qEoznREpfO^3i_w?#xZ!Sn{3}xCpdWjUSQ@267|7Gc*Eh~^q458 z!tpV;8F_PRT-`b-0|W4|x3+Z2MAVG6twqp6lk#c%OP0jZ>qk1kpHC$;Vt8zF*3$?Po9&!F98(w4g7SP0L291e+Hn;p z(29yd{mKxPbJsti(qO5IWQAYUv-t*_fYTyK32&C*7Xb{W)EJGa_{<9$+ zDE7YiRxy^K#+HeeX1! z!q`Gora+|*FNYJ~A`^zscq#PD7$c@)E>nmU@0AtoCA>kmt*h4Nnn_{zE^L@So4I6? z<7Z>pBX&=_cAdKnsE@Zm`=Q{b0m8sX&2IJ|WzX&zX1Rs{^aDiJjAw?JeL13)&i8Xz zbtvc?9L`Y26uOh+$Z8qGf+?Kzcg|XSN-;kbLh=WNe)y^VR)GiaKW-8*-TKr-u}uuS zoU#pjPu~bDfq1$B+xCI=bG#7hD9Nu^a`d=gGpKx>u8!A!9{ac4#1tkw|lEuMy zLx>6q;g}EyU}M|D^C~im7#wj_-eJKGan|(P)G$_RB>>)*mYo<=%e!X{uO`XwA{Uga z?;9Pc0PHgQPy|3?bGx!C3~5bJxp~gubeAVXlPB?0zXyK-WHPP|X$rUF%{^s1;i?*; zD70#VY}6ca^Dyie>-NfN1GgqW@s5e{J0p&X3r*@q?;=eoz>g$M3QUofg#$+#ypjjb zQ87-A7gY$r0dJcX77<=zYli!6gpbj{jq#$?%kbEg5^7RlO7NKM8YoSQ@=H6kY(YmW z`3X-#tT*6xgL+ZDd+&i@SMCYT8o&rRAT|z@*~E^t*zuC;VV=8R1S|jil{yv-Rtsn| zQaM5S=RI_W(M+vb_NfYV;LZ8s(loI=yq+2`fXg<1U5%cQhJ*UqsT7+4{OprPc*=~q zb+};5KWDANj*uq}7yT z#$Xk4M`80ViF&}Dm7=6|ozp?je*&bl<@xgRC&Vo-eZj@gvgA(g2?;H(GaGrzzP)UA z^ywFW4>B;4F=VA2KHw|+46|<=vJ}JZyz-BUtIhyZymy5JsS2{2Xs|F&T&B=VRPkr{ zo5+EWSy;?jA!db#Ag=?e{X(Mrf6w@$loJ^IK$b+grOo?%;K)o!qIlR`&HOhJV&bM~ z8D!AKU<2b{N zXO5R;^0O{hE?;Ph7V&jj&6HlFK_wdn?px<}3n2 zl9CsX$NlEG`jYRAe9pdzlp@^VLK#goIbUm0PI|DUew5IVxD>C%2}4_Hen#k*OciTA zzG;N@{G+|_scsU<*{v1jxsuY=?%t1&FYPFqwTI}m>@0J+=;R>W-c%s|1V-o$fDp|x z`ss*34(AJ@e6i(_@VHwrDfbE(6&O?bBmNSfa{RX8Ys4lbjdUT!T~H~rj)(nIk~Kta zbq?c#Ge+>wHqQ{e4@n%53Xmb72TAb>Y&`E`a0T$lPCpEQ@tNB2aOjsQ$70(5xc!@9 zP%qw#zyXy4A7>xpu*d%yp2QcR-43b*^ck`@PLaL(n!`}8@^tH$xqTo7wiW~Sc#4pD zB!AZ8ZtRlq1B5RswTuglwInmWIVf>G9NtnshNH^|cxav-bWzERs4Uxc4%`V0U2INM zMqf&Z)h;f>S%2ItD@qB=&<}E8ZkJMb0?Z14;MSgE^U)abVhXW7pngyT;7&ry6XGyh z4L>4|qM!c(7ii)>wpt`QPJrf=lK-MsJlmzCR@Y3`balhk|E&~T>7k9H+k^++uj0lZl5<>LF&47OjZ)>C$c_FIi_A?}O9 zLVizxyI*p;mmxf119$oVhw&k4PoDglv}l4ie2>PFsv#M%b9ti9`@)h#cB1-7ji^kQeFW?! zKLDO_at3us8_U4k>%z83Infenvk)2>d_m9!VSJH{Sgz`Tm|6&}dY_BMkwChE{5>+9 zJuqQ@d}THJS6HX-9=zY9t{=*#h5G+^;jYg&2?huipU@k*@?t1@A!WpVu$OHCR~CWH zSwP`PL54SrnH$)}hm^n#V3iP{Y0qie4dcer0*)Inn3*6BskOl48A1_Dk&&a7ImC{) z7d{(7&w(6)0tws5aQZm<@hrLZ2>7DQe{ zEmK8oAr5fRu%TBeHLzG^2V$ayWPhxA8(R>1`OQI7TzMCe92?IqJP-~pvcW{;0^@z93H*LW!>a9N-AEXZsr=Owdd>rTwSd0Rh2}86NjZ|d zZAr7J1WkB-Me0_e146Asra54{A&`bDL1V7YhpS9T6=>wzfe{N@`;uKmBwTRSBrMSn zMD(k$M;N1rYbBVMFR)KCh zb3TML)r2kv48x`|uvMFJm=EEFIK9?Iy2)K+ML;nEEI&%RfUb=31H;&Ne>mWUd3J|* z`Z$hpyYspleIS4>`Hou-Vl~)b($;ZprM%LPvJot$QbNd?*uusMLKnm(V*K{Nv{P9Y z8$97UBmjlgZ=#=nk?CAMZ9#DOWo-oKVh(s;!iM}tyqs4bubCfsaDTYRDNqr1WZuM2oP_V6EXUix z;P!hA{(<}ek(t_>Q<|&T2{YFsUprHhr>i&>R z0({eRG;sQCKE(kuF;5)J!FZ8tMTQ$+C!C1vEJ#+c6NMf(+&fTdJb_CAk{6HY5BLXm zFOz{p`L+%C_~xEwpHoK@@@L?C+AV~ErYasai2N9I_Hb%Zv-(wj{#`5FX1;=dkZ>uu zO?t2#ScZ|AO05Ijh)SKxYm~Fv5PC6XhXzj&qk=hh$zDVn@Odvk6f`EKJyG8eA)T;6 ziPRV}0`lF_)ngWb1GYQGK!Aze-|L2gkYP>N8c( zZWY2GTFoj}dw(bdCUEz%tU{!+ed|Xk2cQB*$p=aXpE6tLjsFoX5~TW{oY&XB8OYbV zA9wv+Zgt&>UQ%o;-kN0^@8QyO?(g#$hK9`oIRYDL;NdEMS+uP;^4$jI!PrYe5PCFF zChLi!qO~-#V6sG?4`{6VTbSY9`Yc)Ap<}-sR0@7 zQ#SE~L(!pCTym~==(M5XIo~;J|C&(U9X;}V$u%G#vauq?SSrQsPBp)(bE@-oTI4`5 zCqc+VP}p(e6ByDZjsVt9W2*?a1YXG;FGm4zJ!pP#MZ*PNQouJP%>xy~>4s|ciOLdm z{bWibG#T(4)J|7AOA=R`7hHwuH3>7u@L}FwR!tC@BIu0*Bhgz6eThPl1W{AOJ84;O zwy>rNbh^~AyPp4Y2gy)Uv#KZfFBsm|f8>969a+Q2(bc!8@kj3uIJoLWqk8;~3GW6&7v<&2E)3Pk1#wT%nF>U+DBIaG~bH1fI1iktAHW~jN)K9^l2xEEnR z3LaNOwHKtUV~Wa1e;s9Q&*B}Kga4O;U850LWR)$%qUOPHnUU(KE`FzsQn;7?i%u+d zp2LrZ0_H@+&le^DJY01^q7pW>pPVPgP(0|=tPUh=xSInXK?0sJ$jsvF8XV+G;OKu+nMqFC2 zM)K9=+EFq^ciM`W0df%E3ULC#L_>|VhEmCj05JZY8h4-`@%(sYT93GLyzOrVFAs2L zQk|pDj>o33^C&(C%WTQ#0%)9s8-Mvz$xEiDwByX3W6GvUBA5;kj1I2&o|oLR`?Qb! zUnLgpSpgm*|K%R4HF z*Ud%d{_vA-2iIrJfX`1t3!A81&h5@Pnf&Zf!Zwfv&sgbVZWvyQ^@V70t25}Km{+Nu z=J#HPDJZ-|MSyR3-bxB2Ax|3ftq>m2s1B_P=I=I%>y$OD6Yzx4#B|Ukj?joW8B|_{ zlN-oaI2Z%PQ2{_j5!5s%7imd+7^V)F3`{Gv!6h#9WBp;m5L!OYtLm;j*TnIbt} zMIJ&M@+#(dpn!ZePVn=X+NZnA&yE4l0T^1pJW1fF6h@q z10cYrF5+yvQ)Q!Cp}+@*)FiYKSjcohgCwt+7=oQ!KyVsJ5C9EmBuaq`V7fpXunpB( z0DNu%@l-(58uhQ?E3Y1JP?f|CQepI`G!St1Jn&)r0DkceF>bw;wgj#Lp>+2ajqE6T zZGLEE?_M0`l@UA7zOdD8ET{o>XKY_Ah+c#%1GTmqXWo&XuOwY+=^TD8J9T{9;P=lxY7SMm`wnS(vz0R@&q%_r;q-&eM_BEdq>n#jtmRkh_1)J)!&oP!VE8 zN&A1lL&h);4Ydq_mmQI`x9g`w#|(v8r#SqHBAA`qcoh=uqr`3rhY*u63*Zg~gA1Z5 z6?_sK4<}CuNw`fJR0_U5|bL-(r7)*E^HTzW81kKeG8f=Td8x1cLztBN)tQ zh~i0Rh*&|qe8uq}l&5j`(q6GA8EEyJ5mRLIuqNtnjwdynihMCif6V$xK9-msl3U9B zFVLyjuuPQQgB6i4Y+6_)x#Y{BM;B#gNvkg#3Kkx&L2@U#BH&?KS~y9LNu{)7{Jf}a zA}aWbCP=Zbe3MsCoDuP-;CzTbb@@N&S^c?hCN%QO5jR1_eEu#ESRpi}7gH^_MYTSSqkl7N(oud_#UqU&! z@lsEi(#RX%CD!5K1^|oLRyF@qk3d{eZ1GNOFjRtry=AtC_;u<8~UXnct#qC29BhRV2x;Z!e>He zJbSf-M}%{PDNe|Qu{3OJEwEJ|)E2i;U_Xp-rtBk6!azR1H?kuwEmc^3qrLKzJEU6< z0-3@1+vN+2E_WhK{rc=X{82q4&6|-xpf@T0+yPk{Ra1#cgn5rk{z$O&HHP|x{ZKwH4B`+72)H+oNGrC?GgR0RfFk4h{j8>6Pu|OT zg&|?yV>FlW{P5W68^@_xX_9UrfZ76UL>v`ZLn*K3jQ9bO;`MK8I zdtkT>;Ep^fkTIRs?x_I5hx1owtvY1&FhVlgE3ezWC}TSXF}nsYzt<6$_X(-6iRSVC zeoy4cjq!MM8aAZgfG;kj3rzyEj;_1J!6RjDG^{&7Zbl^5BBr*?Z6LJwO>2Y{{`!(rdcUnZ$QE zRU_O801C40EcS6g^|%$Aph@}_L?=*$CqT~#V3CD#v#6!k^YWq1vxqbA<}>~i+j&Mo zkw^%%AaleDhHm+wM%A{q?%C;Sv3Z#(*>bpf0_`{j$dNTs<@Y!YoyDaMsAJ#Y25sAL zo}){SzkwmC{^p4~=283Ua?4M0ptOyV0vq$iq{PDe^>U|K_TSA+2}KEeMfWI@BFPb= z%>ufj&ToL^L%lEvi#^8dt;|h%$gH!N0W2znq2mSw2_2qF{*ZKCp*KL_8TV@8+sMK82A(yV26|zlYDkV^; z5Svbhl45#A0a1b5;+R_{t|&`a8A(~3KvF>9-o`TA$U#6%;HD@E$de!tvi;6EpD%Oo z{R@3~9(edZ-}mSJexLVwpA$+yDS(8@8BYsjx8$4?oFcI|#v(GAo1I9GmVDUIaw^B7 zS`2`2ERy0npA19P>&G%S{a$;AD4ej>qy|e)XVx-e#Tx{&pbr*duahn}f=Q?<$M)!KVN&QD4w|J!-MdN}A@gt$Q&Z8C^mMRY&e-`n`@9cbY#6nsDqzBjwxHyU{ zAuD7M?mYjM{{>uZuU5M{fe_2{T?qlbeX!9ORQ-WNPPp!t^)I*>~L zYu(6(fR1J6(<#p5N{0e!VOD8F88O6GI?-^%7U<(R7*}k5BjU^{L=3vF zJFPVhMq{s#8}wi*Y{mK%USa_BmSM>n?_gsdX99|(UlY1I{S*x60B6^jo!!`nJB#}U z#uo?rlxiyxjT7Q#LVON|{BSqcKY~A)gt((>z&wl!obWAnXB%JFuEsOlGp*c%A-_T> zW9`CI2LM^MW4m9y)WAK=WKCb?SZne5R7wX%VDxI0-f%|4xAyhrd_37GU($OtMQs!1 zxC-R%0my6xo&t&BwU2277(4UYnB5X0f$hgjhPsUjrPi-dW`KcdGVO)&<9su^MfSCs z=qfIs%FFuu9LTEhzC_?Nlp+;mc;&9MECLgtGODrqjgY@8(TyoacG?iKU}B@;7DWf7 z8%HS&STICm&?KL*v|Uh!GkT1?KCCUYY)I5`_U=}S*K)M_^}UmV-Mia2%&pyFSg()X zQ=yj;f&ZezD0#vEcK&{zr`@ir3@Sa>d$<%Qdg4d=`Wq_6BFau6}b8qTN5DW+H3jd&SM>Mr zJW)@Ae%8>HxBlwb*2Y5yzuj2qv_kl>EQZ;xzn%moDET4ObEwRk{%JXHl>CB|&88t1 zU>Q~otr}~!d}qzqz+`GQYPA+^oitj!U(_h2&~rij*~51fbx!>wtOi5Y#mhj)0HuZ( zYfyQtFTW>(ZOi`W{F_rw^8K2V3KH)^7E23G_^#OJr-T#o*A)8~c_4lLC3NT5Y#4k5jcT;_m%#^o z&v?$kZxQ4O)E(0^p^J&h!|5{q*E@DL?B1BG-PO3GwB zo@*PGUVT55DSAi7tC)*&b^?!9>B{7t!xzc}c*|U>MH)wwar~-Vo-mH&F;P9fqE!Hn z0GR??li$seDkd>={)FzyzbQP)ji`-v?`Ps=Q~CGXUb!*%2T>>BzgIJ`by`5@ zK+uqo2`Ea&nj2U>PVXd_H$~+`!%5B`-nC2&rnO)XILioR(nKR$4c1BIDb$HH^w3G_ zkYWryELKvqW#_i(_d?ZG8DjDSf`|bT(OUoBLN-uZqeH;Tm6i@$_FwZ68rn}qlD8;@ zYp{`_#3Va6(B0vk)u-}=nXdkTqq2y@Bc<~--3q_AZ*&KcD~&kZkvjk zs7Hi42d&JuBw%8oQXwB5C13LV5Q<@q4WzN-8if{ywIy6-{&8M;w)%dz&?&=-uyt6D zA0Df5T2AwctQ?}+(K%0SihRmaAhhE=V~mmS*ga9ag@7leB>V^QG?9yNw%=`%OAeBQ zjC$)*{Nu}~CW~#67XayxSg)-dC$+DSa=uX*CjQZ~x4e9>Y-8>^bfSr|g&AfuTQL!+ zVY6(U+B!Xbk5p9L?guV@%3*UQh4H~9b83xv=FFpEN^gBXVTQv@8DLC0z@_KwDAZIK z;3Rwupw4kpVZAxC<6?-eCgbqrk=bN5nPvB}LkMUC!kqzJ7H27EQg}v5NVL8?h zk|h<$Lkv(FjUe7=eyh^2W8gm|K5N~cNWJY0vyF92tI*t`M%ZpqMG0QIVX%y>eX9uN zlOUteUz=1WHWKxKuR>XnZS-DMl*IAlSnolV<5piEX2v(D2&e0U=E7vB(~uPvZVuw- zX#RkLsLWa}&68b%`h!2|Xu)QU;8TMrH;dHdV^eCH7q1-;t6yk>Dg{c+Qlst%8iu(8 zv1oHJ%M7{%6e4Xtota@C_`jR*Jv{{|FU_d9C3KL@Y!}BhuYswgny$1YG+lsl^G@=E zL2<{9%r!ehu2mYDmGP7nQ6hALu=Un1$C=lZ@k%aEm&%I=RE(_hS}~?Q;OW_~jrD$) z#Us)(*6vv$K=B;8%Hk^7MxWr7NQODZQM39~`b!=kiHbOXuN#yLYo?FIE`^~i3AihV zkBRU`6MW79!S0;}zsUT2`FAuAy0JCG4nM;KLu$2F;6y+jLJ#@>;LUeYu~ppxFei)UcwhsZ#o_vxV`PNtwsAGNkx`m>Wdx@L2D3aY{I zR2XbM#gEy5-X2pv_~k`jQ|Das6{?i{gj>2PQUMa*zx<>+t>DCrTtlMp@$#L@%+pkx z_O6hp9d05AP$UkTg?kn8>h2Anv(+!TcU^*PobUIJ(&sXeQeplqankf&fCu z!tX2=83|T2P}arFzT`!)v2NRsrX{Ew7djzD6=dhoKZ>A!9WyR9ieQ|Xa4M=f-Cr4Q z?K*walslkh>qDQTtukyEa`6yh`?#2$*wP054{h7Gm>Dv|89Ff#jKDB}i>wup)_9CW zpPJrZtP|zzUQyX)d50x^3LISflUOzYl z6-@)~5!nN$P|zQshT1mPUd49*fL@ToXj?rW@O#Ge>eiqPFWC#2iNxAv&^Sr@8{4lc z+~W+1#IHaaWTwFcn*A5V1f|7^dr$a9NP%TETANyvuZj^%Wr982GkOt?vN)8~TlTll zyX|1ZNV2*~lsk#H2ehEC`$EKn&sFg|#CQGpy2a zQU3;TGrqkgb&Xl}Xjol~CxKM#H7niMqnD)jjyG;%7rFX=&?#`JQR@W)g9G<;q)dm) zTXTK^OGC?p6j(3krXWEGmX4V&nOe%xW)nIDOY7BQMb5jgb;cT1GMDpG7_&D$FYYLd z;bftN(ZiuMf%H6Zk2o38fi5E(RZcF2kU=w6+&@uKcf{_vjm2GZ$3T`Wz=(+zVs>g7 z^%~^R@MLfUYNu-azBL+Eu%zJJktXgKG{N?BF}&bewzN9gecheht?Dn>=LyV;AyB>p zGZFpCxp)qKQGuZJlN3E>0Cr|&8k-!nd;5%NUOQO4Ti@ml(eAoKULMZSx%tEZFU6Gs z=rWkDV!elFLRgtsUw|OanH|$nIL^qHQYO+`8|)U>%xYk+LrJtB01OMiCe+7k7(-Cu z6wMs_zv~``xc+j*G7OZdri219tWhyk%Amq$WXVuOOWh@J=zK%MZ>sMJafs&G3i5-x zsCh_ONHhk*$c~GL8WEEMI1`L0@2eJwp@jxK&yR_XRFVk(B~S-s*;cfrG1HQmwf3LJ zE_eC%hRS&ks1Rhb?f@%s%<@XJVM&j0iD+V{t;Q0dfXI|-yiOg-%Bt;q6p9d=mxNvq zy)lG?Qt0%CHP-Y!%1dpx$o|Q`1&@_1n1E#h!+>!N*sRaF?$eR9$C?m-5Ha(yQn*(g}mKZ+cZnw@Ad%EAUurNOZV5_zIG#LFgalq z+q3&iZnHNQ28dwcP^pDP?Du0IS!N>b9k1AH+yxJTtAAv5NV#)whW6UQmkq*wuE%w! zAtV7!VW2_F2hp-2b!ua?-u1WkuI9)4g^<})_bC>MnXW*3oyzwyyCFtI4SP&K(4nFD zB}I1=H_+Df`8~UY^#;!4Tk6gM)#hCQq>4ozs|X(aBvA0ADp|2MQo3b;WifT3 zV*w(ZPCLL+diP;LM5!xAOCQ)=GuqCu25pAxCCDFmV+CczI((7>ewBEKX^4)mE%Cma}W(dZ4UOhe}ho7_337?}uHWo4J>BgMH9ZT?ZL z4ijchF{H!N9iGNj3dS`8HEQpBx6>t$$AeQOC#eWhLmR{vr=m+@fbhF4X>rhoRc&h> zl&qHF_k_fDWNE!BJG3j|vmTngpm^P@K@v^DUg4FBIz3WSlBivo%Amlba|~>B<~Br- zTM9}wt^GC{F!3l~NKTkC8<*2$e3v@^|7fD4J;{k39L_h4e(H$E$Q~{idlh=Lbkj8k zFzDV=^ujj;~eEOo!1hFw5dFu!1OTgb&6bcj=*^h{8;hL?v<%STywAp$9 ze0amMICmrb(X|8c=Yn8+jMxV}GvE(`#;E?qlZphAK>8`rZ z9>W?*OW;TIUxGVPI)y`92%Ca<51eH#!g0N9T> z+nHrq`c6&DQ}StEvRqI4)UxUtzgXx6Hyq#e#~LFL_8Ufr)9cw~JnPpl*!5R&D9B4V z8l*N0RYCbM(bP-J*Y-UF85hTq>4>EbFpR1h7nLSz`ub$;($_zUJroUtQ%Zcz(o^~( zITe6y+M0SfNvv8N>r7o)KfmQF^di+cEsyEF?)ZrGX30-DJZ%KZ>?NEDGk~wvQMH;m zvN(m1gBX~N%|(7>Ij{93U>uAi`a#44(%9F&#b)P_o!k@tf(WD1aC86=|4DPJ`kpC^ zt|#C=^uR)>Vrc(Og`bpMu(EC_fU%tT;+Fg1-}qsbdHXu9gBfkoB&5O+bG0vwM|;C)h6{G7kZ zxSt&U0yhZhcN-yZ!zLX`{XG1pymt`?fq7M9y~JN)AD}*&m7IXuZ{NkJbH)%zU{Y;) z#3es0zQciKeB%&TgwhE69H}NDtUv1Qjhgunm_xrYBhLsw!)HeQRP9{|ms8z6p>};G zJ9peLVLt$-z$%>=7VW_l5mN3m_^vfkjC_@(hCW98{wi zda-sRabyu6>P-6Fm&>$4qh2!!jN2MUCEX}bpe)*je(UX7XIiI;f&mxD3tpvsJ99(7 zk=)@K<4+G1c{Y!##AFv0mA=)&F8~e#U1zvX!aEW^;Tm4pYjzH!uSHJaYb=fh)WJE$ z>aKX?60$Ogg#JrJB+Pi#5=V(GEt%{fUb_HFd7d0ORpu%RZ-+ z6bzydN&d}Vre5S?F6K1@;Us|Qv@Cit*C-m0XI&8RrPx2^1z6-kpOmCnUOU`6#YEWw za>G&QnVzjV37JzRdqJW94V)W;dNJ&kMtYDNngvOza!!6hvypD)P)?PGB9d5sc zV-~EJM5xHC9lS`!qgbgk&h)OJBJfOEw$PTqd6Z!@N7pQvGYI|G9W!*m!U`o`&~1eA5Wz)a4~7t~-H~Zw z@z5X=TVm0lnVP`L+6@MiU=g8ya`ts^Ih83B;65K@7QJvg*1z}yKWZ#aU-bkn za44Mj)?lMyfB#T7wtmA-KU3)CxT)E4F3G-wz`=|#xL@6wO6!=NGXMeh7Bus}lQ6I1 zxC!ss-c^m%nBS3f8v$tbfpnqmc)$hV#QFmc(E-8uO zhmv~EJ;F2HH9N^4q-SKKtOCkTK!Dmm>(yZ@;Ly34_rXJXL?^h1hIo#TWS;$#ddjx? zQ17AW%YeayI{`=v&OW=&9^~g!92~Vs{B)0tfB>~^hffomKDVm$efR;VEdO_KpLW{a zp2=v69lPgjZiE|LZ;_pr&M+1r&X4Ep#2L#St9;@MSRY9j@ggDTwP>$at-@2^+ANsD zYf74Zu()MQL5TBAjCoG^^sb4#9Y-$K^o>+S?ca30=kM+Nl#ax&z*%H4P%I(K&?X|T zweRKkjE2i=KC3}u$IZ~isI%M?@srFD>js1;bbIdmKu<~$#RBIzJE!pJN7F(P8NmAe zDJ6z@WOI&j_`F)}fnbhgBUu4=r1OU9*xln~xh6O9qar1a3QQhs{yQ@(*;%p5VjKLR+#U|M3@<%k3?d2>An%B%p z9xh#wlf|Ew5jEc6>}XAA6vlyhi*R(Ii@CU)IU!`tIU+KWOYftZKs3rKJzKX&#JwP=0FUMnu}7{6kmxUZ!cU ztsTUws0=j3s`Y>gny>jbe8raR6RTAXmYzTlq5e&BMb7HrEBcwD$?h^tzJrR}E?YpBtB?&Q35KP4#s<)_k}tI4FA+PHi)uiAz9 zL~|oX4_`!k58w-IYAK+j6EG!UyyzC;3r{ui_H>1`$QYGyUeB)YoAd0gXAg&gON-oD zH7tj)UqmMS5c_BA&p;`S$?fY3gQrg_-qikAMHAc|+peId!!2G=_8Xo9vHf#k?9r;$ zi|Y;T>$DX7H28(?2nNHZ*9AAB3yf3q189*wEjtBpg|WQ}%m9WZm?A(3sP#WI=*=Hv z9E`c~(w`Ff*F;4Gzs1@SYbDD+)OA2#X-Si{Et$)oEaRxENY~Vve%&a>?Z+Z~_sw}g3kBx3ijUd#+6mqm<1J*`6Hr&Gxeau)(aDM_E zT3nJ=f`gz>7~|~6y{=-4juY6U5H|*&aPIP3CZ{rLSQ&Elp3i&9 z5>M~iJoHA9SpIWbU?mknJDJ{YJ`YiM>C(CylP#yUe(3mpl7}C{olWD~p0;-Gi7thc%1pOj*Iq}l=mX}krEZY1+~{-NfjMwF6I%2&c*{$AqMcafQalaIGz8L-FxpT5@1Ig~v|`t~ zT82hlF$wGgFTvN1sg}c@$wm&|9eCb#$f%jGorOz~8d+$=1&Z0} z=B9bYBG;CzSRO349}xfbm53Mg0uCX(OIjg!+#zcMNEn&{t%lqKJgN;U;aNIgG>c+Y zdOk@>MTjeCQu#(97fsy0)53T!2KXCQB>A}kkDzQOEITX;g}uDp(E-?xLN&%xw=;C% z^k|#pkkAXXwa~X0_te5mtunpN4n4NET`>7y@(oV{lVodac zF0h7R1fKb@h<7KXlLY6bbC+>Ctl1G^5W0CA2Mix?8oxN$!03|fLs{GNrj|r-NxmfE z3GdK#M~-V;IM1&(lb9M_#&Z01lH9P|g|ZGeK35WvL#l2OkVv4!6IbrZ)Q3{DzO~uN zsuR<|54w`ipzNOWATEQ_PFko|>pz_zWn2f0@f5a`4u^UmwoqSM2;!paYOmbvZl8_d zcxt5dPX@(fK)~m5+MS;dGY`~QaLqjYgTzRT3Tv8Pr#Tu?Xs%>199f%QVRGjT1 zqc>twHaBfhGyA57NuM+P>EKDW1xCa(DoShUW6w2tv#J<;-d-Fyh_y}mnI+5I`VHN4 z{rQ{=>=41L!0T?d2=GI8&f71_I^cw!LwWpMR~0V!t1v#q=&EQyenkJ;vFFd^)jrxQ z0csccSYguAoPtaT%1}T*?6(J|58B#Kt%=Y`(4_{O6kPKi6yI4`{=7v?>X^W36z>S^ z2a*$ro5k||wfikn+%(}FIHrwFY4&H|tAfO}sxJXB&BFwS&d>j>OG4#K6q_m2nA)`E zWVRZ+frUh>G`xzM%}i|J9cm~T1e~7-e|P2fMDWg;&)|=05hZHXEu}UaMn&Zp?443^ z&zm(zvxs`(mrXb;)QsKDHzG^PmH{K_?j%M8$Bic92TgSu>9jZ<@;BB&ST8l2W5A1l z{HKmJL_~NkSG?xiKfAJbHN1-tnFd4rY*MS3bbW@l>Jc?YDye&;+}wO>j07OIY39K3 z7ZINzx~=i|MLq1KP=k4K#|HIYMcyGjzKy(;p`U1F5vt7C__d7@T8@o@g2uiu+x+lx zCA*gW80qoH?vCPjJtF>I67YS?f$^tLr9X4^w*%JgzAyiH!(5*=%=$~ORFPTLmU zfT7H!YXL@a^bm0#)j_PE*oGtoaR@e#hNcA=dPCB&Nw1VQB^j%b4=Dr!qeQ&$`hyFD z^BKzF!Rupm{K6>mE;iOh#lKg+#)~&j@vHY~*6><3bUyW>iZ4=TaF!Vym(L z&QpUBPh55Y>rTRv9tYC+>6IVL0o4Kt?Mf9TGfFyk{=RLBf+jQ#Kjkav+s#g#!C9+e z3nuVL7m4k5{@5|U(}?x4wh5Zvbo4mzwKzsQdv`B=^qwg}C;!rVh}-;-a^AH~d8pyi zH(+SscuDKzIF0gg+9aNaozc^)rt+SFK$b;XqG}9x((EevT)+*tV1nYY(vz2Tky&ws z*(DytM@$tB+!qG+voJr1Zdtlysv0+??tL!$L47)sUMg8ryTjTevXY{k(|Vl3?BAkm zymyN+KMmYOFI>rxT3dTAI8e+a`Fvzsh|6yT!b1&~7p*%#EZPV^qs5bi@2#pG-?bhU ziMeg=4~nzO)GrPHXw2C0{Gk^PX*3k`T~?{s7bU;(+hcqc>zDEg@ME_3clDltHWX_j zIN0$oznDP^I#gcjkoqm;g{&1y+${3uVnEuDa&Nm%i35z0&OITio|x z5Ow%dZL=>!pIq4CI?H77M2s@A;SK^SK=u=67;5BXHqF~3dygh3FWFh447CH;sTyg4 z^9G%j6f;BZD_t2%6om`j?b#c&dNxz!cZ65cKoak9O15Aw!qxv`HVs9irw<2*_ z!i{p-&CS8Z3>%o>9&a?{CI{LTpcAwAm(_>g8w@D);YG!Nm2xIm-p;m;5$gipb0AS7 zLoNSMq>CN9c#Ytkj^2etoSq;Yr>B-!PFLq-yZjtJ#T=J0geH>lAHJo`AM_+ z&?WPas5CDNYm}t~Rq65WF z{M-|>dQaZ|b%oyJD~qm)3IIWt!O9~btmRATx#fi{JOdPnNb)dHH5lEDb7co)txY+T z8c(g|gR{`N>cI!dM2X3NNeud?;LLbJ9oTa!V{~E>ZARYfU_K!$5B_|SoIl{;rUxXl zOXWXRj87lmwZh1mHRtPaT5%&>*dJc`?J6K%#DkjOE}#2@F6q|2aqTI6I)*?#0ko&C z!Dyc?`>HWli~OB~Q-Jp)4H^evjAD3n09aIGMfYf(pks2iX~?!A=|qo`e(-HYpNHe|<*Eh;h~~WvW5g%?*@q?6 ze(-D?SLv)94SjW>ooc*$LqtZM&`tWZ-KHZtPxfn7|n1=%(pslMeF`X&|8_h0_0_25V7sIbeTC%e*08zwy7!wWPXv$iBrqoqX6CsKL! zD?kCdlTx(waqfvUsATbhCi~39KV2rXqN-+ych9ZY`^&_4AxIFke33mNdQ_@kQehr* zJG_Z8-?3>7Y%;T=7SC18LN}wX`e5S5<95vQLSsQ+-!;XmqAx_fZ^0kd*U|5Z`y_e6 zEp_iLvNx-bri&|HFsH^KfK&gTrcAUivI6-ERm5|ChNfft8BVkO47zxt=uIw{?t?Yf?SveDvQS1I~Cn<59>^I&2vbOt=sKgFaD(CoW*!+R6~igZp>| zTtpRffUq*K`G`vL-HK8@RJA`~^1t#A4Ne?&1*y<|()Hf2I=uzTcON>zL9nJ_JT4Ky zgx4R*ZA4_@zsGV!Rf24|%DpM~3FZim8;4(?3Uf?`Uwzjc@d0srHi|XqewqOt%Qvyt0Bty%U zV%+GH1A1Z&(mthg)i-r-C9UeevC$00Y}JhN>33 zqQri3`Vd!J==Ng(L6MvU&L=*6FXy_vIZj#59{VVMfi@UujcwgPFn;nHCupjC=tTEj zA`4h{5-UB%rLd(eSGC@)B0Um*AcnzX%t77-9WSKkqytHT?Z2^KWd9yNOGkhDesC@0 z7(A!&>`irDfekFKfzCgdpiV2Tt9SN3>3(9*1*)EFJsc;F;0*-PrYWYQ+6g8JZyRH# z5gnLbS-`u;94{WZeq(iGM%k^#&kf1C3~Ik1_$bd$*S5`6j2*vsP|DCiKD}0{Mx5F> zf#^I;R%e*q5tw-1=0Oi|6%!p{h_u9;5MPk^USDrIM6DVullrlH2vTGZ%N-$gGf)c{ zPfh7PCy(D##$|?x!?4qN)W&`EiUWMz4jvThk;)K_l661p5UI^7$Yuy(P1n?WwNw1F z%=&N--QHZ|tvwJl!E**+m(h9D@qL%H9MkggF<|phS#{ucGJXf8Ul6C%t3__GtUpqc z{sucU#!mMZUHH1zH}2rDLX50xWcAnL5)xwRL9g?x#)k3iQ8zkN_-wCUdjrf1yzSB* zGR+Z_ydjKdUS7X)ddtUqA6D2XH6fga0Jp#q_~V@tePM%!t~ZVD>cz- z$dOc2_O}_7w#5tARl4I&JehgbXr0B(RI%co{cVr~ z5XV3*KfP-x-iJr-r)l^$RH@cgF7;jojs?cCt-ls)wYExv@KJYKl zHSL&Q3$Y6bRxMpP9n^})OL}{EB8?M#80Js84?YWmL4pb0Z-dE>OArAir{eqJrPmYl z^=8~bNF9sgbSBk21WeV!-f_23X+`PAtTI;huN`uKkvl+@cd3O)Sy!G2K#cgAKFDE~}bZAzwP`G)0v$Px15^cg%Ccr~6464S#H7tH?x7X(uUzjS?_ zZ*X|vjXd)rD8-3TtFrF?z`NNT5p#5G=z{=P+p6woPU;*?tvB>5M)@r2Za!8+O5H}7 zoMU*keb@H!+IoRV&Cs~9=|@eqCI^V@j%o-KGgJ8)_c0T?8|#*5?&>n~7@gaEvqY}N zrq6ZN!`8wt{9MfdxYhZjvD^;XGo~2vv!yOW^z{vO3oJ9n$E3RMyz*Nua)cMXku7m` zRCB-4q^j5-e0Y|3An@iOtBcAWr36+N?dDGb3mjzoLB>!uw9Ax)RQ05G;qcBF?+Uxy zr~`4tY}ktR3DPV)<~7bxXh^+cX#C8?hny_*uU`@4)FCUB#L_u0;%A2L@ou!K>Yh4qxPgKLweE= zjkk9aM;SIF>-HfA5mFYtB5&AyYkhv*u|bQ>)VFHQRJ`Ao!;pDLf?6{xc17+|^=kW) zUHF~wBJWC>(W5?GyT&>m~=rY%2oS{0OTaFomZ36zsY>JNf*avbjuO8a)xE^1j<1Ev)Ow47H(Y2_=aneO!s3n*DpMY zZ6M9F>`9>Fxzsv6-I9>8WTHi0hmvF&5ExB_$1K-o$4OE9;DgCN#(?Ss|3$`-GZikj zp$PI5%BCPRe#)Gb!QsmC#$5g;S_hDxR7)O*bPtAESe&HlCe;{F|6!t%{7iq~WaeTh zLFhbdccB5UexFlrDq$9bxR1`<5S60nUHG^!=BJ2_1U-`F)sdw1Gp6{Y?X6McNBK2{ zFZjN6>DzP@jLP!G;vI{BpRC?BU~>%_{AG%j$_QapBO!T*hx`rFSj7+(fwo%bBF6KE!Zl@PsOhEJ^otTHq?P{crWJOg(b zOMmF`C-F7_b1KLA@1dyLus>3bEL?j0kMR%2+Z;fPb>-0w-9aE?P_MQU7_N@SJ8Ztx z#GQ#T$vvc?Sp(tD+e@8-zKF^IA(R&V7;cT!sr&y-xGv-~oV`jbMb|rx-$zwwQ@%*FA*Tjf$b}hg z+0ew)V>SgQ%hC(bmEJq+jOwYCgZrU^idm{3ljmk53c*lgeKStVwIGAP&%63~W0rIa`}?EnfI_=SW-BWr zd;4DDrw*&nq`wi2sJdZ{lw3C41wT<09U%qWS4jbHqmywSH8TTdqU4YSRv39RsBr*B zhh|LG8Al=P4rULyWQef`1PzIuR6=V^{_a|(pQ_@Vhs;3<6+y;RKkHA%^dsyGlC6h# zZemtiAA8W%c-X^wh!1pI;{90m?WiMdB4l0hp?j-)5;pt=sDQJra_$2$Vp^E;y*O(4 zyv(j8(Lw?)Vx}U=3Q+1q!$|zbc`miGiIN4rQoWn%a!&K^O>bA}WTEEXGA z5IRTWRd<`T7zIBTe8r7b2A&`=`?YR!&)dH(+@bcxD88RhQ1-0mW48*y2roN8ZVZF1N`R0*8`Chx;)s@VL}~lT;I-}fD{5S zL9i>PT~Wu?YI2;V=(Xub6M0-820w6OgP~4T)gBK4$jJ)|VN1iWMYRbHP9|*6H^prq z?6Gp`Pi(nvzGZf6!+gP_I9%wiZf$%7;31hgUm>W9n zWA<>9Ha>3!)-0#~Q?En9xOEHGi;i1c9A`Q)`0Z{*4@cPCwK&)U)G<$yZSpKNvf^dg%egKI%I z&fNg(0!D(fFOz+N>M&B(;qxUTW32d>ccx5S$g_9SXMem;+>1EVyn+o5W(9uiLm$H| zgjdkb-(py-jeDJF9L_bGmEx}6f3uo5jIAtUHxw}rQc4SHayhR@&BEQ1u|PfxGw@+j zQg2zMpp>%QU3j`Ou!EgCZV_?GPEaRx_zQ6<(hE}@v@-|%yL{{(0bbX?&gSjK4wd~F z+L~inE@q(zp(ut6gd(q*;3{tAP*G4s=b{Re35+C6_;#ky9%-)6QJ zudakrp@{i?CsUx~&uoosZ%wFdVcE%f+9AH)6GEIC$-jJ6PQ1c)gb;B6^jdAJ=!6cgZ?$gIJ3}?-7nrl z`==Tu_wp&xAUcfjU$6t5h@tW5YulXh2=e6E#g50D@8>Uc-2g8TeEuD`0t!={RU;NW zB*MhHtvKE77;BK9;BzNMWx#~tAs!|%aj!>vs`7&zouF{=cCzvy8}}4Hoy^`}jXdDo zv3tD9aO%~Ok8i|N)oeJ)Cj1KKZa&SbC^Ny<1!{k`(C4u~1vKG=ujkfRn7(#;Yg96K z%<>c+)n@Gq+OX>D8+#mZ@bIke{Z>kZK1qYa1xs8Qt(PNJ<0mjB*-r!)QW_T6WYMBX z+QcM&1t`Oh`LhOrd01Imt!qt_H-fy-E=(-v28AUF4EEqvfAmIVGo*EGIohTOT81&RRMBJo4CtcM)Y)qCUyjEkHAtg35j}+ zI7BkMxOJ=suwj-K2LA)pf?$4(Z*WxF>1+<3329H}tbW1H>Hru2C3;{!x-MMy9Lp)e z5O-s~J7@i2P>d~nUDUw{(`7;KfWHT9U8})OXS6`MR#rVw9aB@4_im-|iwz;e?qr>j z5e7g+rw!5&_+|gYCyD=Ir?Qk<6$47peOOX3Cq0|YvK6@K?75IG z38Ehsk`{8|Jq@AYO_*I!!&?F8p?GYNdU}vymHgdm+zuoBp(OfIe-VwDl;{-DB^wMT z2F|n0S6_fY4GSFv$_lIPuBeq^HXw{zwofq4Th9CyYg7Ueyn#(iicA#oEA9w1IKC6~ z|xe|en zIlWYKDIg|KuH!%O33#_8;CyO zy6qNvFOGZgZ7jm*+YF)l1>1S#n3ZyFz}+f>mm5fhO_q9kWb_#*ZUSfONE5#S`HjmXuccjRJW<^U-;r^5Ki4uj|8e51B~iAahp{es3fhUtD^b^3^8%a8~RucGUn z-NyI;8KMG|Bqf?%tFdVIi z1{EZq#xM%p6H_Ol{EQjSoD^M~V%jat>+@}D`BN>WJ(AN&$Di~lPqfZy$A<^eHg z3(9?DDh%3)r<(#+esHOYW+C5_x6MlPGy7oQI@paR8?UP7@h4W~R#>L<9bud6OAJc_ zEdd!7QEwc{ix8s%j(u&hEh0M9K0eQGq(~ItvWy;UX;XNt?jen}xDLXm1MljmJYy@% z{W)vFId@Qik_NX3qr3Mhn0@28rhuoQFF6#tg#(Kbc@P3@G+G*yytg!Xw%)4>`*-ww z7PC9}2O{W`D&f_8_W$u9{q8o@oB6lG)<8cAPZW}|Al@@bW9{D;UeWIy8o%(-|3jac zyXwJ;(yfE}Ue^~NT83`ztIy{al#xvW`Je@JD)O10aCaE2Vr2HdYu0-w(hicX+F$(U zs{0MuWU&V=JOKZ`h3Zs=-ol}!B99!IGeAGq=Uir>t?va&P>;=Kl(ei%pQlmqrqpib zD;f^-C~9@8J|1KYxX1Wn#6((UCbU&_zDYq^g5QP%2w{}bxGXKD`)^vyjF_!i^_IBFOLc`3)=NdPWU@JTQl*4nt^S0LlZvVo-o1a=+ zqM`n+$VCbRo43o~a@KaXrE75n1zvs0yi%?i_pi>J8`S1t$KdF!E?C@o%gmyNLKwPx zEZz>R9IGsVBEfb&2QrO3pRgUu@9CZ4{m%({iRTzlcVh-UslMw!gzYhviykyS5{fRo zd~NH67dF{8pEJ)0xv(3G^TH5~z+kxBIpbDs2Ebg;?n}IPuU5Y>n%xn4*HgCzPo!k| z7jr`xqpDM8<;wA0?n3~pA9Dq;;l_}n{bd$!iN&^Y&;bX=NpWd$2fG)0>~0K(fCaF^ zu987K5a3yH#^t<(Dq5C($cF&-#g3jdva|6ySg+_gH34Hwe_6uXuv;NAywmf6Of&Lq zzzB$^_vrVSiSiE2%(b*BAZAyeG?*T!YPP8Typ3lFMcBDq-0y38#z+Y-DfretMFwKki#BGy zO@yS9sK)#T;uvV^j^cZ`L}h!g=(!jGzr?$V+~QK%kYZm2RA!1rFDIoD+gjG>ch$AX zDz;cNf#?VG&iDU;E08Vwx6d=f#puYjm~dhAMYMk3Bi)2Zkzw^4^oDd5*6&7k_-bv)g?(ua*(lRb*TX#>A!&K3KLJa zPL6sAB+x;KPh#aBxnEB}zH!I)ur@{t$|NMcac|Fd1{{1ALIiH)MpW?*8oAr>U!wyI zcP#dR>kFflucE>-uegCnF>(9)N_fkgIkA%qOS^=NM1VNN&==2V&+*2J9kXe7K^c}F z(cT)xK(GQGL!UK0$eS?`4(*&Nw|YeQYW}e>|0*JXmX;Rt1A(psI2^iI>-OA5hi;q5 z7Sy(?>hQY=~4 zA&?_ME^`Mp-TZ@@*h1_qClNz0NmzL>(Q>?^CK)c-irj`^DJWH;C$#|c@whZ0MGGKl z+MiUV+K%Fim>>c;EPIo<%Gg1|EJLr-ay{9$0?rR!FfQ4a-L`n>&|5E%!X|>C&C7uyn&ZUBvtEadOcMU}w&&jA%j-BG%mp-I37jzJM&yTZah`ap7I z1^>~R^*xz#z=m&xYpT+e<>ohzW24HsVGrV^Dnwfhy|!TAlXuEmJ_iRXe0i|A6!m*> zYDk`<;l)Gd>&{L3x^#C4pW%UxCmwdNp~J(N%^8PnvIpdhfbG|JIJButxm8vEd`U8| zL{YyPBMy7S75PK`N=scXf7!`43yW@9EP?tvj8tc*Lc+_FYHNL(M+3nF8Q&H^u9o42 z84lrxq4EDwVN6b;0jk(E^DEp%yXrSU{$l0!xq=2uR{Zfxa~6n|fm-n*$ex(~A~9Co z`s^tT`J_qa9u8K2N)xXlRw6|6lQ#8|*_ssJxN0_vxrfhwQeKLV=oI*Z!=tZs2%)hN z0i>0dEb13bKz30Th>3vT4~?v<0lowphQc|!w)rn0he#&Q`$}Rl}ruv2=IcZ+RbE#E;sJKZBfd zKYqLqMEcpLC46-@ILd9E!UrIw6fN$N?T6r4{-OR0U1+CwW)k%~1LR%pa-wf$?~~gw z5Ut7v$L1;iJRXO5&DE`6nF)HNvsWu7`$1xcPEysI#s zgH8+Y3K=pE_IOKnRk&kR7$%|>HRDoBF(K*>R?6J~a{8r@mGtc-=m&)#Z(|Aoe(@cn zyWZmRF%2ovgcN8wA1bi?-};b(*oPBv2bCO*w42RyLR%pC^!toL)8u!Qn_zS)vR~A% z<8YH;?;mSXSlc*HO~A)XMjM+6EJ{SB3ENvR(IB?rGag0?!RqgOV(|cWfeK{5?c!S$ zqQzTC$x3%!v``nXwJyT|7%kiX3jYI6h?0PZutZ${Z*XB#`!(3?v4Y^p*Yv%Z43SO6 zR>|v1Kx3{T0Z}}k|LaV8D9}d75EP3%(1yl${j_V{KEml}`*Kip>7NBr=TTs2fhuQ| z^+;wxBU9VrCODlifIg*1x?)DGa{$^+pj&GF!#&8FLLIKJ2*xkxg2=;pjBuAH?h;e? zJ6TXENU^G%q(69U16>HFZ3u3<^Sxizyok9zZnx&CLiXzSWz={B_Qi2@v|?VAGoCjg z9)G`T6A1q2ej3VBFx;ikl%EE?rWExe7+y^y-hd%X&kncrcc?@L{||U73)e=#6*dwDjR3{{fn4T&QUg%0^h=rPSY z$lCLWFRT+?E`uu}PGt$1p@|5NLJ_EiE!}UJ=r=9j9#(% zkdS*I7^Z!W^5(nBQ>aq9+#ZMrgy3u!Nn9{B@yUq@#pynoB&KW4-#q;AR7NOdTvzPF zKd6?QqDy*vz%Af+ylZRIZ$jsQ5eBf9^+C`?@cQ^9htmB03@Qr}tcx1G3cBh97Qyr>O>Y4HuZZUeP|D@`cSKR`=780HY)ac$K ze}iY1vIsmes~twG%r!TcXU3W!gZNY)KQfntmgyBbQz3z60Wj7kjrLvE!8i)wcP*x^ z&rz`m(TM|Yz_5!}!iZ=(@8AsT(GCdI6W-@#v@%vp5HNLQ!*uF0P0OAc1|B?v^O9vQ zoHA*p65&xJXG_IsByckWRN&yLPU|1zmH)2Pv7-;iF&+t{rGa~PdOuNJG^W(V_u_cp z^Z-z8_>9FJe#r}bqs`^{%w^CLU{JLWN$XePG`M@>#D->|8Yp`rEowU86EOgBT4ySJ z@*>>kaRnF4Cvf_39yKp2m>=*n*P_G8w#Y6{`+Vuk|0Vv!`oY=aPxSUw9dVcdcEyDF z2Cd(t8ias8IU8V9+}NB^&U60>=qZse|apVJL> znMTg|ymp=C{&)k2AxjJpl|?__4VZ5ke!mW_=V46jBc zWEwe?#Z5{@xWA6?|H24bWL<{(DwCVDZDlr>2yp2KrPn+JfE^X_jEblDu0YIFz)+$V zIL=ev;JqrArY0NA0V27;-8o{!;{d0l!}o;RW$@y6NB#;*5MSMTY`4K@T1w|!XlPz;1KzHZv{vJ9qN8wVbJ)#b5C}!V4_Q%zde_i4Wa?cNYU~% z=uGl_(~P=SgtZw-il$(UOZijmt}*Q(213y`iLr<#38Bi&16_6J|5f#6pZ?NXs)ugQ zO9s?1)f+^EPU44oU>jDr`V?cU7-d#{p}|kQ9h46COgJYNUfQu10f(#nJpv9@s96%m zcC0E8Xls7gI_KX>>KVHj7DA*&K=l6kv=fG4F=v1lRbf6TqD1tmGvro)H$EZ^Po*dF zk%~Lri78|{0iuqraVHwMYR1?OZy0&tPpt&#fID!y|PSF z#BqKNKMiU3+k8=X&4x?Wrchj4i5DRquCGj~xFM0ZSk)it~U&PJ^V`y&9s3_7efF7?kN=s*&^%DJW@2t&v5uioj zXBH{*ocyZsxaPgUa5=ULIh9en;Su~Jb$#+Pyr$K`y!t*-py8_euyE7~qb{mfI=Japm1a!+P9Ky1rgz$h1_^+i_Ui$)PFR6>syiN+U2YoEsMC|7 z^~sRzyaY+gXr3~8*ZM{OCI`4oyrvV%^hrAx{kKG<3~q65xY~PbRE5a}(^B|%gM{1L zWlCghX#B;1!V!%GGp}E&R*SUC<)m@HP*ErExZo&85PEV>spxKDoIkoD-DkFqvag}6N*5W|Bf+N9Ts%cM?SbnQi+If)sopTghcb9?kF2GpaZ#GTJAW?zI?>k@wC zkS&(#C(r`bFqe#vjURTT@#-IOL@)0pM#KY+L?IO)MCWpkpbRB1c8Y^H{ zT--%9_gVdF{AnQz#-E+MUi1f4;g(8zptcIn_58AwSAmnSM2?x@j(+@i<}{blhcgV5 z=AfSD`2q%wA=_Gkgk?+0{XuTS=^dRM$9#|W?nOjr|}7hJ`cvMtGW<)R@}HCUse zoEQ(!Tf+c6>YHa=?fb4tq*r#+L<=VQb@wNjkYN)W*GWS2T+dMVwa^NL0?9nvXk;Xq z!Hii3msz|5Hd)&aU$x3ULss{~ej^XFz;SQEwpEHaBgBfpdLL^!nawjSd}_=(AuI;? zJjk!9wcbhZ4?y7^I}E9!q?nxnMloR!b_u6D0PGwHKCq7My@>>p_L< zjgorw&=3ynknTqMTCS$pxhF$|gCa%~WI=sjH~feXV}%&`dg#A5aa*+=>O>1fpQR|4 z?g_Q?GfnGWg?)&p@nMzR`BxK@by2+`$cz}3U`@6=ByaJ5^M#~`4=JUH1mvgoWz)!h zV`@NP6w(fU=!qDnA1D=mz4S&46(-==mKYhyZ)Shiv7@}JfQeg!}H}*m%+t*y3(8*o0*C*o5GNe*=N!UJ0nH`3xFs2UU2MpeKs6IgT_ZE z)w#NZwI8h?jgg1cAywYN-3K>D-1hfw2Usy8$p#M6o=?)zFkG4LK?Nwa>zbA|83#ibBJlYcD=Z3`Nc*XE*CWCYjf9@ofL9c95WpDPn@}|6-^H%d2}$7@X@kt`+SP=gdV~wx6C|BR&F}#_8z8urjq~*5+Mj z@otfeD+&y0!LM<1TbB6)Y&16=4)qmYw%WADa|^FjiN03uBs<{EOTWUMfNAZ?B z468Yz2-qI%HBx9;+@H$RFw5F|Z!}T!lyUknOr`R>O`?a0+5ujD;?3|O-mLzTJ9b#& z66m*Ht6*!2ZO#mZ6YAsBC1!8U(<#0F5mtK~@MI~rO4fH|VTVe88JRT3&7dwNH$pIG z_O}D2mC#b^AN5xN5L~zj!(c>zL|8Q+_{&Lg?$O`l1i|nI`0xBo{;g_t>^yQU+r=?b zEkpL;2m*=kYh@*|j8J5(Y;c&sNa}C#joKJME}?*a9us&ClLG;>15YXVoG27NlB&iY?c2A)1Y_&DrIU~}q{*=Sp7Q}sxedoL?G(3eAKER2$39L)UVIG<Q5B!XODrC?xus0wT39}#d0}E2R<+Eej!U`CKdcdV7h+~iTC+#VkitFtwRA23 z<840lF^jlP&u7OSl?&Cmw%(mMCKyP)oVugZPMZiA4`687dLB>~KX(065SRs6SfUir zDB}CQGb+7F^>#T^q|QGGvN zh08(D|hUrGRk1_Yt~~04)2d>?u6pbK(1?+i*D&YT?$1UB&l*OJW3O8v3T! z%Fc`2hKHP%>_1k75h}nbkqP?*kFNz#*xiOh2pYiS0jv-^Be?UYuDU=jvqxOcyLE-I zw4>*srJvb+ceg%_3HdZMLBiQ>1SUTHKKzP5VhHMJ$KK!^7cGA5`RDN*SL_HVH(`33 z^73 z)-SdcWnshezK)1uw?LR##PvChipxtKN5u&UO>WMRv*L!-q+oL}OHs z`av^-vX~$FwY{h4`Lj;SQ0H1*{|uD}^Xo#jjfj^y4}YH+?69hgF*C;T0>b)~YP2w4 z{GxJO=*2b5Mm0vM7(DF#E^-x{&@2p2*-qh1+}TQX3FHguKBJ)bIcj!yq0A<?To3CXLNG7Pg>X#Ht z8zeGdO;Ri`b*sm2QwtU#?=$qYWQ;!_j=M%x0?O;99~a{N!#qWda`(a|MGBOEHPbwh zQy1XF!&o3f!kTDvfk#Yl364j7^gmN67&hj;-ctYgYybGuq{WWEaZw3)=!B3c%)`_t zKmW5ePO_CvIlCm(9>0d92R+p*!59I05AP>5y>Ge9TRe#Jab!tg=5q3LoJjm9s)JWw z!dt6LnYS6zse(xSPA7Pvs zJJt#$hXjV0M^l|#obXAa1a ze5e)-k>j{n@0`cbO(o}g4Uqww#cYN?Oqs6;d%9^N2N zv#Vc0_S0PDj`IR1jB0ea+qstLaFr|jlY?Wuz4$q=Si0~O6GYa}5!&-5{(y|}_CF0k z;MBqptHQ0?i0c8}JHtsDih5d?W1U&+RWn6*8->qDVg20#D-SYwbNF$fp3}5e*$A%R zfqp3@0Rn*tGRVX#iO*)2UIqU)6{KGbD}fq;aK2krNMs|ST8POl9I|$- zm4CpNkinurvLD>tMWIdDAzsBYA;}LVPZZ3n40#;ShrR>kCd4BK8l~@bjIS~rrMk^; zWkucQ$J$-v?VFp$HUJ!=lpOx`Xpr+f`1>DAr6m?aF~14iMNEan5rfaisQh_29zL#zp@788iGJyQ|M^mLb~yW= zq!Jle1nRn?N7!e+=1;@=X@u{lg0h8a%R0&;G2b6zYDJT>R?d95I|MhBJdp!9UG);D zO>CY)gyXr8xSjYN3ZxXH3@sIPnn;ZhWd}XC8x>{hOXpW|H42CG9USvvYL?)@>e;H6 zXcG*o7%A^4cVUd_BQC@Nuj3Q-z4LGB=!x>j3_$hVi9U2g+r! z2m*5oL;w^+Wif~f_ZCM`8}yHu{0hs7^sHOj9%S;A!n%Tp{>Fy_S)=!=pWznblO7oMuX^Gx94#R-oQ+asQ6!;^j*_p zn>RALx?FsGAL!}Hgb)zf2ORKSuW!CvEC{o7#WC@a&?i3Oe0k>U!tO z87KuYJe(Ky;0y*_|LT;d?^BG?1ej9QMxZ>H=2Mx^8-p_CDYJPTI6K9#Z1Bkp8p>O2 z8R#$L?&H5udd%$_gY`lPm{;8|yI=SiVC;nDj3+@2)&XtyFcPpNrmaEDWxsFW=TT@Z zK%i5U!&Vxg6onRz3F4}+-xCJ{ zD2=6a>1G7g`>?)kQc~x@Y*o=k=$H1+`a0uj(9-v^n#RVRYQ5knwFDS;%Agk~N3n6K z-~E9Kc;5+zROu1ZTgRA-%r(xWm2e8@lvjH(;+^ySbL4@Ef?3Gcs$8gqh-Dsfd{-z* zM4LjwKvm~M-IG5{13b;w=QCzDgzU)~g`g6)(`>pY+usT>Z#B)Ac<0b3?wP34LL3NW z0FOd_?6!>u8nX1S&Qa-6h757!5t(|#uG<$z^dZ&x1tUd)vA@C!mDu8wl;pzzD`gDv zJD=oHvg>H+yq!^T=N$W^b3+a~d&ueojIV$2^MbB&>HPcX%d!4pz~`tb@4bt%scR0Wt6(a}17GeE$dl?KME7vpGd+QjpM?Tn`scGi-J?$M96S5F|<$IRi!z#WYt zKUQ*Y;Y`#w=Kjzzc*L#Yhbsg-P!j_g$sKh!#{Ilp>%Dgv9}9E$(Zelg zK9$9A$nP==NC7YNzJ|l5!rmAmc4mGLE!ZBL+lIOq_N-s91B+eCQvXx{aD4ZYw?K+2 zm{egaS0_*O8lzb0Q)-62EN=|6x5LG;9~Flba=%<`eMzv|CCp%Rmdkx0q4B43;UUme z3+8DPoo_=t*_E8gOuSUU_X*o}CNa3b^~+KiI2`rJOJ+X{M_`9<`Z5e^aaDVJQtU?e z1+q*(Y-zq&$}W~qaCeTMp37RbT$}UUYrPi?Ep(i<4)=jCAQI6kS$p^tbI6N|(gYP0 z(-eh}FxV<%g6`HsdF#_7czGLLKIxxDKVfVQHcK3=8<8o=X0YnaI?!+YYmHk@VseaX zgO86fm+2$`+UF1~YCOUV6PqPeon`+P9AfFlcH7ba0_*(Tp zk><74JOb2OwN^{8;S?iGw=JP01^BxE{SA0geDxndmB821(Rj^uNRHCR08gxrA{_#U z4#T_7ukn-y1UNtacpjsJ?;lDTv2v*HvQnzS*5HbLIMGKqeoL8ncb`G(t71Reh1K75 zHAfD1_=+Flbr~WOTH814Dd}DM+3xke4UL@ruMB@5(8ZQI0c<7q{$35iJk=b`K70tQs?8X;Wq+WbaZ`lT zSS5iWEWz@bxW(l0Hf>VqGSNMPR%TE2Sk1`*%rDGWd5?-rW7Po7DT&M|ZcyzIlx4t4 zuyxs2K_KfGmeC27H-jo~`lx=VL|Y0-Bb@bOea5KRrsC!&z8<=aOMWZ2pX6Zb`aeNI zA`<}2wP8pYyN35V)>L?PVVrQKN~nbKrVsyeQ#(T}VN8IwJ1~JTPKhUZQ3YAr)=jd^ zt2&dC<{u4mWh9S2^r659Eg-_-7yCxl_p73lx!abyW!B0D0TQQoR3AFs`>PdIPQHbW zfCItbo*`xX*L-FhQ>qxMBmswf1b7|_aT$M%a5$=y`sJ>JhIun*^aS2H%?4O^3;hPN z4oF^<$48+EbpI>BJwx~}_*>*I{_jxRRO%MNZb6+BWQuH*mtcB-T<4TKJa~57jWp5| zr#|8LsO`bw37c1b=Jt?JANVZwFk_PeNZ5I-eCfZaAmQQoM)=EkUP#u8&)>k<4|JXJ zIi|FaDjungM4*G*F0{5zlPgXce}utznc^-P@S%#*Z#v!zq-Fy9q9neN5- zR3)vh!DmT|!0+CM)(W*#R)ILQ*8le3B;9IV%{BW^IS5aFvY_M;LaNc2%jvwLmZZp zNJ7Ab#273h?vB8MCi|B)ttP*NL^AtgJh%Wq63j%}+4` zvi-)*q{^r2Fo1<9P(7GSLGoz@J}>n1!puv#_JP9NtiTt4H>XmzH%+!BlPvvM z+l47;z7<3rFOnB?AW;ZL~V+D zq!Hn7x3luqI%tnsc===_;tUocKO*-eV@|M^sO&(pNf4eCN0!jdcw@zZMTX9LZqEo#(>2NiML@)v}N_%jlQ{K7ZGt722{U@XzvHm zjWGxIdn@EzW!VjI8*{A`R}dAI1u^4|)_M5p!v>!yiu1Lp&I2dHP-v))gd-869(>Ca z(wM^f{ll6!qJ9$vEDfYQs`26dVyZg=$k$nFUt-Vn(1+-vFkk5vLjqT9&Z}&ie9RhI z=rUs?fpui9KvhcWFBNlpe9qH-W;YbFTmwyOa{9VTG9f(LT%s)*2@pWeVe6`N+56gX zcLz=)u0-{Lt4W4p5eXlO4^ls>8(ENPY8~jZk=%L`sY09wSRRNFu{GF@qUPWqi5CH~ zME>ruA65wEiy0092}`EEpH0E}0|w{p1X0n!UN9dO$;|9-Q^&-X@!Tz$uy{hPK)=cB zW1E+W!3Tz#&KqD3nrIe`FuMgUfjw`T3$hM(-U`79(Q7boPDAUv84?!CIsd<@uaB#; z&bI&Dxz(Mn-kdI$qf?q%GG5<)ie@TErIwV!_=bUCpQ|V-j*TFqB6w@^nlG85X&8c| zjHoD%ASxm-pRy8PU>pS^1u78=kdqJ)@%CG5?T5MZmk0;WInR0ackjLS+G}wjiz8vz z>nk~UD*q_b>;k%-q*B60RUfEOl1lZSh?kWP?OfZ>D_wjVLfev$N&(%H*Ax{xlg?4I zlD{00DRn8B88DPu)AAv+{iNNwxKpBgm@-1UZ~PIi`cnyWd&7mD(2hXBuhg)%;1c`> z6;7rgw(i^FjHeMqBAGb5%$uNe~RiScbDh!ETM zqBs(`w+eR+bAV!4y>Cdo&1Z`G;>MvmBk0vfX(bfKC=TU}0v9BsU;*E(ccCHJWO`y) zE+#a-p(|FP9$pZ8Bi!?1>ES{QDe(2?-}gKvrLoy&o=71X-^CO81E9gqPGGcf2yL0; zQ*v>r1Qnaw;44J~H1X|>x1F~4>x!#Q0LhUVgMyMg=lL1+w3I!!q_Pdu=pH^mLOPUR z*Y;`XcoS>hMHk;~>a`)`Ys zUK|}@YID%YmtM;Ek~Ro}SXEqxM}jNSN@4^~Q$XmaMpcTifgW95IH**ms0i;^RsT5Q zOZ)3jB0b&T**oR&L{TH)u+yHf>|FvBMlHvIi{VlaDlY36CZ``aol6)%&qUR#XHILG zqOxvwf{ZMXloxGtw2O?iHz9CoshJ(-a^qIA{5&0LX3gkBS3M!~FLn}P3q>9AZ-?>VL#u1=;kmOg%*m*mGpR{{Yo7zWKVp03?B z)r?ps)uuh>o=iEA_d+6wj_ zQF)p?35<&!?)hvN12i`5Z?)Rk4SCb`)+tk(^<!1lJ5uIgQ zg@Tl2D0CbA;=kn4K7qbdXr@fJ1=eF1`@x_W?+J3R6v4!gQff6@fY6`#eZ?vVfybx- zv>m4%-vQWU)NRMxsOP#AH}}y@!FXdkX+`i}iQq(`<^5|Uoe7>yHyC1tI7u+i_|P1& z)2~EivLfv6lJcC{#*TS~MmNt|dVn+s?4`(3qqMPOL{aum@h09aJqUXYoYz|(^=_;u z0Dl+mZa=k zf1yfPa^M#<uCcsq=85Af}XY!}KKT-cQQi!Dp}a z`hP+Q^IKz&Sg!;#1kj7X@H*pPaeuXP>hT{tEAX1Gmc1kA3o2(9f3~q%fS;Y(#e_X;Tox)fQVBJXmBagAMN^o7Ct~=s*ViI~Xj7+p&bkel=bXI5I?pGj-wm1soH=5K93h|Y z5Y5Z+=crtEqC7Q{i97~uJ^lpLi?xu2Ht0Mf(YB&0rc8IRiEz`%tlIL7`Oe3jJ(9w@ z5Wp0NSm0gFnQr81_~H2v+q<#zcZ3asvf3lki9-NvJLI`)5=s;3h*2^E9ZJ|kv7JMS zQ52lG!n3vcdin~_zPzN+MDqIcQ`l*$%I>Ti_dM*%8gSlqrBroJMkQn;@lr?~Be3v~o*tVh1L9t`+b?jSylC~V%Do@PcI2NCD$dNGZ z-9m*fwRAu{|2|rt>LhX|+MopCtZI1sp+=n2@X}Ttm^es3U{0zVQEMdi-EGuS8!QeH zFnjm9wow$&2;CF1ZYF3BWcd-B0MFuh)q%XO?7@J(rQw32)wPe1-p~??^MN`*Ryxj% z+%W!L$oy)*r79@5?!xil$X)_Xwj2YNsmj0-)4UwuN*|tG%b0U&qCXDv8*!BZb+r}T z5%fH$9L29iRVJk(s4qxH0_kv2ZOTpw9W})v!61_Q_(YoKImB=s&2w0YuqK2o=SRFy zCL%jKv5!`kp%oGpu?@5h!~o%442{@MML;ag1?(G9=dlzJ3CnQ|(g=oggU!=O=>6tsKE3d9%*Ql0a%9X?WDW`+3csgr<0a%{j&|Hwu;~=;1Ct5RynfRD zG$}r)zMTAoDH-$Ag0NX>7A4rRU|40NYv40aSWwJW4P>^m?^P*KP>N+e5gpKfpWrwt z**15>*EbV}fPQ%_WM2T2rkLNgN;mYK65;d7NonP8H+o(OzuQ3n>z7VC9n$21_jp8M z0xc4s8MDn!A{a3w&hJ!3D!voZjCFJ$%sq0{&xL@>|L0CC&%s zX7^t;D|bHpj`uK^1S-bA;K!SL-}7G*#%ulXt>x}3@EyVm2s!RAA!qZ`DoqI_e*#ar z@l}&t#TryO51C;=fn>SSwslV8xsJ|WZ{t==FeXI$*^jUjBot4}w%m*7w&B3iB67`v zs}kzN1xVw66i_#21iehTO0`4NPhlMrjF64&513>A(%y>zDAW$2C5NIwEa8%3oikH z0M^G}*zmOI5okY5XRi(p@*}_FV#ZOR|9qm!P(Pw39|?Sq+eF382C%^Hai#2gJ~kdA zH>L@EVO6w*kPx*|#2BzHc;o_|dP%33C3oYK-6MZnT_&601I#VI6)~!nt5m zW4YHUY*r=3q=sAH%y~uWB{uN_FAD@1b^UBd7uoYs4d=S!t8J@u?855(A7rG( zgN91gF8so+l4Cug4T0h9mPh|tDxw+dLfY*}hGUVG`xgI_1#QTlVd*dNwkT?(h6GnJ zq!`hfI0hTbrY_OHux$Y-wVtDE{7T5o+t6|te$G&Uz`@;B`38fqj7|ekvlt%gvxD&l zN2l1p{>+A{k!Y9o*ox$G|JTXn%6H2Na!tf9pcbv4#2%SZO-Ow*QqWOV4)U-SipR|=_aB`WA!*zae3dm5`7qDB;e+lV(t1$Sv@EOHki^UcV8=1QEPssw+J_8- zvQ)bzV3te&YOCRJvp2!xLwCl-cNtl`cYTN?jq-C@BEtmZ93+?FJY^=ow~XO} z%jR$&@_GdDA>~DnB#By7O-2#kOVrFg5{w-L_n=YwO`p|DSSD!HUUh#riA9LT641$c zep!1IHS;ADU7Rw3*Oex#&Ld@MhW2K91PTv{|Q|@piFPx1N%%DD7d%K1z~4`u(niZ{$fPR@u8Sj-P>Rki72;4{;y-rhm+P2HS-jtK4Ww1=$x>fAGcO z11p!iLSI4HfPupuXCByzuOS!VF&Kr#eTTRL5N@?KIQZrC~=t>{hUjeoNcj}Qww{LWfwu(C_WMj7w?1z_t zE`X%wq$V~>_>{J5!TrK2SU2i2CyBZqULtG{i7ZpLgw}{SEb<9A4Ya#j7C0;_Mag)f z8L|c6>!G9!*#qmh9HS_wu6r~i(VU_nP@4q6kEF8@@P4>cMF*5a!BH!1n(IrYb-%o>{^u-Rl%us0k); zs4G?=MG43%a~edOHOWm125L~B6=tyUJPsV-9t)y)HV0#fqf+nYN@;*$#bv&E0yOoV zY*Q$fdEJUFSo+YgX}m%pt9(g7?0Lpf$mj2Mw?tcm3&k4;0Oz--P=+1xFVCb8JOvuhLqr)uAhW#B^>2OI`qir>y`ESxloebeokf8nGF1hC^w zTWM>PPTK6|3oO3tTgT=M5di|dBOF>!D!H}nqMKol{2{hQoi(8RCQ;PpL5jt;C}u9T+Qv{P19$t~R#?n8_0E=+coV zdHpBnT|ZX#z%ag*NqdYXTlPKvi+VaK#Q4b&<6>n8RC*SO2YgkCE1j!qD9x6Jfskvzd^NQCvNlZKQzk&z9ZYU=u1p*}RdUamn>N#8}4z02> z)+gSk#pe@u$F*iG?7`6v0Yj~guc zh}rvhp3!0I`=1>Ff7(I3`Mnfgbj+WUa%$<@i8X0ZAvTA&x!sdDhqc|9+?sa8wazc! z!zBl8+w~Hfklr^6);P1@^;Hdpp`(O<0IGl~e95bm^Va9q`Z7=p-e}?!7Ed%hAALAj z=9_`eFxs#xipdM~VIc2%XHaWnYpaRujvHKC?YUB1jnohI7VfPeB|IeBd+x$rLLS=S zT&~a2+ifuB(t;eKJ|V^c=rJCZkw#B65WvTo2u1`rlVl;`j>#T}g8)(66}zX}57ry_ zIHjIUjJRjFxfw`KD*fJn&6z4jZR4Ubl5a~0OtKY@d1Vh!m=-|VpwDQXG&Oxoa1>7y zSQQ|jZmS$#IOM0UuHJEZ3mOWyA2&X1d2!ADzbJ~TK>^hQ*pzF1g+D-$oX_B_{Qv=U ztOnL~Rxadq+a8=m(yNn}AZ1|`*oyiesac1ob#G$2g58-ETO?wBYUd1Fvbk9+bNL2v zc&uH*Sn&*eo)j@r)%}(*TT$;WOqYy{xv2h%(kFik930os=ODuk61>P2LNLl8XpWKU z8n+cZi(n^6{2S-0g$DH%_?kh6lEe}7J|zF&!^vUW@f2Xem5-p35qdd38YAm}>tLIu z|-V5vi!#d$>L(f0mJER$`ByPHLUh>>wA zK^}i#W3=uLyr}RU90ta6gu=r+Uc3+BYVFJC&zqE(ejrIWUdPjrMf-rEgC1M|~Cj$_syg5i+e~`Tjb3gXfAU9@4ZYGmYT|oHJF0-;3!{5Xe-Cx#%Lf zuT;7TpltcH@Ru{)n(^!C&YoG%eI?&aO%>JVv`w?G2 zO~S^zTs829f`c*2Ykf804Tv+Wfm%jrckjRyl8FlG#J7=rB=khW*awuRRV-Itid~B( zGBLG=yo7b%hxh>(oFHHdSkUBHwgps}&Gl`XhjMg{-y~!#{7+IXo3LpOAOh#t^-fl& zz#L@{mHkYfA12A75#cQ=`P|6Bo5#T7hrwY5iLEa47eVXG`6vCgWk0f1G8$OIQNGZVVyp8*-+L z+u7DMvKEBhHJLAv1v0=`XXe2ijUiRDYbwDCnrwov%SeLezLknTIc9c<=^10=>Mhe= zfeslgK^P|KuIC(42oM@s5RQT4!S7|h8|#s7qv4maoXH;{ybD0mD7p-qPLEb1ZL=Ac;&&3ty_;_G#ed z^7hFbRLgpiMYB4k(2cj*{T!PYH=gIn&n8d_cPEC%f(a3=)xAT>;ds-=qx^^BL;8yQ#T3;`(uU-}J{7|YRuA?KK& zjvn3*KMu?SKLSFe{1h!IJ>m=3f^z@)At)(NNPPw8CIbb$)XX3FZ^(dCHPP}8#m)jV zRKq9=DWmLQ3CIFAIPvozI~S8ZRrY}LeRl=TB@aH*wGZ4t#tZVDPMC?S+88y-3m4#u zU=P*HZ%?W>p$BN=)~|Q3_huVx6;R%=+Is=;Dg&DQaMGF9&1L3d$P+f2ZpyVrYB+M z_D@`+?3H-Au;3J$hjx+rl?X<^5y0?zVV*+q6GWb5mLbqiGB@Pb3Gl%BtRVo1thl9&;smvpU1Q7>E!Vu z_Kw!8;2P;0yG|qs2?{{Bb8 z?I1n%O8UO%e^O3T@QsYafuU!g>-I|#1<*f5>nTP@v}&D+F5e{wSOO+LCs7Sj0R3bA zQ11Z-`T#~BjJM9up2W!qUU3mNq_8A+?RkTYb^1)1oe*eG3QhV+8=ls&mC1Z((wC_Tn*X9NQ$$-R)DU5t*_&0rL+y9YxkPZo-_u zw%Z}9DNSR7qVrCjW*)p%7*H^#a3J0Y#7ZzTR9;r3c^$;N@1X=g#PXzJRKGTGSjhoG zcp(L_^y3C0URBOC_yzMby_-LZ0asdcBgQVEBw0{TnHfOJ#MUKxSU=2$aa*c|f+tR9 zWVd>9;8}j!rg>T;r7%FY?zQ8j%K`@~M?7|vO-wgBJh?Vu>b=bwApAlSmq&45FTVL8 z$~b;RGGIfi)2Nh`4$Ucog5gt(Q)VcNP?0I^(ry$j%j0w|M}>}tlW-j_$Go^EWAkCf z!^uT{w{sbBAOrCc;GgrZ>|Tg>i2n{D4&^P=RMn%kYXg~^P?mrV0d!*3l`%hb&ILON z^bYAuxZ7|IVvC*dsOfqTx2_bhzDjNKc!s8jL;Ygmw1H;v52hi71yRCu4Xs5t3Igd! zxP$D!Td$&N2}#>LKb8zW=uAYmfEeN0%gNyS8@K!M?gaILCTu77wr``n+oq6}qp#ISc$>oz!$O}ob$XPt?y6snw@W%UAN7xUf>CiRV*E+|s4J1$$euGK_Q2}u`>W~%?A38OO{DyW&Wk+B2pQ>2QL$~1{mqRlhGuyFS}!r{$_ z6Pcx^gr5dEPk3I0M3-sJQn4H@7WzMmpX0-u(}Syf=$L*jV8XR-_67oez%$PV!c$Wj z%4~OsI)bxY7sOmQb$$pFRpsE1PiQw6RgxioV4XSJFr4!kwV)U8PVDVq`MJ=08uHDV z*Ot5>RQy=fz`nd2nttTolk{V;CY$x~Qz}c=e6p1HMN-8Z+E&Ai^Qo(>`p$f`943uvJ(OaaxcKD&nrzt3&o7?WIKJbjBTf8WouzfW!+-)^KliQSm zh1L~5628{xI%vNTz33*U*SvF@B*6T{RfmxOJLZG3bTjNBWi8hax~h)`_&x}M2)wPw zfG*-&(5X8ZQ%26;i=e2afQZ3nYG`$K6$``MFu>=U4;O8@HGx&LR}#d3#DG1YjrtEV zId0D~Q-hs^zKdFxP1J@(7#JlTWm%C2T|%eLF3aZG+vW~k-hFr|*)}%->Af+iti$J7(@2Bia&_sbhuyq+5vgoh%Xu-*2FwYPGAIrF`RPPCfJ|YXrc=34D10R zDS@eQRXB+hKAE*?01Qd``AXUF<2C+yNgy?`3R)@KG}!9p4q^3^wP^3I51|`ICG(ko z6G|p0DchGvKjyIkKgGr3-aGXDMpPeY;mrqr=t2)AJB&A=T)|`n6{#jRQbD7kOylZ$ z3a@|7IInC$gt>Wt<^~5ow9T0M;_t#^i2ufZoQdI_65jqXv!0b~5>}T|62JXB1JUWS zFfvj>(q9MIBUn!683j0z*Eguw$>9Vj2K7xb*~=-j9d4}ywPWLvkAvwF;s|FH@ITnS zqZpQVJc2$|xcYgfFwTkezlw3P!a-ja9bRHaY_kgp2=>aqk?!es#*d-t$pTIhM_X%D zdO(#NTWd@y58nTBdNFm5P%*u|@TA0M9ZY%XX%^!8x{xs6E3OPai5V4Rdy9LR*p-W^LfzvGMSJ z_>^*J8=o=FE8u%w!mo%9e$cHv@isz-tM868XfjB8`c3Me;0^_wUkJ#b%IR8&gca$D zw<*+oPLEMMh^MBCvxi%lB7*u!RwSFO`0Pyov>=p-IF+b6>x2}WF<~BH081BQLsS}k z0=$!Q2z&!OjQJT1x6+qWqdwWeEOc!AHE|EtwU$H)y>LB!9U`ryQD7Lj@3{ZQ?VA5aXG%x{#qq|;(Ht4q zv>WFFV#`|gm=~wNQj}N)M-5cGCMY+qZ87>WFeAo{R^hkQN3wk;igL8XwxN7aLu|HN zSSP>Z2m5I$mPcP#t5ieh!jW+SzUH@43RHWP$P>_sP#I6}kD>D8MS_8a#cJNM<@C1c zABQzJ92X2F^W@!3>&d%MDrDd-!V^fUp;Dj#2Z5wq@E3xo!z5G1pIJzO@sI+cO2!)m zDFv=~CA4du%5lRUycF{2{0Otx+o}e8l1b)gUB+K>h(?id-?h0hL%Q8V-~kdL-Aia5 zPY{l1Z4?N9K$KUS{_R@d=O1h8MDLh#f=1V}B*fb?MHbMppf!QNlm2&BgO@XIz)4{3 z^k*ZB%Tzz+K(r5UwJ~_N3Oz78?yX?FM1OvyW$c>>ARO3-xIB1gP$}wiVic|QL26Rz zLC!L-t}S?v^(;`&272+Ee1FF;bdEOP(Tj1A3uST3SVZ;K_b+52NQb$T#+2}?kZ9kb zagN<|w+-)UZ@cG8uvuWphRK;vxZcUD%2fcX>n_I#=^I*Y%yUoj`Ecx&KER%9XvqPJ zis3^Jz(M$Gw^cb5qK* zQ;7KFh#LXsp#;OseFLS<;%zRKz7yw%sS&%N9E4s%a}uuT;#_Dk4z=;hoBMjtFEb7^ z*y4M!!-0mhIM7IYZNGD-#Y$Q9;FE^Yn@{>VGAxvj<=!ou0mAgCW(yNp@4?e3$?851 zr#3t4p-bz+_f)FB3mP%4uLcdSju0&bLYjT3-{SQf2SRx4LnBSqH0zq%p*uINZ1GY{MUVvFE-}L-C`KlqWHvI@5 zt78S4&UqEoGV}hN&>;^fEb^2|XZG&H%cT5{^?xDlGP#*D+uLwrQ!=VM1+J}%yu=p+ zHC*?s2{x&)3IED?YQ1?mVqZ0528M#hEr&8lxjBfo;GCh!;i?bXC&YYBH6FlDw~B*& z1oB1ydq9&pJYgPY#ox^clZO>(-Yfs#+zphVUc${!B!QG&FVqmrp+6tb=6C8w>w|>d z9enAHqh}bq+yU^;L4j~+NLs}mG_+Z*6kBbkHrp}8QL^ItS^1L{E(6ztP zk^^hqevYI4gvBLBj(~v&>%5~?J z>2BdmGTKEdxZ9v;?VFtz$vk(|7N6jFlv2(V1yNxsi=ksKSNyB-j>5z<@o~53&1DBolqCI&U-=E>54zV5!EAmO1PUNn%fT%s7q+)!P+ed=TDL_`6?is~ZBuLY zJEzl03f!DL94H{Z(Y_{T2#8C#W^;xRfxvX&3_vgDqS(gZS`TVOVE68|NOeyz$V zxe1(u5HZWimA&^WphC-tS^yyRf&?$o4Nq_{AHh4os&Z;mn)jT)bMXmk7*{5IhpA)# zxMTNqavs~#tp6_GT*yZ!vwy%354d=PKyujhKIVe)I%FoPb-1lr=QUqS&xY>?<~hX< zZ}y2A^RB-8#E)>&fRo-pH_Y-_?6nskc-c}(*SGR7ZM}4cI`6g>gBevA3YV#~HR8o~ zw4{__U0M`IJf$w1obN$d)rXhg{ZlAG{xCzKH4}X) zCz&R+7vBa;Kgp@*pUf_<*rj%<#Uz%u&dD?8o?A{VcQPPwa6)bZENHd30z*6;n5Yk* zn-qgSi8Er&_%I1F9)`o;s}D5#5vnMG4vs#Hx+>gGjg_qLpkeTgrVxMJrN-PVEn;Do#An&Ws22<@V0TdfE9^V=MSJF#8G@o3_o;^ zriq#JX*JuUY+-bhVsG3f}(U+YPo+yj8m;eX7SD%Bd=~@$Cs4?h4=2jyK>eMCSFGw<1QesmZzMoec zEbk1DHLYIjn1$lu4O!Gu`7mP&Qg}b~!p1d;6}9gM zf4XjwtP4?eoW|;;zO`}bMvSf*U57G%>xd|OZJN5fM4N@wR&u?_iaEW7J_nMrH2%bx z!cT`K@kj85_YNrN`)htux#fSH?rcE`8Uz92Xw)Y-!T@Hc<->|iFW&fhSX;$vd{^h3 z>7C*Et`bZD6(FG_5jGG_q4Q_|IIowG=J-W;^B3xzw$3W#>J;uC^8E zm0qMLHMkU#tAyWXO&NbR#oupLDKB<+LMsWr!c2u=SX2}t;n^|sUp4*IcHa>am6800 zOaQR(`{vO3A%Mr3&InDk&MZtb9SPenM794Mq+&z}ED3=>JHs;K3$dBmu#`9(AT5=S z(Ej$|Le754>RrccHfJUkkO0~Q!fBpd0xNzTygZk|g$}Xp4zFW06fR>;(!yf{g6~C! zFYCSC4|eGD2gL=W5tJ%a3ySqKbBvt)jMsHmcZLh@29TOK0I#)h`+w-db{6|(W*lrqEDf;sfe(~ot0Z!@@4oGiG!x8Sr z)7At|1#G24=DUp4P}S3T6c4{6) zqo8Nvd0nssANqL@bco>>hZiPYAhwW|xG-~42b(4sdx!dzv* zHF*Lc$2e4(TA~!7fSwIxYAKC6j$@v|5$pEL+IevfB?(JWQHnMrp=wFu78RnfQI?Y9SqwjQcj|5)Fgau+8 z0f?i8CeTuSR%57qlg-OGlC>wzgmtQ;lcYutz9i+g_lJ_fi{a2L;9l3|-GW6z${Dh&F=O|0xfuW}+x zlfRgm1jIQREzibR;x7CBC&CBQq0Zg5&Rh5cJl&%Vog^!$rrZPO!4TjMYPBC YxDWVei4`MSH{R=c`hUJkJO7LS4=8>`vH$=8 literal 0 HcmV?d00001 diff --git a/regression-test/data/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.out b/regression-test/data/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.out index 3404ee00ebf388..18ad24a7652518 100644 --- a/regression-test/data/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.out +++ b/regression-test/data/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.out @@ -1,3 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- 200000 + +-- !sql -- +100000 diff --git a/regression-test/suites/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.groovy b/regression-test/suites/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.groovy index 509f2c98f6068a..75bbd332b760df 100644 --- a/regression-test/suites/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.groovy +++ b/regression-test/suites/load_p1/stream_load/test_stream_load_big_file_with_special_delimiter.groovy @@ -44,4 +44,58 @@ suite("test_stream_load_big_file_with_special_delimiter", "p1") { sql "sync" qt_sql "select count(*) from ${tableName}" + + tableName = "test_csv_big_file_truncate_delimiter"; + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} ( + `measureid` VARCHAR(500) NOT NULL, + `measuretag` VARCHAR(500) NOT NULL, + `timestamp` VARCHAR(500) NOT NULL, + `ds` VARCHAR(255) NULL, + `hh` VARCHAR(255) NULL, + `meter_id` VARCHAR(500) NULL, + `maintenance_team` VARCHAR(1000) NULL, + `psr_class_name` VARCHAR(500) NULL, + `inst_id` VARCHAR(500) NULL, + `location_type` VARCHAR(500) NULL, + `name` VARCHAR(500) NULL, + `depart` VARCHAR(500) NULL, + `measurepoint_id` VARCHAR(500) NULL, + `district` VARCHAR(500) NULL, + `enddevice_psr_class_name` VARCHAR(500) NULL, + `enddevice_psr_id` VARCHAR(500) NULL, + `root_id` VARCHAR(500) NULL, + `rt` VARCHAR(500) NULL, + `measurevalue` VARCHAR(500) NULL, + `dataquality` VARCHAR(500) NULL, + `datatablename` VARCHAR(500) NULL, + `tag` VARCHAR(500) NULL, + `equip_src_id` VARCHAR(500) NULL, + `root_class_name` VARCHAR(500) NULL, + `ssid` VARCHAR(500) NULL, + `sysdate_uep` VARCHAR(500) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`measureid`, `measuretag`, `timestamp`, `ds`) + AUTO PARTITION BY LIST (`ds`)( + ) + DISTRIBUTED BY HASH(`measureid`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + streamLoad { + table "${tableName}" + + set 'column_separator', '@@@' + set 'columns', 'hh,ds,meter_id,maintenance_team,measureid,psr_class_name,inst_id,location_type,name,depart,measurepoint_id,district,enddevice_psr_class_name,enddevice_psr_id,root_id,measuretag,rt,measurevalue,timestamp,dataquality,datatablename,tag,equip_src_id,root_class_name,ssid,sysdate_uep' + set 'enclose', '`' + set 'format', "CSV" + set 'compress_type', 'GZ' + + file 'test_csv_big_file_truncate_delimiter.csv.gz' + } + + sql "sync" + qt_sql "select count(*) from ${tableName}" } From 85d895abf9dcfe7480474326f4228b0dab28ca71 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Sun, 24 Aug 2025 15:19:48 +0800 Subject: [PATCH 498/572] [Cherry-Pick](branch-3.0) Pick "[Enhancement](Compaction) Support auto set compaction task num per round #53408" (#53776) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pick #53408 English: This PR implements adaptive generation of compaction task counts. Before this PR: By default, compaction tasks are generated every 100ms, producing the top-n tasks with the highest compaction scores each time, where n is controlled by the compaction_num_per_round parameter. After this PR: Compaction tasks are still generated every 100ms by default. When compaction_num_per_round is not -1, the number of top-n tasks generated each time is still controlled by this parameter. However, when compaction_num_per_round is set to -1, n is no longer controlled by compaction_num_per_round but becomes adaptive. Implementation details: Initially, 1 task is generated Before generating subsequent tasks, the system checks whether the compaction thread pool queue has been completely consumed (i.e., queue size is 0) If the queue size is 0, it indicates that task generation is too slow, so the number of tasks generated each time is doubled (maximum of 64) If the queue size is greater than half the number of tasks generated per round, it indicates that task generation is too fast, so the number of tasks generated each time is halved (minimum of 1) Summary: This implementation achieves adaptive compaction task count generation. Chinese: 这个pr实现了每次产生compaction task数目的自适应。 在这个pr之前：默认每100ms产生一次compaction task，每次产生compaction score最高的topn个任务，任务数n由compaction_num_per_round参数控制。 这个pr之后：依旧默认每100ms产生一次compaction task，当compaction_num_per_round不为-1时，每次产生topn任务的个数依旧由其控制；当compaction_num_per_round为-1时，n不再由compaction_num_per_round控制而是可以自适应，实现原理：第一次产生1个任务，后续每次产生任务之前会判断compaction线程池的队列任务是否完全被消耗，即是否queue的size是0，如果是，说明产生任务的速度不够，将每次产生任务的个数翻倍，最大为64个；如果queue的size大于每次产生的任务数的一半，说明产生任务的速度过大，这时需要将每次产生任务变为原来的一半，最小为1。 综上所述，这样就实现了compaction task数目的自适应。 ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/common/config.cpp | 7 +- be/src/common/config.h | 1 + be/src/olap/olap_server.cpp | 32 ++++++ be/src/olap/storage_engine.h | 4 + be/src/olap/tablet_manager.cpp | 14 +-- be/test/olap/compaction_task_test.cpp | 153 ++++++++++++++++++++++++++ be/test/olap/tablet_mgr_test.cpp | 19 ++-- 7 files changed, 214 insertions(+), 16 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 0989fd38f18471..34b68cfe0215a4 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1501,7 +1501,12 @@ DEFINE_Bool(enable_table_size_correctness_check, "false"); DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false"); DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false"); -DEFINE_mInt32(compaction_num_per_round, "4"); +// The number of compaction tasks generated each time. +// -1 means automatic number, other values mean fixed number. +DEFINE_mInt32(compaction_num_per_round, "-1"); +// Max automatic compaction task generated num per round. +// Only valid if "compaction_num_per_round = 0" +DEFINE_mInt32(max_automatic_compaction_num_per_round, "64"); DEFINE_mInt32(check_tablet_delete_bitmap_interval_seconds, "300"); DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n, "10"); diff --git a/be/src/common/config.h b/be/src/common/config.h index a36f185a2c2206..cc324c11b6bdc3 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1571,6 +1571,7 @@ DECLARE_Bool(enable_table_size_correctness_check); DECLARE_mBool(enable_sleep_between_delete_cumu_compaction); DECLARE_mInt32(compaction_num_per_round); +DECLARE_mInt32(max_automatic_compaction_num_per_round); DECLARE_mInt32(check_tablet_delete_bitmap_interval_seconds); DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n); diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 36971ba5e9bfd9..fe441522726a35 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -98,6 +98,8 @@ using io::Path; // number of running SCHEMA-CHANGE threads volatile uint32_t g_schema_change_active_threads = 0; +bvar::Status g_cumu_compaction_task_num_per_round("cumu_compaction_task_num_per_round", 0); +bvar::Status g_base_compaction_task_num_per_round("base_compaction_task_num_per_round", 0); static const uint64_t DEFAULT_SEED = 104729; static const uint64_t MOD_PRIME = 7652413; @@ -584,6 +586,7 @@ void StorageEngine::_tablet_path_check_callback() { } void StorageEngine::_adjust_compaction_thread_num() { + TEST_SYNC_POINT_RETURN_WITH_VOID("StorageEngine::_adjust_compaction_thread_num.return_void"); auto base_compaction_threads_num = get_base_compaction_threads_num(_store_map.size()); if (_base_compaction_thread_pool->max_threads() != base_compaction_threads_num) { int old_max_threads = _base_compaction_thread_pool->max_threads(); @@ -687,6 +690,33 @@ void StorageEngine::_compaction_tasks_producer_callback() { last_base_score_update_time = cur_time; } } + std::unique_ptr& thread_pool = + (compaction_type == CompactionType::CUMULATIVE_COMPACTION) + ? _cumu_compaction_thread_pool + : _base_compaction_thread_pool; + bvar::Status& g_compaction_task_num_per_round = + (compaction_type == CompactionType::CUMULATIVE_COMPACTION) + ? g_cumu_compaction_task_num_per_round + : g_base_compaction_task_num_per_round; + if (config::compaction_num_per_round != -1) { + _compaction_num_per_round = config::compaction_num_per_round; + } else if (thread_pool->get_queue_size() == 0) { + // If all tasks in the thread pool queue are executed, + // double the number of tasks generated each time, + // with a maximum of config::max_automatic_compaction_num_per_round tasks per generation. + if (_compaction_num_per_round < config::max_automatic_compaction_num_per_round) { + _compaction_num_per_round *= 2; + g_compaction_task_num_per_round.set_value(_compaction_num_per_round); + } + } else if (thread_pool->get_queue_size() > _compaction_num_per_round / 2) { + // If all tasks in the thread pool is greater than + // half of the tasks submitted in the previous round, + // reduce the number of tasks generated each time by half, with a minimum of 1. + if (_compaction_num_per_round > 1) { + _compaction_num_per_round /= 2; + g_compaction_task_num_per_round.set_value(_compaction_num_per_round); + } + } std::vector tablets_compaction = _generate_compaction_tasks(compaction_type, data_dirs, check_score); if (tablets_compaction.size() == 0) { @@ -946,6 +976,8 @@ bool has_free_compaction_slot(CompactionSubmitRegistry* registry, DataDir* dir, std::vector StorageEngine::_generate_compaction_tasks( CompactionType compaction_type, std::vector& data_dirs, bool check_score) { + TEST_SYNC_POINT_RETURN_WITH_VALUE("olap_server::_generate_compaction_tasks.return_empty", + std::vector {}); _update_cumulative_compaction_policy(); std::vector tablets_compaction; uint32_t max_compaction_score = 0; diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h index 9dd780956f52ec..50b82d67232749 100644 --- a/be/src/olap/storage_engine.h +++ b/be/src/olap/storage_engine.h @@ -342,6 +342,8 @@ class StorageEngine final : public BaseStorageEngine { std::unordered_map>* workers; + int64_t get_compaction_num_per_round() const { return _compaction_num_per_round; } + private: // Instance should be inited from `static open()` // MUST NOT be called in other circumstances. @@ -562,6 +564,8 @@ class StorageEngine final : public BaseStorageEngine { scoped_refptr _check_delete_bitmap_score_thread; int64_t _last_get_peers_replica_backends_time_ms {0}; + + int64_t _compaction_num_per_round {1}; }; // lru cache for create tabelt round robin in disks diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 3e7d48e44af97e..b6b2d3b2293fc2 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -743,6 +743,8 @@ std::vector TabletManager::find_best_tablets_to_compaction( uint32_t single_compact_highest_score = 0; TabletSharedPtr best_tablet; TabletSharedPtr best_single_compact_tablet; + int64_t compaction_num_per_round = + ExecEnv::GetInstance()->storage_engine().to_local().get_compaction_num_per_round(); auto cmp = [](TabletScore left, TabletScore right) { return left.score > right.score; }; std::priority_queue, decltype(cmp)> top_tablets(cmp); @@ -809,20 +811,18 @@ std::vector TabletManager::find_best_tablets_to_compaction( best_single_compact_tablet = tablet_ptr; } - if (config::compaction_num_per_round > 1 && !tablet_ptr->should_fetch_from_peer()) { + if (compaction_num_per_round > 1 && !tablet_ptr->should_fetch_from_peer()) { TabletScore ts; ts.score = current_compaction_score; ts.tablet_ptr = tablet_ptr; - if ((top_tablets.size() >= config::compaction_num_per_round && + if ((top_tablets.size() >= compaction_num_per_round && current_compaction_score > top_tablets.top().score) || - top_tablets.size() < config::compaction_num_per_round) { + top_tablets.size() < compaction_num_per_round) { top_tablets.push(ts); - if (top_tablets.size() > config::compaction_num_per_round) { + if (top_tablets.size() > compaction_num_per_round) { top_tablets.pop(); } - if (current_compaction_score > highest_score) { - highest_score = current_compaction_score; - } + highest_score = std::max(current_compaction_score, highest_score); } } else { if (current_compaction_score > highest_score && !tablet_ptr->should_fetch_from_peer()) { diff --git a/be/test/olap/compaction_task_test.cpp b/be/test/olap/compaction_task_test.cpp index c00406e6d157f4..d70d474236276f 100644 --- a/be/test/olap/compaction_task_test.cpp +++ b/be/test/olap/compaction_task_test.cpp @@ -130,4 +130,157 @@ TEST_F(CompactionTaskTest, TestSubmitCompactionTask) { EXPECT_EQ(executing_task_num, 2); } +TEST_F(CompactionTaskTest, TestAutoSetCompactionIncreaseTaskNum) { + auto st = ThreadPoolBuilder("BaseCompactionTaskThreadPool") + .set_min_threads(2) + .set_max_threads(2) + .build(&_storage_engine->_base_compaction_thread_pool); + EXPECT_TRUE(st.ok()); + st = ThreadPoolBuilder("CumuCompactionTaskThreadPool") + .set_min_threads(2) + .set_max_threads(2) + .build(&_storage_engine->_cumu_compaction_thread_pool); + EXPECT_TRUE(st.ok()); + config::disable_auto_compaction = false; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("olap_server::_generate_compaction_tasks.return_empty", [](auto&& values) { + auto* ret = try_any_cast_ret>(values); + ret->second = true; + }); + sp->set_call_back("StorageEngine::_adjust_compaction_thread_num.return_void", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + sp->set_call_back("StorageEngine::_compaction_tasks_producer_callback", + [](auto&& values) { std::this_thread::sleep_for(std::chrono::seconds(1)); }); + + Defer defer {[&]() { + _storage_engine->_stop_background_threads_latch.count_down(); + sp->clear_all_call_backs(); + }}; + + config::generate_compaction_tasks_interval_ms = 1000; + { + // queue size 1 + // task num 1->1 + _storage_engine->_cumu_compaction_thread_pool->_total_queued_tasks = 1; + // compaction tasks producer thread + st = Thread::create( + "StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_storage_engine->_compaction_tasks_producer_callback(); }, + &_storage_engine->_compaction_tasks_producer_thread); + EXPECT_TRUE(st.ok()); + _storage_engine->_stop_background_threads_latch.count_down(); + sleep(2); + EXPECT_EQ(_storage_engine->get_compaction_num_per_round(), 1); + } + { + // queue size 0 + // task num 4->8 + _storage_engine->_cumu_compaction_thread_pool->_total_queued_tasks = 0; + _storage_engine->_compaction_num_per_round = 4; + // compaction tasks producer thread + st = Thread::create( + "StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_storage_engine->_compaction_tasks_producer_callback(); }, + &_storage_engine->_compaction_tasks_producer_thread); + EXPECT_TRUE(st.ok()); + _storage_engine->_stop_background_threads_latch.count_down(); + sleep(2); + EXPECT_EQ(_storage_engine->get_compaction_num_per_round(), 8); + } + { + // queue size 0 + // task num 64->64 + _storage_engine->_cumu_compaction_thread_pool->_total_queued_tasks = 0; + _storage_engine->_compaction_num_per_round = 64; + // compaction tasks producer thread + st = Thread::create( + "StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_storage_engine->_compaction_tasks_producer_callback(); }, + &_storage_engine->_compaction_tasks_producer_thread); + EXPECT_TRUE(st.ok()); + _storage_engine->_stop_background_threads_latch.count_down(); + sleep(2); + EXPECT_EQ(_storage_engine->get_compaction_num_per_round(), 64); + } +} + +TEST_F(CompactionTaskTest, TestAutoSetCompactionDecreaseTaskNum) { + auto st = ThreadPoolBuilder("BaseCompactionTaskThreadPool") + .set_min_threads(2) + .set_max_threads(2) + .build(&_storage_engine->_base_compaction_thread_pool); + EXPECT_TRUE(st.ok()); + st = ThreadPoolBuilder("CumuCompactionTaskThreadPool") + .set_min_threads(2) + .set_max_threads(2) + .build(&_storage_engine->_cumu_compaction_thread_pool); + EXPECT_TRUE(st.ok()); + config::disable_auto_compaction = false; + + auto* sp = SyncPoint::get_instance(); + sp->enable_processing(); + sp->set_call_back("olap_server::_generate_compaction_tasks.return_empty", [](auto&& values) { + auto* ret = try_any_cast_ret>(values); + ret->second = true; + }); + sp->set_call_back("StorageEngine::_adjust_compaction_thread_num.return_void", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + sp->set_call_back("StorageEngine::_compaction_tasks_producer_callback", + [](auto&& values) { std::this_thread::sleep_for(std::chrono::seconds(1)); }); + + Defer defer {[&]() { + _storage_engine->_stop_background_threads_latch.count_down(); + sp->clear_all_call_backs(); + }}; + + config::generate_compaction_tasks_interval_ms = 1000; + { + // queue size 3 + // task num 8->8 + _storage_engine->_compaction_num_per_round = 8; + _storage_engine->_cumu_compaction_thread_pool->_total_queued_tasks = 3; + // compaction tasks producer thread + st = Thread::create( + "StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_storage_engine->_compaction_tasks_producer_callback(); }, + &_storage_engine->_compaction_tasks_producer_thread); + EXPECT_TRUE(st.ok()); + _storage_engine->_stop_background_threads_latch.count_down(); + sleep(2); + EXPECT_EQ(_storage_engine->get_compaction_num_per_round(), 8); + } + { + // queue size 5 + // task num 8->4 + _storage_engine->_cumu_compaction_thread_pool->_total_queued_tasks = 5; + _storage_engine->_compaction_num_per_round = 8; + // compaction tasks producer thread + st = Thread::create( + "StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_storage_engine->_compaction_tasks_producer_callback(); }, + &_storage_engine->_compaction_tasks_producer_thread); + EXPECT_TRUE(st.ok()); + _storage_engine->_stop_background_threads_latch.count_down(); + sleep(2); + EXPECT_EQ(_storage_engine->get_compaction_num_per_round(), 4); + } + { + // queue size 1 + // task num 1->1 + _storage_engine->_cumu_compaction_thread_pool->_total_queued_tasks = 1; + _storage_engine->_compaction_num_per_round = 1; + // compaction tasks producer thread + st = Thread::create( + "StorageEngine", "compaction_tasks_producer_thread", + [this]() { this->_storage_engine->_compaction_tasks_producer_callback(); }, + &_storage_engine->_compaction_tasks_producer_thread); + EXPECT_TRUE(st.ok()); + _storage_engine->_stop_background_threads_latch.count_down(); + sleep(2); + EXPECT_EQ(_storage_engine->get_compaction_num_per_round(), 1); + } +} + } // namespace doris diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp index 7b940ffa0faf00..4096c9ed86efd9 100644 --- a/be/test/olap/tablet_mgr_test.cpp +++ b/be/test/olap/tablet_mgr_test.cpp @@ -73,7 +73,9 @@ class TabletMgrTest : public testing::Test { EngineOptions options; // won't open engine, options.path is needless options.backend_uid = UniqueId::gen_uid(); - k_engine = std::make_unique(options); + auto engine = std::make_unique(options); + ExecEnv::GetInstance()->set_storage_engine(std::move(engine)); + k_engine = &ExecEnv::GetInstance()->storage_engine().to_local(); _data_dir = new DataDir(*k_engine, _engine_data_path, 1000000000); static_cast(_data_dir->init()); _tablet_mgr = k_engine->tablet_manager(); @@ -82,10 +84,11 @@ class TabletMgrTest : public testing::Test { virtual void TearDown() { SAFE_DELETE(_data_dir); EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_engine_data_path).ok()); + ExecEnv::GetInstance()->set_storage_engine(nullptr); _tablet_mgr = nullptr; config::compaction_num_per_round = 1; } - std::unique_ptr k_engine; + StorageEngine* k_engine; private: DataDir* _data_dir = nullptr; @@ -470,7 +473,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } { - config::compaction_num_per_round = 10; + k_engine->_compaction_num_per_round = 10; for (int64_t i = 1; i <= 100; ++i) { create_tablet(10000 + i, false, i); } @@ -488,7 +491,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { 100 - index - 1); index++; } - config::compaction_num_per_round = 1; + k_engine->_compaction_num_per_round = 1; // drop all tablets for (int64_t id = 10001; id <= 10100; ++id) { Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false); @@ -497,7 +500,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } { - config::compaction_num_per_round = 10; + k_engine->_compaction_num_per_round = 10; for (int64_t i = 1; i <= 100; ++i) { create_tablet(20000 + i, false, i); } @@ -520,7 +523,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { cumulative_compaction_policies[CUMULATIVE_SIZE_BASED_POLICY]), 200 - 1); - config::compaction_num_per_round = 1; + k_engine->_compaction_num_per_round = 1; // drop all tablets for (int64_t id = 20001; id <= 20100; ++id) { Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false); @@ -532,7 +535,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { } { - config::compaction_num_per_round = 10; + k_engine->_compaction_num_per_round = 10; for (int64_t i = 1; i <= 5; ++i) { create_tablet(30000 + i, false, i + 5); } @@ -549,7 +552,7 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) { 10 - i - 1); } - config::compaction_num_per_round = 1; + k_engine->_compaction_num_per_round = 1; // drop all tablets for (int64_t id = 30001; id <= 30005; ++id) { Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false); From b20e0d4497c68e6385b5993d8746876c05fa1c0a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:22:58 +0800 Subject: [PATCH 499/572] branch-3.0: [fix](warmup): refresh BE list on every job execution #54963 (#55207) Cherry-picked from #54963 Co-authored-by: Kaijie Chen --- .../src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java index aacec3f3f997f1..6a99b10f82e862 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CloudWarmUpJob.java @@ -220,6 +220,7 @@ private void fetchBeToThriftAddress() { String clusterName = isEventDriven() ? srcClusterName : dstClusterName; List backends = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) .getBackendsByClusterName(clusterName); + this.beToThriftAddress = new HashMap<>(); for (Backend backend : backends) { beToThriftAddress.put(backend.getId(), backend.getHost() + ":" + backend.getBePort()); } @@ -597,6 +598,10 @@ private void runPendingJob() throws DdlException { this.setJobDone = false; this.lastBatchId = -1; this.startTimeMs = System.currentTimeMillis(); + // reset clients to ensure we have the latest BE info + this.beToThriftAddress = null; + this.beToClient = null; + this.beToAddr = null; MetricRepo.updateClusterWarmUpJobLatestStartTime(String.valueOf(jobId), srcClusterName, dstClusterName, startTimeMs); this.fetchBeToTabletIdBatches(); From 85dd19b1b05c785ca733c5a7dd95c3707155a831 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:24:51 +0800 Subject: [PATCH 500/572] branch-3.0: [fix](case) skip test_set_partition_version in cloud mode #54918 (#55200) Cherry-picked from #54918 Co-authored-by: Yongqiang YANG --- .../suites/version_p0/test_set_partition_version.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/regression-test/suites/version_p0/test_set_partition_version.groovy b/regression-test/suites/version_p0/test_set_partition_version.groovy index b461d01d800eae..8689ce8964c73b 100644 --- a/regression-test/suites/version_p0/test_set_partition_version.groovy +++ b/regression-test/suites/version_p0/test_set_partition_version.groovy @@ -16,6 +16,10 @@ // under the License. suite("test_set_partition_version") { + if (isCloudMode()) { + return + } + def tableName1 = "test_set_partition_version" sql """ DROP TABLE IF EXISTS ${tableName1} """ sql """ From 07ec37d8e178bbbb49b5a9389b005b471cc4e32d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:26:28 +0800 Subject: [PATCH 501/572] branch-3.0: [test](warmup) fix test_warm_up_cluster_event_rename #55145 (#55196) Cherry-picked from #55145 Co-authored-by: Kaijie Chen --- .../test_warm_up_cluster_event_rename.groovy | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy index fc3ac52ae91d29..857eac205f2a58 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_rename.groovy @@ -89,14 +89,26 @@ suite('test_warm_up_cluster_event_rename', 'docker') { def submitted_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_num") def finished_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_num") def failed_segment = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_num") + def submitted_segment_size = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_segment_size") + def finished_segment_size = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_segment_size") + def failed_segment_size = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_segment_size") def submitted_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_num") def finished_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_num") def failed_index = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_num") + def submitted_index_size = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_submitted_index_size") + def finished_index_size = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_finished_index_size") + def failed_index_size = getBrpcMetrics(ip, port, "file_cache_event_driven_warm_up_failed_index_size") logger.info("${cluster} be ${ip}:${port}, submitted_segment=${submitted_segment}" + ", finished_segment=${finished_segment}, failed_segment=${failed_segment}" + + ", submitted_segment_size=${submitted_segment_size}" + + ", finished_segment_size=${finished_segment_size}" + + ", failed_segment_size=${failed_segment_size}" + ", submitted_index=${submitted_index}" + ", finished_index=${finished_index}" - + ", failed_index=${failed_index}") + + ", failed_index=${failed_index}" + + ", submitted_index_size=${submitted_index_size}" + + ", finished_index_size=${finished_index_size}" + + ", failed_index_size=${failed_index_size}") } } @@ -174,7 +186,8 @@ suite('test_warm_up_cluster_event_rename', 'docker') { sql """use @${clusterName1}""" // Simple setup to simulate data load and access - sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 PROPERTIES ("file_cache_ttl_seconds" = "3600")""" + sql """CREATE TABLE IF NOT EXISTS customer (id INT, name STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 3 + PROPERTIES ("file_cache_ttl_seconds" = "3600", "disable_auto_compaction" = "true")""" // Start warm up job def jobId_ = sql """ @@ -199,8 +212,8 @@ suite('test_warm_up_cluster_event_rename', 'docker') { logFileCacheDownloadMetrics(clusterName2) checkTTLCacheSizeSumEqual(clusterName1, clusterName2) - srcSumOld = getClusterTTLCacheSizeSum(clusterName1) - dstSumOld = getClusterTTLCacheSizeSum(clusterName2) + def srcSumOld = getClusterTTLCacheSizeSum(clusterName1) + def dstSumOld = getClusterTTLCacheSizeSum(clusterName2) // rename sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName2} ${clusterName3}""" From 6a9f627a00ea0ede40d2d8bf03193b1ca5f55036 Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Sun, 24 Aug 2025 15:27:05 +0800 Subject: [PATCH 502/572] [branch-3.0] [fix](cloud) Check internal_get_tablet_stats error code #55065 (#55189) pick: https://github.com/apache/doris/pull/55065 --- cloud/src/meta-service/meta_service_job.cpp | 18 ++++++++++++++++++ .../meta-service/meta_service_tablet_stats.cpp | 4 +++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 6b60a0dd81c371..7b1f13462037f1 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -712,6 +712,15 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string // with `config::split_tablet_stats = true` can meet the condition. internal_get_tablet_stats(code, msg, txn.get(), instance_id, request->job().idx(), *stats, detached_stats, config::snapshot_get_tablet_stats); + if (code != MetaServiceCode::OK) { + LOG_WARNING("failed to get tablet stats") + .tag("instance_id", instance_id) + .tag("tablet_id", tablet_id) + .tag("code", code) + .tag("msg", msg); + return; + } + if (compaction.type() == TabletCompactionJobPB::EMPTY_CUMULATIVE) { stats->set_cumulative_compaction_cnt(stats->cumulative_compaction_cnt() + 1); stats->set_cumulative_point(compaction.output_cumulative_point()); @@ -1303,6 +1312,15 @@ void process_schema_change_job(MetaServiceCode& code, std::string& msg, std::str // with `config::split_tablet_stats = true` can meet the condition. internal_get_tablet_stats(code, msg, txn.get(), instance_id, new_tablet_idx, *stats, detached_stats, config::snapshot_get_tablet_stats); + if (code != MetaServiceCode::OK) { + LOG_WARNING("failed to get tablet stats") + .tag("instance_id", instance_id) + .tag("tablet_id", tablet_id) + .tag("code", code) + .tag("msg", msg); + return; + } + // clang-format off // ATTN: cumu point in job is from base tablet which may be fetched long time ago // since the new tablet may have done cumu compactions with alter_version as initial cumu point diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp b/cloud/src/meta-service/meta_service_tablet_stats.cpp index c8bb315f60ebf3..07f86f20d1bc8b 100644 --- a/cloud/src/meta-service/meta_service_tablet_stats.cpp +++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp @@ -173,7 +173,9 @@ void internal_get_tablet_stats(MetaServiceCode& code, std::string& msg, Transact TabletStatsPB& stats, bool snapshot) { TabletStats detached_stats; internal_get_tablet_stats(code, msg, txn, instance_id, idx, stats, detached_stats, snapshot); - merge_tablet_stats(stats, detached_stats); + if (code == MetaServiceCode::OK) { + merge_tablet_stats(stats, detached_stats); + } } MetaServiceResponseStatus parse_fix_tablet_stats_param( From 72ebf34ac92c79914de15ead260dfc600c025248 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:28:28 +0800 Subject: [PATCH 503/572] branch-3.0: [chore](alter job) Fix finished alter job print too much warning log #55132 (#55154) Cherry-picked from #55132 Co-authored-by: xy720 <22125576+xy720@users.noreply.github.com> --- .../apache/doris/alter/MaterializedViewHandler.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 84def3c626680a..9145622871f1fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -162,8 +162,10 @@ private boolean removeAlterJobV2FromTableNotFinalStateJobMap(AlterJobV2 alterJob if (tableNotFinalStateJobIdset == null) { // This could happen when this job is already removed before. // return false, so that we will not set table's to NORMAL again. - LOG.warn("alter job is already removed before. tableId: {}, jobId: {}", - tableId, jobId); + if (LOG.isDebugEnabled()) { + LOG.debug("alter job is already removed before. tableId: {}, jobId: {}", + tableId, jobId); + } return false; } tableNotFinalStateJobIdset.remove(jobId); @@ -1233,8 +1235,10 @@ private void onJobDone(AlterJobV2 alterJob) { LOG.info("set table's state to NORMAL, table id: {}, job id: {}", alterJob.getTableId(), alterJob.getJobId()); } else { - LOG.warn("Failed to remove job from tableNotFinalStateJobMap, table id: {}, job id: {}", - alterJob.getTableId(), alterJob.getJobId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to remove job from tableNotFinalStateJobMap, table id: {}, job id: {}", + alterJob.getTableId(), alterJob.getJobId()); + } } } From cff28e0cbf65341cfd108e9d1eb5379e407275cd Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:31:07 +0800 Subject: [PATCH 504/572] branch-3.0: [test](mtmv) Fix use wrong create async materialized view method which cause regression test unstable #55068 (#55096) Cherry-picked from #55068 Co-authored-by: seawinde --- .../lower_case/test_lower_case_mtmv.out | 2 +- .../lower_case/test_lower_case_mtmv.groovy | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out index f958424e65c626..aaa71b52aaaa91 100644 --- a/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out @@ -1,3 +1,3 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- - +1 lower diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy index 8439e0a56a1a9f..e123e3fb17a846 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy @@ -65,14 +65,8 @@ suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker "connection_pool_max_life_time" = "600000" )""" - - sql """CREATE MATERIALIZED VIEW internal.EXTERNAL_LOWER_MTMV.MTMV_TEST - REFRESH COMPLETE ON SCHEDULE EVERY 1 minute - DISTRIBUTED BY RANDOM BUCKETS 1 - PROPERTIES ( - 'replication_num' = '1' - ) - AS SELECT * FROM test_lower_case_mtmv.external_lower_mtmv.table_test;""" + create_async_mv("EXTERNAL_LOWER_MTMV", "MTMV_TEST", + """SELECT * FROM test_lower_case_mtmv.external_lower_mtmv.table_test""") qt_select """select * from internal.EXTERNAL_LOWER_MTMV.MTMV_TEST""" From 9974cd393a6148853b2a92e067f83e5cc12d3f12 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:32:32 +0800 Subject: [PATCH 505/572] branch-3.0: [opt](docker compose) Support get core dump file from docker compose #54913 (#54968) Cherry-picked from #54913 Co-authored-by: deardeng --- docker/runtime/doris-compose/Readme.md | 75 +++++++++++++++++++ docker/runtime/doris-compose/cluster.py | 23 +++++- docker/runtime/doris-compose/command.py | 3 +- .../doris-compose/resource/entrypoint.sh | 19 +++++ 4 files changed, 117 insertions(+), 3 deletions(-) diff --git a/docker/runtime/doris-compose/Readme.md b/docker/runtime/doris-compose/Readme.md index f304069c3c1f79..c4b60460e49b8a 100644 --- a/docker/runtime/doris-compose/Readme.md +++ b/docker/runtime/doris-compose/Readme.md @@ -213,6 +213,81 @@ steps: Each cluster has logs in Docker in '/tmp/doris/{cluster-name}/{node-xxx}/log/'. For each node, doris compose will also print log in '/tmp/doris/{cluster-name}/{node-xxx}/log/health.out' +### Core Dump + +Doris Compose supports core dump generation for debugging purposes. When a process crashes, it will generate a core dump file that can be analyzed with tools like gdb. + +#### Core Dump Location + +Core dump files are generated in the following locations: + +- **Host System**: `/tmp/doris/{cluster-name}/{node-xxx}/core_dump/` +- **Container**: `/opt/apache-doris/core_dump/` + +The core dump files follow the pattern: `core.{executable}.{pid}.{timestamp}` + +For example: +``` +/tmp/doris/my-cluster/be-1/core_dump/core.doris_be.12345.1755418335 +``` + +#### Core Pattern Configuration + +The system uses the core pattern from `/proc/sys/kernel/core_pattern` on the host system. The default pattern is: +``` +/opt/apache-doris/core_dump/core.%e.%p.%t +``` + +Where: +- `%e`: executable name +- `%p`: process ID +- `%t`: timestamp + +#### Core Dump Settings + +Doris Compose automatically configures the following settings for core dump generation: + +1. **Container Settings**: + - `ulimits.core = -1` (unlimited core file size) + - `cap_add: ["SYS_ADMIN"]` (required capabilities) + - `privileged: true` (privileged mode) + +2. **Directory Permissions**: + - Core dump directory is created with 777 permissions + - Ownership is set to the host user for non-root containers + +3. **Non-Root User Support**: + - Core dump directory permissions are automatically configured + - Works with both root and non-root user containers + +#### Troubleshooting + +If core dumps are not being generated: + +1. **Check ulimit settings**: + ```bash + ulimit -c + # Should return "unlimited" or a positive number + ``` + +2. **Check directory permissions**: + ```bash + ls -la /tmp/doris/{cluster-name}/{node-xxx}/core_dump/ + # Should show 777 permissions + ``` + +3. **Check core pattern**: + ```bash + cat /proc/sys/kernel/core_pattern + # Should show the expected pattern + ``` + +4. **Check container logs**: + ```bash + docker logs {container-name} + # Look for core dump related messages + ``` + ### Up cluster using non-detach mode ```shell diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py index 8d4d65097e9a2f..cf63f403240fce 100644 --- a/docker/runtime/doris-compose/cluster.py +++ b/docker/runtime/doris-compose/cluster.py @@ -371,6 +371,14 @@ def service_name(self): return utils.with_doris_prefix("{}-{}".format(self.cluster.name, self.get_name())) + def get_system_core_pattern(self): + """Get system core pattern from /proc/sys/kernel/core_pattern""" + try: + with open("/proc/sys/kernel/core_pattern", "r") as f: + return f.read().strip() + except: + return "core" + def docker_env(self): enable_coverage = self.cluster.coverage_dir @@ -433,7 +441,19 @@ def docker_home_dir(self): raise Exception("No implemented") def compose(self): + # Get system core pattern to determine core dump directory + core_pattern = self.get_system_core_pattern() + + # Extract directory from core pattern if it's an absolute path + core_dump_dir = "/opt/apache-doris/core_dump" # default + if core_pattern.startswith("/"): + # Extract directory part (everything before the filename) + core_dump_dir = os.path.dirname(core_pattern) + volumes = [ + "{}:{}".format(os.path.join(self.get_path(), "core_dump"), + core_dump_dir) + ] + [ "{}:{}/{}".format(os.path.join(self.get_path(), sub_dir), self.docker_home_dir(), sub_dir) for sub_dir in self.expose_sub_dirs() @@ -452,7 +472,8 @@ def compose(self): DOCKER_DORIS_PATH)) content = { - "cap_add": ["SYS_PTRACE"], + "cap_add": ["SYS_ADMIN"], + "privileged": "true", "container_name": self.service_name(), "environment": self.docker_env(), "image": self.get_image(), diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py index 84febcf415ff74..35adfacf7aae92 100644 --- a/docker/runtime/doris-compose/command.py +++ b/docker/runtime/doris-compose/command.py @@ -1258,8 +1258,7 @@ def parse_cluster_compose_file(cluster_name): if services is None: return COMPOSE_BAD, {} return COMPOSE_GOOD, { - service: - ComposeService( + service: ComposeService( service, ip_for_host_mode if ip_for_host_mode else list(service_conf["networks"].values())[0] ["ipv4_address"], service_conf["image"]) diff --git a/docker/runtime/doris-compose/resource/entrypoint.sh b/docker/runtime/doris-compose/resource/entrypoint.sh index a3cdaaae8f1b52..0446004d9c2ff6 100644 --- a/docker/runtime/doris-compose/resource/entrypoint.sh +++ b/docker/runtime/doris-compose/resource/entrypoint.sh @@ -58,6 +58,25 @@ create_host_user() { create_host_user +# Setup core dump directory permissions for non-root users +setup_core_dump() { + # Set fs.suid_dumpable to allow core dumps from setuid programs + echo 2 >/proc/sys/fs/suid_dumpable + # Set core pattern to save core dumps in the mounted directory + echo "/opt/apache-doris/core_dump/core-%e-%p-%t" >/proc/sys/kernel/core_pattern + if [ ! -z ${HOST_USER} ]; then + # Create core dump directory if it doesn't exist + mkdir -p /opt/apache-doris/core_dump + # Set permissions to allow core dumps from non-root users + chmod 777 /opt/apache-doris/core_dump + # Set ownership to the host user + chown ${HOST_USER}:${HOST_USER} /opt/apache-doris/core_dump + health_log "setup core dump directory for user ${HOST_USER}" + fi +} + +setup_core_dump + if command -v gosu 2>&1 >/dev/null; then if [ -f ${LOG_FILE} ]; then chown ${RUN_USER}:${RUN_USER} ${LOG_FILE} From b5da89ebaad2f58cdd31f1b1fb07a1a5e4b84465 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:33:18 +0800 Subject: [PATCH 506/572] branch-3.0: [test](warmup) increase wait time to fix flaky test #54964 (#55032) Cherry-picked from #54964 Co-authored-by: Kaijie Chen --- .../warm_up/cluster/test_warm_up_cluster_periodic.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy index 87938bd60af998..3ad8c820f7d254 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy @@ -164,7 +164,7 @@ suite('test_warm_up_cluster_periodic', 'docker') { sql """SELECT * FROM customer""" } - sleep(5000) + sleep(10000) def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" logger.info("hotspot: {}", hotspot) From b4768500f29a8b26cacc438afb44522e3e886b5d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:33:56 +0800 Subject: [PATCH 507/572] branch-3.0: [fix](asan) clear global storage resource map before exiting #54814 (#54826) Cherry-picked from #54814 Co-authored-by: Yongqiang YANG --- be/src/olap/storage_policy.cpp | 5 +++++ be/src/olap/storage_policy.h | 2 ++ be/src/runtime/exec_env_init.cpp | 2 ++ 3 files changed, 9 insertions(+) diff --git a/be/src/olap/storage_policy.cpp b/be/src/olap/storage_policy.cpp index 837e9bed178e3a..b3c132b68e89bb 100644 --- a/be/src/olap/storage_policy.cpp +++ b/be/src/olap/storage_policy.cpp @@ -128,6 +128,11 @@ void delete_storage_resource(int64_t resource_id) { s_storage_resource_mgr.map.erase(id_str); } +void clear_storage_resource() { + std::lock_guard lock(s_storage_resource_mgr.mtx); + s_storage_resource_mgr.map.clear(); +} + std::vector> get_storage_resource_ids() { std::vector> res; res.reserve(s_storage_resource_mgr.map.size()); diff --git a/be/src/olap/storage_policy.h b/be/src/olap/storage_policy.h index f79b1a052095ca..e83a447aa6a60e 100644 --- a/be/src/olap/storage_policy.h +++ b/be/src/olap/storage_policy.h @@ -104,6 +104,8 @@ void put_storage_resource(int64_t resource_id, StorageResource resource, int64_t void delete_storage_resource(int64_t resource_id); +void clear_storage_resource(); + // return [id, version] of all resources std::vector> get_storage_resource_ids(); diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp index 3ec7fed673f6da..d2f91c6664e4ae 100644 --- a/be/src/runtime/exec_env_init.cpp +++ b/be/src/runtime/exec_env_init.cpp @@ -55,6 +55,7 @@ #include "olap/schema_cache.h" #include "olap/segment_loader.h" #include "olap/storage_engine.h" +#include "olap/storage_policy.h" #include "olap/tablet_column_object_pool.h" #include "olap/tablet_meta.h" #include "olap/tablet_schema_cache.h" @@ -843,6 +844,7 @@ void ExecEnv::destroy() { _s_tracking_memory = false; + clear_storage_resource(); LOG(INFO) << "Doris exec envorinment is destoried."; } From 55fd0fe2f623cae424a570c87ba290271973bb59 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:38:33 +0800 Subject: [PATCH 508/572] branch-3.0: [Fix](case) Fix case `test_skip_calc_between_segments` in cloud mode #55089 (#55205) Cherry-picked from #55089 Co-authored-by: bobhan1 --- .../test_skip_calc_between_segments.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/fault_injection_p0/test_skip_calc_between_segments.groovy b/regression-test/suites/fault_injection_p0/test_skip_calc_between_segments.groovy index 8fee3051b87e71..5a127335d25298 100644 --- a/regression-test/suites/fault_injection_p0/test_skip_calc_between_segments.groovy +++ b/regression-test/suites/fault_injection_p0/test_skip_calc_between_segments.groovy @@ -156,11 +156,12 @@ suite("test_skip_calc_between_segments", "nonConcurrent") { unblock_publish() t1.join() - // ensure that we really write multi segments - checkSegmentNum(4, 3) + sql "set disable_nereids_rules='ELIMINATE_GROUP_BY';" qt_sql "select count() from (select k1,count() as cnt from ${table1} group by k1 having cnt > 1) A;" + // ensure that we really write multi segments + checkSegmentNum(4, 3) } catch(Exception e) { logger.info(e.getMessage()) throw e From e8839ea2e60c07cd52015903e33aaf6ecc3908a6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:39:27 +0800 Subject: [PATCH 509/572] =?UTF-8?q?branch-3.0:=20[opt][editlog]=20Added=20?= =?UTF-8?q?the=20ability=20to=20skip=20certain=20editlog=20exceptions=20w?= =?UTF-8?q?=E2=80=A6=20#54090=20(#55202)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #54090 Co-authored-by: deardeng --- .../java/org/apache/doris/common/Config.java | 9 ++++++++- .../java/org/apache/doris/persist/EditLog.java | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 1a33cc1d72f3f3..c09f3f315103a3 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1325,6 +1325,13 @@ public class Config extends ConfigBase { @ConfField(mutable = false, masterOnly = false) public static String[] force_skip_journal_ids = {}; + @ConfField(description = {"当回放 editlog 时遇到特定操作类型的异常导致 FE 无法启动时，可以配置需要忽略的 editlog 操作类型枚举值，" + + "从而跳过这些异常，让 replay 线程可以继续回放其他日志", + "When replaying editlog encounters exceptions with specific operation types that prevent FE from starting, " + + "you can configure the editlog operation type enum values to be ignored, " + + "thereby skipping these exceptions and allowing the replay thread to continue replaying other logs"}) + public static short[] skip_operation_types_on_replay_exception = {-1, -1}; + /** * Decide how often to check dynamic partition */ @@ -2416,7 +2423,7 @@ public class Config extends ConfigBase { /** * To prevent different types (V1, V2, V3) of behavioral inconsistencies, * we may delete the DecimalV2 and DateV1 types in the future. - * At this stage, we use ‘disable_decimalv2’ and ‘disable_datev1’ + * At this stage, we use 'disable_decimalv2' and 'disable_datev1' * to determine whether these two types take effect. */ @ConfField(mutable = true) diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index a06e2c4b693a67..81280575248cc1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -1274,8 +1274,20 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { */ LOG.warn("[INCONSISTENT META] replay log {} failed, journal {}: {}", logId, journal, e.getMessage(), e); } catch (Exception e) { - LOG.error("replay Operation Type {}, log id: {}", opCode, logId, e); - System.exit(-1); + short[] ignoreExceptionLogIds = Config.skip_operation_types_on_replay_exception; + boolean skip = false; + for (short ignoreLogId : ignoreExceptionLogIds) { + if (ignoreLogId == opCode) { + skip = true; + break; + } + } + if (!skip) { + LOG.error("replay Operation Type {}, log id: {}", opCode, logId, e); + System.exit(-1); + } else { + LOG.warn("Skip replay Operation Type {} due to exception, log id: {}", opCode, logId, e); + } } } From 1958bd3bd32f95c7815052035bac23b5f54cbe7e Mon Sep 17 00:00:00 2001 From: yujun Date: Sun, 24 Aug 2025 15:40:03 +0800 Subject: [PATCH 510/572] branch-3.0: [test](nereids) add test case for continuous filter or project in plan #40176 #54872 (#55142) cherry pick from #54872 --- .../test_physical_translator.out | 24 +++++++++ .../test_physical_translator.groovy | 51 +++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 regression-test/data/nereids_p0/physical_translator/test_physical_translator.out create mode 100644 regression-test/suites/nereids_p0/physical_translator/test_physical_translator.groovy diff --git a/regression-test/data/nereids_p0/physical_translator/test_physical_translator.out b/regression-test/data/nereids_p0/physical_translator/test_physical_translator.out new file mode 100644 index 00000000000000..d5166d53ce7031 --- /dev/null +++ b/regression-test/data/nereids_p0/physical_translator/test_physical_translator.out @@ -0,0 +1,24 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !continue_project_shape -- +PhysicalResultSink +--PhysicalProject[t.a, x AS `k1`, x AS `k2`] +----PhysicalProject[random(100) AS `x`, t.a] +------PhysicalOlapScan[tbl_test_physical_translator] + +-- !continue_project_result -- +1 0.9616644308453555 0.9616644308453555 +200 0.32358402105146866 0.32358402105146866 + +-- !continue_filter_shape -- +PhysicalResultSink +--PhysicalProject[(a + 2) AS `x`] +----filter((cast(a as BIGINT) < 9998) and (cast(a as BIGINT) > 3)) +------PhysicalLimit[GLOBAL] +--------PhysicalLimit[LOCAL] +----------PhysicalProject[tbl_test_physical_translator.a] +------------filter((tbl_test_physical_translator.b > 10)) +--------------PhysicalOlapScan[tbl_test_physical_translator] + +-- !continue_filter_result -- +202 + diff --git a/regression-test/suites/nereids_p0/physical_translator/test_physical_translator.groovy b/regression-test/suites/nereids_p0/physical_translator/test_physical_translator.groovy new file mode 100644 index 00000000000000..3e5001a5ea3fdf --- /dev/null +++ b/regression-test/suites/nereids_p0/physical_translator/test_physical_translator.groovy @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_physical_translator") { + def tbl = "tbl_test_physical_translator" + sql 'SET enable_nereids_planner=true' + sql 'SET runtime_filter_mode=OFF' + sql 'SET enable_fallback_to_original_planner=false' + sql "SET ignore_shape_nodes='PhysicalDistribute'" + sql "SET detail_shape_nodes='PhysicalProject'" + sql 'SET disable_nereids_rules=PRUNE_EMPTY_PARTITION' + sql "drop table if exists ${tbl} force" + sql "create table ${tbl} (a int, b int) properties('replication_num' = '1')" + sql "insert into ${tbl} values(1, 10), (200, 300)" + + def sql1 = """ + SELECT a, x as k1, x as k2 FROM (SELECT a, random(100) as x FROM ${tbl}) t + """ + + explainAndOrderResult "continue_project", sql1 + explain { + sql sql1 + contains "VSELECT" + } + + def sql2 = """ + select * from (select a + 2 as x from ${tbl} where b > 10 limit 100)s where x > 5 and x < 10000 + """ + + explainAndOrderResult "continue_filter", sql2 + explain { + sql sql2 + contains "VSELECT" + } + + sql "drop table if exists ${tbl} force" +} From 3182d26083245a7724457c2765cce959e5a44aa2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 15:40:54 +0800 Subject: [PATCH 511/572] branch-3.0: [fix](warmup): fix passive cancellation of event-driven jobs #54962 (#55087) Cherry-picked from #54962 Co-authored-by: Kaijie Chen --- .../org/apache/doris/service/FrontendServiceImpl.java | 2 +- .../test_warm_up_cluster_event_cancel_passive.groovy | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 9169f560309d7b..49ef6e5e6358c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2752,7 +2752,7 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos CloudWarmUpJob job = ((CloudEnv) Env.getCurrentEnv()) .getCacheHotspotMgr() .getCloudWarmUpJob(request.getWarmUpJobId()); - if (job == null) { + if (job == null || job.isDone()) { LOG.info("warmup job {} is not running, notify caller BE {} to cancel job", job.getJobId(), clientAddr); // notify client to cancel this job diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy index 5c1cdb8ce0594f..e4def0b7d30ae9 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_cancel_passive.groovy @@ -18,12 +18,14 @@ import org.apache.doris.regression.suite.ClusterOptions import groovy.json.JsonSlurper -suite('test_warm_up_cluster_event_cancel', 'docker') { +suite('test_warm_up_cluster_event_cancel_passive', 'docker') { def options = new ClusterOptions() options.feConfigs += [ + 'enable_debug_points=true', 'cloud_cluster_check_interval_second=1', ] options.beConfigs += [ + 'enable_debug_points=true', 'file_cache_enter_disk_resource_limit_mode_percent=99', 'enable_evict_file_cache_in_advance=false', 'file_cache_background_monitor_interval_ms=1000', @@ -43,9 +45,10 @@ suite('test_warm_up_cluster_event_cancel', 'docker') { // Read response def responseText = conn.inputStream.text + logger.info("Response from ${urlStr}: ${responseText}") def json = new JsonSlurper().parseText(responseText) - return json?.msg == "OK" && json?.code == 0 + return json?.msg == "OK" } def setDebugPointsForCluster = { cluster, debug_point, enable -> @@ -226,7 +229,7 @@ suite('test_warm_up_cluster_event_cancel', 'docker') { } sleep(15000) def cacheSize1 = getClusterTTLCacheSizeSum(clusterName2); - assertTrue(cacheSize1 > cacheSize0, "some more syncs before cache expire is expected") + assertTrue(cacheSize1 >= cacheSize0, "some more syncs before cache expire is expected") // At this point, cache should be expired, so we expect no more syncs From d781f8c35dce7c499bef42e9595c3ef5b4f2703c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 22:21:08 +0800 Subject: [PATCH 512/572] branch-3.0: [fix](docker case) disable `PluginQueryTimeoutDebugger` in docker case #54984 (#55217) Cherry-picked from #54984 Co-authored-by: deardeng --- .../plugins/plugin_query_timeout_debugger.groovy | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/regression-test/plugins/plugin_query_timeout_debugger.groovy b/regression-test/plugins/plugin_query_timeout_debugger.groovy index 5d863b9a754a9d..743582419ee6fc 100644 --- a/regression-test/plugins/plugin_query_timeout_debugger.groovy +++ b/regression-test/plugins/plugin_query_timeout_debugger.groovy @@ -37,6 +37,7 @@ class PluginQueryTimeoutDebuggerHolder { PluginQueryTimeoutDebugger.jdbcUrl = context.config.jdbcUrl PluginQueryTimeoutDebugger.jdbcUser = context.config.jdbcUser PluginQueryTimeoutDebugger.jdbcPassword = context.config.jdbcPassword +PluginQueryTimeoutDebugger.skip = !context.config.excludeDockerTest PluginQueryTimeoutDebugger.logger = logger PluginQueryTimeoutDebuggerHolder.staticResource.startWorker() @@ -54,6 +55,7 @@ class PluginQueryTimeoutDebugger { static public String jdbcUser static public String jdbcPassword static public Logger logger + static public Boolean skip private ScheduledExecutorService scheduler private List backendUrls = [] @@ -61,6 +63,10 @@ class PluginQueryTimeoutDebugger { // catch all exceptions in timer function. private void startWorker() { + if (skip) { + logger.info("docker case skip this plugin") + return + } if (scheduler?.isShutdown() == false) { logger.warn("worker already started") return From 4ed7a95fb38c4107d69580cf3b93f287f7a0023c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 24 Aug 2025 22:26:26 +0800 Subject: [PATCH 513/572] =?UTF-8?q?branch-3.0:=20[fix](cloud)=20Not=20log?= =?UTF-8?q?=20npe=20in=20unprotectUpdateCloudReplica=20when=20partitio?= =?UTF-8?q?=E2=80=A6=20#55165=20(#55220)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #55165 Co-authored-by: deardeng --- .../apache/doris/cloud/datasource/CloudInternalCatalog.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ca1052afeeb236..ad7596c18fa251 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -1016,6 +1016,10 @@ public void replayUpdateCloudReplica(UpdateCloudReplicaInfo info) throws MetaNot private void unprotectUpdateCloudReplica(OlapTable olapTable, UpdateCloudReplicaInfo info) { LOG.debug("replay update a cloud replica {}", info); Partition partition = olapTable.getPartition(info.getPartitionId()); + if (partition == null) { + LOG.warn("replay update cloud replica, unknown partition {}, may be dropped", info.toString()); + return; + } MaterializedIndex materializedIndex = partition.getIndex(info.getIndexId()); try { From 18371debadb01615411f6390e8aa6c1423e6f961 Mon Sep 17 00:00:00 2001 From: TengJianPing Date: Mon, 25 Aug 2025 10:47:29 +0800 Subject: [PATCH 514/572] [fix](regression) fix regression (#55230) --- .../test_routine_load_schedule.out | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/regression-test/data/load_p0/routine_load/test_routine_load_schedule.out b/regression-test/data/load_p0/routine_load/test_routine_load_schedule.out index 58613e1bdfb886..34035ee3c5ac36 100644 --- a/regression-test/data/load_p0/routine_load/test_routine_load_schedule.out +++ b/regression-test/data/load_p0/routine_load/test_routine_load_schedule.out @@ -1,23 +1,23 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- -8 2023-08-14 true 109 -31573 -1362465190 3990845741226497177 2732763251146840270 -25698.553 1.312831962567818E9 99999999.9 99999999.9 2023-03-07T14:13:19 2022-10-18 2023-07-16T05:03:13 D PBn1wa6X8WneZYLMac11zzyhGl7tPXB5XgjmOV8L6uav9ja5oY433ktb2yhyQQIqBveZPkme {"animal":"lion","weight":200,"habitat":["savannah","grassland"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -20 2023-08-17 false -5 18158 784479801 1485484354598941738 -6632681928222776815 9708.431 -3.30432620706069E8 -99999999.9 99999999.9 2022-09-15T21:40:55 2023-02-23 2023-08-13T21:31:54 O X 2pYmX2vAhfEEHZZYPsgAmda1G7otnwx5TmUC879FPhDeIjvWI79ksBZpfFG2gp7jhCSbpZiecKGklB5SvG8tm31i5SUqe1xrWgLt4HSq7lMJWp75tx2kxD7pRIOpn {"name":"Sarah","age":30,"city":"London","isMarried":false} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -21 2023-08-18 false 63 -27847 -35409596 8638201997392767650 4919963231735304178 -23382.541 -1.803403621426313E9 -22009767.0 99999999.9 2023-03-31T10:56:14 2023-01-20 2023-02-18T13:37:52 N T PSiFwUEx3eVFNtjlnQ70YkgZNvKrGmQ2DN5K9yYHiSdFWeEDB1UpL3Frt8z1kEAIWRDWqXZuyi {"city":"Sydney","population":5312000,"area":2058.7} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -31 2023-08-27 false 17 -18849 1728109133 3266501886640700374 527195452623418935 -24062.328 -1.514348021262435E9 -99999999.9 -99999999.9 2022-10-07T03:24:23 2022-09-25 \N 0 8 yKMiAntORoRa8svnMfcxlOPwwND1m5s2fdS26Xu6cfs6HK5SAibqIp9h8sZcpjHy4 {"team":"Manchester United","players":["Ronaldo","Rooney","Giggs"],"coach":"Ole Gunnar Solskjaer"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -41 2023-08-27 true -104 22750 \N 8527773271030840740 5554497317268279215 -5296.828 -1.71564688801304E9 -99999999.9 99999999.9 2022-12-02T17:56:44 2022-10-12 2023-02-19T07:02:54 V \N E9GzQdTwX1ITUQz27IVznAs6Ca4WwprKk6Odjs6SH75D2F1089QiY3HQ52LXRD1V6xAWjhLE2hWgW3EdHuAOnUDVrb5V {"food":"Sushi","price":10,"restaurant":"Sushi King"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -49 2023-08-08 false \N 16275 -2144851675 -2303421957908954634 -46526938720058765 -13141.143 -6.866322332302E8 99999999.9 -99999999.9 2022-09-01T00:16:01 2023-03-25 2022-09-07T14:59:03 s yvuILR2iNxfe8RRml {"student":true,"name":"Alice","grade":9,"subjects":["math","science","history"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -50 2023-08-06 true 109 -6330 1479023892 -8630800697573159428 -1645095773540208759 17880.96 -1.453844792013949E9 -99999999.9 -99999999.9 2022-09-22T02:03:21 2023-05-14 2023-03-25T02:18:34 m JKnIgXvGVidGiWl9YRSi3mFI7wHKt1sBpWSadKF8VX3LAuElm4sdc9gtxREaUr57oikSYlU8We8h1MWqQlYNiJObl {"city":"Tokyo","temperature":20.5,"humidity":75} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -50 2023-08-24 true 15 14403 \N -6418906115745394180 9205303779366462513 -4331.549 -6.15112179557648E8 99999999.9 -99999999.9 2022-12-29T02:27:20 2023-06-01 2023-08-12T04:50:04 a eCl38sztIvBQvGvGKyYZmyMXy9vIJx197iu3JwP9doJGcrYUl9Uova0rz4iCCgrjlAiZU18Fs9YtCq830nhM {"band":"The Beatles","members":["John Lennon","Paul McCartney","George Harrison","Ringo Starr"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -57 2023-08-19 true 2 -25462 -74112029 6458082754318544493 -7910671781690629051 -15205.859 -3.06870797484914E8 99999999.9 -99999999.9 2023-07-10T18:39:10 2023-02-12 2023-01-27T07:26:06 y Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ {"name":"John","age":25,"city":"New York"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -58 2023-08-22 \N 0 -18231 1832867360 6997858407575297145 2480714305422728023 -5450.489 1.475901032138386E9 -99999999.9 -99999999.9 2023-02-02T05:13:24 2022-09-18 2023-04-23T10:51:15 k LdFXF7Kmfzgmnn2R6zLsXdmi3A2cLBLq4G4WDVNDhxvH7dYH8Kga2WA47uSIxp6NSrwPSdw0ssB1TS8RFJTDJAB0Uba3e05NL2Aiw0ja {"restaurant":"Pizza Hut","menu":["pizza","pasta","salad"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -60 2023-08-27 false -52 -2338 -757056972 1047567408607120856 6541476642780646552 6614.0894 -1.204448798517855E9 99999999.9 99999999.9 2022-12-29T14:47:30 2022-09-24 2023-08-01T12:41:59 O F RM4F1Ke7lkcnuxF2nK0j9VBW3MDcgyHR4pseBjtFnqS6GUkVFuzF6u3Cp9Nv7ab0O6UYrpP4DhU {"game":"Chess","players":2,"time":"1 hour"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -62 2023-08-21 false 81 20302 -200761532 6365479976421007608 \N -29916.533 1.709141750828478E9 99999999.9 -99999999.9 2023-05-04T01:14:51 2022-09-17 2022-12-04T19:30:09 d v BKWy9dTNg1aZW7ancEJAmEDOPK5TwFsNSHbI78emu9gymeIlx5NoLmyii0QAqdzRvSQPZKiqKkwInGCTIBnK1yYkK7zD {"username":"user123","password":"pass123","email":"user123@example.com"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -65 2023-08-09 false 94 31514 814994517 -297697460695940343 734910652450318597 -13061.892 6.2750847041706E7 -9808654.0 \N 2023-08-14T22:01:27 2023-05-19 2022-11-13T13:44:28 V aGeMsI24O12chGlP5ak0AHghAz7bu5MargJBStHnt0yMnChH0JnfYhsfH1u59XIHkJKMsHYktBqORkGlovu8V47E74KeFpaqxn5yLyXfDbhhzUKf {"language":"Python","version":3.9,"frameworks":["Django","Flask"]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -66 2023-08-15 true -91 28378 609923317 4872185586197131212 1207709464099378591 \N -1.863683325985123E9 -99999999.9 -99999999.9 2022-09-24T10:39:23 2022-09-24 2022-10-16T18:36:43 Y z AI1BSPQdKiHJiQH1kguyLSWsDXkC7zwy7PwgWnyGSaa9tBKRex8vHBdxg2QSKZKL2mV2lHz7iI1PnsTd4MXDcIKhqiHyPuQPt2tEtgt0UgF6 {"book":{"title":"The Great Gatsby","author":"F. Scott Fitzgerald"},"year":1925} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -68 2023-08-23 true -73 20117 1737338128 795638676048937749 -5551546237562433901 -30627.04 6.8589475684545E7 99999999.9 99999999.9 2022-12-28T20:26:51 2022-10-04 2023-07-30T00:20:06 y keZ3JlWWpdnPBejf0cuiCQCVBBTd5gjvO08NVdcAFewqL7nRT4N9lnvSU6pWmletA5VbPQCeQapJdcnQCHfZUDCf4ulCnczyqr7SGrbGRT0XYcd7iktKM {"country":"Brazil","continent":"South America","population":211049527} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -80 2023-08-18 false -18 -8971 679027874 6535956962935330265 3960889045799757165 -13219.76 1.187161924505394E9 -99999999.9 -99999999.9 2023-03-11T07:40 2022-11-29 2023-01-14T07:24:07 \N D 3Nhx6xX1qdwaq7lxwLRSKMtJFbC03swWv12mpySSVysH3igGZTiGPuKMsYW7HAkf6CWc7c0nzqDsjuH3FYVMNCWRmfxMrmY8rykQCC4Ve {"car":"BMW","model":"X5","year":2020,"color":"black"} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -81 2023-08-23 false 106 11492 -667795397 4480250461471356146 -5346660566234294101 9082.75 3.85167225902608E8 -99999999.9 99999999.9 2023-03-20T03:33:16 2022-11-24 2023-02-16T18:29:41 G 9 Lk3eNVQNjucbekD1rZmUlGPiXS5JvcWr2LQzRU8GSGIbSag {"flower":"rose","color":"red","fragrance":true} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -85 2023-08-11 true -7 24304 -2043877415 -2024144417867729183 \N 5363.0244 -5.78615669042831E8 -99999999.9 -99999999.9 2023-07-15T01:07:41 2023-08-13 2023-01-20T11:57:48 i WQ9dh9ajPu0y {"country":"France","capital":"Paris","population":67081000} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -90 2023-08-27 true 22 16456 -1476824962 -3279894870153540825 8990195191470116763 26651.906 2.06860148942546E8 -99999999.9 -99999999.9 2022-10-07T03:11:03 2023-03-18 2023-04-15T00:38:33 T L QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD {"company":"Apple","products":[{"name":"iPhone","price":1000},{"name":"MacBook","price":1500}]} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N -91 2023-08-27 true 90 2465 702240964 6373830997821598984 305860046137409400 15991.356 1.599972327386147E9 -99999999.9 \N 2023-04-26T19:31:10 2023-07-21 \N 2 B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK {"fruit":"apple","color":"red","qty":5,"price":2.5} true 1 2 3 4 5 6.0 7.0 888888888 999999999 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +8 2023-08-14 true 109 -31573 -1362465190 3990845741226497177 2732763251146840270 -25698.553 1.312831962567818E9 771983879.0 173937916.0 2023-03-07T14:13:19 2022-10-18 2023-07-16T05:03:13 D PBn1wa6X8WneZYLMac11zzyhGl7tPXB5XgjmOV8L6uav9ja5oY433ktb2yhyQQIqBveZPkme {"animal":"lion","weight":200,"habitat":["savannah","grassland"]} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +20 2023-08-17 false -5 18158 784479801 1485484354598941738 -6632681928222776815 9708.431 -3.30432620706069E8 -816424174.0 571112646.0 2022-09-15T21:40:55 2023-02-23 2023-08-13T21:31:54 O X 2pYmX2vAhfEEHZZYPsgAmda1G7otnwx5TmUC879FPhDeIjvWI79ksBZpfFG2gp7jhCSbpZiecKGklB5SvG8tm31i5SUqe1xrWgLt4HSq7lMJWp75tx2kxD7pRIOpn {"name":"Sarah","age":30,"city":"London","isMarried":false} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +21 2023-08-18 false 63 -27847 -35409596 8638201997392767650 4919963231735304178 -23382.541 -1.803403621426313E9 -22009767.0 661750756.0 2023-03-31T10:56:14 2023-01-20 2023-02-18T13:37:52 N T PSiFwUEx3eVFNtjlnQ70YkgZNvKrGmQ2DN5K9yYHiSdFWeEDB1UpL3Frt8z1kEAIWRDWqXZuyi {"city":"Sydney","population":5312000,"area":2058.7} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +31 2023-08-27 false 17 -18849 1728109133 3266501886640700374 527195452623418935 -24062.328 -1.514348021262435E9 -322205854.0 -278237157.0 2022-10-07T03:24:23 2022-09-25 \N 0 8 yKMiAntORoRa8svnMfcxlOPwwND1m5s2fdS26Xu6cfs6HK5SAibqIp9h8sZcpjHy4 {"team":"Manchester United","players":["Ronaldo","Rooney","Giggs"],"coach":"Ole Gunnar Solskjaer"} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +41 2023-08-27 true -104 22750 \N 8527773271030840740 5554497317268279215 -5296.828 -1.71564688801304E9 -306075962.0 897769189.0 2022-12-02T17:56:44 2022-10-12 2023-02-19T07:02:54 V \N E9GzQdTwX1ITUQz27IVznAs6Ca4WwprKk6Odjs6SH75D2F1089QiY3HQ52LXRD1V6xAWjhLE2hWgW3EdHuAOnUDVrb5V {"food":"Sushi","price":10,"restaurant":"Sushi King"} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +49 2023-08-08 false \N 16275 -2144851675 -2303421957908954634 -46526938720058765 -13141.143 -6.866322332302E8 229942298.0 -152553823.0 2022-09-01T00:16:01 2023-03-25 2022-09-07T14:59:03 s yvuILR2iNxfe8RRml {"student":true,"name":"Alice","grade":9,"subjects":["math","science","history"]} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +50 2023-08-06 true 109 -6330 1479023892 -8630800697573159428 -1645095773540208759 17880.96 -1.453844792013949E9 -158871820.0 -862940384.0 2022-09-22T02:03:21 2023-05-14 2023-03-25T02:18:34 m JKnIgXvGVidGiWl9YRSi3mFI7wHKt1sBpWSadKF8VX3LAuElm4sdc9gtxREaUr57oikSYlU8We8h1MWqQlYNiJObl {"city":"Tokyo","temperature":20.5,"humidity":75} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +50 2023-08-24 true 15 14403 \N -6418906115745394180 9205303779366462513 -4331.549 -6.15112179557648E8 367305015.0 -551652958.0 2022-12-29T02:27:20 2023-06-01 2023-08-12T04:50:04 a eCl38sztIvBQvGvGKyYZmyMXy9vIJx197iu3JwP9doJGcrYUl9Uova0rz4iCCgrjlAiZU18Fs9YtCq830nhM {"band":"The Beatles","members":["John Lennon","Paul McCartney","George Harrison","Ringo Starr"]} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +57 2023-08-19 true 2 -25462 -74112029 6458082754318544493 -7910671781690629051 -15205.859 -3.06870797484914E8 759730669.0 -628556336.0 2023-07-10T18:39:10 2023-02-12 2023-01-27T07:26:06 y Xi9nDVrLv8m6AwEpUxmtzFAuK48sQ {"name":"John","age":25,"city":"New York"} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +58 2023-08-22 \N 0 -18231 1832867360 6997858407575297145 2480714305422728023 -5450.489 1.475901032138386E9 -893480655.0 -607891858.0 2023-02-02T05:13:24 2022-09-18 2023-04-23T10:51:15 k LdFXF7Kmfzgmnn2R6zLsXdmi3A2cLBLq4G4WDVNDhxvH7dYH8Kga2WA47uSIxp6NSrwPSdw0ssB1TS8RFJTDJAB0Uba3e05NL2Aiw0ja {"restaurant":"Pizza Hut","menu":["pizza","pasta","salad"]} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +60 2023-08-27 false -52 -2338 -757056972 1047567408607120856 6541476642780646552 6614.0894 -1.204448798517855E9 236657733.0 731515433.0 2022-12-29T14:47:30 2022-09-24 2023-08-01T12:41:59 O F RM4F1Ke7lkcnuxF2nK0j9VBW3MDcgyHR4pseBjtFnqS6GUkVFuzF6u3Cp9Nv7ab0O6UYrpP4DhU {"game":"Chess","players":2,"time":"1 hour"} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +62 2023-08-21 false 81 20302 -200761532 6365479976421007608 \N -29916.533 1.709141750828478E9 549873536.0 -119205359.0 2023-05-04T01:14:51 2022-09-17 2022-12-04T19:30:09 d v BKWy9dTNg1aZW7ancEJAmEDOPK5TwFsNSHbI78emu9gymeIlx5NoLmyii0QAqdzRvSQPZKiqKkwInGCTIBnK1yYkK7zD {"username":"user123","password":"pass123","email":"user123@example.com"} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +65 2023-08-09 false 94 31514 814994517 -297697460695940343 734910652450318597 -13061.892 6.2750847041706E7 -9808654.0 \N 2023-08-14T22:01:27 2023-05-19 2022-11-13T13:44:28 V aGeMsI24O12chGlP5ak0AHghAz7bu5MargJBStHnt0yMnChH0JnfYhsfH1u59XIHkJKMsHYktBqORkGlovu8V47E74KeFpaqxn5yLyXfDbhhzUKf {"language":"Python","version":3.9,"frameworks":["Django","Flask"]} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +66 2023-08-15 true -91 28378 609923317 4872185586197131212 1207709464099378591 \N -1.863683325985123E9 -783792012.0 -708986976.0 2022-09-24T10:39:23 2022-09-24 2022-10-16T18:36:43 Y z AI1BSPQdKiHJiQH1kguyLSWsDXkC7zwy7PwgWnyGSaa9tBKRex8vHBdxg2QSKZKL2mV2lHz7iI1PnsTd4MXDcIKhqiHyPuQPt2tEtgt0UgF6 {"book":{"title":"The Great Gatsby","author":"F. Scott Fitzgerald"},"year":1925} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +68 2023-08-23 true -73 20117 1737338128 795638676048937749 -5551546237562433901 -30627.04 6.8589475684545E7 585022347.0 513722420.0 2022-12-28T20:26:51 2022-10-04 2023-07-30T00:20:06 y keZ3JlWWpdnPBejf0cuiCQCVBBTd5gjvO08NVdcAFewqL7nRT4N9lnvSU6pWmletA5VbPQCeQapJdcnQCHfZUDCf4ulCnczyqr7SGrbGRT0XYcd7iktKM {"country":"Brazil","continent":"South America","population":211049527} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +80 2023-08-18 false -18 -8971 679027874 6535956962935330265 3960889045799757165 -13219.76 1.187161924505394E9 -526615878.0 -947410627.0 2023-03-11T07:40 2022-11-29 2023-01-14T07:24:07 \N D 3Nhx6xX1qdwaq7lxwLRSKMtJFbC03swWv12mpySSVysH3igGZTiGPuKMsYW7HAkf6CWc7c0nzqDsjuH3FYVMNCWRmfxMrmY8rykQCC4Ve {"car":"BMW","model":"X5","year":2020,"color":"black"} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +81 2023-08-23 false 106 11492 -667795397 4480250461471356146 -5346660566234294101 9082.75 3.85167225902608E8 -717553011.0 649146853.0 2023-03-20T03:33:16 2022-11-24 2023-02-16T18:29:41 G 9 Lk3eNVQNjucbekD1rZmUlGPiXS5JvcWr2LQzRU8GSGIbSag {"flower":"rose","color":"red","fragrance":true} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +85 2023-08-11 true -7 24304 -2043877415 -2024144417867729183 \N 5363.0244 -5.78615669042831E8 -378574346.0 -810302932.0 2023-07-15T01:07:41 2023-08-13 2023-01-20T11:57:48 i WQ9dh9ajPu0y {"country":"France","capital":"Paris","population":67081000} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +90 2023-08-27 true 22 16456 -1476824962 -3279894870153540825 8990195191470116763 26651.906 2.06860148942546E8 -580959198.0 -210329147.0 2022-10-07T03:11:03 2023-03-18 2023-04-15T00:38:33 T L QW0GQ3GoMtHgxPQOWGfVaveynahNpsNs09siMFA1OtO6QEDBQTdivmGyq7bFzejAqwbbVQQpREAmeLjcFSXLnQuou2KbwYD {"company":"Apple","products":[{"name":"iPhone","price":1000},{"name":"MacBook","price":1500}]} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N +91 2023-08-27 true 90 2465 702240964 6373830997821598984 305860046137409400 15991.356 1.599972327386147E9 -165530947.0 \N 2023-04-26T19:31:10 2023-07-21 \N 2 B7YKYBYT8w0YC926bZ8Yz1VzyiWw2NWDAiTlEoPVyz9AXGti2Npg1FxWqWk4hEaALw0ZBSuiAIPj41lq36g5QRpPmAjNPK {"fruit":"apple","color":"red","qty":5,"price":2.5} true 1 2 3 4 5 6.0 7.0 888888888.000000000 999999999.000000000 2023-08-24 2023-08-24T12:00 2023-08-24 2023-08-24T12:00 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 我能吞下玻璃而不伤身体 \N From 75af5c267d4abc02f48d56596b979048bd700ef8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 15:25:42 +0800 Subject: [PATCH 515/572] branch-3.0: [fix](case) do not use global var in groovy #55216 (#55225) Cherry-picked from #55216 Co-authored-by: Yongqiang YANG --- .../compaction10/test_schema_change_with_compaction10.groovy | 2 +- .../compaction11/test_schema_change_with_compaction11.groovy | 5 ++++- .../compaction6/test_schema_change_with_compaction6.groovy | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy index db3da9aa044241..7e3d9d233cbcae 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction10/test_schema_change_with_compaction10.groovy @@ -121,7 +121,7 @@ suite('test_schema_change_with_compaction10', 'docker') { // cu compaction trigger_and_wait_compaction("date", "cumulative") } catch (Exception e) { - logger.error("Exception: " + e) + logger.info("Exception: " + e) } finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) diff --git a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy index 310c10b4dbcb55..cb32e99a266535 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction11/test_schema_change_with_compaction11.groovy @@ -139,7 +139,10 @@ suite('test_schema_change_with_compaction11', 'docker') { logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) assertTrue(out.contains("invalid tablet state.")) - } finally { + } catch (Exception e) { + logger.info("Exception: " + e) + } + finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) } diff --git a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy index 31a5760c62db46..894f06bee131d8 100644 --- a/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy +++ b/regression-test/suites/cloud_p0/schema_change/compaction6/test_schema_change_with_compaction6.groovy @@ -124,7 +124,7 @@ suite('test_schema_change_with_compaction6', 'docker') { // base compaction logger.info("run compaction:" + originTabletId) - (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) + def (code, out, err) = be_run_base_compaction(injectBe.Host, injectBe.HttpPort, originTabletId) logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) @@ -169,7 +169,7 @@ suite('test_schema_change_with_compaction6', 'docker') { } } catch (Exception e) { - logger.error("Exception: " + e) + logger.info("Exception: " + e) } finally { if (injectBe != null) { GetDebugPoint().disableDebugPointForAllBEs(injectName) From 826a6e41f95d8a2bf7e9428c9b1f63d21637b82a Mon Sep 17 00:00:00 2001 From: James Date: Mon, 25 Aug 2025 15:53:28 +0800 Subject: [PATCH 516/572] branch-3.0: [fix](constant fold)Remove double boundary check. (#54939) (#55077) backport: https://github.com/apache/doris/pull/54939 --- .../apache/doris/analysis/FloatLiteral.java | 7 +- .../executable/NumericArithmetic.java | 45 +-- .../trees/expressions/literal/Literal.java | 8 + .../doris/analysis/ArrayLiteralTest.java | 16 +- .../apache/doris/analysis/MapLiteralTest.java | 8 +- .../doris/analysis/StructLiteralTest.java | 8 +- .../rules/expression/FoldConstantTest.java | 278 ++++++++++++++++-- .../cast_function/test_cast_struct.out | 2 +- .../doris/regression/suite/Suite.groovy | 3 +- .../fold_constant_numeric_arithmatic.groovy | 10 + 10 files changed, 303 insertions(+), 82 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index 074b0eea5ec727..eb402c22e760d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -170,10 +170,11 @@ public String getStringValueForQuery(FormatOptions options) { String timeStr = getStringValue(); return timeStr.substring(1, timeStr.length() - 1); } else { - if (Double.isInfinite(getValue())) { - return Double.toString(getValue()); + double value = getValue(); + if (Double.isInfinite(value)) { + return value > 0 ? "inf" : "-inf"; } - return BigDecimal.valueOf(getValue()).toPlainString(); + return BigDecimal.valueOf(getValue()).stripTrailingZeros().toPlainString(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java index abf6363c03597a..741eb03fb40865 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java @@ -689,8 +689,8 @@ public static Boolean inputOutOfBound(Literal input, double lowerBound, double u private static Expression checkOutputBoundary(Literal input) { if (input instanceof DoubleLiteral) { - if (((DoubleLiteral) input).getValue().isNaN() || ((DoubleLiteral) input).getValue().isInfinite()) { - return new NullLiteral(DoubleType.INSTANCE); + if (((DoubleLiteral) input).getValue().isNaN()) { + throw new IllegalArgumentException(); } } return input; @@ -819,7 +819,7 @@ public static Expression exp(DoubleLiteral first) { */ @ExecFunction(name = "ln") public static Expression ln(DoubleLiteral first) { - if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true)) { + if (inputOutOfBound(first, 0.0d, Double.POSITIVE_INFINITY, false, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.log(first.getValue()))); @@ -831,7 +831,8 @@ public static Expression ln(DoubleLiteral first) { @ExecFunction(name = "log") public static Expression log(DoubleLiteral first, DoubleLiteral second) { if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true) - || first.getValue().equals(1.0d)) { + || first.getValue().equals(1.0d) + || inputOutOfBound(second, 0.0d, Double.POSITIVE_INFINITY, false, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.log(second.getValue()) / Math.log(first.getValue()))); @@ -842,7 +843,7 @@ public static Expression log(DoubleLiteral first, DoubleLiteral second) { */ @ExecFunction(name = "log2") public static Expression log2(DoubleLiteral first) { - if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true)) { + if (inputOutOfBound(first, 0.0d, Double.POSITIVE_INFINITY, false, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.log(first.getValue()) / Math.log(2.0))); @@ -853,7 +854,7 @@ public static Expression log2(DoubleLiteral first) { */ @ExecFunction(name = "log10") public static Expression log10(DoubleLiteral first) { - if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true)) { + if (inputOutOfBound(first, 0.0d, Double.POSITIVE_INFINITY, false, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.log10(first.getValue()))); @@ -864,7 +865,7 @@ public static Expression log10(DoubleLiteral first) { */ @ExecFunction(name = "sqrt") public static Expression sqrt(DoubleLiteral first) { - if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, true, true)) { + if (inputOutOfBound(first, 0.0d, Double.POSITIVE_INFINITY, true, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.sqrt(first.getValue()))); @@ -875,10 +876,6 @@ public static Expression sqrt(DoubleLiteral first) { */ @ExecFunction(name = "power") public static Expression power(DoubleLiteral first, DoubleLiteral second) { - if (inputOutOfBound(second, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false) - || (first.getValue() < 0 && second.getValue() % 1 != 0)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(Math.pow(first.getValue(), second.getValue()))); } @@ -887,9 +884,6 @@ public static Expression power(DoubleLiteral first, DoubleLiteral second) { */ @ExecFunction(name = "sin") public static Expression sin(DoubleLiteral first) { - if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(Math.sin(first.getValue()))); } @@ -898,9 +892,6 @@ public static Expression sin(DoubleLiteral first) { */ @ExecFunction(name = "cos") public static Expression cos(DoubleLiteral first) { - if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(Math.cos(first.getValue()))); } @@ -909,9 +900,6 @@ public static Expression cos(DoubleLiteral first) { */ @ExecFunction(name = "tan") public static Expression tan(DoubleLiteral first) { - if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(Math.tan(first.getValue()))); } @@ -920,9 +908,6 @@ public static Expression tan(DoubleLiteral first) { */ @ExecFunction(name = "cot") public static Expression cot(DoubleLiteral first) { - if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(1.0 / Math.tan(first.getValue()))); } @@ -931,9 +916,6 @@ public static Expression cot(DoubleLiteral first) { */ @ExecFunction(name = "sec") public static Expression sec(DoubleLiteral first) { - if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(1.0 / Math.cos(first.getValue()))); } @@ -942,9 +924,6 @@ public static Expression sec(DoubleLiteral first) { */ @ExecFunction(name = "cosec") public static Expression cosec(DoubleLiteral first) { - if (inputOutOfBound(first, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false, false)) { - return new NullLiteral(DoubleType.INSTANCE); - } return checkOutputBoundary(new DoubleLiteral(1.0 / Math.sin(first.getValue()))); } @@ -1102,7 +1081,7 @@ public static Expression dexp(DoubleLiteral first) { */ @ExecFunction(name = "dlog10") public static Expression dlog10(DoubleLiteral first) { - if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true)) { + if (inputOutOfBound(first, 0.0d, Double.POSITIVE_INFINITY, false, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.log10(first.getValue()))); @@ -1113,7 +1092,7 @@ public static Expression dlog10(DoubleLiteral first) { */ @ExecFunction(name = "dsqrt") public static Expression dsqrt(DoubleLiteral first) { - if (inputOutOfBound(first, 0.0d, Double.MAX_VALUE, false, true)) { + if (inputOutOfBound(first, 0.0d, Double.POSITIVE_INFINITY, true, true)) { return new NullLiteral(DoubleType.INSTANCE); } return checkOutputBoundary(new DoubleLiteral(Math.sqrt(first.getValue()))); @@ -1124,7 +1103,7 @@ public static Expression dsqrt(DoubleLiteral first) { */ @ExecFunction(name = "dpow") public static Expression dpow(DoubleLiteral first, DoubleLiteral second) { - return checkOutputBoundary(new DoubleLiteral(Math.pow(first.getValue(), second.getValue()))); + return power(first, second); } /** @@ -1148,7 +1127,7 @@ public static Expression fmod(FloatLiteral first, FloatLiteral second) { */ @ExecFunction(name = "fpow") public static Expression fpow(DoubleLiteral first, DoubleLiteral second) { - return checkOutputBoundary(new DoubleLiteral(Math.pow(first.getValue(), second.getValue()))); + return power(first, second); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 932bf6ba83de0e..0eba30a70f8ecb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -259,8 +259,16 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti } else if (targetType.isLargeIntType()) { return Literal.of(new BigDecimal(desc).toBigInteger()); } else if (targetType.isFloatType()) { + float f = Double.valueOf(desc).floatValue(); + if (Float.isNaN(f)) { + return new NullLiteral(targetType); + } return Literal.of(Double.valueOf(desc).floatValue()); } else if (targetType.isDoubleType()) { + double d = Double.parseDouble(desc); + if (Double.isNaN(d)) { + return new NullLiteral(targetType); + } return Literal.of(Double.parseDouble(desc)); } else if (targetType.isCharType()) { if (((CharType) targetType).getLen() >= desc.length()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java index 2085a407388cce..b880905ef7000c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ArrayLiteralTest.java @@ -40,7 +40,7 @@ public void testGetStringForQuery() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForQuery(options)); + Assert.assertEquals("[1, 2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForQuery(options)); @@ -88,7 +88,7 @@ public void testGetStringForQuery() throws AnalysisException { MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", + Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForQuery(options)); @@ -112,7 +112,7 @@ public void testGetStringForQueryForPresto() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForQuery(options)); + Assert.assertEquals("[1, 2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); Assert.assertEquals("[11:22:33]", arrayLiteralWithTime.getStringValueForQuery(options)); @@ -160,7 +160,7 @@ public void testGetStringForQueryForPresto() throws AnalysisException { MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", arrayLiteral13.getStringValueForQuery(options)); + Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); Assert.assertEquals("[{1=2.15}]", arrayLiteral14.getStringValueForQuery(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); @@ -182,7 +182,7 @@ public void testGetStringForQueryForHive() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0,2.15]", arrayLiteral1.getStringValueForQuery(options)); + Assert.assertEquals("[1,2.15]", arrayLiteral1.getStringValueForQuery(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForQuery(options)); @@ -231,7 +231,7 @@ public void testGetStringForQueryForHive() throws AnalysisException { MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1.0,2.15],[1.0,2.15]]", arrayLiteral13.getStringValueForQuery(options)); + Assert.assertEquals("[[1,2.15],[1,2.15]]", arrayLiteral13.getStringValueForQuery(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForQuery(options)); ArrayLiteral arrayLiteral15 = new ArrayLiteral(structLiteral); @@ -254,7 +254,7 @@ public void testGetStringForStreamLoad() throws AnalysisException { DateLiteral dateLiteral = new DateLiteral("2022-10-10", Type.DATE); DateLiteral datetimeLiteral = new DateLiteral("2022-10-10 12:10:10", Type.DATETIME); ArrayLiteral arrayLiteral1 = new ArrayLiteral(intLiteral1, floatLiteral); - Assert.assertEquals("[1.0, 2.15]", arrayLiteral1.getStringValueForStreamLoad(options)); + Assert.assertEquals("[1, 2.15]", arrayLiteral1.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteralWithTime = new ArrayLiteral(floatLiteral1); Assert.assertEquals("[\"11:22:33\"]", arrayLiteralWithTime.getStringValueForStreamLoad(options)); @@ -303,7 +303,7 @@ public void testGetStringForStreamLoad() throws AnalysisException { MapLiteral mapLiteral = new MapLiteral(intLiteral1, floatLiteral); StructLiteral structLiteral = new StructLiteral(intLiteral1, floatLiteral, dateLiteral); ArrayLiteral arrayLiteral13 = new ArrayLiteral(arrayLiteral, arrayLiteral); - Assert.assertEquals("[[1.0, 2.15], [1.0, 2.15]]", + Assert.assertEquals("[[1, 2.15], [1, 2.15]]", arrayLiteral13.getStringValueForStreamLoad(options)); ArrayLiteral arrayLiteral14 = new ArrayLiteral(mapLiteral); Assert.assertEquals("[{1:2.15}]", arrayLiteral14.getStringValueForStreamLoad(options)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java index 9e483e5177117d..782f0d1ed3cb15 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/MapLiteralTest.java @@ -87,7 +87,7 @@ public void testGetStringForQuery() throws AnalysisException { Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueForQuery(options)); + Assertions.assertEquals("{1:[1, 2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -142,7 +142,7 @@ public void testGetStringForQueryForPresto() throws AnalysisException { Assertions.assertEquals("{1=NULL}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1=[1.0, 2.15]}", mapLiteral10.getStringValueForQuery(options)); + Assertions.assertEquals("{1=[1, 2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -198,7 +198,7 @@ public void testGetStringForQueryForHive() throws AnalysisException { Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForQuery(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1:[1.0,2.15]}", mapLiteral10.getStringValueForQuery(options)); + Assertions.assertEquals("{1:[1,2.15]}", mapLiteral10.getStringValueForQuery(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { @@ -255,7 +255,7 @@ public void testGetStringForStreamLoad() throws AnalysisException { Assertions.assertEquals("{1:null}", mapLiteral9.getStringValueForStreamLoad(options)); MapLiteral mapLiteral10 = new MapLiteral(intLiteral1, arrayLiteral); - Assertions.assertEquals("{1:[1.0, 2.15]}", mapLiteral10.getStringValueForStreamLoad(options)); + Assertions.assertEquals("{1:[1, 2.15]}", mapLiteral10.getStringValueForStreamLoad(options)); try { new MapLiteral(arrayLiteral, floatLiteral); } catch (Exception e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java index 5df4dc38e88021..4afee613f6a09a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StructLiteralTest.java @@ -69,7 +69,7 @@ public void testGetStringInFe() throws AnalysisException { + "\"shortstring\", \"col6\":1000000000000000000000, \"col7\":1.0, \"col8\":2, \"col9\":\"2022-10-10\", \"col10\":\"2022-10-10 12:10:10\"}", structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{\"col1\":[1.0, 2.15], \"col2\":{1:2.15}, \"col3\":" + Assert.assertEquals("{\"col1\":[1, 2.15], \"col2\":{1:2.15}, \"col3\":" + "{\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); @@ -91,7 +91,7 @@ public void testGetStringInFeForPresto() throws AnalysisException { + "shortstring, col6=1000000000000000000000, col7=1.0, col8=2, col9=2022-10-10, col10=2022-10-10 12:10:10}", structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); - Assert.assertEquals("{col1=[1.0, 2.15], col2={1=2.15}, col3=" + Assert.assertEquals("{col1=[1, 2.15], col2={1=2.15}, col3=" + "{col1=1, col2=2.15, col3=1.0, col4=2022-10-10}}", structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); @@ -114,7 +114,7 @@ public void testGetStringInFeForHive() throws AnalysisException { structLiteral1.getStringValueForQuery(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); Assert.assertEquals( - "{\"col1\":[1.0,2.15],\"col2\":{1:2.15},\"col3\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", + "{\"col1\":[1,2.15],\"col2\":{1:2.15},\"col3\":{\"col1\":1,\"col2\":2.15,\"col3\":1.0,\"col4\":\"2022-10-10\"}}", structLiteral2.getStringValueForQuery(options)); StructLiteral structLiteral3 = new StructLiteral(); Assert.assertEquals("{}", structLiteral3.getStringValueForQuery(options)); @@ -136,7 +136,7 @@ public void testGetStringForStreamLoad() throws AnalysisException { structLiteral1.getStringValueForStreamLoad(options)); StructLiteral structLiteral2 = new StructLiteral(arrayLiteral, mapLiteral, structLiteral); Assert.assertEquals( - "{[1.0, 2.15], {1:2.15}, {\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", + "{[1, 2.15], {1:2.15}, {\"col1\":1, \"col2\":2.15, \"col3\":1.0, \"col4\":\"2022-10-10\"}}", structLiteral2.getStringValueForStreamLoad(options)); StructLiteral structLiteral3 = new StructLiteral(); Assert.assertEquals("{}", structLiteral3.getStringValueForStreamLoad(options)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java index 21934a7ed355a7..9ccc75fc797118 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/FoldConstantTest.java @@ -26,12 +26,16 @@ import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule; import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE; import org.apache.doris.nereids.rules.expression.rules.SimplifyConditionalFunction; +import org.apache.doris.nereids.trees.expressions.Add; import org.apache.doris.nereids.trees.expressions.CaseWhen; import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.Multiply; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.TimestampArithmetic; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; @@ -39,31 +43,44 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.Acos; import org.apache.doris.nereids.trees.expressions.functions.scalar.AppendTrailingCharIfAbsent; import org.apache.doris.nereids.trees.expressions.functions.scalar.Asin; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Atan; import org.apache.doris.nereids.trees.expressions.functions.scalar.Bin; import org.apache.doris.nereids.trees.expressions.functions.scalar.BitCount; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cbrt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ceil; import org.apache.doris.nereids.trees.expressions.functions.scalar.CharacterLength; import org.apache.doris.nereids.trees.expressions.functions.scalar.Coalesce; import org.apache.doris.nereids.trees.expressions.functions.scalar.ConvertTz; import org.apache.doris.nereids.trees.expressions.functions.scalar.Cos; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cosh; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Cot; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateFormat; import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Degrees; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Dexp; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Dlog10; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Dsqrt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Exp; import org.apache.doris.nereids.trees.expressions.functions.scalar.Floor; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Fmod; import org.apache.doris.nereids.trees.expressions.functions.scalar.FromUnixtime; import org.apache.doris.nereids.trees.expressions.functions.scalar.HoursAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.Left; import org.apache.doris.nereids.trees.expressions.functions.scalar.Ln; import org.apache.doris.nereids.trees.expressions.functions.scalar.Locate; import org.apache.doris.nereids.trees.expressions.functions.scalar.Log; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Log10; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Log2; import org.apache.doris.nereids.trees.expressions.functions.scalar.MinutesAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.MonthsBetween; import org.apache.doris.nereids.trees.expressions.functions.scalar.NextDay; import org.apache.doris.nereids.trees.expressions.functions.scalar.Overlay; import org.apache.doris.nereids.trees.expressions.functions.scalar.Power; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Radians; import org.apache.doris.nereids.trees.expressions.functions.scalar.ReplaceEmpty; import org.apache.doris.nereids.trees.expressions.functions.scalar.Right; import org.apache.doris.nereids.trees.expressions.functions.scalar.Round; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Sec; import org.apache.doris.nereids.trees.expressions.functions.scalar.SecondsAdd; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sign; import org.apache.doris.nereids.trees.expressions.functions.scalar.Sin; @@ -71,6 +88,7 @@ import org.apache.doris.nereids.trees.expressions.functions.scalar.StrToDate; import org.apache.doris.nereids.trees.expressions.functions.scalar.Substring; import org.apache.doris.nereids.trees.expressions.functions.scalar.Tan; +import org.apache.doris.nereids.trees.expressions.functions.scalar.Tanh; import org.apache.doris.nereids.trees.expressions.functions.scalar.ToDays; import org.apache.doris.nereids.trees.expressions.functions.scalar.UnixTimestamp; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; @@ -574,20 +592,51 @@ void testleFoldNumeric() { Assertions.assertEquals(new DoubleLiteral(1.0), rewritten); Expression exExp = new Exp(new DoubleLiteral(1000d)); rewritten = executor.rewrite(exExp, context); - Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + + Dexp dexp = new Dexp(new DoubleLiteral(0d)); + rewritten = executor.rewrite(dexp, context); + Assertions.assertEquals(new DoubleLiteral(1.0), rewritten); + dexp = new Dexp(new DoubleLiteral(1000d)); + rewritten = executor.rewrite(dexp, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); Ln ln = new Ln(new DoubleLiteral(1d)); rewritten = executor.rewrite(ln, context); Assertions.assertEquals(new DoubleLiteral(0.0), rewritten); - exExp = new Ln(new DoubleLiteral(0.0d)); - rewritten = executor.rewrite(exExp, context); + ln = new Ln(new DoubleLiteral(0.0d)); + rewritten = executor.rewrite(ln, context); Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + ln = new Ln(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(ln, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); - exExp = new Ln(new DoubleLiteral(-1d)); - rewritten = executor.rewrite(exExp, context); + ln = new Ln(new DoubleLiteral(-1d)); + rewritten = executor.rewrite(ln, context); Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); - exExp = new Log(new DoubleLiteral(1.0d), new DoubleLiteral(1.0d)); - rewritten = executor.rewrite(exExp, context); + Log log = new Log(new DoubleLiteral(1.0d), new DoubleLiteral(1.0d)); + rewritten = executor.rewrite(log, context); + Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + log = new Log(new DoubleLiteral(10d), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(log, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + Log2 log2 = new Log2(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(log2, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + log2 = new Log2(new DoubleLiteral(-1d)); + rewritten = executor.rewrite(log2, context); + Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + Log10 log10 = new Log10(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(log10, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + log10 = new Log10(new DoubleLiteral(-1d)); + rewritten = executor.rewrite(log10, context); + Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + Dlog10 dlog10 = new Dlog10(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(dlog10, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + dlog10 = new Dlog10(new DoubleLiteral(-1d)); + rewritten = executor.rewrite(dlog10, context); Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); Sqrt sqrt = new Sqrt(new DoubleLiteral(16d)); @@ -596,19 +645,50 @@ void testleFoldNumeric() { sqrt = new Sqrt(new DoubleLiteral(0d)); rewritten = executor.rewrite(sqrt, context); Assertions.assertEquals(new DoubleLiteral(0d), rewritten); - exExp = new Sqrt(new DoubleLiteral(-1d)); - rewritten = executor.rewrite(exExp, context); + sqrt = new Sqrt(new DoubleLiteral(-1d)); + rewritten = executor.rewrite(sqrt, context); Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + sqrt = new Sqrt(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(sqrt, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + + Dsqrt dsqrt = new Dsqrt(new DoubleLiteral(16d)); + rewritten = executor.rewrite(dsqrt, context); + Assertions.assertEquals(new DoubleLiteral(4d), rewritten); + dsqrt = new Dsqrt(new DoubleLiteral(0d)); + rewritten = executor.rewrite(dsqrt, context); + Assertions.assertEquals(new DoubleLiteral(0d), rewritten); + dsqrt = new Dsqrt(new DoubleLiteral(-1d)); + rewritten = executor.rewrite(dsqrt, context); + Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + dsqrt = new Dsqrt(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(dsqrt, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); Power power = new Power(new DoubleLiteral(2d), new DoubleLiteral(3)); rewritten = executor.rewrite(power, context); Assertions.assertEquals(new DoubleLiteral(8d), rewritten); - exExp = new Power(new DoubleLiteral(2d), new DoubleLiteral(10000d)); - rewritten = executor.rewrite(exExp, context); - Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); - exExp = new Power(new DoubleLiteral(-1d), new DoubleLiteral(1.1d)); - rewritten = executor.rewrite(exExp, context); - Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + power = new Power(new DoubleLiteral(2d), new DoubleLiteral(10000d)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + power = new Power(new DoubleLiteral(-1d), new DoubleLiteral(1.1d)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(power, rewritten); + power = new Power(new DoubleLiteral(1d), new DoubleLiteral(0d)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(new DoubleLiteral(1d), rewritten); + power = new Power(new DoubleLiteral(0d), new DoubleLiteral(1d)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(new DoubleLiteral(0d), rewritten); + power = new Power(new DoubleLiteral(1.1), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + power = new Power(new DoubleLiteral(-1d), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(power, rewritten); + power = new Power(new DoubleLiteral(-1d), new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(power, context); + Assertions.assertEquals(power, rewritten); Sin sin = new Sin(new DoubleLiteral(Math.PI / 2)); rewritten = executor.rewrite(sin, context); @@ -616,38 +696,104 @@ void testleFoldNumeric() { sin = new Sin(new DoubleLiteral(0d)); rewritten = executor.rewrite(sin, context); Assertions.assertEquals(new DoubleLiteral(0d), rewritten); - exExp = new Sin(new DoubleLiteral(Double.POSITIVE_INFINITY)); - rewritten = executor.rewrite(exExp, context); - Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + sin = new Sin(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(sin, context); + Assertions.assertEquals(sin, rewritten); + sin = new Sin(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(sin, context); + Assertions.assertEquals(sin, rewritten); + sin = new Sin(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(sin, context); + Assertions.assertEquals(sin, rewritten); Cos cos = new Cos(new DoubleLiteral(0d)); rewritten = executor.rewrite(cos, context); Assertions.assertEquals(new DoubleLiteral(1d), rewritten); - exExp = new Cos(new DoubleLiteral(Double.POSITIVE_INFINITY)); - rewritten = executor.rewrite(exExp, context); - Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + cos = new Cos(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(cos, context); + Assertions.assertEquals(cos, rewritten); + cos = new Cos(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(cos, context); + Assertions.assertEquals(cos, rewritten); Tan tan = new Tan(new DoubleLiteral(0d)); rewritten = executor.rewrite(tan, context); Assertions.assertEquals(new DoubleLiteral(0d), rewritten); - exExp = new Tan(new DoubleLiteral(Double.POSITIVE_INFINITY)); - rewritten = executor.rewrite(exExp, context); - Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + tan = new Tan(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(tan, context); + Assertions.assertEquals(tan, rewritten); + tan = new Tan(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(tan, context); + Assertions.assertEquals(tan, rewritten); + + Cot cot = new Cot(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(cot, context); + Assertions.assertEquals(cot, rewritten); + cot = new Cot(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(cot, context); + Assertions.assertEquals(cot, rewritten); + + Sec sec = new Sec(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(sec, context); + Assertions.assertEquals(sec, rewritten); + sec = new Sec(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(sec, context); + Assertions.assertEquals(sec, rewritten); Asin asin = new Asin(new DoubleLiteral(1d)); rewritten = executor.rewrite(asin, context); Assertions.assertEquals(new DoubleLiteral(Math.PI / 2), rewritten); - exExp = new Asin(new DoubleLiteral(2d)); - rewritten = executor.rewrite(exExp, context); + asin = new Asin(new DoubleLiteral(2d)); + rewritten = executor.rewrite(asin, context); Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); Acos acos = new Acos(new DoubleLiteral(1d)); rewritten = executor.rewrite(acos, context); Assertions.assertEquals(new DoubleLiteral(0), rewritten); - exExp = new Acos(new DoubleLiteral(2d)); - rewritten = executor.rewrite(exExp, context); + acos = new Acos(new DoubleLiteral(2d)); + rewritten = executor.rewrite(acos, context); Assertions.assertEquals(new NullLiteral(DoubleType.INSTANCE), rewritten); + Atan atan = new Atan(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(atan, context); + Assertions.assertEquals(new DoubleLiteral(1.5707963267948966), rewritten); + atan = new Atan(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(atan, context); + Assertions.assertEquals(new DoubleLiteral(-1.5707963267948966), rewritten); + atan = new Atan(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(atan, context); + Assertions.assertEquals(atan, rewritten); + + Cbrt cbrt = new Cbrt(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(cbrt, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + cbrt = new Cbrt(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(cbrt, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + cbrt = new Cbrt(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(cbrt, context); + Assertions.assertEquals(cbrt, rewritten); + + Cosh cosh = new Cosh(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(cosh, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + cosh = new Cosh(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(cosh, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + cosh = new Cosh(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(cosh, context); + Assertions.assertEquals(cosh, rewritten); + + Tanh tanh = new Tanh(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(tanh, context); + Assertions.assertEquals(new DoubleLiteral(1d), rewritten); + tanh = new Tanh(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(tanh, context); + Assertions.assertEquals(new DoubleLiteral(-1d), rewritten); + tanh = new Tanh(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(tanh, context); + Assertions.assertEquals(tanh, rewritten); + Sign sign = new Sign(new DoubleLiteral(1d)); rewritten = executor.rewrite(sign, context); Assertions.assertEquals(new TinyIntLiteral((byte) 1), rewritten); @@ -668,6 +814,82 @@ void testleFoldNumeric() { bitCount = new BitCount(new BigIntLiteral(-1)); rewritten = executor.rewrite(bitCount, context); Assertions.assertEquals(new TinyIntLiteral((byte) 64), rewritten); + + Add add = new Add(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(100)); + rewritten = executor.rewrite(add, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + add = new Add(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(add, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + add = new Add(new DoubleLiteral(Double.NEGATIVE_INFINITY), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(add, context); + Assertions.assertEquals(add, rewritten); + add = new Add(new DoubleLiteral(Double.NaN), new DoubleLiteral(1)); + rewritten = executor.rewrite(add, context); + Assertions.assertEquals(add, rewritten); + + Subtract subtract = new Subtract(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(100)); + rewritten = executor.rewrite(subtract, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + subtract = new Subtract(new DoubleLiteral(1), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(subtract, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + subtract = new Subtract(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(subtract, context); + Assertions.assertEquals(subtract, rewritten); + + Multiply multiply = new Multiply(new DoubleLiteral(1e300), new DoubleLiteral(1e100)); + rewritten = executor.rewrite(multiply, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + multiply = new Multiply(new DoubleLiteral(-1e300), new DoubleLiteral(1e100)); + rewritten = executor.rewrite(multiply, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + multiply = new Multiply(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(100)); + rewritten = executor.rewrite(multiply, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + multiply = new Multiply(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(multiply, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + + Divide divide = new Divide(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(1e100)); + rewritten = executor.rewrite(divide, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + divide = new Divide(new DoubleLiteral(Double.NEGATIVE_INFINITY), new DoubleLiteral(1e100)); + rewritten = executor.rewrite(divide, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + divide = new Divide(new DoubleLiteral(Double.NEGATIVE_INFINITY), new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(divide, context); + Assertions.assertEquals(divide, rewritten); + + Fmod fmod = new Fmod(new DoubleLiteral(Double.POSITIVE_INFINITY), new DoubleLiteral(1)); + rewritten = executor.rewrite(fmod, context); + Assertions.assertEquals(fmod, rewritten); + fmod = new Fmod(new DoubleLiteral(Double.NEGATIVE_INFINITY), new DoubleLiteral(1)); + rewritten = executor.rewrite(fmod, context); + Assertions.assertEquals(fmod, rewritten); + fmod = new Fmod(new DoubleLiteral(Double.NaN), new DoubleLiteral(1)); + rewritten = executor.rewrite(fmod, context); + Assertions.assertEquals(fmod, rewritten); + + Radians radians = new Radians(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(radians, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + radians = new Radians(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(radians, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + radians = new Radians(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(radians, context); + Assertions.assertEquals(rewritten, rewritten); + + Degrees degrees = new Degrees(new DoubleLiteral(Double.POSITIVE_INFINITY)); + rewritten = executor.rewrite(degrees, context); + Assertions.assertEquals(new DoubleLiteral(Double.POSITIVE_INFINITY), rewritten); + degrees = new Degrees(new DoubleLiteral(Double.NEGATIVE_INFINITY)); + rewritten = executor.rewrite(degrees, context); + Assertions.assertEquals(new DoubleLiteral(Double.NEGATIVE_INFINITY), rewritten); + degrees = new Degrees(new DoubleLiteral(Double.NaN)); + rewritten = executor.rewrite(degrees, context); + Assertions.assertEquals(degrees, rewritten); } @Test diff --git a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_struct.out b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_struct.out index fa63c0da504a35..5f40301cb15747 100644 --- a/regression-test/data/query_p0/sql_functions/cast_function/test_cast_struct.out +++ b/regression-test/data/query_p0/sql_functions/cast_function/test_cast_struct.out @@ -39,5 +39,5 @@ {"f1":1, "f2":"2022-10-10"} -- !sql14 -- -{"f1":1.0, "f2":"2022-10-10 00:00:00"} +{"f1":1, "f2":"2022-10-10 00:00:00"} diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index b5a3f78dd9da6c..c23079331b48e3 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1793,7 +1793,8 @@ class Suite implements GroovyInterceptable { String closeFoldConstant = "set debug_skip_fold_constant=true"; sql(closeFoldConstant) logger.info(foldSql) - List> resultExpected = sql(foldSql) + Tuple2>, ResultSetMetaData> tupleResult2 = JdbcUtils.executeToStringList(context.getConnection(), foldSql) + List> resultExpected = tupleResult2.first logger.info("result expected: " + resultExpected.toString()) String errorMsg = null diff --git a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy index 97a70ac3c0c23a..5ded6898d9f552 100644 --- a/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy +++ b/regression-test/suites/nereids_p0/expression/fold_constant/fold_constant_numeric_arithmatic.groovy @@ -65,6 +65,11 @@ suite("fold_constant_numeric_arithmatic") { testFoldConst("SELECT ASIN(1E308)") testFoldConst("SELECT ASIN(-1E308)") + // Add function cases + testFoldConst("SELECT ADD(1e1000, 1)"); + testFoldConst("SELECT ADD(-1e1000, 1)"); + testFoldConst("SELECT ADD(cast(\"nan\" as double), 1)"); + //Atan function cases testFoldConst("SELECT ATAN(1) AS atan_case_1") //atan(1) = π/4 testFoldConst("SELECT ATAN(0) AS atan_case_2") //atan(0) = 0 @@ -251,6 +256,7 @@ suite("fold_constant_numeric_arithmatic") { testFoldConst("SELECT EXP(-1E-308)") // Very small negative number testFoldConst("SELECT EXP(709.782712893384)") // Near overflow boundary testFoldConst("SELECT EXP(-709.782712893384)") // Near underflow boundary + testFoldConst("SELECT EXP(1959859681)") // Result overflow become infinity //Floor function cases testFoldConst("SELECT FLOOR(3.7) AS floor_case_1") @@ -301,6 +307,8 @@ suite("fold_constant_numeric_arithmatic") { // testFoldConst("SELECT POWER(2, 1E308)") testFoldConst("SELECT POWER(1E-308, 2)") // Very small base testFoldConst("SELECT POWER(2, -1E308)") // Very small negative exponent + testFoldConst("SELECT POWER(-1.1, 3.2)") // NaN + testFoldConst("SELECT POWER(1, 1e1000)") //Ln function cases testFoldConst("SELECT LN(1) AS ln_case_1") //ln(1) = 0 @@ -459,6 +467,8 @@ suite("fold_constant_numeric_arithmatic") { testFoldConst("SELECT SQRT(16) AS sqrt_case_1") //sqrt(16) = 4 testFoldConst("SELECT SQRT(0) AS sqrt_case_2") //sqrt(0) = 0 testFoldConst("SELECT SQRT(2) AS sqrt_case_3") //sqrt(2) + //testFoldConst("SELECT SQRT(1e1000)") //sqrt(2) + testFoldConst("SELECT SQRT(-1e1000)") //sqrt(2) //Tan function cases testFoldConst("SELECT TAN(PI() / 4) AS tan_case_1") //tan(π/4) = 1 From 9d0b957d1f21c538054b8f3d0e38a591e3e596fc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Aug 2025 16:27:59 +0800 Subject: [PATCH 517/572] branch-3.0: [fix](warmup) fix NPE in getTabletReplicaInfos #55130 (#55198) Cherry-picked from #55130 Co-authored-by: Kaijie Chen --- .../org/apache/doris/service/FrontendServiceImpl.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 49ef6e5e6358c7..f3140d490c1d83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2761,6 +2761,14 @@ public TGetTabletReplicaInfosResult getTabletReplicaInfos(TGetTabletReplicaInfos } clusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) .getCloudClusterIdByName(job.getDstClusterName()); + if (clusterId == null) { + LOG.warn("cluster {} is not found, cannot get primary backend for warmup job {}", + job.getDstClusterName(), request.getWarmUpJobId()); + result.setTabletReplicaInfos(tabletReplicaInfos); + result.setToken(Env.getCurrentEnv().getToken()); + result.setStatus(new TStatus(TStatusCode.OK)); + return result; + } } for (Long tabletId : tabletIds) { if (DebugPointUtil.isEnable("getTabletReplicaInfos.returnEmpty")) { From 19c69ceaea8c489b021d893f9bae9752385dcfdc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 09:46:01 +0800 Subject: [PATCH 518/572] branch-3.0: [fix](regression) fix alter_ttl_1 regression #55257 (#55270) Cherry-picked from #55257 Co-authored-by: zhengyu --- regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy index 8f34a04a640859..8f5d070e091c39 100644 --- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy +++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_1.groovy @@ -25,6 +25,7 @@ suite("alter_ttl_1") { setBeConfigTemporary(custoBeConfig) { sql "set global enable_auto_analyze = false" + sql "set global enable_audit_plugin = false" def clusters = sql " SHOW CLUSTERS; " assertTrue(!clusters.isEmpty()) def validCluster = clusters[0][0] @@ -110,7 +111,6 @@ suite("alter_ttl_1") { sleep(30000) load_customer_ttl_once("customer_ttl") - sql """ select count(*) from customer_ttl """ sleep(30000) long ttl_cache_size = 0 long normal_cache_size = 0 From 38491c5f37b9761ea89297a8029c3c15e17616d1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 10:57:19 +0800 Subject: [PATCH 519/572] branch-3.0: [Fix](case) Fix some show data p2 cases #55244 (#55266) Cherry-picked from #55244 Co-authored-by: abmdocrt --- regression-test/plugins/aliyun_oss_sdk.groovy | 99 +++++++++++++++---- .../plugins/cloud_show_data_plugin.groovy | 10 +- .../test_cloud_follower_show_data.groovy | 14 +-- ...t_cloud_delete_table_rows_show_data.groovy | 1 + .../test_cloud_drop_table_show_data.groovy | 8 +- ..._cloud_disable_compaction_show_data.groovy | 8 +- 6 files changed, 103 insertions(+), 37 deletions(-) diff --git a/regression-test/plugins/aliyun_oss_sdk.groovy b/regression-test/plugins/aliyun_oss_sdk.groovy index 6b0c096d7e5992..a23924530d1bbf 100644 --- a/regression-test/plugins/aliyun_oss_sdk.groovy +++ b/regression-test/plugins/aliyun_oss_sdk.groovy @@ -91,29 +91,86 @@ Suite.metaClass.listOssObjectWithPrefix = { OSS client, String bucketName, Strin } -// get file size in a specific directory +/** + * 计算指定文件夹的总大小（递归计算所有文件） + * @param client OSS客户端实例 + * @param bucketName OSS存储桶名称 + * @param folder 文件夹路径前缀 + * @return 文件夹总大小（字节） + */ Suite.metaClass.calculateFolderLength = { OSS client, String bucketName, String folder -> - long size = 0L; + logger.info("[calculateFolderLength] 开始计算文件夹大小 - Bucket: ${bucketName}, Folder: ${folder}") + + long size = 0L; // 累计文件大小 ObjectListing objectListing = null; - do { - // The default value for MaxKey is 100, and the maximum value is 1000 - logger.info("debug:" + folder) - ListObjectsRequest request = new ListObjectsRequest(bucketName).withPrefix(folder).withMaxKeys(1000); - if (objectListing != null) { - request.setMarker(objectListing.getNextMarker()); - } - objectListing = client.listObjects(request); - List sums = objectListing.getObjectSummaries(); - for (OSSObjectSummary s : sums) { - logger.info("Object Key: ${s.getKey()}") - logger.info("Size: ${s.getSize()} bytes") - logger.info("Last Modified: ${s.getLastModified()}") - logger.info("Storage Class: ${s.getStorageClass()}") - logger.info("Owner: ${s.getOwner()?.getId()}") - logger.info("-------------------") - size += s.getSize(); - } - } while (objectListing.isTruncated()); + int pageCount = 0; // 分页计数器 + int totalObjects = 0; // 总文件数量计数器 + + try { + // 使用分页方式遍历所有对象，避免一次性加载过多数据 + do { + pageCount++; + + // 创建列表对象请求，设置最大返回数量为1000（OSS限制的最大值） + ListObjectsRequest request = new ListObjectsRequest(bucketName) + .withPrefix(folder) + .withMaxKeys(1000); + + // 如果不是第一页，设置分页标记 + if (objectListing != null) { + String nextMarker = objectListing.getNextMarker(); + request.setMarker(nextMarker); + } + + // 执行OSS请求获取对象列表 + objectListing = client.listObjects(request); + + // 获取当前页的对象摘要列表 + List sums = objectListing.getObjectSummaries(); + + + // 遍历当前页的所有对象，累加大小 + for (OSSObjectSummary s : sums) { + totalObjects++; + long objSize = s.getSize(); + + // 详细记录每个对象的信息 + logger.info("📄 [OBJECT #${totalObjects}] 单个对象详情:") + logger.info(" ├─ Key: ${s.getKey()}") + logger.info(" ├─ Size: ${objSize} bytes (${String.format('%.2f', objSize / 1024.0 / 1024.0)} MB)") + logger.info(" ├─ Last Modified: ${s.getLastModified()}") + logger.info(" ├─ Storage Class: ${s.getStorageClass()}") + logger.info(" ├─ Owner: ${s.getOwner()?.getId() ?: 'N/A'}") + logger.info(" └─ ETag: ${s.getETag()}") + + // 累加到总大小 + size += objSize; + logger.info("🔢 [RUNNING TOTAL] 当前累计: ${size} bytes (${String.format('%.2f', size / 1024.0 / 1024.0)} MB)") + logger.info("─────────────────────────────────────────") + } + + + } while (objectListing.isTruncated()); // 继续处理下一页，直到所有数据处理完毕 + + // 记录最终统计结果 + logger.info("📊 [FOLDER SUMMARY] 文件夹统计完成:") + logger.info(" ╔══════════════════════════════════════════╗") + logger.info(" ║ 📁 文件夹路径: ${folder}") + logger.info(" ║ 📝 总文件数: ${totalObjects}") + logger.info(" ║ 📏 总大小: ${size} bytes") + logger.info(" ║ 📏 总大小: ${String.format('%.2f', size / 1024.0 / 1024.0)} MB") + logger.info(" ║ 📏 总大小: ${String.format('%.2f', size / 1024.0 / 1024.0 / 1024.0)} GB") + logger.info(" ╚══════════════════════════════════════════╝") + + } catch (Exception e) { + logger.error("[calculateFolderLength] 计算文件夹大小时发生异常:", e) + logger.error(" - Bucket: ${bucketName}") + logger.error(" - Folder: ${folder}") + logger.error(" - 已处理对象数: ${totalObjects}") + logger.error(" - 当前累计大小: ${size} bytes") + throw e // 重新抛出异常 + } + return size; } diff --git a/regression-test/plugins/cloud_show_data_plugin.groovy b/regression-test/plugins/cloud_show_data_plugin.groovy index 3180d452dc6982..25019937ced8ea 100644 --- a/regression-test/plugins/cloud_show_data_plugin.groovy +++ b/regression-test/plugins/cloud_show_data_plugin.groovy @@ -186,8 +186,10 @@ import org.codehaus.groovy.runtime.IOGroovyMethods } Suite.metaClass.show_table_data_size_through_mysql = { String table -> + logger.info("[show_table_data_size_through_mysql] 表名: ${table}") def mysqlShowDataSize = 0L def res = sql_return_maparray " show data from ${table}" + logger.info("[show_table_data_size_through_mysql] show data结果: " + res.toString()) def tableSizeInfo = res[0] def fields = tableSizeInfo["Size"].split(" ") if (fields.length == 2 ){ @@ -196,13 +198,16 @@ import org.codehaus.groovy.runtime.IOGroovyMethods mysqlShowDataSize = translate_different_unit_to_MB(sizeField, unitField) } def round_size = new BigDecimal(mysqlShowDataSize).setScale(0, BigDecimal.ROUND_FLOOR); + logger.info("[show_table_data_size_through_mysql] 最终结果: ${round_size} MB") return round_size } Suite.metaClass.caculate_table_data_size_through_api = { List> tablets -> Double apiCaculateSize = 0 for (HashMap tablet in tablets) { + def tabletId = tablet.TabletId def tabletStatus = show_tablet_compaction(tablet) + logger.info("[caculate_table_data_size_through_api] tablet ID: ${tabletId}, status: " + tabletStatus.toString()) for(String rowset: tabletStatus.rowsets){ def fields = rowset.split(" ") @@ -215,6 +220,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods } } def round_size = new BigDecimal(apiCaculateSize).setScale(0, BigDecimal.ROUND_FLOOR); + logger.info("[caculate_table_data_size_through_api] 最终结果: ${round_size} MB") return round_size } @@ -274,10 +280,10 @@ import org.codehaus.groovy.runtime.IOGroovyMethods getBackendIpHttpPort(backendIdToBackendIP, backendIdToBackendHttpPort); def backendId = backendIdToBackendIP.keySet()[0] - + def code, out, err def get_be_param = { paramName -> // assuming paramName on all BEs have save value - def (code, out, err) = show_be_config(backendIdToBackendIP.get(backendId), backendIdToBackendHttpPort.get(backendId)) + (code, out, err) = show_be_config(backendIdToBackendIP.get(backendId), backendIdToBackendHttpPort.get(backendId)) assertEquals(code, 0) def configList = parseJson(out.trim()) assert configList instanceof List diff --git a/regression-test/suites/show_data_p2/test_cloud_follower_show_data.groovy b/regression-test/suites/show_data_p2/test_cloud_follower_show_data.groovy index 671191a963dd8d..026420dbb1d9c7 100644 --- a/regression-test/suites/show_data_p2/test_cloud_follower_show_data.groovy +++ b/regression-test/suites/show_data_p2/test_cloud_follower_show_data.groovy @@ -91,25 +91,27 @@ suite("test_cloud_follower_show_data","p2, nonConcurrent") { assertEquals(sizeRecords["apiSize"][0], sizeRecords["apiSize"][1]) assertEquals(sizeRecords["cbsSize"][0], sizeRecords["cbsSize"][1]) - + sleep(10 * 1000) def result = sql """show frontends;""" logger.info("result:" + result) for (int i = 0; i < result.size(); i++) { if (result[i][8] == "false" && result[i][11] == "true") { - def tokens = context.config.jdbcUrl.split('/') - url = tokens[0] + "//" + tokens[2] + "/" + "information_schema" + "?" - def new_jdbc_url = url.replaceAll(/\/\/[0-9.]+:/, "//${switch_ip}:") + def follower_ip = result[i][1] + //jdbc:mysql://127.0.0.1:9030/?useLocalSessionState=true&allowLoadLocalInfile=true&zeroDateTimeBehavior=round + def old_url = context.config.jdbcUrl + def new_jdbc_url = old_url.replaceAll("://[^:]+:", "://${follower_ip}:") logger.info("new_jdbc_url: " + new_jdbc_url) connect('root', '', new_jdbc_url) { - sql "select count(*) from ${tableName}" + sql "select count(*) from regression_test_show_data_p2.${tableName}" + sql "use regression_test_show_data_p2" sizeRecords["mysqlSize"].add(show_table_data_size_through_mysql(tableName)) } } } - for (int i = 0; i < sizeRecords["mysqlSize"].size(); i++) { + for (int i = 0; i < sizeRecords["mysqlSize"].size(); i++) { if (i > 0) { assertEquals(sizeRecords["mysqlSize"][i], sizeRecords["mysqlSize"][i-1]) } diff --git a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_delete_table_rows_show_data.groovy b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_delete_table_rows_show_data.groovy index 9cef5a53712d4e..a5a49618b468ed 100644 --- a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_delete_table_rows_show_data.groovy +++ b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_delete_table_rows_show_data.groovy @@ -162,6 +162,7 @@ suite("test_cloud_delete_table_rows_show_data","p2, nonConcurrent") { assertEquals(sizeRecords["apiSize"][0], sizeRecords["apiSize"][1]) assertEquals(sizeRecords["cbsSize"][0], sizeRecords["cbsSize"][1]) + sql """set delete_without_partition=true;""" sql """delete from ${tableName} where L_ORDERKEY >=0;""" // 加一下触发compaction的机制 diff --git a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy index d80295d802f8cc..e80deb85834713 100644 --- a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy +++ b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy @@ -21,7 +21,7 @@ import org.codehaus.groovy.runtime.IOGroovyMethods // loading one data 10 times, expect data size not rising -suite("test_cloud_drop_and_recover_table_show_data","p2, nonConcurrent") { +suite("test_cloud_drop_and_show_data","p2, nonConcurrent") { //cloud-mode if (!isCloudMode()) { logger.info("not cloud mode, not run") @@ -124,15 +124,15 @@ suite("test_cloud_drop_and_recover_table_show_data","p2, nonConcurrent") { sizeRecords["cbsSize"].add(caculate_table_data_size_in_backend_storage(tablets)) logger.info("after drop table force, storageSize is: ${sizeRecords["cbsSize"][-1]}") - assertEquals(sizeRecords["cbsSize"][2], 0.0) + assertTrue(sizeRecords["cbsSize"][2] == 0) } } def main = { def tableName = "test_cloud_drop_and_recover_table_show_data" - //create_normal_table(tableName) - //check(tableName, 1) + create_normal_table(tableName) + check(tableName, 1) tableName = "test_cloud_drop_and_recover_table_force_show_data" create_normal_table(tableName) diff --git a/regression-test/suites/show_data_p2/test_table_property/test_cloud_disable_compaction_show_data.groovy b/regression-test/suites/show_data_p2/test_table_property/test_cloud_disable_compaction_show_data.groovy index f04c66133343a5..4b1c0a639daf17 100644 --- a/regression-test/suites/show_data_p2/test_table_property/test_cloud_disable_compaction_show_data.groovy +++ b/regression-test/suites/show_data_p2/test_table_property/test_cloud_disable_compaction_show_data.groovy @@ -81,11 +81,11 @@ suite("test_cloud_disable_compaction_show_data","p2, nonConcurrent") { // expect mysqlSize == apiSize == storageSize assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["apiSize"][0]) assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["cbsSize"][0]) - // expect load 1 times == load 10 times + // expect load 10 times >= 10 x load 1 times logger.info("after 1 time stream load, size is ${sizeRecords["mysqlSize"][0]}, after 10 times stream load, size is ${sizeRecords["mysqlSize"][1]}") - assertEquals(sizeRecords["mysqlSize"][0], sizeRecords["mysqlSize"][1]) - assertEquals(sizeRecords["apiSize"][0], sizeRecords["apiSize"][1]) - assertEquals(sizeRecords["cbsSize"][0], sizeRecords["cbsSize"][1]) + assertTrue(sizeRecords["mysqlSize"][1] >= 10 * sizeRecords["mysqlSize"][0]) + assertTrue(sizeRecords["apiSize"][1] >= 10 * sizeRecords["apiSize"][0]) + assertTrue(sizeRecords["cbsSize"][1] >= 10 * sizeRecords["cbsSize"][0]) } set_config_before_show_data_test() From fe30bbe6919fbfd000ad3e036c12fd483c76cd12 Mon Sep 17 00:00:00 2001 From: Uniqueyou Date: Tue, 26 Aug 2025 10:58:16 +0800 Subject: [PATCH 520/572] [branch-3.0] [fix](recycler) Concurrent recycle cause txn commit conflict #54849 (#55252) pick: https://github.com/apache/doris/pull/54849 https://github.com/apache/doris/pull/54908 --- cloud/src/recycler/recycler.cpp | 37 +++- cloud/test/recycler_test.cpp | 323 ++++++++++++++++++++++++++++++++ 2 files changed, 359 insertions(+), 1 deletion(-) diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index 33d5ec62952e18..e7144e44afa0f7 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -2893,6 +2894,7 @@ int InstanceRecycler::recycle_expired_txn_label() { return 0; }; + // int 0 for success, 1 for conflict, -1 for error auto delete_recycle_txn_kv = [&](const std::string& k) -> int { std::string_view k1 = k; //RecycleTxnKeyInfo 0:instance_id 1:db_id 2:txn_id @@ -2958,17 +2960,33 @@ int InstanceRecycler::recycle_expired_txn_label() { } if (txn_label.txn_ids().empty()) { txn->remove(label_key); + TEST_SYNC_POINT_CALLBACK( + "InstanceRecycler::recycle_expired_txn_label.remove_label_before"); } else { if (!txn_label.SerializeToString(&label_val)) { LOG(WARNING) << "failed to serialize txn label, key=" << hex(label_key); return -1; } + TEST_SYNC_POINT_CALLBACK( + "InstanceRecycler::recycle_expired_txn_label.update_label_before"); txn->atomic_set_ver_value(label_key, label_val); + TEST_SYNC_POINT_CALLBACK( + "InstanceRecycler::recycle_expired_txn_label.update_label_after"); } // Remove recycle txn kv txn->remove(k); + TEST_SYNC_POINT_CALLBACK("InstanceRecycler::recycle_expired_txn_label.before_commit"); err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { + if (err == TxnErrorCode::TXN_CONFLICT) { + TEST_SYNC_POINT_CALLBACK( + "InstanceRecycler::recycle_expired_txn_label.txn_conflict"); + // log the txn_id and label + LOG(WARNING) << "txn conflict, txn_id=" << txn_id + << " txn_label_pb=" << txn_label.ShortDebugString() + << " txn_label=" << txn_info.label(); + return 1; + } LOG(WARNING) << "failed to delete expired txn, err=" << err << " key=" << hex(k); return -1; } @@ -2988,7 +3006,24 @@ int InstanceRecycler::recycle_expired_txn_label() { &recycle_txn_info_keys); for (const auto& k : recycle_txn_info_keys) { concurrent_delete_executor.add([&]() { - if (delete_recycle_txn_kv(k) != 0) { + int ret = delete_recycle_txn_kv(k); + if (ret == 1) { + constexpr int MAX_RETRY = 10; + for (size_t i = 1; i <= MAX_RETRY; ++i) { + LOG(WARNING) << "txn conflict, retry times=" << i << " key=" << hex(k); + ret = delete_recycle_txn_kv(k); + // clang-format off + TEST_SYNC_POINT_CALLBACK( + "InstanceRecycler::recycle_expired_txn_label.delete_recycle_txn_kv_error", &ret); + // clang-format off + if (ret != 1) { + break; + } + // random sleep 0-100 ms to retry + std::this_thread::sleep_for(std::chrono::milliseconds(rand() % 100)); + } + } + if (ret != 0) { LOG_WARNING("failed to delete recycle txn kv") .tag("instance id", instance_id_) .tag("key", hex(k)); diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index 0d4f10e15f4a7d..85ad3ef947e4d0 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -5116,4 +5116,327 @@ TEST(RecyclerTest, concurrent_recycle_txn_label_failure_test) { << "ms" << std::endl; check_multiple_txn_info_kvs(txn_kv, 5000); } +TEST(RecyclerTest, concurrent_recycle_txn_label_conflict_test) { + config::label_keep_max_second = 0; + config::recycle_pool_parallelism = 20; + + doris::cloud::RecyclerThreadPoolGroup recycle_txn_label_thread_group; + auto recycle_txn_label_s3_producer_pool = + std::make_shared(config::recycle_pool_parallelism); + recycle_txn_label_s3_producer_pool->start(); + auto recycle_txn_label_recycle_tablet_pool = + std::make_shared(config::recycle_pool_parallelism); + recycle_txn_label_recycle_tablet_pool->start(); + auto recycle_txn_label_group_recycle_function_pool = + std::make_shared(config::recycle_pool_parallelism); + recycle_txn_label_group_recycle_function_pool->start(); + recycle_txn_label_thread_group = + RecyclerThreadPoolGroup(std::move(recycle_txn_label_s3_producer_pool), + std::move(recycle_txn_label_recycle_tablet_pool), + std::move(recycle_txn_label_group_recycle_function_pool)); + + auto mem_txn_kv = std::make_shared(); + ASSERT_EQ(mem_txn_kv->init(), 0); + + std::string shared_label = "shared_conflict_label"; + int64_t shared_db_id = 1000; + std::vector shared_txn_ids = {2001, 2002, 2003, 2004, 2005, + 2006, 2007, 2008, 2009, 2010}; + + // create shared TxnLabel + { + std::unique_ptr txn; + ASSERT_EQ(mem_txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + + std::string label_key; + std::string label_val; + txn_label_key({instance_id, shared_db_id, shared_label}, &label_key); + + TxnLabelPB txn_label_pb; + for (auto txn_id : shared_txn_ids) { + txn_label_pb.add_txn_ids(txn_id); + } + + if (!txn_label_pb.SerializeToString(&label_val)) { + FAIL() << "Failed to serialize txn label"; + } + + MemTxnKv::gen_version_timestamp(123456790, 0, &label_val); + txn->put(label_key, label_val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + } + + int64_t current_time = duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + for (auto txn_id : shared_txn_ids) { + std::unique_ptr txn; + ASSERT_EQ(mem_txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + + // RecycleTxnKeyInfo -> RecycleTxnPB (set to expired) + std::string recycle_txn_info_key; + std::string recycle_txn_info_val; + RecycleTxnKeyInfo recycle_txn_key_info {instance_id, shared_db_id, txn_id}; + recycle_txn_key(recycle_txn_key_info, &recycle_txn_info_key); + RecycleTxnPB recycle_txn_pb; + recycle_txn_pb.set_creation_time(current_time - 300000); + recycle_txn_pb.set_label(shared_label); + if (!recycle_txn_pb.SerializeToString(&recycle_txn_info_val)) { + FAIL() << "Failed to serialize recycle txn"; + } + txn->put(recycle_txn_info_key, recycle_txn_info_val); + + // TxnIndexKey -> TxnIndexPB + std::string txn_idx_key = txn_index_key({instance_id, txn_id}); + std::string txn_idx_val; + TxnIndexPB txn_index_pb; + if (!txn_index_pb.SerializeToString(&txn_idx_val)) { + FAIL() << "Failed to serialize txn index"; + } + txn->put(txn_idx_key, txn_idx_val); + + // TxnInfoKey -> TxnInfoPB + std::string info_key = txn_info_key({instance_id, shared_db_id, txn_id}); + std::string info_val; + TxnInfoPB txn_info_pb; + txn_info_pb.set_label(shared_label); + if (!txn_info_pb.SerializeToString(&info_val)) { + FAIL() << "Failed to serialize txn info"; + } + txn->put(info_key, info_val); + + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + } + + auto* sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + + std::atomic update_label_before_count {0}; + std::atomic remove_label_before_count {0}; + std::atomic update_label_after_count {0}; + std::atomic txn_conflict_count {0}; + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.remove_label_before", + [&](auto&& args) { + remove_label_before_count++; + std::this_thread::sleep_for(std::chrono::milliseconds(60)); + }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.update_label_before", + [&](auto&& args) { + update_label_before_count++; + std::this_thread::sleep_for(std::chrono::milliseconds(80)); + }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.update_label_after", + [&](auto&& args) { update_label_after_count++; }); + + sp->set_call_back( + "InstanceRecycler::recycle_expired_txn_label.before_commit", + [&](auto&& args) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.txn_conflict", [&](auto&& args) { + txn_conflict_count++; + LOG(WARNING) << "Transaction conflict detected in test"; + }); + + sp->enable_processing(); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + InstanceRecycler recycler(mem_txn_kv, instance, recycle_txn_label_thread_group, + std::make_shared(mem_txn_kv)); + ASSERT_EQ(recycler.init(), 0); + + auto start = std::chrono::steady_clock::now(); + ASSERT_EQ(recycler.recycle_expired_txn_label(), 0); + auto finish = std::chrono::steady_clock::now(); + + std::cout << "Concurrent recycle cost=" + << std::chrono::duration_cast(finish - start).count() + << "ms" << std::endl; + std::cout << "Update label before count: " << update_label_before_count << std::endl; + std::cout << "Update label after count: " << update_label_after_count << std::endl; + std::cout << "Transaction conflict count: " << txn_conflict_count << std::endl; + + EXPECT_GT(txn_conflict_count, 0) << "txn_conflict sync point should be triggered"; + + std::unique_ptr verify_txn; + ASSERT_EQ(mem_txn_kv->create_txn(&verify_txn), TxnErrorCode::TXN_OK); + + RecycleTxnKeyInfo recycle_txn_key_info0 {instance_id, 0, 0}; + RecycleTxnKeyInfo recycle_txn_key_info1 {instance_id, INT64_MAX, INT64_MAX}; + std::string begin_key = recycle_txn_key(recycle_txn_key_info0); + std::string end_key = recycle_txn_key(recycle_txn_key_info1); + + std::unique_ptr it; + ASSERT_EQ(verify_txn->get(begin_key, end_key, &it), TxnErrorCode::TXN_OK); + EXPECT_EQ(it->size(), 0) << "All recycle txn keys should be deleted"; + + std::string label_key; + std::string label_val; + txn_label_key({instance_id, shared_db_id, shared_label}, &label_key); + EXPECT_EQ(verify_txn->get(label_key, &label_val), TxnErrorCode::TXN_KEY_NOT_FOUND) + << "Shared label should be deleted"; + + for (auto txn_id : shared_txn_ids) { + std::string info_key = txn_info_key({instance_id, shared_db_id, txn_id}); + std::string info_val; + EXPECT_EQ(verify_txn->get(info_key, &info_val), TxnErrorCode::TXN_KEY_NOT_FOUND) + << "TxnInfo for txn_id " << txn_id << " should be deleted"; + } +} + +TEST(RecyclerTest, recycle_txn_label_deal_with_conflict_error_test) { + config::label_keep_max_second = 0; + config::recycle_pool_parallelism = 20; + + doris::cloud::RecyclerThreadPoolGroup recycle_txn_label_thread_group; + auto recycle_txn_label_s3_producer_pool = + std::make_shared(config::recycle_pool_parallelism); + recycle_txn_label_s3_producer_pool->start(); + auto recycle_txn_label_recycle_tablet_pool = + std::make_shared(config::recycle_pool_parallelism); + recycle_txn_label_recycle_tablet_pool->start(); + auto recycle_txn_label_group_recycle_function_pool = + std::make_shared(config::recycle_pool_parallelism); + recycle_txn_label_group_recycle_function_pool->start(); + recycle_txn_label_thread_group = + RecyclerThreadPoolGroup(std::move(recycle_txn_label_s3_producer_pool), + std::move(recycle_txn_label_recycle_tablet_pool), + std::move(recycle_txn_label_group_recycle_function_pool)); + + auto mem_txn_kv = std::make_shared(); + ASSERT_EQ(mem_txn_kv->init(), 0); + + std::string shared_label = "shared_conflict_label"; + int64_t shared_db_id = 1000; + std::vector shared_txn_ids = {2001, 2002, 2003, 2004, 2005, + 2006, 2007, 2008, 2009, 2010}; + + // create shared TxnLabel + { + std::unique_ptr txn; + ASSERT_EQ(mem_txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + + std::string label_key; + std::string label_val; + txn_label_key({instance_id, shared_db_id, shared_label}, &label_key); + + TxnLabelPB txn_label_pb; + for (auto txn_id : shared_txn_ids) { + txn_label_pb.add_txn_ids(txn_id); + } + + if (!txn_label_pb.SerializeToString(&label_val)) { + FAIL() << "Failed to serialize txn label"; + } + + MemTxnKv::gen_version_timestamp(123456790, 0, &label_val); + txn->put(label_key, label_val); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + } + + int64_t current_time = duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + + for (auto txn_id : shared_txn_ids) { + std::unique_ptr txn; + ASSERT_EQ(mem_txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + + // RecycleTxnKeyInfo -> RecycleTxnPB (set to expired) + std::string recycle_txn_info_key; + std::string recycle_txn_info_val; + RecycleTxnKeyInfo recycle_txn_key_info {instance_id, shared_db_id, txn_id}; + recycle_txn_key(recycle_txn_key_info, &recycle_txn_info_key); + RecycleTxnPB recycle_txn_pb; + recycle_txn_pb.set_creation_time(current_time - 300000); + recycle_txn_pb.set_label(shared_label); + if (!recycle_txn_pb.SerializeToString(&recycle_txn_info_val)) { + FAIL() << "Failed to serialize recycle txn"; + } + txn->put(recycle_txn_info_key, recycle_txn_info_val); + + // TxnIndexKey -> TxnIndexPB + std::string txn_idx_key = txn_index_key({instance_id, txn_id}); + std::string txn_idx_val; + TxnIndexPB txn_index_pb; + if (!txn_index_pb.SerializeToString(&txn_idx_val)) { + FAIL() << "Failed to serialize txn index"; + } + txn->put(txn_idx_key, txn_idx_val); + + // TxnInfoKey -> TxnInfoPB + std::string info_key = txn_info_key({instance_id, shared_db_id, txn_id}); + std::string info_val; + TxnInfoPB txn_info_pb; + txn_info_pb.set_label(shared_label); + if (!txn_info_pb.SerializeToString(&info_val)) { + FAIL() << "Failed to serialize txn info"; + } + txn->put(info_key, info_val); + + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + } + + auto* sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->clear_all_call_backs(); + }; + + std::atomic update_label_before_count {0}; + std::atomic remove_label_before_count {0}; + std::atomic update_label_after_count {0}; + std::atomic txn_conflict_count {0}; + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.remove_label_before", + [&](auto&& args) { + remove_label_before_count++; + std::this_thread::sleep_for(std::chrono::milliseconds(60)); + }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.update_label_before", + [&](auto&& args) { + update_label_before_count++; + std::this_thread::sleep_for(std::chrono::milliseconds(80)); + }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.update_label_after", + [&](auto&& args) { update_label_after_count++; }); + + sp->set_call_back( + "InstanceRecycler::recycle_expired_txn_label.before_commit", + [&](auto&& args) { std::this_thread::sleep_for(std::chrono::milliseconds(20)); }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.txn_conflict", [&](auto&& args) { + txn_conflict_count++; + LOG(WARNING) << "Transaction conflict detected in test"; + }); + + sp->set_call_back("InstanceRecycler::recycle_expired_txn_label.delete_recycle_txn_kv_error", + [&](auto&& args) { + auto ret = try_any_cast(args[0]); + *ret = -1; + LOG(WARNING) + << "Simulating delete recycle txn kv error in deal with conflict"; + }); + + sp->enable_processing(); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + InstanceRecycler recycler(mem_txn_kv, instance, recycle_txn_label_thread_group, + std::make_shared(mem_txn_kv)); + ASSERT_EQ(recycler.init(), 0); + + // deal with conflict but error during recycle + ASSERT_EQ(recycler.recycle_expired_txn_label(), -1); + + EXPECT_GT(txn_conflict_count, 0) << "txn_conflict sync point should be triggered"; +} + } // namespace doris::cloud From 980a86bea83fe687f15ab3d139603e61067d3ec2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 19:09:32 +0800 Subject: [PATCH 521/572] branch-3.0: [test](inverted index) use inner function instead of load_from_hdfs #55282 (#55304) Cherry-picked from #55282 Co-authored-by: Jack --- .../suites/inverted_index_p0/load/test_spark_load.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy b/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy index 488b7c6b35bde7..4dff6264a2e1a8 100644 --- a/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy +++ b/regression-test/suites/inverted_index_p0/load/test_spark_load.groovy @@ -154,7 +154,7 @@ suite("test_spark_load_with_index_p0", "p0") { create_test_table.call(testTable) create_test_table.call(testTable2) def test_load_label = UUID.randomUUID().toString().replaceAll("-", "") - load_from_hdfs.call(testTable, testTable2, test_load_label, hdfs_txt_file_path1, hdfs_txt_file_path2) + load_from_hdfs_use_spark.call(testTable, testTable2, test_load_label, hdfs_txt_file_path1, hdfs_txt_file_path2) check_load_result.call(test_load_label, testTable, testTable2) } finally { From 10791efa61b9bb676fd3abeb3159b7cf1cad9f10 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Aug 2025 09:07:51 +0800 Subject: [PATCH 522/572] branch-3.0: [chore](case) do not use global var #55280 (#55314) Cherry-picked from #55280 Co-authored-by: Yongqiang YANG --- .../suites/export/test_array_export.groovy | 24 +++++++++---------- .../broker_load/test_array_load.groovy | 6 ++--- .../broker_load/test_broker_load_seq.groovy | 6 ++--- .../test_broker_load_with_merge.groovy | 6 ++--- .../test_broker_load_with_partition.groovy | 10 ++++---- .../test_broker_load_with_where.groovy | 8 +++---- .../test_broker_load_without_filepath.groovy | 6 ++--- .../test_broker_load_without_seq.groovy | 6 ++--- 8 files changed, 36 insertions(+), 36 deletions(-) diff --git a/regression-test/suites/export/test_array_export.groovy b/regression-test/suites/export/test_array_export.groovy index 4ce9786e9972bb..a6c834cefc6ccf 100644 --- a/regression-test/suites/export/test_array_export.groovy +++ b/regression-test/suites/export/test_array_export.groovy @@ -210,20 +210,20 @@ suite("test_array_export", "export") { if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() - hdfsDataDir = getHdfsDataDir() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() + def hdfsDataDir = getHdfsDataDir() // case2: test "select ...into outfile 'hdfs_path'" try { create_test_table.call(tableName) - resultCount = sql "select count(*) from ${tableName}" - currentTotalRows = resultCount[0][0] + def resultCount = sql "select count(*) from ${tableName}" + def currentTotalRows = resultCount[0][0] - label = UUID.randomUUID().toString().replaceAll("-", "") - select_out_file(tableName, hdfsDataDir + "/" + label + "/export-data", "csv", brokerName, hdfsUser, hdfsPasswd) + def label = UUID.randomUUID().toString().replaceAll("-", "") + def result = select_out_file(tableName, hdfsDataDir + "/" + label + "/export-data", "csv", brokerName, hdfsUser, hdfsPasswd) result = downloadExportFromHdfs(label + "/export-data") check_download_result(result, currentTotalRows) } finally { @@ -234,13 +234,13 @@ suite("test_array_export", "export") { try { create_test_table.call(tableName) - resultCount = sql "select count(*) from ${tableName}" - currentTotalRows = resultCount[0][0] + def resultCount = sql "select count(*) from ${tableName}" + def currentTotalRows = resultCount[0][0] - label = UUID.randomUUID().toString().replaceAll("-", "") + def label = UUID.randomUUID().toString().replaceAll("-", "") export_to_hdfs.call(tableName, label, hdfsDataDir + "/" + label, '', brokerName, hdfsUser, hdfsPasswd) check_export_result(label) - result = downloadExportFromHdfs(label + "/export-data") + def result = downloadExportFromHdfs(label + "/export-data") check_download_result(result, currentTotalRows) } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") diff --git a/regression-test/suites/load_p0/broker_load/test_array_load.groovy b/regression-test/suites/load_p0/broker_load/test_array_load.groovy index fe2985aff4aa0c..6ec6a601d0f77f 100644 --- a/regression-test/suites/load_p0/broker_load/test_array_load.groovy +++ b/regression-test/suites/load_p0/broker_load/test_array_load.groovy @@ -246,9 +246,9 @@ suite("test_array_load", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName =getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName =getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_json_file_path = uploadToHdfs "load_p0/broker_load/simple_object_array.json" def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/simple_array.csv" def hdfs_orc_file_path = uploadToHdfs "load_p0/broker_load/simple_array.orc" diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_seq.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_seq.groovy index 2529226382c4ab..b9851c11789cc7 100644 --- a/regression-test/suites/load_p0/broker_load/test_broker_load_seq.groovy +++ b/regression-test/suites/load_p0/broker_load/test_broker_load_seq.groovy @@ -95,9 +95,9 @@ suite("test_broker_load_seq", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load.csv" //def hdfs_csv_file_path = "hdfs://ip:port/testfile" diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_with_merge.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_with_merge.groovy index eb76e4d389aaba..24bdee9761342a 100644 --- a/regression-test/suites/load_p0/broker_load/test_broker_load_with_merge.groovy +++ b/regression-test/suites/load_p0/broker_load/test_broker_load_with_merge.groovy @@ -174,9 +174,9 @@ suite("test_broker_load_with_merge", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load_with_merge.csv" // case1: has delete on condition and without merge diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy index 45f0cc50be7eed..3b1620e1358bda 100644 --- a/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy +++ b/regression-test/suites/load_p0/broker_load/test_broker_load_with_partition.groovy @@ -108,7 +108,7 @@ suite("test_broker_load_with_partition", "load_p0") { } def check_load_result = {checklabel, testTablex -> - max_try_milli_secs = 10000 + def max_try_milli_secs = 10000 while(max_try_milli_secs) { result = sql "show load where label = '${checklabel}'" log.info("result: ${result}") @@ -127,7 +127,7 @@ suite("test_broker_load_with_partition", "load_p0") { } def check_load_tmp_partition_result = {checklabel, testTablex -> - max_try_milli_secs = 10000 + def max_try_milli_secs = 10000 while(max_try_milli_secs) { result = sql "show load where label = '${checklabel}'" log.info("result: ${result}") @@ -148,9 +148,9 @@ suite("test_broker_load_with_partition", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load_with_partition.csv" //def hdfs_csv_file_path = "hdfs://ip:port/testfile" diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_with_where.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_with_where.groovy index dd05905e4bbdf2..a058f10349ec9c 100644 --- a/regression-test/suites/load_p0/broker_load/test_broker_load_with_where.groovy +++ b/regression-test/suites/load_p0/broker_load/test_broker_load_with_where.groovy @@ -99,7 +99,7 @@ suite("test_broker_load_with_where", "load_p0") { } def check_load_result = {checklabel, testTablex -> - max_try_milli_secs = 10000 + def max_try_milli_secs = 10000 while(max_try_milli_secs) { result = sql "show load where label = '${checklabel}'" if(result[0][2] == "FINISHED") { @@ -125,9 +125,9 @@ suite("test_broker_load_with_where", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load_with_where.csv" //def hdfs_csv_file_path = "hdfs://ip:port/testfile" diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_without_filepath.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_without_filepath.groovy index 2dae37c615b6a7..c89ba78fc32671 100644 --- a/regression-test/suites/load_p0/broker_load/test_broker_load_without_filepath.groovy +++ b/regression-test/suites/load_p0/broker_load/test_broker_load_without_filepath.groovy @@ -78,9 +78,9 @@ suite("test_broker_load_without_filepath", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load_without_filepath.csv" try { diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load_without_seq.groovy b/regression-test/suites/load_p0/broker_load/test_broker_load_without_seq.groovy index 9d34c0c09788e3..b5a487ae562ea4 100644 --- a/regression-test/suites/load_p0/broker_load/test_broker_load_without_seq.groovy +++ b/regression-test/suites/load_p0/broker_load/test_broker_load_without_seq.groovy @@ -79,9 +79,9 @@ suite("test_broker_load_without_seq", "load_p0") { // if 'enableHdfs' in regression-conf.groovy has been set to true, // the test will run these case as below. if (enableHdfs()) { - brokerName = getBrokerName() - hdfsUser = getHdfsUser() - hdfsPasswd = getHdfsPasswd() + def brokerName = getBrokerName() + def hdfsUser = getHdfsUser() + def hdfsPasswd = getHdfsPasswd() def hdfs_csv_file_path = uploadToHdfs "load_p0/broker_load/broker_load_without_seq.csv" //def hdfs_csv_file_path = "hdfs://ip:port/testfile" From deccd3ac0ea553a68d5ee52d5a8307935ae8dc0a Mon Sep 17 00:00:00 2001 From: minghong Date: Thu, 28 Aug 2025 12:09:30 +0800 Subject: [PATCH 523/572] branch-3.0 [fix](regression) unstable case push_down_min_through_join (#55306) ### What problem does this PR solve? pick #55307 the join order is not fixed in those cases --- .../push_down_min_through_join.out | 20 +++++++++---------- .../push_down_min_through_join.groovy | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out index d2129c0b417f1f..707c24b3754bff 100644 --- a/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out +++ b/regression-test/data/nereids_rules_p0/eager_aggregate/push_down_min_through_join.out @@ -68,9 +68,9 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[min_t] --------filter((min_t.score > 10)) ----------PhysicalOlapScan[min_t] +--------PhysicalOlapScan[min_t] -- !groupby_pushdown_outer_join -- PhysicalResultSink @@ -85,9 +85,9 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[min_t] --------filter((min_t.score > 10)) ----------PhysicalOlapScan[min_t] +--------PhysicalOlapScan[min_t] -- !groupby_pushdown_having -- PhysicalResultSink @@ -163,9 +163,9 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[min_t] --------filter((t1.score > 50)) ----------PhysicalOlapScan[min_t] +--------PhysicalOlapScan[min_t] -- !groupby_pushdown_varied_aggregates -- PhysicalResultSink @@ -214,10 +214,10 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------filter((min_t.id < 100)) -----------PhysicalOlapScan[min_t] --------filter((min_t.score > 20) and (t1.id < 100)) ----------PhysicalOlapScan[min_t] +--------filter((min_t.id < 100)) +----------PhysicalOlapScan[min_t] -- !with_hint_groupby_pushdown_basic -- PhysicalResultSink @@ -328,9 +328,9 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[min_t] --------filter((min_t.score > 10)) ----------PhysicalOlapScan[min_t] +--------PhysicalOlapScan[min_t] Hint log: Used: @@ -355,9 +355,9 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[min_t] --------filter((min_t.score > 10)) ----------PhysicalOlapScan[min_t] +--------PhysicalOlapScan[min_t] Hint log: Used: @@ -478,9 +478,9 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------PhysicalOlapScan[min_t] --------filter((t1.score > 50)) ----------PhysicalOlapScan[min_t] +--------PhysicalOlapScan[min_t] Hint log: Used: @@ -559,10 +559,10 @@ PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] ------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=() ---------filter((min_t.id < 100)) -----------PhysicalOlapScan[min_t] --------filter((min_t.score > 20) and (t1.id < 100)) ----------PhysicalOlapScan[min_t] +--------filter((min_t.id < 100)) +----------PhysicalOlapScan[min_t] Hint log: Used: diff --git a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy index fb728057f57750..fc2a22af82291e 100644 --- a/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy +++ b/regression-test/suites/nereids_rules_p0/eager_aggregate/push_down_min_through_join.groovy @@ -22,7 +22,7 @@ suite("push_down_min_through_join") { sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" sql "set be_number_for_test=1" sql "set DISABLE_NEREIDS_RULES='ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT, ONE_PHASE_AGGREGATE_SINGLE_DISTINCT_TO_MULTI'" - + sql "set disable_join_reorder=true;" sql """ DROP TABLE IF EXISTS min_t; """ From 734f09060a024fddfb80c51155271bfef3582f85 Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 28 Aug 2025 17:47:38 +0800 Subject: [PATCH 524/572] branch-3.0: [fix](cloud) commit txn with sub txns should consider lazy txn commiter (#54653) (#55351) pick https://github.com/apache/doris/pull/54653 --- cloud/src/meta-service/meta_service_txn.cpp | 881 +++++++++--------- cloud/src/meta-service/txn_lazy_committer.cpp | 1 + cloud/test/txn_lazy_commit_test.cpp | 279 ++++++ 3 files changed, 745 insertions(+), 416 deletions(-) diff --git a/cloud/src/meta-service/meta_service_txn.cpp b/cloud/src/meta-service/meta_service_txn.cpp index 2f3f0d1554531e..a800b41f83e2b7 100644 --- a/cloud/src/meta-service/meta_service_txn.cpp +++ b/cloud/src/meta-service/meta_service_txn.cpp @@ -2005,11 +2005,14 @@ void commit_txn_eventually( * t2: t2_p3(4), t2_p4(4) */ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* response, - std::shared_ptr& txn_kv, MetaServiceCode& code, - std::string& msg, const std::string& instance_id, KVStats& stats) { + std::shared_ptr& txn_kv, + std::shared_ptr& txn_lazy_committer, + MetaServiceCode& code, std::string& msg, + const std::string& instance_id, KVStats& stats) { std::stringstream ss; int64_t txn_id = request->txn_id(); auto sub_txn_infos = request->sub_txn_infos(); + // Create a readonly txn for scan tmp rowset std::unique_ptr txn; TxnErrorCode err = txn_kv->create_txn(&txn); @@ -2020,15 +2023,6 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* LOG(WARNING) << msg; return; } - DORIS_CLOUD_DEFER { - if (txn == nullptr) return; - stats.get_bytes += txn->get_bytes(); - stats.put_bytes += txn->put_bytes(); - stats.del_bytes += txn->delete_bytes(); - stats.get_counter += txn->num_get_keys(); - stats.put_counter += txn->num_put_keys(); - stats.del_counter += txn->num_del_keys(); - }; // Get db id with txn id std::string index_val; @@ -2122,468 +2116,522 @@ void commit_txn_with_sub_txn(const CommitTxnRequest* request, CommitTxnResponse* } stats.get_bytes += txn->get_bytes(); stats.get_counter += txn->num_get_keys(); - // Create a read/write txn for guarantee consistency txn.reset(); - err = txn_kv->create_txn(&txn); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - ss << "filed to create txn, txn_id=" << txn_id << " err=" << err; - msg = ss.str(); - LOG(WARNING) << msg; - return; - } - - // Get txn info with db_id and txn_id - std::string info_val; // Will be reused when saving updated txn - const std::string info_key = txn_info_key({instance_id, db_id, txn_id}); - err = txn->get(info_key, &info_val); - if (err != TxnErrorCode::TXN_OK) { - code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND - : cast_as(err); - ss << "failed to get txn_info, db_id=" << db_id << " txn_id=" << txn_id << " err=" << err; - msg = ss.str(); - LOG(WARNING) << msg; - return; - } - - TxnInfoPB txn_info; - if (!txn_info.ParseFromString(info_val)) { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - ss << "failed to parse txn_info, db_id=" << db_id << " txn_id=" << txn_id; - msg = ss.str(); - LOG(WARNING) << msg; - return; - } - // TODO: do more check like txn state - DCHECK(txn_info.txn_id() == txn_id); - if (txn_info.status() == TxnStatusPB::TXN_STATUS_ABORTED) { - code = MetaServiceCode::TXN_ALREADY_ABORTED; - ss << "transaction is already aborted: db_id=" << db_id << " txn_id=" << txn_id; - msg = ss.str(); - LOG(WARNING) << msg; - return; - } - - if (txn_info.status() == TxnStatusPB::TXN_STATUS_VISIBLE) { - code = MetaServiceCode::OK; - ss << "transaction is already visible: db_id=" << db_id << " txn_id=" << txn_id; - msg = ss.str(); - LOG(INFO) << msg; - response->mutable_txn_info()->CopyFrom(txn_info); - return; - } - - LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString(); + do { + TEST_SYNC_POINT_CALLBACK("commit_txn_with_sub_txn:begin", &txn_id); + // Create a read/write txn for guarantee consistency + err = txn_kv->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "filed to create txn, txn_id=" << txn_id << " err=" << err; + msg = ss.str(); + LOG(WARNING) << msg; + return; + } + DORIS_CLOUD_DEFER { + if (txn == nullptr) return; + stats.get_bytes += txn->get_bytes(); + stats.put_bytes += txn->put_bytes(); + stats.del_bytes += txn->delete_bytes(); + stats.get_counter += txn->num_get_keys(); + stats.put_counter += txn->num_put_keys(); + stats.del_counter += txn->num_del_keys(); + }; - // Prepare rowset meta and new_versions - // Read tablet indexes in batch. - std::map tablet_id_to_idx; - std::vector tablet_idx_keys; - std::vector partition_ids; - auto idx = 0; - for (auto& [_, tmp_rowsets_meta] : sub_txn_to_tmp_rowsets_meta) { - for (auto& [_, i] : tmp_rowsets_meta) { - auto tablet_id = i.tablet_id(); - if (tablet_id_to_idx.count(tablet_id) == 0) { - tablet_id_to_idx.emplace(tablet_id, idx); - tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id, i.tablet_id()})); - partition_ids.push_back(i.partition_id()); - idx++; + // Get txn info with db_id and txn_id + std::string info_val; // Will be reused when saving updated txn + const std::string info_key = txn_info_key({instance_id, db_id, txn_id}); + err = txn->get(info_key, &info_val); + if (err != TxnErrorCode::TXN_OK) { + code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TXN_ID_NOT_FOUND + : cast_as(err); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + ss << "transaction [" << txn_id << "] not found, db_id=" << db_id; + } else { + ss << "failed to get txn_info, db_id=" << db_id << " txn_id=" << txn_id + << " err=" << err; } + msg = ss.str(); + LOG(WARNING) << msg; + return; } - } - std::vector> tablet_idx_values; - err = txn->batch_get(&tablet_idx_values, tablet_idx_keys, Transaction::BatchGetOptions(false)); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - ss << "failed to get tablet table index ids, err=" << err; - msg = ss.str(); - LOG(WARNING) << msg << " txn_id=" << txn_id; - return; - } - // tablet_id -> {table/index/partition}_id - std::unordered_map tablet_ids; - // table_id -> tablets_ids - std::unordered_map> table_id_tablet_ids; - for (auto [tablet_id, i] : tablet_id_to_idx) { - if (!tablet_idx_values[i].has_value()) [[unlikely]] { - // The value must existed - code = MetaServiceCode::KV_TXN_GET_ERR; - ss << "failed to get tablet table index ids, err=not found" - << " tablet_id=" << tablet_id << " key=" << hex(tablet_idx_keys[i]); + TxnInfoPB txn_info; + if (!txn_info.ParseFromString(info_val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + ss << "failed to parse txn_info, db_id=" << db_id << " txn_id=" << txn_id; msg = ss.str(); - LOG(WARNING) << msg << " err=" << err << " txn_id=" << txn_id; + LOG(WARNING) << msg; return; } - if (!tablet_ids[tablet_id].ParseFromString(tablet_idx_values[i].value())) [[unlikely]] { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - ss << "malformed tablet index value tablet_id=" << tablet_id << " txn_id=" << txn_id; + + // TODO: do more check like txn state + DCHECK(txn_info.txn_id() == txn_id); + if (txn_info.status() == TxnStatusPB::TXN_STATUS_ABORTED) { + code = MetaServiceCode::TXN_ALREADY_ABORTED; + ss << "transaction is already aborted: db_id=" << db_id << " txn_id=" << txn_id; msg = ss.str(); LOG(WARNING) << msg; return; } - table_id_tablet_ids[tablet_ids[tablet_id].table_id()].push_back(tablet_id); - VLOG_DEBUG << "tablet_id:" << tablet_id - << " value:" << tablet_ids[tablet_id].ShortDebugString(); - } - tablet_idx_keys.clear(); - tablet_idx_values.clear(); + if (txn_info.status() == TxnStatusPB::TXN_STATUS_VISIBLE) { + code = MetaServiceCode::OK; + ss << "transaction is already visible: db_id=" << db_id << " txn_id=" << txn_id; + msg = ss.str(); + LOG(INFO) << msg; + response->mutable_txn_info()->CopyFrom(txn_info); + return; + } - // {table/partition} -> version - std::unordered_map new_versions; - std::vector version_keys; - for (auto& [tablet_id, i] : tablet_id_to_idx) { - int64_t table_id = tablet_ids[tablet_id].table_id(); - int64_t partition_id = partition_ids[i]; - std::string ver_key = partition_version_key({instance_id, db_id, table_id, partition_id}); - if (new_versions.count(ver_key) == 0) { - new_versions.insert({ver_key, 0}); - LOG(INFO) << "xxx add a partition_version_key=" << hex(ver_key) << " txn_id=" << txn_id - << ", db_id=" << db_id << ", table_id=" << table_id - << ", partition_id=" << partition_id; - version_keys.push_back(std::move(ver_key)); - } - } - std::vector> version_values; - err = txn->batch_get(&version_values, version_keys); - if (err != TxnErrorCode::TXN_OK) { - code = cast_as(err); - ss << "failed to get partition versions, err=" << err; - msg = ss.str(); - LOG(WARNING) << msg << " txn_id=" << txn_id; - return; - } - size_t total_versions = version_keys.size(); - for (size_t i = 0; i < total_versions; i++) { - int64_t version; - if (version_values[i].has_value()) { - VersionPB version_pb; - if (!version_pb.ParseFromString(version_values[i].value())) { + LOG(INFO) << "txn_id=" << txn_id << " txn_info=" << txn_info.ShortDebugString(); + + // Prepare rowset meta and new_versions + // Read tablet indexes in batch. + std::map tablet_id_to_idx; + std::vector tablet_idx_keys; + std::vector partition_ids; + auto idx = 0; + for (auto& [_, tmp_rowsets_meta] : sub_txn_to_tmp_rowsets_meta) { + for (auto& [_, i] : tmp_rowsets_meta) { + auto tablet_id = i.tablet_id(); + if (tablet_id_to_idx.count(tablet_id) == 0) { + tablet_id_to_idx.emplace(tablet_id, idx); + tablet_idx_keys.push_back(meta_tablet_idx_key({instance_id, i.tablet_id()})); + partition_ids.push_back(i.partition_id()); + idx++; + } + } + } + std::vector> tablet_idx_values; + err = txn->batch_get(&tablet_idx_values, tablet_idx_keys, + Transaction::BatchGetOptions(false)); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to get tablet table index ids, err=" << err; + msg = ss.str(); + LOG(WARNING) << msg << " txn_id=" << txn_id; + return; + } + + // tablet_id -> {table/index/partition}_id + std::unordered_map tablet_ids; + // table_id -> tablets_ids + std::unordered_map> table_id_tablet_ids; + for (auto [tablet_id, i] : tablet_id_to_idx) { + if (!tablet_idx_values[i].has_value()) [[unlikely]] { + // The value must existed + code = MetaServiceCode::KV_TXN_GET_ERR; + ss << "failed to get tablet table index ids, err=not found" + << " tablet_id=" << tablet_id << " key=" << hex(tablet_idx_keys[i]); + msg = ss.str(); + LOG(WARNING) << msg << " err=" << err << " txn_id=" << txn_id; + return; + } + if (!tablet_ids[tablet_id].ParseFromString(tablet_idx_values[i].value())) [[unlikely]] { code = MetaServiceCode::PROTOBUF_PARSE_ERR; - ss << "failed to parse version pb txn_id=" << txn_id - << " key=" << hex(version_keys[i]); + ss << "malformed tablet index value tablet_id=" << tablet_id + << " txn_id=" << txn_id; msg = ss.str(); + LOG(WARNING) << msg; return; } - version = version_pb.version(); - } else { - version = 1; + table_id_tablet_ids[tablet_ids[tablet_id].table_id()].push_back(tablet_id); + VLOG_DEBUG << "tablet_id:" << tablet_id + << " value:" << tablet_ids[tablet_id].ShortDebugString(); } - new_versions[version_keys[i]] = version; - LOG(INFO) << "xxx get partition_version_key=" << hex(version_keys[i]) - << " version:" << version << " txn_id=" << txn_id; - } - version_keys.clear(); - version_values.clear(); - std::vector> rowsets; - std::unordered_map tablet_stats; // tablet_id -> stats - for (const auto& sub_txn_info : sub_txn_infos) { - auto sub_txn_id = sub_txn_info.sub_txn_id(); - auto tmp_rowsets_meta = sub_txn_to_tmp_rowsets_meta[sub_txn_id]; - std::unordered_map partition_id_to_version; - for (auto& [_, i] : tmp_rowsets_meta) { - int64_t tablet_id = i.tablet_id(); + tablet_idx_keys.clear(); + tablet_idx_values.clear(); + + // {table/partition} -> version + std::unordered_map new_versions; + std::vector version_keys; + for (auto& [tablet_id, i] : tablet_id_to_idx) { int64_t table_id = tablet_ids[tablet_id].table_id(); - int64_t partition_id = i.partition_id(); + int64_t partition_id = partition_ids[i]; std::string ver_key = partition_version_key({instance_id, db_id, table_id, partition_id}); - if (new_versions.count(ver_key) == 0) [[unlikely]] { - // it is impossible. - code = MetaServiceCode::UNDEFINED_ERR; - ss << "failed to get partition version key, the target version not exists in " - "new_versions." - << " txn_id=" << txn_id << ", db_id=" << db_id << ", table_id=" << table_id - << ", partition_id=" << partition_id; - msg = ss.str(); - LOG(ERROR) << msg; - return; + if (new_versions.count(ver_key) == 0) { + new_versions.insert({ver_key, 0}); + LOG(INFO) << "xxx add a partition_version_key=" << hex(ver_key) + << " txn_id=" << txn_id << ", db_id=" << db_id + << ", table_id=" << table_id << ", partition_id=" << partition_id; + version_keys.push_back(std::move(ver_key)); } - - // Update rowset version - int64_t new_version = new_versions[ver_key]; - if (partition_id_to_version.count(partition_id) == 0) { - new_versions[ver_key] = new_version + 1; - new_version = new_versions[ver_key]; - partition_id_to_version[partition_id] = new_version; + } + std::vector> version_values; + err = txn->batch_get(&version_values, version_keys); + if (err != TxnErrorCode::TXN_OK) { + code = cast_as(err); + ss << "failed to get partition versions, err=" << err; + msg = ss.str(); + LOG(WARNING) << msg << " txn_id=" << txn_id; + return; + } + size_t total_versions = version_keys.size(); + int64_t last_pending_txn_id = 0; + for (size_t i = 0; i < total_versions; i++) { + int64_t version; + if (version_values[i].has_value()) { + VersionPB version_pb; + if (!version_pb.ParseFromString(version_values[i].value())) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + ss << "failed to parse version pb txn_id=" << txn_id + << " key=" << hex(version_keys[i]); + msg = ss.str(); + return; + } + if (version_pb.pending_txn_ids_size() > 0) { + DCHECK(version_pb.pending_txn_ids_size() == 1); + last_pending_txn_id = version_pb.pending_txn_ids(0); + DCHECK(last_pending_txn_id > 0); + break; + } + version = version_pb.version(); + } else { + version = 1; } - i.set_start_version(new_version); - i.set_end_version(new_version); - LOG(INFO) << "xxx update rowset version, txn_id=" << txn_id - << ", sub_txn_id=" << sub_txn_id << ", table_id=" << table_id - << ", partition_id=" << partition_id << ", tablet_id=" << tablet_id - << ", new_version=" << new_version; + new_versions[version_keys[i]] = version; + last_pending_txn_id = 0; + LOG(INFO) << "xxx get partition_version_key=" << hex(version_keys[i]) + << " version:" << version << " txn_id=" << txn_id; + } + version_keys.clear(); + version_values.clear(); - std::string key = meta_rowset_key({instance_id, tablet_id, i.end_version()}); - std::string val; - if (!i.SerializeToString(&val)) { - code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; - ss << "failed to serialize rowset_meta, txn_id=" << txn_id; - msg = ss.str(); + if (last_pending_txn_id > 0) { + stats.get_bytes += txn->get_bytes(); + stats.get_counter += txn->num_get_keys(); + txn.reset(); + TEST_SYNC_POINT_CALLBACK("commit_txn_with_sub_txn::advance_last_pending_txn_id", + &last_pending_txn_id); + std::shared_ptr task = + txn_lazy_committer->submit(instance_id, last_pending_txn_id); + + std::tie(code, msg) = task->wait(); + if (code != MetaServiceCode::OK) { + LOG(WARNING) << "advance_last_txn failed last_txn=" << last_pending_txn_id + << " code=" << code << " msg=" << msg; return; } - rowsets.emplace_back(std::move(key), std::move(val)); + last_pending_txn_id = 0; + continue; + } - // Accumulate affected rows - auto& stats = tablet_stats[tablet_id]; - stats.data_size += i.total_disk_size(); - stats.num_rows += i.num_rows(); - ++stats.num_rowsets; - stats.num_segs += i.num_segments(); - stats.index_size += i.index_disk_size(); - stats.segment_size += i.data_disk_size(); - } // for tmp_rowsets_meta - } + std::vector> rowsets; + std::unordered_map tablet_stats; // tablet_id -> stats + for (const auto& sub_txn_info : sub_txn_infos) { + auto sub_txn_id = sub_txn_info.sub_txn_id(); + auto tmp_rowsets_meta = sub_txn_to_tmp_rowsets_meta[sub_txn_id]; + std::unordered_map partition_id_to_version; + for (auto& [_, i] : tmp_rowsets_meta) { + int64_t tablet_id = i.tablet_id(); + int64_t table_id = tablet_ids[tablet_id].table_id(); + int64_t partition_id = i.partition_id(); + std::string ver_key = + partition_version_key({instance_id, db_id, table_id, partition_id}); + if (new_versions.count(ver_key) == 0) [[unlikely]] { + // it is impossible. + code = MetaServiceCode::UNDEFINED_ERR; + ss << "failed to get partition version key, the target version not exists in " + "new_versions." + << " txn_id=" << txn_id << ", db_id=" << db_id << ", table_id=" << table_id + << ", partition_id=" << partition_id; + msg = ss.str(); + LOG(ERROR) << msg; + return; + } - // Save rowset meta - for (auto& i : rowsets) { - size_t rowset_size = i.first.size() + i.second.size(); - txn->put(i.first, i.second); - LOG(INFO) << "xxx put rowset_key=" << hex(i.first) << " txn_id=" << txn_id - << " rowset_size=" << rowset_size; - } + // Update rowset version + int64_t new_version = new_versions[ver_key]; + if (partition_id_to_version.count(partition_id) == 0) { + new_versions[ver_key] = new_version + 1; + new_version = new_versions[ver_key]; + partition_id_to_version[partition_id] = new_version; + } + i.set_start_version(new_version); + i.set_end_version(new_version); + LOG(INFO) << "xxx update rowset version, txn_id=" << txn_id + << ", sub_txn_id=" << sub_txn_id << ", table_id=" << table_id + << ", partition_id=" << partition_id << ", tablet_id=" << tablet_id + << ", new_version=" << new_version; + + std::string key = meta_rowset_key({instance_id, tablet_id, i.end_version()}); + std::string val; + if (!i.SerializeToString(&val)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + ss << "failed to serialize rowset_meta, txn_id=" << txn_id; + msg = ss.str(); + return; + } + rowsets.emplace_back(std::move(key), std::move(val)); - // Save versions - for (auto& i : new_versions) { - std::string ver_val; - VersionPB version_pb; - version_pb.set_version(i.second); - if (!version_pb.SerializeToString(&ver_val)) { - code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; - ss << "failed to serialize version_pb when saving, txn_id=" << txn_id; - msg = ss.str(); - return; + // Accumulate affected rows + auto& stats = tablet_stats[tablet_id]; + stats.data_size += i.total_disk_size(); + stats.num_rows += i.num_rows(); + ++stats.num_rowsets; + stats.num_segs += i.num_segments(); + stats.index_size += i.index_disk_size(); + stats.segment_size += i.data_disk_size(); + } // for tmp_rowsets_meta } - txn->put(i.first, ver_val); - LOG(INFO) << "xxx put partition_version_key=" << hex(i.first) << " version:" << i.second - << " txn_id=" << txn_id; - - std::string_view ver_key = i.first; - ver_key.remove_prefix(1); // Remove key space - // PartitionVersionKeyInfo {instance_id, db_id, table_id, partition_id} - std::vector, int, int>> out; - int ret = decode_key(&ver_key, &out); - if (ret != 0) [[unlikely]] { - // decode version key error means this is something wrong, - // we can not continue this txn - LOG(WARNING) << "failed to decode key, ret=" << ret << " key=" << hex(ver_key); - code = MetaServiceCode::UNDEFINED_ERR; - msg = "decode version key error"; - return; + // Save rowset meta + for (auto& i : rowsets) { + size_t rowset_size = i.first.size() + i.second.size(); + txn->put(i.first, i.second); + LOG(INFO) << "xxx put rowset_key=" << hex(i.first) << " txn_id=" << txn_id + << " rowset_size=" << rowset_size; } - int64_t table_id = std::get(std::get<0>(out[4])); - int64_t partition_id = std::get(std::get<0>(out[5])); - VLOG_DEBUG << "txn_id=" << txn_id << " table_id=" << table_id - << " partition_id=" << partition_id << " version=" << i.second; + // Save versions + for (auto& i : new_versions) { + std::string ver_val; + VersionPB version_pb; + version_pb.set_version(i.second); + if (!version_pb.SerializeToString(&ver_val)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + ss << "failed to serialize version_pb when saving, txn_id=" << txn_id; + msg = ss.str(); + return; + } - response->add_table_ids(table_id); - response->add_partition_ids(partition_id); - response->add_versions(i.second); - } + txn->put(i.first, ver_val); + LOG(INFO) << "xxx put partition_version_key=" << hex(i.first) << " version:" << i.second + << " txn_id=" << txn_id; - // Save table versions - for (auto& i : table_id_tablet_ids) { - std::string ver_key = table_version_key({instance_id, db_id, i.first}); - txn->atomic_add(ver_key, 1); - LOG(INFO) << "xxx atomic add table_version_key=" << hex(ver_key) << " txn_id=" << txn_id; - } + std::string_view ver_key = i.first; + ver_key.remove_prefix(1); // Remove key space + // PartitionVersionKeyInfo {instance_id, db_id, table_id, partition_id} + std::vector, int, int>> out; + int ret = decode_key(&ver_key, &out); + if (ret != 0) [[unlikely]] { + // decode version key error means this is something wrong, + // we can not continue this txn + LOG(WARNING) << "failed to decode key, ret=" << ret << " key=" << hex(ver_key); + code = MetaServiceCode::UNDEFINED_ERR; + msg = "decode version key error"; + return; + } - LOG(INFO) << " before update txn_info=" << txn_info.ShortDebugString(); + int64_t table_id = std::get(std::get<0>(out[4])); + int64_t partition_id = std::get(std::get<0>(out[5])); + VLOG_DEBUG << "txn_id=" << txn_id << " table_id=" << table_id + << " partition_id=" << partition_id << " version=" << i.second; - // Update txn_info - txn_info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE); + response->add_table_ids(table_id); + response->add_partition_ids(partition_id); + response->add_versions(i.second); + } - auto now_time = system_clock::now(); - uint64_t commit_time = duration_cast(now_time.time_since_epoch()).count(); - if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) { - code = MetaServiceCode::UNDEFINED_ERR; - msg = fmt::format("txn is expired, not allow to commit txn_id={}", txn_id); - LOG(INFO) << msg << " prepare_time=" << txn_info.prepare_time() - << " timeout_ms=" << txn_info.timeout_ms() << " commit_time=" << commit_time; - return; - } - txn_info.set_commit_time(commit_time); - txn_info.set_finish_time(commit_time); - if (request->has_commit_attachment()) { - txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment()); - } - LOG(INFO) << "after update txn_info=" << txn_info.ShortDebugString(); - info_val.clear(); - if (!txn_info.SerializeToString(&info_val)) { - code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; - ss << "failed to serialize txn_info when saving, txn_id=" << txn_id; - msg = ss.str(); - return; - } - txn->put(info_key, info_val); - LOG(INFO) << "xxx put info_key=" << hex(info_key) << " txn_id=" << txn_id; + // Save table versions + for (auto& i : table_id_tablet_ids) { + std::string ver_key = table_version_key({instance_id, db_id, i.first}); + txn->atomic_add(ver_key, 1); + LOG(INFO) << "xxx atomic add table_version_key=" << hex(ver_key) + << " txn_id=" << txn_id; + } - // Update stats of affected tablet - std::deque kv_pool; - std::function update_tablet_stats; - if (config::split_tablet_stats) { - update_tablet_stats = [&](const StatsTabletKeyInfo& info, const TabletStats& stats) { - if (stats.num_segs > 0) { - auto& data_size_key = kv_pool.emplace_back(); - stats_tablet_data_size_key(info, &data_size_key); - txn->atomic_add(data_size_key, stats.data_size); - auto& num_rows_key = kv_pool.emplace_back(); - stats_tablet_num_rows_key(info, &num_rows_key); - txn->atomic_add(num_rows_key, stats.num_rows); - auto& num_segs_key = kv_pool.emplace_back(); - stats_tablet_num_segs_key(info, &num_segs_key); - txn->atomic_add(num_segs_key, stats.num_segs); - auto& index_size_key = kv_pool.emplace_back(); - stats_tablet_index_size_key(info, &index_size_key); - txn->atomic_add(index_size_key, stats.index_size); - auto& segment_size_key = kv_pool.emplace_back(); - stats_tablet_segment_size_key(info, &segment_size_key); - txn->atomic_add(segment_size_key, stats.segment_size); - } - auto& num_rowsets_key = kv_pool.emplace_back(); - stats_tablet_num_rowsets_key(info, &num_rowsets_key); - txn->atomic_add(num_rowsets_key, stats.num_rowsets); - }; - } else { - update_tablet_stats = [&](const StatsTabletKeyInfo& info, const TabletStats& stats) { - auto& key = kv_pool.emplace_back(); - stats_tablet_key(info, &key); - auto& val = kv_pool.emplace_back(); - TxnErrorCode err = txn->get(key, &val); - if (err != TxnErrorCode::TXN_OK) { - code = err == TxnErrorCode::TXN_KEY_NOT_FOUND ? MetaServiceCode::TABLET_NOT_FOUND - : cast_as(err); - msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err, - std::get<4>(info)); - return; - } - TabletStatsPB stats_pb; - if (!stats_pb.ParseFromString(val)) { - code = MetaServiceCode::PROTOBUF_PARSE_ERR; - msg = fmt::format("malformed tablet stats value, key={}", hex(key)); - return; + LOG(INFO) << " before update txn_info=" << txn_info.ShortDebugString(); + + // Update txn_info + txn_info.set_status(TxnStatusPB::TXN_STATUS_VISIBLE); + + auto now_time = system_clock::now(); + uint64_t commit_time = duration_cast(now_time.time_since_epoch()).count(); + if ((txn_info.prepare_time() + txn_info.timeout_ms()) < commit_time) { + code = MetaServiceCode::UNDEFINED_ERR; + msg = fmt::format("txn is expired, not allow to commit txn_id={}", txn_id); + LOG(INFO) << msg << " prepare_time=" << txn_info.prepare_time() + << " timeout_ms=" << txn_info.timeout_ms() << " commit_time=" << commit_time; + return; + } + txn_info.set_commit_time(commit_time); + txn_info.set_finish_time(commit_time); + if (request->has_commit_attachment()) { + txn_info.mutable_commit_attachment()->CopyFrom(request->commit_attachment()); + } + LOG(INFO) << "after update txn_info=" << txn_info.ShortDebugString(); + info_val.clear(); + if (!txn_info.SerializeToString(&info_val)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + ss << "failed to serialize txn_info when saving, txn_id=" << txn_id; + msg = ss.str(); + return; + } + txn->put(info_key, info_val); + LOG(INFO) << "xxx put info_key=" << hex(info_key) << " txn_id=" << txn_id; + + // Update stats of affected tablet + std::deque kv_pool; + std::function update_tablet_stats; + if (config::split_tablet_stats) { + update_tablet_stats = [&](const StatsTabletKeyInfo& info, const TabletStats& stats) { + if (stats.num_segs > 0) { + auto& data_size_key = kv_pool.emplace_back(); + stats_tablet_data_size_key(info, &data_size_key); + txn->atomic_add(data_size_key, stats.data_size); + auto& num_rows_key = kv_pool.emplace_back(); + stats_tablet_num_rows_key(info, &num_rows_key); + txn->atomic_add(num_rows_key, stats.num_rows); + auto& num_segs_key = kv_pool.emplace_back(); + stats_tablet_num_segs_key(info, &num_segs_key); + txn->atomic_add(num_segs_key, stats.num_segs); + auto& index_size_key = kv_pool.emplace_back(); + stats_tablet_index_size_key(info, &index_size_key); + txn->atomic_add(index_size_key, stats.index_size); + auto& segment_size_key = kv_pool.emplace_back(); + stats_tablet_segment_size_key(info, &segment_size_key); + txn->atomic_add(segment_size_key, stats.segment_size); + } + auto& num_rowsets_key = kv_pool.emplace_back(); + stats_tablet_num_rowsets_key(info, &num_rowsets_key); + txn->atomic_add(num_rowsets_key, stats.num_rowsets); + }; + } else { + update_tablet_stats = [&](const StatsTabletKeyInfo& info, const TabletStats& stats) { + auto& key = kv_pool.emplace_back(); + stats_tablet_key(info, &key); + auto& val = kv_pool.emplace_back(); + TxnErrorCode err = txn->get(key, &val); + if (err != TxnErrorCode::TXN_OK) { + code = err == TxnErrorCode::TXN_KEY_NOT_FOUND + ? MetaServiceCode::TABLET_NOT_FOUND + : cast_as(err); + msg = fmt::format("failed to get tablet stats, err={} tablet_id={}", err, + std::get<4>(info)); + return; + } + TabletStatsPB stats_pb; + if (!stats_pb.ParseFromString(val)) { + code = MetaServiceCode::PROTOBUF_PARSE_ERR; + msg = fmt::format("malformed tablet stats value, key={}", hex(key)); + return; + } + stats_pb.set_data_size(stats_pb.data_size() + stats.data_size); + stats_pb.set_num_rows(stats_pb.num_rows() + stats.num_rows); + stats_pb.set_num_rowsets(stats_pb.num_rowsets() + stats.num_rowsets); + stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs); + stats_pb.set_index_size(stats_pb.index_size() + stats.index_size); + stats_pb.set_segment_size(stats_pb.segment_size() + stats.segment_size); + stats_pb.SerializeToString(&val); + txn->put(key, val); + LOG(INFO) << "put stats_tablet_key, key=" << hex(key); + }; + } + for (auto& [tablet_id, stats] : tablet_stats) { + DCHECK(tablet_ids.count(tablet_id)); + auto& tablet_idx = tablet_ids[tablet_id]; + StatsTabletKeyInfo info {instance_id, tablet_idx.table_id(), tablet_idx.index_id(), + tablet_idx.partition_id(), tablet_id}; + update_tablet_stats(info, stats); + if (code != MetaServiceCode::OK) return; + } + // Remove tmp rowset meta + for (auto& [_, tmp_rowsets_meta] : sub_txn_to_tmp_rowsets_meta) { + for (auto& [k, _] : tmp_rowsets_meta) { + txn->remove(k); + LOG(INFO) << "xxx remove tmp_rowset_key=" << hex(k) << " txn_id=" << txn_id; } - stats_pb.set_data_size(stats_pb.data_size() + stats.data_size); - stats_pb.set_num_rows(stats_pb.num_rows() + stats.num_rows); - stats_pb.set_num_rowsets(stats_pb.num_rowsets() + stats.num_rowsets); - stats_pb.set_num_segments(stats_pb.num_segments() + stats.num_segs); - stats_pb.set_index_size(stats_pb.index_size() + stats.index_size); - stats_pb.set_segment_size(stats_pb.segment_size() + stats.segment_size); - stats_pb.SerializeToString(&val); - txn->put(key, val); - LOG(INFO) << "put stats_tablet_key, key=" << hex(key); - }; - } - for (auto& [tablet_id, stats] : tablet_stats) { - DCHECK(tablet_ids.count(tablet_id)); - auto& tablet_idx = tablet_ids[tablet_id]; - StatsTabletKeyInfo info {instance_id, tablet_idx.table_id(), tablet_idx.index_id(), - tablet_idx.partition_id(), tablet_id}; - update_tablet_stats(info, stats); - if (code != MetaServiceCode::OK) return; - } - // Remove tmp rowset meta - for (auto& [_, tmp_rowsets_meta] : sub_txn_to_tmp_rowsets_meta) { - for (auto& [k, _] : tmp_rowsets_meta) { - txn->remove(k); - LOG(INFO) << "xxx remove tmp_rowset_key=" << hex(k) << " txn_id=" << txn_id; } - } - const std::string running_key = txn_running_key({instance_id, db_id, txn_id}); - LOG(INFO) << "xxx remove running_key=" << hex(running_key) << " txn_id=" << txn_id; - txn->remove(running_key); + const std::string running_key = txn_running_key({instance_id, db_id, txn_id}); + LOG(INFO) << "xxx remove running_key=" << hex(running_key) << " txn_id=" << txn_id; + txn->remove(running_key); - std::string recycle_val; - std::string recycle_key = recycle_txn_key({instance_id, db_id, txn_id}); - RecycleTxnPB recycle_pb; - recycle_pb.set_creation_time(commit_time); - recycle_pb.set_label(txn_info.label()); + std::string recycle_val; + std::string recycle_key = recycle_txn_key({instance_id, db_id, txn_id}); + RecycleTxnPB recycle_pb; + recycle_pb.set_creation_time(commit_time); + recycle_pb.set_label(txn_info.label()); - if (!recycle_pb.SerializeToString(&recycle_val)) { - code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; - ss << "failed to serialize recycle_pb, txn_id=" << txn_id; - msg = ss.str(); - return; - } - txn->put(recycle_key, recycle_val); - LOG(INFO) << "xxx commit_txn put recycle_txn_key key=" << hex(recycle_key) - << " txn_id=" << txn_id; + if (!recycle_pb.SerializeToString(&recycle_val)) { + code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR; + ss << "failed to serialize recycle_pb, txn_id=" << txn_id; + msg = ss.str(); + return; + } + txn->put(recycle_key, recycle_val); + LOG(INFO) << "xxx commit_txn put recycle_txn_key key=" << hex(recycle_key) + << " txn_id=" << txn_id; - LOG(INFO) << "commit_txn put_size=" << txn->put_bytes() << " del_size=" << txn->delete_bytes() - << " num_put_keys=" << txn->num_put_keys() << " num_del_keys=" << txn->num_del_keys() - << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id; + LOG(INFO) << "commit_txn put_size=" << txn->put_bytes() + << " del_size=" << txn->delete_bytes() << " num_put_keys=" << txn->num_put_keys() + << " num_del_keys=" << txn->num_del_keys() + << " txn_size=" << txn->approximate_bytes() << " txn_id=" << txn_id; - // Finally we are done... - err = txn->commit(); - if (err != TxnErrorCode::TXN_OK) { - if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE || err == TxnErrorCode::TXN_BYTES_TOO_LARGE) { - size_t max_size = 0, max_num_segments = 0, - min_num_segments = std::numeric_limits::max(), avg_num_segments = 0; - std::pair* max_rowset_meta = nullptr; - for (auto& sub_txn : sub_txn_infos) { - auto it = sub_txn_to_tmp_rowsets_meta.find(sub_txn.sub_txn_id()); - if (it == sub_txn_to_tmp_rowsets_meta.end()) { - continue; - } - for (auto& rowset_meta : it->second) { - if (rowset_meta.second.ByteSizeLong() > max_size) { - max_size = rowset_meta.second.ByteSizeLong(); - max_rowset_meta = &rowset_meta; + TEST_SYNC_POINT_RETURN_WITH_VOID("commit_txn_with_sub_txn::before_commit", &err, &code); + err = txn->commit(); + if (err != TxnErrorCode::TXN_OK) { + if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE || + err == TxnErrorCode::TXN_BYTES_TOO_LARGE) { + size_t max_size = 0, max_num_segments = 0, + min_num_segments = std::numeric_limits::max(), avg_num_segments = 0; + std::pair* max_rowset_meta = nullptr; + for (auto& sub_txn : sub_txn_infos) { + auto it = sub_txn_to_tmp_rowsets_meta.find(sub_txn.sub_txn_id()); + if (it == sub_txn_to_tmp_rowsets_meta.end()) { + continue; } - if (rowset_meta.second.num_segments() > max_num_segments) { - max_num_segments = rowset_meta.second.num_segments(); + for (auto& rowset_meta : it->second) { + if (rowset_meta.second.ByteSizeLong() > max_size) { + max_size = rowset_meta.second.ByteSizeLong(); + max_rowset_meta = &rowset_meta; + } + if (rowset_meta.second.num_segments() > max_num_segments) { + max_num_segments = rowset_meta.second.num_segments(); + } + if (rowset_meta.second.num_segments() < min_num_segments) { + min_num_segments = rowset_meta.second.num_segments(); + } + avg_num_segments += rowset_meta.second.num_segments(); } - if (rowset_meta.second.num_segments() < min_num_segments) { - min_num_segments = rowset_meta.second.num_segments(); + if (!it->second.empty()) { + avg_num_segments /= it->second.size(); } - avg_num_segments += rowset_meta.second.num_segments(); } - if (!it->second.empty()) { - avg_num_segments /= it->second.size(); + if (max_rowset_meta) { + LOG(WARNING) << "failed to commit kv txn with sub txn" + << ", err=" << err << ", txn_id=" << txn_id + << ", total_rowsets=" << rowsets.size() + << ", avg_num_segments=" << avg_num_segments + << ", min_num_segments=" << min_num_segments + << ", max_num_segments=" << max_num_segments + << ", largest_rowset_size=" << max_size + << ", largest_rowset_key=" << hex(max_rowset_meta->first) + << ", largest_rowset_value=" + << max_rowset_meta->second.ShortDebugString(); } } - if (max_rowset_meta) { - LOG(WARNING) << "failed to commit kv txn with sub txn" - << ", err=" << err << ", txn_id=" << txn_id - << ", total_rowsets=" << rowsets.size() - << ", avg_num_segments=" << avg_num_segments - << ", min_num_segments=" << min_num_segments - << ", max_num_segments=" << max_num_segments - << ", largest_rowset_size=" << max_size - << ", largest_rowset_key=" << hex(max_rowset_meta->first) - << ", largest_rowset_value=" - << max_rowset_meta->second.ShortDebugString(); - } + code = cast_as(err); + ss << "failed to commit kv txn with sub txn, txn_id=" << txn_id << " err=" << err; + msg = ss.str(); + return; } - code = cast_as(err); - ss << "failed to commit kv txn with sub txn, txn_id=" << txn_id << " err=" << err; - msg = ss.str(); - return; - } - // calculate table stats from tablets stats - std::map table_stats; - std::vector base_tablet_ids(request->base_tablet_ids().begin(), - request->base_tablet_ids().end()); - calc_table_stats(tablet_ids, tablet_stats, table_stats, base_tablet_ids); - for (const auto& pair : table_stats) { - TableStatsPB* stats_pb = response->add_table_stats(); - auto table_id = pair.first; - auto stats = pair.second; - get_pb_from_tablestats(stats, stats_pb); - stats_pb->set_table_id(table_id); - VLOG_DEBUG << "Add TableStats to CommitTxnResponse. txn_id=" << txn_id - << " table_id=" << table_id - << " updated_row_count=" << stats_pb->updated_row_count(); - } + // calculate table stats from tablets stats + std::map table_stats; + std::vector base_tablet_ids(request->base_tablet_ids().begin(), + request->base_tablet_ids().end()); + calc_table_stats(tablet_ids, tablet_stats, table_stats, base_tablet_ids); + for (const auto& pair : table_stats) { + TableStatsPB* stats_pb = response->add_table_stats(); + auto table_id = pair.first; + auto stats = pair.second; + get_pb_from_tablestats(stats, stats_pb); + stats_pb->set_table_id(table_id); + VLOG_DEBUG << "Add TableStats to CommitTxnResponse. txn_id=" << txn_id + << " table_id=" << table_id + << " updated_row_count=" << stats_pb->updated_row_count(); + } - response->mutable_txn_info()->CopyFrom(txn_info); + response->mutable_txn_info()->CopyFrom(txn_info); + TEST_SYNC_POINT_CALLBACK("commit_txn_with_sub_txn::finish", &code); + break; + } while (true); } // end commit_txn_with_sub_txn static bool force_txn_lazy_commit() { @@ -2614,7 +2662,8 @@ void MetaServiceImpl::commit_txn(::google::protobuf::RpcController* controller, RPC_RATE_LIMIT(commit_txn) if (request->has_is_txn_load() && request->is_txn_load()) { - commit_txn_with_sub_txn(request, response, txn_kv_, code, msg, instance_id, stats); + commit_txn_with_sub_txn(request, response, txn_kv_, txn_lazy_committer_, code, msg, + instance_id, stats); return; } diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index c4e67b2ef017e0..795f2f21cb75b7 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -451,6 +451,7 @@ void TxnLazyCommitTask::commit() { << " txn_id=" << txn_id_; } + TEST_SYNC_POINT_CALLBACK("TxnLazyCommitter::commit"); err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { code_ = cast_as(err); diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 2e018839e4e1b7..62f959167e64a9 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -1825,6 +1825,285 @@ TEST(TxnLazyCommitTest, ConcurrentCommitTxnEventuallyCase4Test) { ASSERT_EQ(txn_id, txn_info_pb.txn_id()); } +TEST(TxnLazyCommitTest, ConcurrentCommitTxnEventuallyCase5Test) { + // =========================================================================== + // threads concurrent execution flow: + // + // thread1 thread2 + // | | + // commit_txn_eventually begin commit_txn_with_sub_txn begin + // | | + // lazy commit wait | + // | | + // | advance last txn + // | | + // | finish + // | | + // finish | + // | | + // | | + // v v + + auto txn_kv = get_mem_txn_kv(); + int64_t db_id = 134179142; + int64_t table_id = 3243264; + int64_t index_id = 8098394; + int64_t partition_id = 32895361; + + std::mutex go_mutex; + std::condition_variable go_cv; + bool go = false; + + std::atomic commit_txn_immediately_begin_count = {0}; + std::atomic last_pending_txn_id_count = {0}; + std::atomic txn_lazy_committer_wait_count = {0}; + std::atomic immediately_finish_count = {0}; + std::atomic eventually_finish_count = {0}; + + auto sp = SyncPoint::get_instance(); + + int64_t first_txn_id = 0; + sp->set_call_back("commit_txn_with_sub_txn:begin", [&](auto&& args) { + std::unique_lock _lock(go_mutex); + commit_txn_immediately_begin_count++; + if (commit_txn_immediately_begin_count == 1) { + { + first_txn_id = *try_any_cast(args[0]); + go_cv.wait(_lock, [&] { return txn_lazy_committer_wait_count == 1; }); + go_cv.notify_all(); + } + } + }); + + int64_t second_txn_id = 0; + sp->set_call_back("commit_txn_eventually::txn_lazy_committer_wait", [&](auto&& args) { + std::unique_lock _lock(go_mutex); + txn_lazy_committer_wait_count++; + if (txn_lazy_committer_wait_count == 1) { + int64_t txn_id = *try_any_cast(args[0]); + second_txn_id = txn_id; + go_cv.notify_all(); + } + }); + + sp->set_call_back("commit_txn_with_sub_txn::advance_last_pending_txn_id", [&](auto&& args) { + std::unique_lock _lock(go_mutex); + last_pending_txn_id_count++; + if (last_pending_txn_id_count == 1) { + int64_t last_pending_txn_id = *try_any_cast(args[0]); + ASSERT_EQ(last_pending_txn_id, second_txn_id); + } + go_cv.notify_all(); + }); + + sp->set_call_back("commit_txn_with_sub_txn::finish", [&](auto&& args) { + MetaServiceCode code = *try_any_cast(args[0]); + ASSERT_EQ(code, MetaServiceCode::OK); + std::unique_lock _lock(go_mutex); + immediately_finish_count++; + if (immediately_finish_count == 1) { + go_cv.notify_all(); + } + }); + + sp->set_call_back("commit_txn_eventually::finish", [&](auto&& args) { + MetaServiceCode code = *try_any_cast(args[0]); + ASSERT_EQ(code, MetaServiceCode::OK); + eventually_finish_count++; + }); + + sp->set_call_back("TxnLazyCommitter::commit", [&](auto&& args) { + std::unique_lock _lock(go_mutex); + go_cv.wait(_lock, [&] { return last_pending_txn_id_count == 1; }); + }); + + sp->enable_processing(); + + auto meta_service = get_meta_service(txn_kv, true); + // mock rowset and tablet + int64_t tablet_id_base = 1908562; + for (int i = 0; i < 10; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id, + tablet_id_base + i); + } + + int64_t txn_id1 = 0; + std::thread thread1([&] { + { + std::unique_lock _lock(go_mutex); + go_cv.wait(_lock, [&] { return go; }); + } + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label("test_label_concurrent_commit_txn_eventually3442"); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id1 = res.txn_id(); + ASSERT_GT(txn_id1, 0); + } + { + for (int i = 0; i < 10; ++i) { + auto tmp_rowset = + create_rowset(txn_id1, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + } + + { + brpc::Controller cntl; + CommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.set_txn_id(txn_id1); + req.set_is_2pc(false); + req.set_enable_txn_lazy_commit(true); + CommitTxnResponse res; + meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + }); + + int64_t txn_id2 = 0; + std::thread thread2([&] { + { + std::unique_lock _lock(go_mutex); + go_cv.wait(_lock, [&] { return go; }); + } + { + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label("test_label_concurrent_commit_txn_eventually5"); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + txn_id2 = res.txn_id(); + ASSERT_GT(txn_id2, 0); + } + { + for (int i = 0; i < 10; ++i) { + auto tmp_rowset = + create_rowset(txn_id2, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + } + int64_t sub_txn_id1 = txn_id2; + + // begin sub_txn1 + int64_t sub_txn_id2 = -1; + { + brpc::Controller cntl; + BeginSubTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_txn_id(txn_id2); + req.set_sub_txn_num(0); + req.set_db_id(db_id); + req.set_label("test_label_concurrent_commit_txn_eventually5_sub"); + req.mutable_table_ids()->Add(table_id); + req.mutable_table_ids()->Add(table_id); + BeginSubTxnResponse res; + meta_service->begin_sub_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + ASSERT_EQ(res.txn_info().table_ids().size(), 2); + ASSERT_EQ(res.txn_info().sub_txn_ids().size(), 1); + ASSERT_TRUE(res.has_sub_txn_id()); + sub_txn_id2 = res.sub_txn_id(); + ASSERT_EQ(sub_txn_id2, res.txn_info().sub_txn_ids()[0]); + } + { + for (int i = 0; i < 10; ++i) { + auto tmp_rowset = + create_rowset(sub_txn_id2, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + } + + { + brpc::Controller cntl; + CommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.set_txn_id(txn_id2); + req.set_is_txn_load(true); + + SubTxnInfo sub_txn_info1; + sub_txn_info1.set_sub_txn_id(sub_txn_id1); + sub_txn_info1.set_table_id(table_id); + for (int i = 0; i < 10; ++i) { + sub_txn_info1.mutable_base_tablet_ids()->Add(tablet_id_base + i); + } + + SubTxnInfo sub_txn_info2; + sub_txn_info2.set_sub_txn_id(sub_txn_id2); + sub_txn_info2.set_table_id(table_id); + for (int i = 0; i < 10; ++i) { + sub_txn_info1.mutable_base_tablet_ids()->Add(tablet_id_base + i); + } + + req.mutable_sub_txn_infos()->Add(std::move(sub_txn_info1)); + req.mutable_sub_txn_infos()->Add(std::move(sub_txn_info2)); + CommitTxnResponse res; + meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + }); + + std::unique_lock go_lock(go_mutex); + go = true; + go_lock.unlock(); + go_cv.notify_all(); + + thread1.join(); + thread2.join(); + + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); + ASSERT_EQ(commit_txn_immediately_begin_count, 2); + ASSERT_EQ(last_pending_txn_id_count, 1); + ASSERT_EQ(immediately_finish_count, 1); + ASSERT_EQ(eventually_finish_count, 1); + + { + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + for (int i = 0; i < 10; ++i) { + int64_t tablet_id = tablet_id_base + i; + check_tablet_idx_db_id(txn, db_id, tablet_id); + + check_tmp_rowset_not_exist(txn, tablet_id, first_txn_id); + check_rowset_meta_exist(txn, tablet_id, 2); + + check_tmp_rowset_not_exist(txn, tablet_id, second_txn_id); + check_rowset_meta_exist(txn, tablet_id, 4); + } + } +} + TEST(TxnLazyCommitTest, RowsetMetaSizeExceedTest) { auto txn_kv = get_mem_txn_kv(); From c82e418724d05113f64f2b7579a7fefdd6607976 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Thu, 28 Aug 2025 17:54:29 +0800 Subject: [PATCH 525/572] branch-3.0: [fix](warmup) remove duplicate adding of periodic job metrics (#55283) ### What problem does this PR solve? Problem Summary: Some metrics were added twice, remove duplicated adding. This is caused by #53189 PR merge issue. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/cloud/cloud_warm_up_manager.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/be/src/cloud/cloud_warm_up_manager.cpp b/be/src/cloud/cloud_warm_up_manager.cpp index 5b23f8ebe88244..06a7689a147433 100644 --- a/be/src/cloud/cloud_warm_up_manager.cpp +++ b/be/src/cloud/cloud_warm_up_manager.cpp @@ -122,14 +122,6 @@ void CloudWarmUpManager::submit_download_tasks(io::Path path, int64_t file_size, g_file_cache_once_or_periodic_warm_up_submitted_segment_size << file_size; } - if (is_index) { - g_file_cache_once_or_periodic_warm_up_submitted_index_num << 1; - g_file_cache_once_or_periodic_warm_up_submitted_index_size << file_size; - } else { - g_file_cache_once_or_periodic_warm_up_submitted_segment_num << 1; - g_file_cache_once_or_periodic_warm_up_submitted_segment_size << file_size; - } - const int64_t chunk_size = 10 * 1024 * 1024; // 10MB int64_t offset = 0; int64_t remaining_size = file_size; From ea9e703b4713ae58de3102a6d0185c4bff694c7b Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Thu, 28 Aug 2025 18:03:59 +0800 Subject: [PATCH 526/572] [fix](case) fix test_cloud_version_already_merged (#55332) --- .../cloud/test_cloud_version_already_merged.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy index e1c6669b11b8ea..f7453c93300fcc 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy @@ -101,7 +101,7 @@ suite("test_cloud_version_already_merged", "nonConcurrent") { test { sql """ SELECT * from ${tblName} ORDER BY k1 """ - exception "version already merged, meet error during remote capturing rowsets" + exception "versions are already compacted" } } finally { From 04d9e1c5f5aed3f589ee48ff188e8bf70ba3e426 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Thu, 28 Aug 2025 18:13:24 +0800 Subject: [PATCH 527/572] [fix](case) log info for test_alter_colocate_group (#55276) --- .../suites/alter_p2/test_alter_colocate_group.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/alter_p2/test_alter_colocate_group.groovy b/regression-test/suites/alter_p2/test_alter_colocate_group.groovy index 848f4146dcc99b..3b8aa4068650ec 100644 --- a/regression-test/suites/alter_p2/test_alter_colocate_group.groovy +++ b/regression-test/suites/alter_p2/test_alter_colocate_group.groovy @@ -134,8 +134,8 @@ suite ("test_alter_colocate_group") { def checkTableReplicaAlloc = { tableName, hasDynamicPart, replicaNum -> def result = sql """ show create table ${tableName} """ def createTbl = result[0][1].toString() - assertTrue(createTbl.indexOf("\"replication_allocation\" = \"tag.location.default: ${replicaNum}\"") > 0) log.info("createTbl: ${createTbl}") + assertTrue(createTbl.indexOf("\"replication_allocation\" = \"tag.location.default: ${replicaNum}\"") > 0) if (hasDynamicPart) { assertTrue(createTbl.indexOf( "\"dynamic_partition.replication_allocation\" = \"tag.location.default: ${replicaNum}\"") > 0) From de3862f910dac5fd64f92f9c5a4d6564ace0f7fe Mon Sep 17 00:00:00 2001 From: Wen Zhenghu Date: Fri, 29 Aug 2025 17:10:16 +0800 Subject: [PATCH 528/572] [refactor](file-cache) refactor statistics collection logic in VFileScanner (#53385) ### What problem does this PR solve? Add query statistics collection for scan bytes from local and remote storage when enable file cache. This change enhances the file cache statistics collection mechanism and provides better visibility into storage access patterns during file scanning operations. Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [x] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/exec/scan/vfile_scanner.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 7cbe6116b639d8..d6059347f026cf 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -1186,10 +1186,18 @@ void VFileScanner::try_stop() { void VFileScanner::_collect_profile_before_close() { VScanner::_collect_profile_before_close(); - if (config::enable_file_cache && _state->query_options().enable_file_cache && - _profile != nullptr) { - io::FileCacheProfileReporter cache_profile(_profile); - cache_profile.update(_file_cache_statistics.get()); + if (config::enable_file_cache && _state->query_options().enable_file_cache) { + if (_profile != nullptr) { + io::FileCacheProfileReporter cache_profile(_profile); + cache_profile.update(_file_cache_statistics.get()); + } + + if (_query_statistics) { + _query_statistics->add_scan_bytes_from_local_storage( + _file_cache_statistics->bytes_read_from_local); + _query_statistics->add_scan_bytes_from_remote_storage( + _file_cache_statistics->bytes_read_from_remote); + } } if (_cur_reader != nullptr) { From 2f6f4236f9ec7428661352ee8dd2aaf964077bc9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 29 Aug 2025 21:30:04 +0800 Subject: [PATCH 529/572] branch-3.0: [fix](test) Failed to show create table for async mv #55278 (#55478) Cherry-picked from #55278 Co-authored-by: Uniqueyou --- .../suites/data_reliability/check_meta.groovy | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/data_reliability/check_meta.groovy b/regression-test/suites/data_reliability/check_meta.groovy index 50e887c3e9b51d..2f788ae140db33 100644 --- a/regression-test/suites/data_reliability/check_meta.groovy +++ b/regression-test/suites/data_reliability/check_meta.groovy @@ -28,7 +28,23 @@ suite("check_meta", "data_reliability,p3") { List> tableRes = sql """ show tables from ${db} """ for (tableRow : tableRes) { def table = tableRow[0] - def createTableSql = sql """ show create table ${db}.`${table}` """ + def createTableSql + try { + createTableSql = sql "show create table ${db}.${table}" + } catch (Exception e) { + if (e.getMessage().contains("not support async materialized view")) { + try { + createTableSql = sql "show create materialized view ${db}.${table}" + } catch (Exception e2) { + if (e2.getMessage().contains("table not found")) { + continue + } + } + } else { + logger.warn("Failed to show create materialized view ${db}.${table}: ${e.getMessage()}") + continue + } + } if (createTableSql[0][1].contains("CREATE VIEW")) { continue } From 09b0cc49a60ffdd444df3e40e5f3dc180299b561 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Fri, 29 Aug 2025 21:31:02 +0800 Subject: [PATCH 530/572] [chore] bump to 3.0.8 (#55477) --- gensrc/script/gen_build_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 2b3a7f7ce3c10d..cd539bb26b38c3 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -35,7 +35,7 @@ fi build_version_prefix="${vendor}" build_version_major=3 build_version_minor=0 -build_version_patch=7 +build_version_patch=8 build_version_hotfix=0 build_version_rc_version="rc01" From 96abadcc8fd3887847b45ee3858e5e8e62907af8 Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Sun, 31 Aug 2025 19:10:23 +0800 Subject: [PATCH 531/572] [chore](cloud) Optimize RPC log #53341 (#54217) #53341 --- be/src/cloud/cloud_meta_mgr.cpp | 38 ++++--- cloud/src/meta-service/meta_service.cpp | 20 ++-- cloud/src/meta-service/meta_service_helper.h | 113 +++++++++---------- cloud/test/meta_service_test.cpp | 8 +- 4 files changed, 92 insertions(+), 87 deletions(-) diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index fe47bb7c8e0628..efa4ce92aa3a79 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -491,6 +491,9 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, MetaServiceProxy* proxy; RETURN_IF_ERROR(MetaServiceProxy::get_proxy(&proxy)); + std::string tablet_info = + fmt::format("tablet_id={} table_id={} index_id={} partition_id={}", tablet->tablet_id(), + tablet->table_id(), tablet->index_id(), tablet->partition_id()); int tried = 0; while (true) { std::shared_ptr stub; @@ -540,12 +543,8 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, std::uniform_int_distribution u1(500, 1000); uint32_t duration_ms = tried >= 100 ? u(rng) : u1(rng); bthread_usleep(duration_ms * 1000); - LOG_INFO("failed to get rowset meta") + LOG_INFO("failed to get rowset meta, " + tablet_info) .tag("reason", cntl.ErrorText()) - .tag("tablet_id", tablet_id) - .tag("table_id", table_id) - .tag("index_id", index_id) - .tag("partition_id", tablet->partition_id()) .tag("tried", tried) .tag("sleep", duration_ms); continue; @@ -553,18 +552,26 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, return Status::RpcError("failed to get rowset meta: {}", cntl.ErrorText()); } if (resp.status().code() == MetaServiceCode::TABLET_NOT_FOUND) { - return Status::NotFound("failed to get rowset meta: {}", resp.status().msg()); + LOG(WARNING) << "failed to get rowset meta, err=" << resp.status().msg() << " " + << tablet_info; + return Status::NotFound("failed to get rowset meta: {}, {}", resp.status().msg(), + tablet_info); } if (resp.status().code() != MetaServiceCode::OK) { - return Status::InternalError("failed to get rowset meta: {}", resp.status().msg()); + LOG(WARNING) << " failed to get rowset meta, err=" << resp.status().msg() << " " + << tablet_info; + return Status::InternalError("failed to get rowset meta: {}, {}", resp.status().msg(), + tablet_info); } if (latency > 100 * 1000) { // 100ms LOG(INFO) << "finish get_rowset rpc. rowset_meta.size()=" << resp.rowset_meta().size() - << ", latency=" << latency << "us"; + << ", latency=" << latency << "us" + << " " << tablet_info; } else { LOG_EVERY_N(INFO, 100) << "finish get_rowset rpc. rowset_meta.size()=" << resp.rowset_meta().size() - << ", latency=" << latency << "us"; + << ", latency=" << latency << "us" + << " " << tablet_info; } int64_t now = duration_cast(system_clock::now().time_since_epoch()).count(); @@ -586,16 +593,13 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, resp.stats(), req.idx(), &delete_bitmap, full_sync, sync_stats); if (st.is() && tried++ < retry_times) { - LOG_INFO("rowset meta is expired, need to retry") - .tag("tablet", tablet->tablet_id()) + LOG_INFO("rowset meta is expired, need to retry, " + tablet_info) .tag("tried", tried) .error(st); continue; } if (!st.ok()) { - LOG_WARNING("failed to get delete bitmap") - .tag("tablet", tablet->tablet_id()) - .error(st); + LOG_WARNING("failed to get delete bitmap, " + tablet_info).error(st); return st; } tablet->tablet_meta()->delete_bitmap()->merge(delete_bitmap); @@ -653,9 +657,7 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, } } - LOG_INFO("[verbose] sync tablet delete bitmap") - .tag("tablet_id", tablet->tablet_id()) - .tag("table_id", tablet->table_id()) + LOG_INFO("[verbose] sync tablet delete bitmap " + tablet_info) .tag("full_sync", full_sync) .tag("old_max_version", old_max_version) .tag("new_max_version", new_max_version) @@ -703,7 +705,7 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, stats.cumulative_compaction_cnt() < tablet->cumulative_compaction_cnt()) [[unlikely]] { // stale request, ignore - LOG_WARNING("stale get rowset meta request") + LOG_WARNING("stale get rowset meta request " + tablet_info) .tag("resp_base_compaction_cnt", stats.base_compaction_cnt()) .tag("base_compaction_cnt", tablet->base_compaction_cnt()) .tag("resp_cumulative_compaction_cnt", stats.cumulative_compaction_cnt()) diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 87a6cf68e33915..533d01afb02e8d 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1542,7 +1542,8 @@ void MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, const std::string& instance_id, int64_t tablet_id, MetaServiceCode& code, std::string& msg, GetRowsetResponse* response) { - LOG(INFO) << "get_rowset start=" << start << ", end=" << end; + LOG(INFO) << "get_rowset start=" << start << ", end=" + << " tablet_id=" << tablet_id; MetaRowsetKeyInfo key_info0 {instance_id, tablet_id, start}; MetaRowsetKeyInfo key_info1 {instance_id, tablet_id, end + 1}; std::string key0; @@ -1554,7 +1555,8 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, int num_rowsets = 0; DORIS_CLOUD_DEFER_COPY(key0, key1) { LOG(INFO) << "get rowset meta, num_rowsets=" << num_rowsets << " range=[" << hex(key0) - << "," << hex(key1) << "]"; + << "," << hex(key1) << "]" + << " tablet_id=" << tablet_id; }; std::stringstream ss; @@ -1562,7 +1564,7 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, TxnErrorCode err = txn->get(key0, key1, &it); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); - ss << "internal error, failed to get rowset, err=" << err; + ss << "internal error, failed to get rowset, err=" << err << " tablet_id=" << tablet_id; msg = ss.str(); LOG(WARNING) << msg; return; @@ -1576,14 +1578,16 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, if (byte_size + v.size() > std::numeric_limits::max()) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; msg = fmt::format( - "rowset meta exceeded 2G, unable to serialize, key={}. byte_size={}", - hex(k), byte_size); + "rowset meta exceeded 2G, unable to serialize, key={}. byte_size={} " + "tablet_id={}", + hex(k), byte_size, tablet_id); LOG(WARNING) << msg; return; } if (!rs->ParseFromArray(v.data(), v.size())) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; - msg = "malformed rowset meta, unable to serialize"; + msg = "malformed rowset meta, unable to serialize, tablet_id=" + + std::to_string(tablet_id); LOG(WARNING) << msg << " key=" << hex(k); return; } @@ -1712,7 +1716,9 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, if (!request->has_base_compaction_cnt() || !request->has_cumulative_compaction_cnt() || !request->has_cumulative_point()) { code = MetaServiceCode::INVALID_ARGUMENT; - msg = "no valid compaction_cnt or cumulative_point given"; + msg = "no valid compaction_cnt or cumulative_point given, tablet_id=" + + std::to_string(tablet_id); + LOG(WARNING) << msg; return; } int64_t req_bc_cnt = request->base_compaction_cnt(); diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index 321b0ceb3a67f2..c35873ed94933f 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -94,107 +94,104 @@ inline std::string encryt_sk(std::string debug_string) { return debug_string; } -template -void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) { +template +void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req, + Response* res) { + res->Clear(); // clear response in case of this is call is a local retry in MS if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip(); + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip(); + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() << " table_id=" << req->table_id() + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " table_id=" << req->table_id() << " tablet_id=" << req->tablet_id() << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() << " delete_bitmap_size=" << req->segment_delete_bitmaps_size(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " tablet_id=" << req->tablet_id() << " rowset_size=" << req->rowset_ids_size(); } else if constexpr (std::is_same_v) { - VLOG_DEBUG << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() - << " tablet size: " << req->tablet_idx().size(); + VLOG_DEBUG << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() + << " num_tablets: " << req->tablet_idx().size(); } else if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { - VLOG_DEBUG << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() + VLOG_DEBUG << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " request=" << req->ShortDebugString(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " tablet_id=" << req->tablet_id() << " rowset_size=" << req->rowset_ids_size(); } else if constexpr (std::is_same_v) { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() << " table_id=" << req->table_id() + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " table_id=" << req->table_id() << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() << " expiration=" << req->expiration() << " require_compaction_stats=" << req->require_compaction_stats(); } else { - LOG(INFO) << "begin " << func_name << " remote caller: " << ctrl->remote_side() - << " original client ip: " << req->request_ip() + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " request=" << req->ShortDebugString(); } } -template -void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* res) { +template +void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req, + Response* res) { if constexpr (std::is_same_v) { - if (res->status().code() != MetaServiceCode::OK) { - res->clear_table_ids(); - res->clear_partition_ids(); - res->clear_versions(); - } - LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { - if (res->status().code() != MetaServiceCode::OK) { - res->clear_rowset_meta(); - } - VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() - << " status=" << res->status().ShortDebugString(); + LOG_IF(INFO, res->status().code() != MetaServiceCode::OK) + << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() + << " request=" << req->ShortDebugString() + << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v) { - VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() - << " status=" << res->status().ShortDebugString() - << " tablet size: " << res->tablet_stats().size(); + LOG_IF(INFO, res->status().code() != MetaServiceCode::OK) + << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() + << " request=" << req->ShortDebugString() + << " status=" << res->status().ShortDebugString() + << " num_tablets: " << res->tablet_stats().size(); } else if constexpr (std::is_same_v || std::is_same_v) { - VLOG_DEBUG << "finish " << func_name << " remote caller: " << ctrl->remote_side() - << " response=" << res->ShortDebugString(); + LOG_IF(INFO, res->status().code() != MetaServiceCode::OK) + << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " request=" << req->ShortDebugString() + << " original_client_ip=" << req->request_ip() + << " response=" << res->ShortDebugString(); } else if constexpr (std::is_same_v) { - if (res->status().code() != MetaServiceCode::OK) { - res->clear_rowset_ids(); - res->clear_segment_ids(); - res->clear_versions(); - res->clear_segment_delete_bitmaps(); - } - LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " status=" << res->status().ShortDebugString() << " tablet=" << res->tablet_id() << " delete_bitmap_count=" << res->segment_delete_bitmaps_size(); } else if constexpr (std::is_same_v) { - if (res->status().code() != MetaServiceCode::OK) { - res->clear_base_compaction_cnts(); - res->clear_cumulative_compaction_cnts(); - res->clear_cumulative_points(); - } - LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v || std::is_same_v) { std::string debug_string = encryt_sk(res->DebugString()); TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string); - LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() - << " response=" << debug_string; + LOG(INFO) << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " response=" << debug_string; } else { - LOG(INFO) << "finish " << func_name << " remote caller: " << ctrl->remote_side() + LOG(INFO) << "finish " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " response=" << res->ShortDebugString(); } } -enum ErrCategory { CREATE, READ, COMMIT }; +enum class ErrCategory { CREATE, READ, COMMIT }; template inline MetaServiceCode cast_as(TxnErrorCode code) { @@ -262,7 +259,7 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { #define RPC_PREPROCESS(func_name, ...) \ StopWatch sw; \ auto ctrl = static_cast(controller); \ - begin_rpc(#func_name, ctrl, request); \ + begin_rpc(#func_name, ctrl, request, response); \ brpc::ClosureGuard closure_guard(done); \ [[maybe_unused]] std::stringstream ss; \ [[maybe_unused]] MetaServiceCode code = MetaServiceCode::OK; \ @@ -274,7 +271,7 @@ inline MetaServiceCode cast_as(TxnErrorCode code) { DORIS_CLOUD_DEFER { \ response->mutable_status()->set_code(code); \ response->mutable_status()->set_msg(msg); \ - finish_rpc(#func_name, ctrl, response); \ + finish_rpc(#func_name, ctrl, request, response); \ closure_guard.reset(nullptr); \ if (txn != nullptr) { \ stats.get_bytes += txn->get_bytes(); \ diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index ad1c054e639352..8150dced5bd9d1 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -4782,10 +4782,10 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLockTabletStatsLockExpired) { GetDeleteBitmapUpdateLockResponse res; get_delete_bitmap_update_lock(meta_service.get(), res, db_id, table_id, index_id, tablet_idxes, 5, 999999, -1, true); - ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED); - ASSERT_EQ(res.base_compaction_cnts().size(), 0); - ASSERT_EQ(res.cumulative_compaction_cnts().size(), 0); - ASSERT_EQ(res.cumulative_points().size(), 0); + EXPECT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED); + EXPECT_EQ(res.base_compaction_cnts().size(), 3); + EXPECT_EQ(res.cumulative_compaction_cnts().size(), 3); + EXPECT_EQ(res.cumulative_points().size(), 3); } for (bool val : enable_batch_get_mow_tablet_stats_and_meta_vals) { From 66f1001b1e16e82772698b934a91cb6af3c8f9c2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Sep 2025 09:54:38 +0800 Subject: [PATCH 532/572] branch-3.0: [Fix](case) Fix show data p2 cases #55449 (#55492) Cherry-picked from #55449 Co-authored-by: abmdocrt --- .../test_cloud_drop_and_recover_partition_show_data.groovy | 4 ++++ .../test_cloud_drop_table_show_data.groovy | 2 ++ 2 files changed, 6 insertions(+) diff --git a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_and_recover_partition_show_data.groovy b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_and_recover_partition_show_data.groovy index b067ad5d9ad25d..9534066b8996f6 100644 --- a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_and_recover_partition_show_data.groovy +++ b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_and_recover_partition_show_data.groovy @@ -162,6 +162,8 @@ suite("test_cloud_drop_and_recover_partition_show_data","p2, nonConcurrent") { assertEquals(sizeRecords["apiSize"][0], sizeRecords["apiSize"][1]) assertEquals(sizeRecords["cbsSize"][0], sizeRecords["cbsSize"][1]) + sql """admin set frontend config ("catalog_trash_expire_second" = "30")""" + if (op == 1){ sql """alter table ${tableName} drop partition p1;""" } else if(op == 2){ @@ -192,6 +194,8 @@ suite("test_cloud_drop_and_recover_partition_show_data","p2, nonConcurrent") { } else if(op == 2){ sql """recover partition p19920101000000 from ${tableName};""" } + + sql """admin set frontend config ("catalog_trash_expire_second" = "1")""" // after drop partition，tablets will changed，need get new tablets tablets = get_tablets_from_table(tableName) diff --git a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy index e80deb85834713..8eb20b916fb5af 100644 --- a/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy +++ b/regression-test/suites/show_data_p2/test_table_operation/test_cloud_drop_table_show_data.groovy @@ -91,11 +91,13 @@ suite("test_cloud_drop_and_show_data","p2, nonConcurrent") { if(op == 1){ + sql """admin set frontend config ("catalog_trash_expire_second" = "30")""" sql """drop table ${tableName}""" sleep(10 * 1000) sql """recover table ${tableName}""" + sql """admin set frontend config ("catalog_trash_expire_second" = "1")""" // 加一下触发compaction的机制 trigger_compaction(tablets) From 5052f9029bc27f327e907d127069b8f3229ce893 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 1 Sep 2025 19:10:04 +0800 Subject: [PATCH 533/572] branch-3.0: [fix](load): fix incorrect progress on finished loads #55509 (#55529) Cherry-picked from #55509 Co-authored-by: Kaijie Chen --- be/src/runtime/fragment_mgr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 3f51331346d932..9ba558be76d7d8 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -437,7 +437,7 @@ void FragmentMgr::coordinator_callback(const ReportStatusRequest& req) { } else if (!req.runtime_states.empty()) { for (auto* rs : req.runtime_states) { if (rs->num_rows_load_total() > 0 || rs->num_rows_load_filtered() > 0 || - req.runtime_state->num_finished_range() > 0) { + rs->num_finished_range() > 0) { params.__isset.load_counters = true; num_rows_load_success += rs->num_rows_load_success(); num_rows_load_filtered += rs->num_rows_load_filtered(); From b9737ae11acd32957c6b81bf2bbc48d13843c5ad Mon Sep 17 00:00:00 2001 From: hui lai Date: Mon, 1 Sep 2025 21:52:41 +0800 Subject: [PATCH 534/572] branch-3.0: [enhance](transaction) add completeness of commit tablet info check in cloud mode (#53979) (#55055) pick #53979 Add completeness of commit tablet info checker: check if all tablets in each partition are committed (except for the index added during writing), dealing with scenarios where part of commit info may be missed ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/vec/sink/writer/vtablet_writer.cpp | 10 +++ .../CloudGlobalTransactionMgr.java | 69 +++++++++++++++++++ .../test_incomplete_commit_info.groovy | 63 +++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp index 14e9d3206169a3..296620e56316cc 100644 --- a/be/src/vec/sink/writer/vtablet_writer.cpp +++ b/be/src/vec/sink/writer/vtablet_writer.cpp @@ -804,7 +804,17 @@ void VNodeChannel::_add_block_success_callback(const PTabletWriterAddBlockResult if (!st.ok()) { _cancel_with_msg(st.to_string()); } else if (ctx._is_last_rpc) { + bool skip_tablet_info = false; + DBUG_EXECUTE_IF("VNodeChannel.add_block_success_callback.incomplete_commit_info", + { skip_tablet_info = true; }); for (const auto& tablet : result.tablet_vec()) { + DBUG_EXECUTE_IF("VNodeChannel.add_block_success_callback.incomplete_commit_info", { + if (skip_tablet_info) { + LOG(INFO) << "skip tablet info: " << tablet.tablet_id(); + skip_tablet_info = false; + continue; + } + }); TTabletCommitInfo commit_info; commit_info.tabletId = tablet.tablet_id(); commit_info.backendId = _node_id; diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index 0f50ec8704f8f4..d6517e0b122010 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -72,6 +72,7 @@ import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.LabelAlreadyUsedException; +import org.apache.doris.common.LoadException; import org.apache.doris.common.MarkedCountDownLatch; import org.apache.doris.common.MetaNotFoundException; import org.apache.doris.common.Pair; @@ -633,6 +634,7 @@ private void commitTransactionWithoutLock(long dbId, List

tableList, long private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long transactionId, boolean is2PC, long dbId, List
tableList) throws UserException { + checkCommitInfo(commitTxnRequest); CommitTxnResponse commitTxnResponse = null; TransactionState txnState = null; int retryTime = 0; @@ -694,6 +696,73 @@ private TransactionState commitTxn(CommitTxnRequest commitTxnRequest, long trans return txnState; } + private void checkCommitInfo(CommitTxnRequest commitTxnRequest) throws UserException { + List commitTabletIds = Lists.newArrayList(); + List commitIndexIds = Lists.newArrayList(); + commitTabletIds.addAll(commitTxnRequest.getBaseTabletIdsList()); + for (SubTxnInfo subTxnInfo : commitTxnRequest.getSubTxnInfosList()) { + commitTabletIds.addAll(subTxnInfo.getBaseTabletIdsList()); + } + if (commitTabletIds.isEmpty()) { + return; + } + + TabletInvertedIndex tabletInvertedIndex = Env.getCurrentEnv().getTabletInvertedIndex(); + List tabletMetaList = tabletInvertedIndex.getTabletMetaList(commitTabletIds); + Map> tableToPartition = Maps.newHashMap(); + for (int i = 0; i < tabletMetaList.size(); i++) { + TabletMeta tabletMeta = tabletMetaList.get(i); + if (tabletMeta == null) { + continue; + } + long tableId = tabletMeta.getTableId(); + long dbId = tabletMeta.getDbId(); + Database db = Env.getCurrentEnv().getInternalCatalog().getDbNullable(dbId); + if (db == null) { + // this can happen when dbId == -1 (tablet being dropping) or db really not exist. + continue; + } + OlapTable tbl = (OlapTable) db.getTableNullable(tableId); + if (tbl == null) { + // this can happen when tableId == -1 (tablet being dropping) or table really not exist. + continue; + } + // check relative partition restore here + long partitionId = tabletMeta.getPartitionId(); + if (tbl.getPartition(partitionId) == null) { + // this can happen when partitionId == -1 (tablet being dropping) or partition really not exist. + continue; + } + tableToPartition.computeIfAbsent(tbl, k -> Sets.newHashSet()).add(partitionId); + commitIndexIds.add(tabletMeta.getIndexId()); + } + + for (OlapTable tbl : tableToPartition.keySet()) { + for (Partition partition : tbl.getAllPartitions()) { + if (!tableToPartition.get(tbl).contains(partition.getId())) { + continue; + } + List allIndices + = partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL); + for (MaterializedIndex index : allIndices) { + // Schema change during load will increase partition index number, + // and we need to skip these indexes. + // TODO: judge by transactionState.getLoadedTblIndexes() is better + if (!commitIndexIds.contains(index.getId())) { + continue; + } + for (Tablet tablet : index.getTablets()) { + if (!commitTabletIds.contains(tablet.getId())) { + throw new LoadException("Table [" + tbl.getName() + "], Index [" + + index.getId() + "], Partition [" + partition.getName() + + "], tablet " + tablet.getId() + " should be committed"); + } + } + } + } + } + } + // return mow tables with contains tablet commit info private List getMowTableList(List
tableList, List tabletCommitInfos) { if (tabletCommitInfos == null || tabletCommitInfos.isEmpty()) { diff --git a/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy b/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy new file mode 100644 index 00000000000000..2b6503a929e3fa --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_incomplete_commit_info.groovy @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods +import org.apache.doris.regression.util.Http + +suite("test_incomplete_commit_info", "nonConcurrent") { + try { + def tableName = "test_incomplete_commit_info" + sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k0` boolean null comment "", + `k1` tinyint(4) null comment "", + `k2` smallint(6) null comment "", + `k3` int(11) null comment "", + `k4` bigint(20) null comment "", + `k5` decimal(9, 3) null comment "", + `k6` char(5) null comment "", + `k10` date null comment "", + `k11` datetime null comment "", + `k7` varchar(20) null comment "", + `k8` double max null comment "", + `k9` float sum null comment "", + `k12` string replace null comment "", + `k13` largeint(40) replace null comment "" + ) engine=olap + DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1") + """ + GetDebugPoint().enableDebugPointForAllBEs("VNodeChannel.add_block_success_callback.incomplete_commit_info") + streamLoad { + table "${tableName}" + db "regression_test_fault_injection_p0" + set 'column_separator', ',' + file "baseall.txt" + + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + } + } + } finally { + GetDebugPoint().disableDebugPointForAllBEs("VNodeChannel.add_block_success_callback.incomplete_commit_info") + } +} \ No newline at end of file From 65a65e7a3327b3a770f6bf9a7b126552b2f66464 Mon Sep 17 00:00:00 2001 From: hui lai Date: Tue, 2 Sep 2025 21:19:20 +0800 Subject: [PATCH 535/572] branch-3.0: [fix](job) use cluster name rather than cluster id to find available be (#52911) (#55563) pick #52911 In multi cluster scenario, users perceive the cluster name, so it should be used cluster name rather than cluster id to find available be node. --- .../cloud/load/CloudRoutineLoadManager.java | 12 ++--- .../load/routineload/RoutineLoadJob.java | 52 +++++++++---------- 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java index eff1c345e5a0ae..399b33e32d4761 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudRoutineLoadManager.java @@ -51,14 +51,8 @@ public void addRoutineLoadJob(RoutineLoadJob routineLoadJob, String dbName, Stri @Override protected List getAvailableBackendIds(long jobId) throws LoadException { RoutineLoadJob routineLoadJob = getJob(jobId); - String cloudClusterId = routineLoadJob.getCloudClusterId(); - if (Strings.isNullOrEmpty(cloudClusterId)) { - LOG.warn("cluster id is empty"); - throw new LoadException("cluster id is empty"); - } - return ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getBackendsByClusterId(cloudClusterId) + .getBackendsByClusterName(routineLoadJob.getCloudCluster()) .stream() .filter(Backend::isAlive) .map(Backend::getId) @@ -67,13 +61,13 @@ protected List getAvailableBackendIds(long jobId) throws LoadException { @Override public void replayCreateRoutineLoadJob(RoutineLoadJob routineLoadJob) { - routineLoadJob.setCloudClusterById(); + routineLoadJob.setCloudCluster(); super.replayCreateRoutineLoadJob(routineLoadJob); } @Override public void replayChangeRoutineLoadJob(RoutineLoadOperation operation) { - getJob(operation.getId()).setCloudClusterById(); + getJob(operation.getId()).setCloudCluster(); super.replayChangeRoutineLoadJob(operation); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java index 9c83bb6e9f9971..46460e1b82ad31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java @@ -292,6 +292,8 @@ public boolean isFinalState() { protected byte escape = 0; // use for cloud cluster mode + protected String qualifiedUser; + @SerializedName("ccn") protected String cloudCluster; public void setTypeRead(boolean isTypeRead) { @@ -303,6 +305,13 @@ public RoutineLoadJob(long id, LoadDataSourceType type) { this.dataSourceType = type; if (ConnectContext.get() != null) { this.memtableOnSinkNode = ConnectContext.get().getSessionVariable().enableMemtableOnSinkNode; + if (Config.isCloudMode()) { + try { + this.cloudCluster = ConnectContext.get().getCloudCluster(); + } catch (ComputeGroupException e) { + LOG.warn("failed to get cloud cluster", e); + } + } } } @@ -320,6 +329,13 @@ public RoutineLoadJob(Long id, String name, SessionVariable var = ConnectContext.get().getSessionVariable(); sessionVariables.put(SessionVariable.SQL_MODE, Long.toString(var.getSqlMode())); this.memtableOnSinkNode = ConnectContext.get().getSessionVariable().enableMemtableOnSinkNode; + if (Config.isCloudMode()) { + try { + this.cloudCluster = ConnectContext.get().getCloudCluster(); + } catch (ComputeGroupException e) { + LOG.warn("failed to get cloud cluster", e); + } + } } else { sessionVariables.put(SessionVariable.SQL_MODE, String.valueOf(SqlModeHelper.MODE_DEFAULT)); } @@ -1010,21 +1026,13 @@ public TPipelineFragmentParams plan(StreamLoadPlanner planner, TUniqueId loadId, table.readLock(); try { if (Config.isCloudMode()) { - String clusterName = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getClusterNameByClusterId(cloudClusterId); - if (Strings.isNullOrEmpty(clusterName)) { - String err = String.format("cluster name is empty, cluster id is %s", cloudClusterId); - LOG.warn(err); - throw new UserException(err); - } - if (ConnectContext.get() == null) { ConnectContext ctx = new ConnectContext(); ctx.setThreadLocalInfo(); - ctx.setCloudCluster(clusterName); + ctx.setCloudCluster(cloudCluster); needCleanCtx = true; } else { - ConnectContext.get().setCloudCluster(clusterName); + ConnectContext.get().setCloudCluster(cloudCluster); } } @@ -1599,27 +1607,15 @@ public void setOrigStmt(OriginStatement origStmt) { this.origStmt = origStmt; } - public void setCloudCluster(String cloudClusterName) throws UserException { - if (Strings.isNullOrEmpty(cloudClusterName)) { - LOG.warn("cluster name is empty"); - throw new UserException("cluster name is empty"); + public void setCloudCluster() { + if (Strings.isNullOrEmpty(cloudCluster)) { + this.cloudCluster = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) + .getClusterNameByClusterId(cloudClusterId); } - - this.cloudClusterId = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getCloudClusterIdByName(cloudClusterName); - if (Strings.isNullOrEmpty(this.cloudClusterId)) { - LOG.warn("cluster id is empty, cluster name {}", cloudClusterName); - throw new UserException("cluster id is empty, cluster name: " + cloudClusterName); - } - } - - public String getCloudClusterId() { - return cloudClusterId; } - public void setCloudClusterById() { - this.cloudCluster = ((CloudSystemInfoService) Env.getCurrentSystemInfo()) - .getClusterNameByClusterId(cloudClusterId); + public void setCloudCluster(String cloudCluster) { + this.cloudCluster = cloudCluster; } // check the correctness of commit info From 10ddc8ba67f98d430f4e46252a7014f65122854d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 3 Sep 2025 14:44:19 +0800 Subject: [PATCH 536/572] branch-3.0: [opt](nereids) optimize normalize window (#54947) (#55045) cherry pick from #54947 --- .../ExtractAndNormalizeWindowExpression.java | 39 +++++++++++ .../PushDownFilterThroughWindowTest.java | 64 ++++++++++++++++++- .../nereids_hint_tpcds_p0/shape/query51.out | 31 +++++---- .../cte/test_cte_filter_pushdown.out | 2 +- .../cte/test_cte_filter_pushdown.out | 2 +- .../shape/query51.out | 31 +++++---- .../noStatsRfPrune/query51.out | 31 +++++---- .../no_stats_shape/query51.out | 31 +++++---- .../rf_prune/query51.out | 31 +++++---- .../shape/query51.out | 31 +++++---- .../shape/query51.out | 31 +++++---- .../tpch/push_filter_window_eqset.out | 7 +- .../tpcds_sf100/noStatsRfPrune/query51.out | 31 +++++---- .../tpcds_sf100/no_stats_shape/query51.out | 31 +++++---- .../tpcds_sf100/rf_prune/query51.out | 31 +++++---- .../tpcds_sf100/shape/query51.out | 31 +++++---- .../tpcds_sf1000/shape/query51.out | 31 +++++---- 17 files changed, 287 insertions(+), 199 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java index 0216570956418c..e96cf3e105c9cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ExtractAndNormalizeWindowExpression.java @@ -180,6 +180,45 @@ private Set collectExpressionsToBePushedDown(List e inputSlots.stream() ).distinct(); } + + // for this sql: + // select + // SUBSTR(orderdate,1,10) AS dt, + // ROW_NUMBER() OVER(PARTITION BY orderdate ORDER BY orderid DESC) AS rn + // from lineorders + // having dt = '2025-01-01' + // + // we not push down the `dt` slot under LogicalWindow, but push down [orderdate, orderid] + // to the bottom projects, because if we push down `dt`, the plan tree will be: + // + // LogicalFilter(substr(dt#3, 1, 10) = '2025-01-01') + // | + // LogicalWindow(rowNumber(partition by orderdate#2, order by orderid#1)) + // | + // LogicalProject(orderid#1, orderdate#2, substr(orderdate#1, 1, 10) as dt#3) + // + // and can not push down filter by `PushDownFilterThroughWindow`, causing inefficiency, + // because dt#3 in LogicalFilter not contains in the partition key in LogicalWindow: [orderdate#2]. + // + // so we only push down orderdate in the LogicalFilter, not push down `dt`: + // + // LogicalFilter(substr(orderdate#2, 1, 10) = '2025-01-01') + // | + // LogicalWindow(rowNumber(partition by orderdate#2, order by orderid#1)) + // | + // LogicalProject(orderid#1, orderdate#2) + // + // and then, `PushDownFilterThroughWindow` found the LogicalFilter's `orderdate#2` contains + // in the LogicalWindow's partition key: [orderdate#2], and can push down filter to: + // + // LogicalWindow(rowNumber(partition by orderdate#2, order by orderid#1)) + // | + // LogicalProject(orderid#1, orderdate#2) + // | + // LogicalFilter(substr(orderdate#2, 1, 10) = '2025-01-01') + if (expression instanceof Alias) { + return expression.getInputSlots().stream(); + } return ImmutableList.of(expression).stream(); }) .collect(ImmutableSet.toImmutableSet()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindowTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindowTest.java index 6edcd01a8ea595..3c367550cfe543 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindowTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownFilterThroughWindowTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.rewrite; +import org.apache.doris.nereids.rules.RuleType; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Expression; @@ -35,6 +36,7 @@ import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.nereids.util.PlanConstructor; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -42,7 +44,7 @@ import java.util.List; -class PushDownFilterThroughWindowTest implements MemoPatternMatchSupported { +class PushDownFilterThroughWindowTest extends TestWithFeService implements MemoPatternMatchSupported { private final LogicalOlapScan scan = new LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), PlanConstructor.student, ImmutableList.of("")); @@ -89,4 +91,64 @@ void pushDownFilterThroughWindowTest() { ) ); } + + @Test + public void testPushDownFilter() throws Exception { + String db = "test"; + createDatabase(db); + useDatabase(db); + createTable("CREATE TABLE lineorders (\n" + + "orderdate varchar(100) NOT NULL,\n" + + "orderid int NOT NULL,\n" + + "country_id int NOT NULL,\n" + + "vender_id int NOT NULL,\n" + + "ordernum int NOT NULL,\n" + + "ordemoney int NOT NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(orderdate, orderid, country_id)\n" + + "COMMENT 'OLAP'\n" + + "PARTITION BY LIST(orderdate)\n" + + "(PARTITION p1992 VALUES IN (\"0-2020\"),\n" + + "PARTITION p1993 VALUES IN (\"0-2021\"),\n" + + "PARTITION p1994 VALUES IN (\"0-2022\"),\n" + + "PARTITION p1995 VALUES IN (\"0-2023\"),\n" + + "PARTITION p1996 VALUES IN (\"0-2024\"),\n" + + "PARTITION p1997 VALUES IN (\"0-2025\"))\n" + + "DISTRIBUTED BY HASH(orderid) BUCKETS 48\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\"\n" + + ")"); + + connectContext.getSessionVariable() + .setDisableNereidsRules( + RuleType.OLAP_SCAN_PARTITION_PRUNE.name() + "," + RuleType.PRUNE_EMPTY_PARTITION.name()); + + PlanChecker.from(connectContext) + .analyze("select * from ( \n" + + " select \n" + + " orderid,\n" + + " orderdate,\n" + + " country_id,\n" + + " ordernum,\n" + + " ordemoney,\n" + + " SUBSTR(lineorders.orderdate,3,4) AS dt,\n" + + " ROW_NUMBER() OVER(PARTITION BY lineorders.orderid,lineorders.orderdate ORDER BY lineorders.country_id DESC) AS rn\n" + + " from lineorders\n" + + ") a \n" + + "where SUBSTR(a.dt, 1, 4) = SUBSTR(curdate(), 1, 4)") + .rewrite() + .matchesFromRoot( + logicalResultSink( + logicalProject( + logicalWindow( + logicalProject( + logicalFilter( + logicalOlapScan() + ) + ) + ) + ) + ) + ); + } } diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query51.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query51.out index a10c39885ad110..ad962e7d114470 100644 --- a/regression-test/data/nereids_hint_tpcds_p0/shape/query51.out +++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1223) and (date_dim.d_month_seq >= 1212)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out b/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out index 0bbae0dc25f3a1..edcbfcb6022c5f 100644 --- a/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out +++ b/regression-test/data/nereids_p0/cte/test_cte_filter_pushdown.out @@ -30,7 +30,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) -- !cte_filter_pushdown_3 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((k3 = dd.k3)) otherCondition=() -----filter((tmp2.k3 = 0)) +----filter((tmp.k3 = 0)) ------PhysicalWindow --------PhysicalQuickSort[LOCAL_SORT] ----------filter((tmp.k1 = 1)) diff --git a/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out b/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out index 7dd6492aa12499..db4dacb4b6ed51 100644 --- a/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out +++ b/regression-test/data/nereids_rules_p0/cte/test_cte_filter_pushdown.out @@ -30,7 +30,7 @@ PhysicalCteAnchor ( cteId=CTEId#0 ) -- !cte_filter_pushdown_3 -- PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((k3 = dd.k3)) otherCondition=() -----filter((tmp2.k3 = 0)) +----filter((tmp.k3 = 0)) ------PhysicalWindow --------PhysicalQuickSort[LOCAL_SORT] ----------filter((tmp.k1 = 1)) diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query51.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query51.out index 6c22d2df3086e3..98b43bfdfc171f 100644 --- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1223) and (date_dim.d_month_seq >= 1212)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query51.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query51.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query51.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query51.out b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query51.out index 5635b5dbabf1b6..5d049b5a6bb2dc 100644 --- a/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query51.out +++ b/regression-test/data/nereids_tpcds_shape_sf10t_orc/shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1225) and (date_dim.d_month_seq >= 1214)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/nereids_tpch_p0/tpch/push_filter_window_eqset.out b/regression-test/data/nereids_tpch_p0/tpch/push_filter_window_eqset.out index 0a14dd5f07c181..bb38206fc48490 100644 --- a/regression-test/data/nereids_tpch_p0/tpch/push_filter_window_eqset.out +++ b/regression-test/data/nereids_tpch_p0/tpch/push_filter_window_eqset.out @@ -5,8 +5,7 @@ PhysicalResultSink ----PhysicalProject ------PhysicalWindow --------PhysicalQuickSort[LOCAL_SORT] -----------PhysicalDistribute[DistributionSpecHash] -------------PhysicalProject ---------------filter((region.r_regionkey = 1)) -----------------PhysicalOlapScan[region] +----------PhysicalProject +------------filter((region.r_regionkey = 1)) +--------------PhysicalOlapScan[region] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.out b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/noStatsRfPrune/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.out b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/no_stats_shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query51.out b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query51.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/rf_prune/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query51.out b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query51.out index 470fabc0f31e81..cec684574edf4b 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query51.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf100/shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1227) and (date_dim.d_month_seq >= 1216)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] diff --git a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query51.out b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query51.out index 6c22d2df3086e3..98b43bfdfc171f 100644 --- a/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query51.out +++ b/regression-test/data/new_shapes_p0/tpcds_sf1000/shape/query51.out @@ -4,17 +4,17 @@ PhysicalResultSink --PhysicalTopN[MERGE_SORT] ----PhysicalDistribute[DistributionSpecGather] ------PhysicalTopN[LOCAL_SORT] ---------filter((web_cumulative > store_cumulative)) -----------PhysicalWindow -------------PhysicalQuickSort[LOCAL_SORT] ---------------PhysicalDistribute[DistributionSpecHash] -----------------PhysicalProject -------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +--------PhysicalProject +----------filter((web_cumulative > store_cumulative)) +------------PhysicalWindow +--------------PhysicalQuickSort[LOCAL_SORT] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalProject +--------------------hashJoin[FULL_OUTER_JOIN colocated] hashCondition=((web.d_date = store.d_date) and (web.item_sk = store.item_sk)) otherCondition=() +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] @@ -25,11 +25,10 @@ PhysicalResultSink ----------------------------------------PhysicalProject ------------------------------------------filter((date_dim.d_month_seq <= 1223) and (date_dim.d_month_seq >= 1212)) --------------------------------------------PhysicalOlapScan[date_dim] ---------------------PhysicalProject -----------------------PhysicalWindow -------------------------PhysicalQuickSort[LOCAL_SORT] ---------------------------PhysicalDistribute[DistributionSpecHash] -----------------------------PhysicalProject +----------------------PhysicalProject +------------------------PhysicalWindow +--------------------------PhysicalQuickSort[LOCAL_SORT] +----------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------hashAgg[GLOBAL] --------------------------------PhysicalDistribute[DistributionSpecHash] ----------------------------------hashAgg[LOCAL] From 565c42aaaf2eabd9cb1270ea62de7a1d8a89513c Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Wed, 3 Sep 2025 15:01:29 +0800 Subject: [PATCH 537/572] [opt](txn lazy commit) Make convert tmp rowsets batch more adaptive (#55035) (#55574) * default txn_lazy_max_rowsets_per_batch is 1000, when RowsetMetaCloudPB size exceeds `10MB / 1000`, fdb will refuse kv_txn, so we calculate batch size dynamically ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- cloud/src/common/config.h | 2 +- cloud/src/meta-service/txn_lazy_committer.cpp | 22 ++- cloud/test/txn_lazy_commit_test.cpp | 139 ++++++++++++++++++ 3 files changed, 155 insertions(+), 8 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index 6c3dd03c1da2a7..ffdf3837ee61d2 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -278,7 +278,7 @@ CONF_mInt64(max_txn_commit_byte, "7340032"); CONF_Bool(enable_cloud_txn_lazy_commit, "true"); CONF_Int32(txn_lazy_commit_rowsets_thresold, "1000"); CONF_Int32(txn_lazy_commit_num_threads, "8"); -CONF_Int32(txn_lazy_max_rowsets_per_batch, "1000"); +CONF_mInt64(txn_lazy_max_rowsets_per_batch, "1000"); // max TabletIndexPB num for batch get CONF_Int32(max_tablet_index_num_per_batch, "1000"); diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index 795f2f21cb75b7..aaff8e948566b8 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -315,7 +315,6 @@ void TxnLazyCommitTask::commit() { LOG(WARNING) << "scan_tmp_rowset failed, txn_id=" << txn_id_ << " code=" << code_; break; } - VLOG_DEBUG << "txn_id=" << txn_id_ << " tmp_rowset_metas.size()=" << all_tmp_rowset_metas.size(); if (all_tmp_rowset_metas.size() == 0) { @@ -325,23 +324,32 @@ void TxnLazyCommitTask::commit() { // std::map>> partition_to_tmp_rowset_metas; + size_t max_rowset_meta_size = 0; for (auto& [tmp_rowset_key, tmp_rowset_pb] : all_tmp_rowset_metas) { partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].emplace_back(); partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].back().first = tmp_rowset_key; partition_to_tmp_rowset_metas[tmp_rowset_pb.partition_id()].back().second = tmp_rowset_pb; + max_rowset_meta_size = std::max(max_rowset_meta_size, tmp_rowset_pb.ByteSizeLong()); + } + + // fdb txn limit 10MB, we use 4MB as the max size for each batch. + size_t max_rowsets_per_batch = config::txn_lazy_max_rowsets_per_batch; + if (max_rowset_meta_size > 0) { + max_rowsets_per_batch = std::min((4UL << 20) / max_rowset_meta_size, + size_t(config::txn_lazy_max_rowsets_per_batch)); + TEST_SYNC_POINT_CALLBACK("TxnLazyCommitTask::commit::max_rowsets_per_batch", + &max_rowsets_per_batch, &max_rowset_meta_size); } for (auto& [partition_id, tmp_rowset_metas] : partition_to_tmp_rowset_metas) { // tablet_id -> TabletIndexPB std::map tablet_ids; - for (size_t i = 0; i < tmp_rowset_metas.size(); - i += config::txn_lazy_max_rowsets_per_batch) { - size_t end = - (i + config::txn_lazy_max_rowsets_per_batch) > tmp_rowset_metas.size() - ? tmp_rowset_metas.size() - : i + config::txn_lazy_max_rowsets_per_batch; + for (size_t i = 0; i < tmp_rowset_metas.size(); i += max_rowsets_per_batch) { + size_t end = (i + max_rowsets_per_batch) > tmp_rowset_metas.size() + ? tmp_rowset_metas.size() + : i + max_rowsets_per_batch; std::vector> sub_partition_tmp_rowset_metas(tmp_rowset_metas.begin() + i, tmp_rowset_metas.begin() + end); diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 62f959167e64a9..5b1686eed28745 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -178,6 +178,33 @@ static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, return rowset; } +static doris::RowsetMetaCloudPB create_huge_rowset(int64_t txn_id, int64_t tablet_id, int index_id, + int partition_id, int64_t version = -1, + int num_rows = 100) { + doris::RowsetMetaCloudPB rowset; + rowset.set_rowset_id(0); // required + rowset.set_rowset_id_v2(next_rowset_id()); + rowset.set_tablet_id(tablet_id); + rowset.set_partition_id(partition_id); + rowset.set_index_id(index_id); + rowset.set_txn_id(txn_id); + if (version > 0) { + rowset.set_start_version(version); + rowset.set_end_version(version); + } + rowset.set_num_segments(600); + for (int i = 0; i < 600; i++) { + auto ptr = rowset.add_segments_key_bounds(); + ptr->set_min_key("xxsqewqeqweeqwewqeqeq"); + ptr->set_max_key("dase23452rr234ewdw534523"); + } + rowset.set_num_rows(0); + rowset.set_data_disk_size(0); + rowset.mutable_tablet_schema()->set_schema_version(0); + rowset.set_txn_expiration(::time(nullptr)); // Required by DCHECK + return rowset; +} + static void commit_rowset(MetaServiceProxy* meta_service, const doris::RowsetMetaCloudPB& rowset, CreateRowsetResponse& res) { brpc::Controller cntl; @@ -202,6 +229,18 @@ static std::shared_ptr get_mem_txn_kv() { return txn_kv; } +static std::shared_ptr get_fdb_txn_kv() { + int ret = 0; + cloud::config::fdb_cluster_file_path = "fdb.cluster"; + auto fdb_txn_kv = std::dynamic_pointer_cast(std::make_shared()); + if (fdb_txn_kv != nullptr) { + ret = fdb_txn_kv->init(); + [&] { ASSERT_EQ(ret, 0); }(); + } + [&] { ASSERT_NE(fdb_txn_kv.get(), nullptr); }(); + return fdb_txn_kv; +} + static void check_tablet_idx_db_id(std::unique_ptr& txn, int64_t db_id, int64_t tablet_id) { std::string mock_instance = "test_instance"; @@ -2949,4 +2988,104 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithMultiTableTest) { sp->disable_processing(); } +TEST(TxnLazyCommitTest, CommitTxnEventuallyWithHugeRowsetMetaTest) { + config::txn_lazy_max_rowsets_per_batch = 1000; + auto txn_kv = get_fdb_txn_kv(); + int64_t db_id = 14135425; + int64_t table_id = 31245456; + int64_t index_id = 434324; + int64_t partition_id = 3215764; + + int64_t table_id2 = 213476; + int64_t index_id2 = 126765; + int64_t partition_id2 = 214567; + bool commit_txn_eventually_finish_hit = false; + + auto sp = SyncPoint::get_instance(); + sp->set_call_back("commit_txn_eventually::task->wait", [&](auto&& args) { + auto [code, msg] = *try_any_cast*>(args[0]); + ASSERT_EQ(code, MetaServiceCode::OK); + commit_txn_eventually_finish_hit = true; + }); + + sp->set_call_back("TxnLazyCommitTask::commit::max_rowsets_per_batch", [&](auto&& args) { + size_t max_rowsets_per_batch = *try_any_cast(args[0]); + size_t max_rowset_meta_size = *try_any_cast(args[1]); + LOG(INFO) << "max_rowsets_per_batch:" << max_rowsets_per_batch + << " max_rowset_meta_size:" << max_rowset_meta_size; + ASSERT_EQ(max_rowsets_per_batch, 134); + }); + + sp->enable_processing(); + + auto meta_service = get_meta_service(txn_kv, true); + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label("test_label_multi_table_commit_txn"); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.add_table_ids(table_id2); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + int64_t txn_id = res.txn_id(); + + // mock rowset and tablet + int64_t tablet_id_base = 3131124; + for (int i = 0; i < 1000; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id, + tablet_id_base + i); + auto tmp_rowset = create_huge_rowset(txn_id, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + for (int i = 1000; i < 2000; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id2, index_id2, partition_id2, + tablet_id_base + i); + auto tmp_rowset = create_huge_rowset(txn_id, tablet_id_base + i, index_id2, partition_id2); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + { + brpc::Controller cntl; + CommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.set_txn_id(txn_id); + req.set_is_2pc(false); + req.set_enable_txn_lazy_commit(true); + CommitTxnResponse res; + meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, + &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + ASSERT_TRUE(commit_txn_eventually_finish_hit); + } + + { + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string mock_instance = "test_instance"; + for (int i = 0; i < 2000; ++i) { + int64_t tablet_id = tablet_id_base + i; + check_tablet_idx_db_id(txn, db_id, tablet_id); + check_tmp_rowset_not_exist(txn, tablet_id, txn_id); + check_rowset_meta_exist(txn, tablet_id, 2); + } + } + + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); + config::txn_lazy_max_rowsets_per_batch = 2; +} + } // namespace doris::cloud From 783b165ced23eae94b4601319501c7bd3f2c8a06 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 10:21:53 +0800 Subject: [PATCH 538/572] branch-3.0: [fix](docker) update kafka docker image to internal src #55460 (#55489) Cherry-picked from #55460 Co-authored-by: MoanasDaddyXu --- docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl b/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl index af5822c573fcbe..d2629694b1cd4d 100644 --- a/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl +++ b/docker/thirdparties/docker-compose/kafka/kafka.yaml.tpl @@ -25,7 +25,7 @@ networks: - subnet: 192.168.0.0/24 services: doris--zookeeper: - image: bitnami/zookeeper + image: doristhirdpartydocker/zookeeper restart: always container_name: doris--zookeeper ports: @@ -43,7 +43,7 @@ services: networks: - doris--kafka--network doris--kafka: - image: bitnami/kafka:2 + image: doristhirdpartydocker/kafka restart: always container_name: doris--kafka depends_on: From b3dc3a07a770e821bf3f4d37583032cfc91da1e5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 10:44:08 +0800 Subject: [PATCH 539/572] branch-3.0: [bugfix](qe) Fix query error in ssl mode #53134 (#55627) Cherry-picked from #53134 Co-authored-by: lw112 <131352377+felixwluo@users.noreply.github.com> --- .../org/apache/doris/mysql/MysqlChannel.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java index aa640c57eb70ac..4a32fc8db138d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java @@ -327,12 +327,20 @@ public ByteBuffer fetchOnePacket() throws IOException { readLen = readAll(result, false); if (isSslMode && remainingBuffer.position() == 0 && result.hasRemaining()) { byte[] header = result.array(); - int packetId = header[3] & 0xFF; - if (packetId != sequenceId) { - LOG.warn("receive packet sequence id[" + packetId() + "] want to get[" + sequenceId + "]"); - throw new IOException("Bad packet sequence."); + int mysqlPacketLength = (header[0] & 0xFF) | ((header[1] & 0xFF) << 8) | ((header[2] & 0xFF) << 16); + if (result.position() >= 4 && mysqlPacketLength > 0 && mysqlPacketLength + <= MAX_PHYSICAL_PACKET_LENGTH) { + int packetId = header[3] & 0xFF; + if (packetId != sequenceId) { + LOG.warn("receive packet sequence id[" + packetId + "] want to get[" + sequenceId + "]"); + throw new IOException("Bad packet sequence."); + } + } else { + if (LOG.isDebugEnabled()) { + LOG.debug("SSL mode: skipping sequence check, packet length: " + mysqlPacketLength + + ", buffer position: " + result.position()); + } } - int mysqlPacketLength = (header[0] & 0xFF) | ((header[1] & 0XFF) << 8) | ((header[2] & 0XFF) << 16); // remove mysql packet header result.position(4); result.compact(); From 1e973723dfa22bfbfba49ead67368428f27d9bbe Mon Sep 17 00:00:00 2001 From: yulihua Date: Thu, 4 Sep 2025 13:57:09 +0800 Subject: [PATCH 540/572] branch-3.0: [enhancement](backup) handle dropped tables and partitions during backup #52935 (#54976) fix unit test compile error for branch-3.0 ### What problem does this PR solve? Issue Number: close #52274 Related PR: #52935 Problem Summary: ### Release note ### Check List (For Author) - Test - [ ] Regression test - [X] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --------- Co-authored-by: ylh --- .../org/apache/doris/backup/BackupJob.java | 141 ++++++++++- .../org/apache/doris/backup/BackupMeta.java | 9 + .../apache/doris/backup/BackupJobTest.java | 234 +++++++++++++++++- .../doris/common/util/UnitTestUtil.java | 18 +- 4 files changed, 381 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 3b9c177297bc6f..1b42dbbaa9daca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -78,6 +78,8 @@ import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; @@ -133,6 +135,14 @@ public enum BackupJobState { @SerializedName("prop") private Map properties = Maps.newHashMap(); + // Record table IDs that were dropped during backup + @SerializedName("dt") + private Set droppedTables = ConcurrentHashMap.newKeySet(); + + // Record partition IDs that were dropped during backup (tableId -> set of partitionIds) + @SerializedName("dp") + private Map> droppedPartitionsByTable = Maps.newConcurrentMap(); + private long commitSeq = 0; public BackupJob() { @@ -236,6 +246,39 @@ private synchronized boolean tryNewTabletSnapshotTask(SnapshotTask task) { return true; } + private boolean handleTabletMissing(SnapshotTask task) { + LOG.info("handleTabletMissing task: {}", task); + Table table = env.getInternalCatalog().getTableByTableId(task.getTableId()); + if (table == null) { + // Table was dropped (including cases where database was dropped) + droppedTables.add(task.getTableId()); + LOG.info("table {} marked as dropped during backup. {}", task.getTableId(), this); + return true; + } + + if (!(table instanceof OlapTable)) { + return false; + } + + OlapTable olapTable = (OlapTable) table; + olapTable.readLock(); + try { + Partition partition = olapTable.getPartition(task.getPartitionId()); + if (partition == null) { + // Partition was dropped or truncated (partition ID changed) + droppedPartitionsByTable.computeIfAbsent(task.getTableId(), k -> ConcurrentHashMap.newKeySet()) + .add(task.getPartitionId()); + LOG.info("partition {} from table {} marked as dropped during backup (dropped or truncated). {}", + task.getPartitionId(), task.getTableId(), this); + return true; + } + + // If partition still exists, tablet missing is caused by other reasons + return false; + } finally { + olapTable.readUnlock(); + } + } public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishTaskRequest request) { Preconditions.checkState(task.getJobId() == jobId); @@ -250,11 +293,21 @@ public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishT cancelInternal(); } - if (request.getTaskStatus().getStatusCode() == TStatusCode.TABLET_MISSING - && !tryNewTabletSnapshotTask(task)) { - status = new Status(ErrCode.NOT_FOUND, - "make snapshot failed, failed to ge tablet, table will be dropped or truncated"); - cancelInternal(); + if (request.getTaskStatus().getStatusCode() == TStatusCode.TABLET_MISSING) { + if (handleTabletMissing(task)) { + // Successfully handled drop case, remove from task queue + taskProgress.remove(task.getSignature()); + taskErrMsg.remove(task.getSignature()); + Long oldValue = unfinishedTaskIds.remove(task.getSignature()); + return oldValue != null; + } else { + // Not caused by drop, follow original logic + if (!tryNewTabletSnapshotTask(task)) { + status = new Status(ErrCode.NOT_FOUND, + "make snapshot failed, failed to get tablet, table will be dropped or truncated"); + cancelInternal(); + } + } } if (request.getTaskStatus().getStatusCode() == TStatusCode.NOT_IMPLEMENTED_ERROR) { @@ -499,13 +552,18 @@ private void prepareAndSendSnapshotTask() { List
copiedTables = Lists.newArrayList(); List copiedResources = Lists.newArrayList(); AgentBatchTask batchTask = new AgentBatchTask(Config.backup_restore_batch_task_num_per_rpc); + // Track if we have any valid tables for backup + boolean hasValidTables = false; for (TableRef tableRef : tableRefs) { String tblName = tableRef.getName().getTbl(); Table tbl = db.getTableNullable(tblName); if (tbl == null) { - status = new Status(ErrCode.NOT_FOUND, "table " + tblName + " does not exist"); - return; + // Table was dropped, skip it and continue with other tables + LOG.info("table {} does not exist, it was dropped during backup preparation, skip it. {}", + tblName, this); + continue; } + hasValidTables = true; tbl.readLock(); try { switch (tbl.getType()) { @@ -539,7 +597,11 @@ private void prepareAndSendSnapshotTask() { return; } } - + // If no valid tables found, cancel the job + if (!hasValidTables) { + status = new Status(ErrCode.NOT_FOUND, "no valid tables found for backup"); + return; + } // Limit the max num of tablets involved in a backup job, to avoid OOM. if (unfinishedTaskIds.size() > Config.max_backup_tablets_per_job) { String msg = String.format("the num involved tablets %d exceeds the limit %d, " @@ -826,6 +888,43 @@ private void waitingAllUploadingFinished() { } } + private void cleanupDroppedTablesAndPartitions() { + if (backupMeta == null) { + return; + } + + // Remove dropped partitions first (before removing tables) + for (Map.Entry> entry : droppedPartitionsByTable.entrySet()) { + Long tableId = entry.getKey(); + Set droppedPartitionIds = entry.getValue(); + + Table table = backupMeta.getTable(tableId); + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + + // Directly get partitions by ID instead of iterating all partitions + for (Long droppedPartitionId : droppedPartitionIds) { + Partition partition = olapTable.getPartition(droppedPartitionId); + if (partition != null) { + LOG.info("remove dropped partition {} from table {} (id: {}) in backup meta. {}", + partition.getName(), table.getName(), tableId, this); + olapTable.dropPartitionAndReserveTablet(partition.getName()); + } + } + } + } + + // Remove dropped tables after processing partitions + for (Long tableId : droppedTables) { + Table removedTable = backupMeta.getTable(tableId); + if (removedTable != null) { + LOG.info("remove dropped table {} (id: {}) from backup meta. {}", + removedTable.getName(), tableId, this); + backupMeta.removeTable(tableId); + } + } + } + private void saveMetaInfo(boolean replay) { String createTimeStr = TimeUtils.longToTimeString(createTime, TimeUtils.getDatetimeFormatWithHyphenWithTimeZone()); @@ -847,7 +946,10 @@ private void saveMetaInfo(boolean replay) { return; } - // 2. save meta info file + // 2. Clean up dropped tables and partitions from backup metadata + cleanupDroppedTablesAndPartitions(); + + // 3. save meta info file File metaInfoFile = new File(jobDir, Repository.FILE_META_INFO); if (!metaInfoFile.createNewFile()) { status = new Status(ErrCode.COMMON_ERROR, @@ -857,7 +959,7 @@ private void saveMetaInfo(boolean replay) { backupMeta.writeToFile(metaInfoFile); localMetaInfoFilePath = metaInfoFile.getAbsolutePath(); - // 3. save job info file + // 4. save job info file Map tableCommitSeqMap = Maps.newHashMap(); // iterate properties, convert key, value from string to long // key is "${TABLE_COMMIT_SEQ_PREFIX}{tableId}", only need tableId to long @@ -870,8 +972,21 @@ private void saveMetaInfo(boolean replay) { tableCommitSeqMap.put(tableId, commitSeq); } } + // Filter out snapshot infos for dropped tables and partitions + Map filteredSnapshotInfos = Maps.newHashMap(); + for (Map.Entry entry : snapshotInfos.entrySet()) { + SnapshotInfo info = entry.getValue(); + boolean isDroppedTable = droppedTables.contains(info.getTblId()); + boolean isDroppedPartition = droppedPartitionsByTable.getOrDefault(info.getTblId(), + Collections.emptySet()).contains(info.getPartitionId()); + + if (!isDroppedTable && !isDroppedPartition) { + filteredSnapshotInfos.put(entry.getKey(), info); + } + } + jobInfo = BackupJobInfo.fromCatalog(createTime, label, dbName, dbId, - getContent(), backupMeta, snapshotInfos, tableCommitSeqMap); + getContent(), backupMeta, filteredSnapshotInfos, tableCommitSeqMap); if (LOG.isDebugEnabled()) { LOG.debug("job info: {}. {}", jobInfo, this); } @@ -904,6 +1019,10 @@ private void saveMetaInfo(boolean replay) { snapshotInfos.clear(); + // Clean up temporary records to reduce editlog size + droppedPartitionsByTable.clear(); + droppedTables.clear(); + // log env.getEditLog().logBackupJob(this); LOG.info("finished to save meta the backup job info file to local.[{}], [{}] {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index 0f1a043bdada3b..850d6f92e83c0f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -86,6 +86,15 @@ public Table getTable(Long tblId) { return tblIdMap.get(tblId); } + public boolean removeTable(Long tableId) { + Table removedTable = tblIdMap.remove(tableId); + if (removedTable != null) { + tblNameMap.remove(removedTable.getName()); + return true; + } + return false; + } + public static BackupMeta fromFile(String filePath, int metaVersion) throws IOException { return fromInputStream(new FileInputStream(filePath), metaVersion); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java index cbee4a8bcecd5b..621093200694f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.FsBroker; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; @@ -75,6 +76,7 @@ public class BackupJobTest { private BackupJob job; private Database db; + private OlapTable table2; private long dbId = 1; private long tblId = 2; @@ -84,6 +86,12 @@ public class BackupJobTest { private long backendId = 10000; private long version = 6; + private long tblId2 = 3; + private long partId2 = 4; + private long idxId2 = 5; + private long tabletId2 = 6; + private String table2Name = "testTable2"; + private long repoId = 20000; private AtomicLong id = new AtomicLong(50000); @@ -153,6 +161,10 @@ public void setUp() { Deencapsulation.setField(env, "backupHandler", backupHandler); db = UnitTestUtil.createDb(dbId, tblId, partId, idxId, tabletId, backendId, version); + + // Create second table in setUp to avoid Env initialization issues + table2 = UnitTestUtil.createTable(db, tblId2, table2Name, partId2, idxId2, tabletId2, backendId, version); + catalog = Deencapsulation.newInstance(InternalCatalog.class); new Expectations(env) { { @@ -164,13 +176,26 @@ public void setUp() { minTimes = 0; result = db; + catalog.getTableByTableId(anyLong); + minTimes = 0; + result = new Delegate
() { + public Table getTableByTableId(Long tableId) { + // Check if table exists in the database + return db.getTableNullable(tableId); + } + }; + Env.getCurrentEnvJournalVersion(); minTimes = 0; result = FeConstants.meta_version; env.getNextId(); minTimes = 0; - result = id.getAndIncrement(); + result = new Delegate() { + public Long getNextId() { + return id.getAndIncrement(); + } + }; env.getEditLog(); minTimes = 0; @@ -210,6 +235,7 @@ Status getBrokerAddress(Long beId, Env env, List brokerAddrs) { } }; + // Only include first table to ensure other tests are not affected List tableRefs = Lists.newArrayList(); tableRefs.add(new TableRef( new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, UnitTestUtil.TABLE_NAME), @@ -218,9 +244,20 @@ Status getBrokerAddress(Long beId, Env env, List brokerAddrs) { env, repo.getId(), 0); } + /** + * Test normal backup job execution flow + * + * Scenario: Backup a single table with all content + * Expected Results: + * 1. Job should progress through all states: PENDING -> SNAPSHOTING -> UPLOAD_SNAPSHOT -> UPLOADING -> SAVE_META -> UPLOAD_INFO -> FINISHED + * 2. Backup meta should contain the correct table information + * 3. Snapshot and upload tasks should be created and executed successfully + * 4. Meta files should be saved and uploaded correctly + * 5. Job should complete successfully with OK status + */ @Test public void testRunNormal() { - // 1.pending + // 1. pending Assert.assertEquals(BackupJobState.PENDING, job.getState()); job.run(); Assert.assertEquals(Status.OK, job.getStatus()); @@ -341,9 +378,18 @@ public void testRunNormal() { Assert.assertEquals(BackupJobState.FINISHED, job.getState()); } + /** + * Test backup job execution with non-existent table + * + * Scenario: Attempt to backup a table that does not exist + * Expected Results: + * 1. Job should fail with NOT_FOUND error code + * 2. Job state should be CANCELLED + * 3. No backup tasks should be created + */ @Test public void testRunAbnormal() { - // 1.pending + // 1. pending AgentTaskQueue.clearAllTasks(); List tableRefs = Lists.newArrayList(); @@ -357,6 +403,188 @@ public void testRunAbnormal() { Assert.assertEquals(BackupJobState.CANCELLED, job.getState()); } + /** + * Test backup job execution with mixed existing and non-existent tables + * + * Scenario: Backup two tables - one existing table and one non-existent table + * Expected Results: + * 1. Job should succeed and proceed to SNAPSHOTING state + * 2. Backup meta should only contain the existing table + * 3. Only snapshot tasks for the existing table should be created + * 4. Non-existent table should be skipped without causing job failure + */ + @Test + public void testRunAbnormalWithMixedTables() { + // Test backup two tables: one normal table and one non-existent table + // Verify backup succeeds, backs up the normal table, and skips the non-existent table + AgentTaskQueue.clearAllTasks(); + + List tableRefs = Lists.newArrayList(); + // Add normal table + tableRefs.add(new TableRef( + new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, UnitTestUtil.TABLE_NAME), + null)); + // Add non-existent table + tableRefs.add( + new TableRef(new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, "unknown_tbl"), + null)); + + job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, BackupStmt.BackupContent.ALL, + env, repo.getId(), 0); + + // 1. pending + Assert.assertEquals(BackupJobState.PENDING, job.getState()); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.SNAPSHOTING, job.getState()); + + // Verify backup meta only contains the normal table + BackupMeta backupMeta = job.getBackupMeta(); + Assert.assertEquals(1, backupMeta.getTables().size()); + OlapTable backupTbl = (OlapTable) backupMeta.getTable(UnitTestUtil.TABLE_NAME); + Assert.assertNotNull(backupTbl); + Assert.assertNull(backupMeta.getTable("unknown_tbl")); + + // Verify only snapshot tasks for the normal table are created + Assert.assertEquals(1, AgentTaskQueue.getTaskNum()); + AgentTask task = AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, id.get() - 1); + Assert.assertTrue(task instanceof SnapshotTask); + SnapshotTask snapshotTask = (SnapshotTask) task; + Assert.assertEquals(tblId, snapshotTask.getTableId()); + Assert.assertEquals(dbId, snapshotTask.getDbId()); + Assert.assertEquals(partId, snapshotTask.getPartitionId()); + Assert.assertEquals(idxId, snapshotTask.getIndexId()); + Assert.assertEquals(tabletId, snapshotTask.getTabletId()); + } + + /** + * Test backup job execution when a table is dropped during SNAPSHOTING phase + * + * Scenario: Start backup with two normal tables, then drop one table during SNAPSHOTING phase + * Expected Results: + * 1. Job should start with two tables and create snapshot tasks for both + * 2. When one table is dropped during SNAPSHOTING, the dropped table should be marked as dropped + * 3. Backup should continue successfully with only the remaining table + * 4. Final backup meta should only contain the non-dropped table + * 5. Job should complete successfully with FINISHED state + */ + @Test + public void testRunWithTableDroppedDuringSnapshoting() { + try { + AgentTaskQueue.clearAllTasks(); + + List tableRefs = Lists.newArrayList(); + tableRefs.add(new TableRef( + new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, UnitTestUtil.TABLE_NAME), + null)); + tableRefs.add(new TableRef( + new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, table2Name), + null)); + + job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, BackupStmt.BackupContent.ALL, + env, repo.getId(), 0); + + // 1. pending - should create snapshot tasks for both tables + Assert.assertEquals(BackupJobState.PENDING, job.getState()); + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.SNAPSHOTING, job.getState()); + + // Verify backup meta contains both tables initially + BackupMeta backupMeta = job.getBackupMeta(); + Assert.assertEquals(2, backupMeta.getTables().size()); + Assert.assertNotNull(backupMeta.getTable(UnitTestUtil.TABLE_NAME)); + Assert.assertNotNull(backupMeta.getTable(table2Name)); + + // Verify snapshot tasks are created for both tables + Assert.assertEquals(2, AgentTaskQueue.getTaskNum()); + + // 2. Simulate dropping the second table during SNAPSHOTING phase + db.unregisterTable(table2Name); + + // 3. Finish snapshot tasks + SnapshotTask taskForDroppedTable = null; + SnapshotTask taskForExistingTable = null; + long taskTabletId1 = AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, id.get() - 2).getTabletId(); + if (taskTabletId1 == tabletId) { + taskForExistingTable = (SnapshotTask) AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, id.get() - 2); + taskForDroppedTable = (SnapshotTask) AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, id.get() - 1); + } else { + taskForDroppedTable = (SnapshotTask) AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, id.get() - 2); + taskForExistingTable = (SnapshotTask) AgentTaskQueue.getTask(backendId, TTaskType.MAKE_SNAPSHOT, id.get() - 1); + } + + TBackend tBackend = new TBackend("", 0, 1); + + // Finish task for dropped table + TStatus taskStatusMissing = new TStatus(TStatusCode.TABLET_MISSING); + taskStatusMissing.setErrorMsgs(Lists.newArrayList("Tablet missing")); + TFinishTaskRequest requestMissing = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, + taskForDroppedTable.getSignature(), taskStatusMissing); + Assert.assertTrue(job.finishTabletSnapshotTask(taskForDroppedTable, requestMissing)); + + // Finish task for existing table + String snapshotPath = "/path/to/snapshot"; + List snapshotFiles = Lists.newArrayList("1.dat", "1.idx", "1.hdr"); + TStatus taskStatusOK = new TStatus(TStatusCode.OK); + TFinishTaskRequest requestOK = new TFinishTaskRequest(tBackend, TTaskType.MAKE_SNAPSHOT, + taskForExistingTable.getSignature(), taskStatusOK); + requestOK.setSnapshotFiles(snapshotFiles); + requestOK.setSnapshotPath(snapshotPath); + Assert.assertTrue(job.finishTabletSnapshotTask(taskForExistingTable, requestOK)); + + // 4. Continue the backup process + job.run(); + Assert.assertEquals(Status.OK, job.getStatus()); + Assert.assertEquals(BackupJobState.UPLOAD_SNAPSHOT, job.getState()); + + AgentTaskQueue.clearAllTasks(); + job.run(); // UPLOAD_SNAPSHOT -> UPLOADING + Assert.assertEquals(1, AgentTaskQueue.getTaskNum()); + UploadTask upTask = (UploadTask) AgentTaskQueue.getTask(backendId, TTaskType.UPLOAD, id.get() - 1); + + // Finish upload task + Map> tabletFileMap = Maps.newHashMap(); + List tabletFiles = Lists.newArrayList(); + tabletFiles.add("1.dat.4f158689243a3d6030352fec3cfd3798"); + tabletFiles.add("1.idx.4f158689243a3d6030352fec3cfd3798"); + tabletFiles.add("1.hdr.4f158689243a3d6030352fec3cfd3798"); + tabletFileMap.put(taskForExistingTable.getTabletId(), tabletFiles); + TFinishTaskRequest requestUpload = new TFinishTaskRequest(tBackend, TTaskType.UPLOAD, + upTask.getSignature(), taskStatusOK); + requestUpload.setTabletFiles(tabletFileMap); + Assert.assertTrue(job.finishSnapshotUploadTask(upTask, requestUpload)); + + job.run(); // UPLOADING -> SAVE_META + Assert.assertEquals(BackupJobState.SAVE_META, job.getState()); + + job.run(); // SAVE_META -> UPLOAD_INFO + Assert.assertEquals(BackupJobState.UPLOAD_INFO, job.getState()); + + job.run(); // UPLOAD_INFO -> FINISHED + Assert.assertEquals(BackupJobState.FINISHED, job.getState()); + + } catch (Throwable e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } finally { + // Clean up: re-register the second table if it was removed + if (db.getTableNullable(table2Name) == null && table2 != null) { + db.registerTable(table2); + } + } + } + + /** + * Test backup job serialization and deserialization + * + * Scenario: Write backup job to file and read it back + * Expected Results: + * 1. Backup job should be successfully written to file + * 2. Backup job should be successfully read from file + * 3. All job properties should be preserved during serialization/deserialization + * 4. Temporary files should be cleaned up + */ @Test public void testSerialization() throws IOException, AnalysisException { // 1. Write objects to file diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java index 9963479f6b846c..0c85a2e7655688 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/UnitTestUtil.java @@ -66,8 +66,14 @@ public class UnitTestUtil { public static Database createDb(long dbId, long tableId, long partitionId, long indexId, long tabletId, long backendId, long version) { - // Catalog.getCurrentInvertedIndex().clear(); + Database db = new Database(dbId, DB_NAME); + createTable(db, tableId, TABLE_NAME, partitionId, indexId, tabletId, backendId, version); + + return db; + } + public static OlapTable createTable(Database db, long tableId, String tableName, long partitionId, long indexId, + long tabletId, long backendId, long version) { // replica long replicaId = 0; Replica replica1 = new Replica(replicaId, backendId, ReplicaState.NORMAL, version, 0); @@ -79,7 +85,7 @@ public static Database createDb(long dbId, long tableId, long partitionId, long // index MaterializedIndex index = new MaterializedIndex(indexId, IndexState.NORMAL); - TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, 0, TStorageMedium.HDD); + TabletMeta tabletMeta = new TabletMeta(db.getId(), tableId, partitionId, indexId, 0, TStorageMedium.HDD); index.addTablet(tablet, tabletMeta); tablet.addReplica(replica1); @@ -115,17 +121,15 @@ public static Database createDb(long dbId, long tableId, long partitionId, long partitionInfo.setIsInMemory(partitionId, false); partitionInfo.setIsMutable(partitionId, true); partitionInfo.setTabletType(partitionId, TTabletType.TABLET_TYPE_DISK); - OlapTable table = new OlapTable(tableId, TABLE_NAME, columns, + OlapTable table = new OlapTable(tableId, tableName, columns, KeysType.AGG_KEYS, partitionInfo, distributionInfo); Deencapsulation.setField(table, "baseIndexId", indexId); table.addPartition(partition); - table.setIndexMeta(indexId, TABLE_NAME, columns, 0, SCHEMA_HASH, (short) 1, TStorageType.COLUMN, + table.setIndexMeta(indexId, tableName, columns, 0, SCHEMA_HASH, (short) 1, TStorageType.COLUMN, KeysType.AGG_KEYS); - // db - Database db = new Database(dbId, DB_NAME); db.registerTable(table); - return db; + return table; } public static Backend createBackend(long id, String host, int heartPort, int bePort, int httpPort) { From 6c279262303b0ebfe1ad8ab420eb51bea7d74a3e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 14:21:55 +0800 Subject: [PATCH 541/572] branch-3.0: [fix](compaction) skip tablets with compaction score 0 during compaction #55550 (#55569) Cherry-picked from #55550 Co-authored-by: Luwei <814383175@qq.com> --- be/src/olap/tablet_manager.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index b6b2d3b2293fc2..556ef14076b9e1 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -804,6 +804,10 @@ std::vector TabletManager::find_best_tablets_to_compaction( tablet_ptr->set_skip_compaction(true, compaction_type, UnixSeconds()); } + if (current_compaction_score <= 0) { + return; + } + // tablet should do single compaction if (current_compaction_score > single_compact_highest_score && tablet_ptr->should_fetch_from_peer()) { From 0b04cc9fe2311a77aaeb1eedd0a73ca836e3cdef Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 4 Sep 2025 15:22:28 +0800 Subject: [PATCH 542/572] branch-3.0: [feature](nereids) support explain command return empty string to test planner performance (#54422) (#55038) cherry pick from #54422 --- .../apache/doris/nereids/NereidsPlanner.java | 4 ++++ .../org/apache/doris/planner/Planner.java | 5 +++++ .../org/apache/doris/qe/SessionVariable.java | 8 ++++++++ .../trees/plans/ExplainInsertCommandTest.java | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 584b0e67c1330f..ca2b17ac77543c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -653,6 +653,10 @@ public String getHintExplainString(List hints) { @Override public String getExplainString(ExplainOptions explainOptions) { + ConnectContext context = cascadesContext == null ? ConnectContext.get() : cascadesContext.getConnectContext(); + if (context.getSessionVariable().enableExplainNone) { + return ""; + } ExplainLevel explainLevel = getExplainLevel(explainOptions); String plan = ""; String mvSummary = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index cfcd27af8fa5d2..717558407266ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -24,6 +24,7 @@ import org.apache.doris.common.profile.PlanTreeBuilder; import org.apache.doris.common.profile.PlanTreePrinter; import org.apache.doris.nereids.trees.plans.physical.TopnFilter; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ResultSet; import org.apache.doris.thrift.TQueryOptions; @@ -53,6 +54,10 @@ public abstract void plan(StatementBase queryStmt, public String getExplainString(ExplainOptions explainOptions) { Preconditions.checkNotNull(explainOptions); + ConnectContext connectContext = ConnectContext.get(); + if (connectContext != null && connectContext.getSessionVariable().enableExplainNone) { + return ""; + } if (explainOptions.isGraph()) { // print the plan graph PlanTreeBuilder builder = new PlanTreeBuilder(fragments); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index e05ca844f58960..3bc7c92ca8bbf6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -2216,6 +2216,8 @@ public void setIgnoreRuntimeFilterIds(String ignoreRuntimeFilterIds) { public static final String IGNORE_SHAPE_NODE = "ignore_shape_nodes"; + public static final String ENABLE_EXPLAIN_NONE = "enable_explain_none"; + public static final String DETAIL_SHAPE_NODES = "detail_shape_nodes"; public static final String ENABLE_SEGMENT_CACHE = "enable_segment_cache"; @@ -2238,6 +2240,12 @@ public void setIgnoreShapePlanNodes(String ignoreShapePlanNodes) { "the plan node type show detail in 'explain shape plan' command"}) public String detailShapePlanNodes = ""; + @VariableMgr.VarAttr(name = ENABLE_EXPLAIN_NONE, needForward = true, description = { + "执行explain命令，但不打印explain结果", + "execute explain command and return nothing" + }) + public boolean enableExplainNone = false; + private Set detailShapePlanNodesSet = ImmutableSet.of(); public Set getDetailShapePlanNodesSet() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java index d4756746e98953..94e5bae2c75a0c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.trees.plans; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -26,10 +27,12 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.planner.PlanFragment; +import org.apache.doris.qe.StmtExecutor; import org.apache.doris.utframe.TestWithFeService; import org.junit.jupiter.api.Assertions; @@ -134,6 +137,22 @@ public void testWithMV() throws Exception { Assertions.assertEquals(8, getOutputFragment(sql).getOutputExprs().size()); } + @Test + public void explainNone() throws Exception { + String sql = "explain insert into agg_have_dup_base select -4, -4, -4, 'd'"; + connectContext.getSessionVariable().enableExplainNone = false; + StmtExecutor sqlStmtExecutor = getSqlStmtExecutor(sql); + String explainString = sqlStmtExecutor.planner() + .getExplainString(new ExplainOptions(ExplainLevel.VERBOSE, false)); + Assertions.assertNotEquals("", explainString); + + connectContext.getSessionVariable().enableExplainNone = true; + sqlStmtExecutor = getSqlStmtExecutor(sql); + explainString = sqlStmtExecutor.planner() + .getExplainString(new ExplainOptions(ExplainLevel.VERBOSE, false)); + Assertions.assertEquals("", explainString); + } + private PlanFragment getOutputFragment(String sql) throws Exception { StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); From dd64463a37e2e96848bfeb1cf8cd048352e739b2 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 4 Sep 2025 15:23:15 +0800 Subject: [PATCH 543/572] branch-3.0: [opt](nereids) support simplify string range (#55378) (#55454) cherry pick from #55378 --- .../nereids/rules/expression/rules/SimplifyRange.java | 7 +++++-- .../doris/nereids/rules/expression/SimplifyRangeTest.java | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java index 434f7a6f5bda41..4cf3e57e38e64f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java @@ -38,6 +38,7 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.BooleanType; +import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.ExpressionUtils; import com.google.common.collect.BoundType; @@ -113,8 +114,10 @@ private ValueDesc buildRange(ExpressionRewriteContext context, ComparisonPredica if (right.isNullLiteral()) { return new UnknownValue(context, predicate); } - // only handle `NumericType` and `DateLikeType` - if (right.isLiteral() && (right.getDataType().isNumericType() || right.getDataType().isDateLikeType())) { + // only handle `NumericType` and `DateLikeType` and `StringLikeType` + DataType rightDataType = right.getDataType(); + if (right.isLiteral() && (rightDataType.isNumericType() || rightDataType.isDateLikeType() + || rightDataType.isStringLikeType())) { return ValueDesc.range(context, predicate); } return new UnknownValue(context, predicate); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java index ca2cab9905b1e9..899f342f8a3f54 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/SimplifyRangeTest.java @@ -198,6 +198,8 @@ public void testSimplify() { assertRewrite("(TA + TC > 3 and TA + TC < 1) or TB < 5", "((TA + TC) is null and null) OR TB < 5"); assertRewrite("(TA + TC > 3 OR TA < 1) AND TB = 2) AND IA =1", "(TA + TC > 3 OR TA < 1) AND TB = 2) AND IA =1"); + assertRewrite("SA = '20250101' and SA < '20200101'", "SA is null and null"); + assertRewrite("SA > '20250101' and SA > '20260110'", "SA > '20260110'"); } From 51f64494b4f7ff2569af668fabb310b054b649d3 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 4 Sep 2025 15:27:40 +0800 Subject: [PATCH 544/572] branch-3.0: [Fix](mow) Fix mow coreudmp in `BaseTablet::get_rowset_by_ids()` (#55539) (#55639) pick https://github.com/apache/doris/pull/55539 --- be/src/cloud/cloud_rowset_builder.cpp | 4 ++-- be/src/olap/olap_common.h | 6 +++--- be/src/olap/rowset/beta_rowset_writer.cpp | 5 +++-- be/src/olap/rowset/segment_v2/segment_writer.cpp | 2 +- .../rowset/segment_v2/vertical_segment_writer.cpp | 2 +- be/src/olap/rowset_builder.cpp | 14 +++++++------- be/src/olap/rowset_builder.h | 2 +- be/test/olap/segcompaction_mow_test.cpp | 8 ++++---- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/be/src/cloud/cloud_rowset_builder.cpp b/be/src/cloud/cloud_rowset_builder.cpp index b495ae953caf7e..3da6a55aa44241 100644 --- a/be/src/cloud/cloud_rowset_builder.cpp +++ b/be/src/cloud/cloud_rowset_builder.cpp @@ -126,7 +126,7 @@ Status CloudRowsetBuilder::set_txn_related_delete_bitmap() { if (_tablet->enable_unique_key_merge_on_write()) { if (config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0) { auto st = _tablet->check_delete_bitmap_correctness( - _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, _rowset_ids); + _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids); if (!st.ok()) { LOG(WARNING) << fmt::format( "[tablet_id:{}][txn_id:{}][load_id:{}][partition_id:{}] " @@ -137,7 +137,7 @@ Status CloudRowsetBuilder::set_txn_related_delete_bitmap() { } } _engine.txn_delete_bitmap_cache().set_tablet_txn_info( - _req.txn_id, _tablet->tablet_id(), _delete_bitmap, _rowset_ids, _rowset, + _req.txn_id, _tablet->tablet_id(), _delete_bitmap, *_rowset_ids, _rowset, _req.txn_expiration, _partial_update_info); } return Status::OK(); diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index f792e3ac6f90f2..264250636da2f7 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -540,16 +540,16 @@ inline RowsetId extract_rowset_id(std::string_view filename) { class DeleteBitmap; // merge on write context struct MowContext { - MowContext(int64_t version, int64_t txnid, const RowsetIdUnorderedSet& ids, + MowContext(int64_t version, int64_t txnid, std::shared_ptr ids, std::vector rowset_ptrs, std::shared_ptr db) : max_version(version), txn_id(txnid), - rowset_ids(ids), + rowset_ids(std::move(ids)), rowset_ptrs(std::move(rowset_ptrs)), delete_bitmap(std::move(db)) {} int64_t max_version; int64_t txn_id; - const RowsetIdUnorderedSet& rowset_ids; + std::shared_ptr rowset_ids; std::vector rowset_ptrs; std::shared_ptr delete_bitmap; }; diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index e230a25c874bad..a71eba21acbc77 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -326,7 +326,8 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { std::vector specified_rowsets; { std::shared_lock meta_rlock(_context.tablet->get_header_lock()); - specified_rowsets = _context.tablet->get_rowset_by_ids(&_context.mow_context->rowset_ids); + specified_rowsets = + _context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get()); } OlapStopWatch watch; RETURN_IF_ERROR(BaseTablet::calc_delete_bitmap( @@ -336,7 +337,7 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) { segments.begin(), segments.end(), 0, [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); }); LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id() - << ", rowset_ids: " << _context.mow_context->rowset_ids.size() + << ", rowset_ids: " << _context.mow_context->rowset_ids->size() << ", cur max_version: " << _context.mow_context->max_version << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: " << _context.mow_context->delete_bitmap->get_delete_bitmap_count() diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index be6c1dc45b8c81..8fd68b171d0961 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -687,7 +687,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* if (config::enable_merge_on_write_correctness_check) { _tablet->add_sentinel_mark_to_delete_bitmap(_mow_context->delete_bitmap.get(), - _mow_context->rowset_ids); + *_mow_context->rowset_ids); } // read and fill block diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 6339c7db4bfc63..7bdc2aa3c2120f 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -548,7 +548,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da if (config::enable_merge_on_write_correctness_check) { _tablet->add_sentinel_mark_to_delete_bitmap(_mow_context->delete_bitmap.get(), - _mow_context->rowset_ids); + *_mow_context->rowset_ids); } // read and fill block diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index dfe4da0d55fdd2..0cd0df391be51c 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -135,11 +135,11 @@ Status BaseRowsetBuilder::init_mow_context(std::shared_ptr& mow_cont "Unable to do 'partial_update' when " "the tablet is undergoing a 'schema changing process'"); } - _rowset_ids.clear(); + _rowset_ids->clear(); } else { RETURN_IF_ERROR( - tablet()->get_all_rs_id_unlocked(_max_version_in_flush_phase, &_rowset_ids)); - rowset_ptrs = tablet()->get_rowset_by_ids(&_rowset_ids); + tablet()->get_all_rs_id_unlocked(_max_version_in_flush_phase, _rowset_ids.get())); + rowset_ptrs = tablet()->get_rowset_by_ids(_rowset_ids.get()); } _delete_bitmap = std::make_shared(tablet()->tablet_id()); mow_context = std::make_shared(_max_version_in_flush_phase, _req.txn_id, @@ -289,7 +289,7 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() { "partial update calc delete bitmap summary before commit: tablet({}), txn_id({}), " "rowset_ids({}), cur max_version({}), bitmap num({}), bitmap_cardinality({}), num " "rows updated({}), num rows new added({}), num rows deleted({}), total rows({})", - tablet()->tablet_id(), _req.txn_id, _rowset_ids.size(), + tablet()->tablet_id(), _req.txn_id, _rowset_ids->size(), rowset_writer()->context().mow_context->max_version, _delete_bitmap->get_delete_bitmap_count(), _delete_bitmap->cardinality(), rowset_writer()->num_rows_updated(), rowset_writer()->num_rows_new_added(), @@ -299,7 +299,7 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task() { LOG(INFO) << "submit calc delete bitmap task to executor, tablet_id: " << tablet()->tablet_id() << ", txn_id: " << _req.txn_id; - return BaseTablet::commit_phase_update_delete_bitmap(_tablet, _rowset, _rowset_ids, + return BaseTablet::commit_phase_update_delete_bitmap(_tablet, _rowset, *_rowset_ids, _delete_bitmap, segments, _req.txn_id, _calc_delete_bitmap_token.get(), nullptr); } @@ -319,7 +319,7 @@ Status RowsetBuilder::commit_txn() { config::enable_merge_on_write_correctness_check && _rowset->num_rows() != 0 && tablet()->tablet_state() != TABLET_NOTREADY) { auto st = tablet()->check_delete_bitmap_correctness( - _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, _rowset_ids); + _delete_bitmap, _rowset->end_version() - 1, _req.txn_id, *_rowset_ids); if (!st.ok()) { LOG(WARNING) << fmt::format( "[tablet_id:{}][txn_id:{}][load_id:{}][partition_id:{}] " @@ -364,7 +364,7 @@ Status RowsetBuilder::commit_txn() { if (_tablet->enable_unique_key_merge_on_write()) { _engine.txn_manager()->set_txn_related_delete_bitmap( _req.partition_id, _req.txn_id, tablet()->tablet_id(), tablet()->tablet_uid(), true, - _delete_bitmap, _rowset_ids, _partial_update_info); + _delete_bitmap, *_rowset_ids, _partial_update_info); } _is_committed = true; diff --git a/be/src/olap/rowset_builder.h b/be/src/olap/rowset_builder.h index d87e2a9efa4a8f..8da8fe8ad60897 100644 --- a/be/src/olap/rowset_builder.h +++ b/be/src/olap/rowset_builder.h @@ -104,7 +104,7 @@ class BaseRowsetBuilder { DeleteBitmapPtr _delete_bitmap; std::unique_ptr _calc_delete_bitmap_token; // current rowset_ids, used to do diff in publish_version - RowsetIdUnorderedSet _rowset_ids; + std::shared_ptr _rowset_ids {std::make_shared()}; int64_t _max_version_in_flush_phase {-1}; std::shared_ptr _partial_update_info; diff --git a/be/test/olap/segcompaction_mow_test.cpp b/be/test/olap/segcompaction_mow_test.cpp index efe40dcb859bca..18a0f241a2a95b 100644 --- a/be/test/olap/segcompaction_mow_test.cpp +++ b/be/test/olap/segcompaction_mow_test.cpp @@ -314,7 +314,7 @@ TEST_P(SegCompactionMoWTest, SegCompactionThenRead) { RowsetWriterContext writer_context; int raw_rsid = rand(); create_rowset_writer_context(raw_rsid, tablet_schema, &writer_context); - RowsetIdUnorderedSet rsids; + std::shared_ptr rsids {std::make_shared()}; std::vector rowset_ptrs; writer_context.mow_context = std::make_shared(1, 1, rsids, rowset_ptrs, delete_bitmap); @@ -417,7 +417,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_ooooOOoOooooooooO) { { // write `num_segments * rows_per_segment` rows to rowset RowsetWriterContext writer_context; create_rowset_writer_context(20048, tablet_schema, &writer_context); - RowsetIdUnorderedSet rsids; + std::shared_ptr rsids {std::make_shared()}; std::vector rowset_ptrs; writer_context.mow_context = std::make_shared(1, 1, rsids, rowset_ptrs, delete_bitmap); @@ -648,7 +648,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionInterleaveWithBig_OoOoO) { { // write `num_segments * rows_per_segment` rows to rowset RowsetWriterContext writer_context; create_rowset_writer_context(20049, tablet_schema, &writer_context); - RowsetIdUnorderedSet rsids; + std::shared_ptr rsids {std::make_shared()}; std::vector rowset_ptrs; writer_context.mow_context = std::make_shared(1, 1, rsids, rowset_ptrs, delete_bitmap); @@ -838,7 +838,7 @@ TEST_F(SegCompactionMoWTest, SegCompactionNotTrigger) { { // write `num_segments * rows_per_segment` rows to rowset RowsetWriterContext writer_context; create_rowset_writer_context(20050, tablet_schema, &writer_context); - RowsetIdUnorderedSet rsids; + std::shared_ptr rsids {std::make_shared()}; std::vector rowset_ptrs; writer_context.mow_context = std::make_shared(1, 1, rsids, rowset_ptrs, delete_bitmap); From 5f1032661c96f33e78ba4293675ad553af92fb04 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:36:01 +0800 Subject: [PATCH 545/572] branch-3.0: [fix](case) fix case: variant build index #55613 (#55640) Cherry-picked from #55613 Co-authored-by: Sun Chenyang --- .../variant_github_events_nonConcurrent_p2/load.groovy | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy index 1aac1d1d94dbc0..46a7ae0f83f414 100644 --- a/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy +++ b/regression-test/suites/variant_github_events_nonConcurrent_p2/load.groovy @@ -179,10 +179,13 @@ suite("regression_test_variant_github_events_p2", "nonConcurrent,p2"){ load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2022-11-07-23.json'}""") // BUILD INDEX - test { + if (!isCloudMode()) { + test { sql """ BUILD INDEX idx_var ON github_events""" - exception "The idx_var index can not be built on the v column, because it is a variant type column" + exception "The idx_var index can not be built on the v column, because it is a variant type column" + } } + // // add bloom filter at the end of loading data From 5593057f427bd4bea3d19689acdf062651e52258 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:36:49 +0800 Subject: [PATCH 546/572] branch-3.0: [chore](sk) Encrypt `secret key` and hide `access key` for log #55241 (#55618) Cherry-picked from #55241 Co-authored-by: Uniqueyou --- be/src/cloud/cloud_meta_mgr.cpp | 9 ++ be/src/util/s3_util.cpp | 26 ++++ be/src/util/s3_util.h | 8 +- be/test/util/s3_util_test.cpp | 79 ++++++++++++ cloud/src/meta-service/meta_service.cpp | 26 ++++ cloud/src/meta-service/meta_service_helper.h | 84 +++++++++---- cloud/src/meta-service/meta_service_http.cpp | 11 +- .../meta-service/meta_service_resource.cpp | 13 +- .../src/resource-manager/resource_manager.cpp | 6 +- cloud/test/meta_service_http_test.cpp | 118 ++++++++++++++++++ .../cloud/system/CloudSystemInfoService.java | 57 +++++++++ 11 files changed, 405 insertions(+), 32 deletions(-) create mode 100644 be/test/util/s3_util_test.cpp diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index efa4ce92aa3a79..904e8e2e099405 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -1219,6 +1219,15 @@ Status CloudMetaMgr::get_storage_vault_info(StorageVaultInfos* vault_infos, bool j->mutable_obj_info()->set_sk(j->obj_info().sk().substr(0, 2) + "xxx"); } + for (int i = 0; i < resp.obj_info_size(); ++i) { + resp.mutable_obj_info(i)->set_ak(hide_access_key(resp.obj_info(i).sk())); + } + for (int i = 0; i < resp.storage_vault_size(); ++i) { + auto* j = resp.mutable_storage_vault(i); + if (!j->has_obj_info()) continue; + j->mutable_obj_info()->set_sk(hide_access_key(j->obj_info().sk())); + } + LOG(INFO) << "get storage vault, enable_storage_vault=" << *is_vault_mode << " response=" << resp.ShortDebugString(); return Status::OK(); diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 174b6f8a5e6d59..d4c4e4e3d342cd 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -574,4 +574,30 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) { return ret; } +std::string hide_access_key(const std::string& ak) { + std::string key = ak; + size_t key_len = key.length(); + size_t reserved_count; + if (key_len > 7) { + reserved_count = 6; + } else if (key_len > 2) { + reserved_count = key_len - 2; + } else { + reserved_count = 0; + } + + size_t x_count = key_len - reserved_count; + size_t left_x_count = (x_count + 1) / 2; + + if (left_x_count > 0) { + key.replace(0, left_x_count, left_x_count, 'x'); + } + + if (x_count - left_x_count > 0) { + key.replace(key_len - (x_count - left_x_count), x_count - left_x_count, + x_count - left_x_count, 'x'); + } + return key; +} + } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 8b96fb0776b4cd..7f72bd3af26c30 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -61,6 +61,8 @@ extern bvar::LatencyRecorder s3_get_bucket_version_latency; extern bvar::LatencyRecorder s3_copy_object_latency; }; // namespace s3_bvar +std::string hide_access_key(const std::string& ak); + class S3URI; struct S3ClientConf { std::string endpoint; @@ -107,9 +109,9 @@ struct S3ClientConf { "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " "cred_provider_type={},role_arn={}, external_id={}", - ak, token, endpoint, region, bucket, max_connections, request_timeout_ms, - connect_timeout_ms, use_virtual_addressing, cred_provider_type, role_arn, - external_id); + hide_access_key(ak), token, endpoint, region, bucket, max_connections, + request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, + role_arn, external_id); } }; diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp new file mode 100644 index 00000000000000..cd8eeb5cb68027 --- /dev/null +++ b/be/test/util/s3_util_test.cpp @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/s3_util.h" + +#include + +#include + +#include "gtest/gtest_pred_impl.h" +#include "util/s3_uri.h" + +namespace doris { + +class S3UTILTest : public testing::Test { +public: + S3UTILTest() = default; + ~S3UTILTest() = default; +}; // end class S3UTILTest + +TEST_F(S3UTILTest, hide_access_key_empty) { + EXPECT_EQ("", hide_access_key("")); +} + +TEST_F(S3UTILTest, hide_access_key_single_char) { + EXPECT_EQ("x", hide_access_key("A")); +} + +TEST_F(S3UTILTest, hide_access_key_two_chars) { + EXPECT_EQ("xx", hide_access_key("AB")); +} + +TEST_F(S3UTILTest, hide_access_key_three_chars) { + EXPECT_EQ("xBx", hide_access_key("ABC")); +} + +TEST_F(S3UTILTest, hide_access_key_four_chars) { + EXPECT_EQ("xBCx", hide_access_key("ABCD")); +} + +TEST_F(S3UTILTest, hide_access_key_six_chars) { + EXPECT_EQ("xBCDEx", hide_access_key("ABCDEF")); +} + +TEST_F(S3UTILTest, hide_access_key_seven_chars) { + EXPECT_EQ("xBCDEFx", hide_access_key("ABCDEFG")); +} + +TEST_F(S3UTILTest, hide_access_key_normal_length) { + EXPECT_EQ("xxxDEFGHIxxx", hide_access_key("ABCDEFGHIJKL")); +} + +TEST_F(S3UTILTest, hide_access_key_long_key) { + std::string long_key = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; + std::string result = hide_access_key(long_key); + EXPECT_EQ("xxxxxxxxxxxxxxxPQRSTUxxxxxxxxxxxxxxx", result); +} + +TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { + std::string aws_key = "AKIAIOSFODNN7EXAMPLE"; + std::string result = hide_access_key(aws_key); + EXPECT_EQ("xxxxxxxFODNN7xxxxxxx", result); +} + +} // end namespace doris diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 533d01afb02e8d..2fa746e5f69f18 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -3112,4 +3112,30 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control response->mutable_schema_dict()->Swap(&schema_dict); } +std::string hide_access_key(const std::string& ak) { + std::string key = ak; + size_t key_len = key.length(); + size_t reserved_count; + if (key_len > 6) { + reserved_count = 6; + } else if (key_len > 2) { + reserved_count = key_len - 2; + } else { + reserved_count = 0; + } + + size_t x_count = key_len - reserved_count; + size_t left_x_count = x_count / 2; + + if (left_x_count > 0) { + key.replace(0, left_x_count, left_x_count, 'x'); + } + + if (x_count - left_x_count > 0) { + key.replace(key_len - (x_count - left_x_count), x_count - left_x_count, + x_count - left_x_count, 'x'); + } + return key; +} + } // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service_helper.h b/cloud/src/meta-service/meta_service_helper.h index c35873ed94933f..38f32b2d4f8ea9 100644 --- a/cloud/src/meta-service/meta_service_helper.h +++ b/cloud/src/meta-service/meta_service_helper.h @@ -54,6 +54,45 @@ inline std::string md5(const std::string& str) { return ss.str(); } +std::string hide_access_key(const std::string& ak); + +// is_handle_sk: true for encrypting sk, false for hiding ak +inline void process_ak_sk_pattern(std::string& str, const std::string& pattern, bool is_handle_sk) { + size_t pos = 0; + while ((pos = str.find(pattern, pos)) != std::string::npos) { + size_t colon_pos = str.find(':', pos); + if (colon_pos == std::string::npos) { + pos += pattern.length(); + continue; + } + + size_t quote_pos = str.find('\"', colon_pos); + if (quote_pos == std::string::npos) { + pos += pattern.length(); + continue; + } + + size_t value_start = quote_pos + 1; + size_t value_end = str.find('\"', value_start); + if (value_end == std::string::npos) { + pos = value_start; + continue; + } + + std::string key_value = str.substr(value_start, value_end - value_start); + + if (is_handle_sk) { + key_value = "md5: " + md5(key_value); + } else { + key_value = hide_access_key(key_value); + } + + str.replace(value_start, value_end - value_start, key_value); + + pos = value_end + (key_value.length() - key_value.length()); + } +}; + /** * Encrypts all "sk" values in the given debug string with MD5 hashes. * @@ -61,36 +100,26 @@ inline std::string md5(const std::string& str) { * - Input string contains one or more occurrences of "sk: " followed by a value in double quotes. * - An md5() function exists that takes a std::string and returns its MD5 hash as a string. * - * @param debug_string Input string containing "sk: " fields to be encrypted. + * @param debug_string Input string containing "sk: " or ""sk": " fields to be encrypted. * @return A new string with all "sk" values replaced by their MD5 hashes. * - * Behavior: + * Behavior for "sk: " format: * 1. Searches for all occurrences of "sk: " in the input string. * 2. For each occurrence, extracts the value between double quotes. * 3. Replaces the original value with "md5: " followed by its MD5 hash. * 4. Returns the modified string with all "sk" values encrypted. */ inline std::string encryt_sk(std::string debug_string) { - // Start position for searching "sk" fields - size_t pos = 0; - // Iterate through the string and find all occurrences of "sk: " - while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) { - // Find the start and end of the "sk" value (assumed to be within quotes) - // Start after the quote - size_t sk_value_start = debug_string.find('\"', pos) + 1; - // End at the next quote - size_t sk_value_end = debug_string.find('\"', sk_value_start); - - // Extract the "sk" value - std::string sk_value = debug_string.substr(sk_value_start, sk_value_end - sk_value_start); - // Encrypt the "sk" value with MD5 - std::string encrypted_sk = "md5: " + md5(sk_value); - - // Replace the original "sk" value with the encrypted MD5 value - debug_string.replace(sk_value_start, sk_value_end - sk_value_start, encrypted_sk); - // Move the position to the end of the current "sk" field and continue searching - pos = sk_value_end; - } + process_ak_sk_pattern(debug_string, "sk: ", true); + process_ak_sk_pattern(debug_string, "\"sk\"", true); + + return debug_string; +} + +inline std::string hide_ak(std::string debug_string) { + process_ak_sk_pattern(debug_string, "ak: ", false); + process_ak_sk_pattern(debug_string, "\"ak\"", false); + return debug_string; } @@ -134,6 +163,13 @@ void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request << " lock_id=" << req->lock_id() << " initiator=" << req->initiator() << " expiration=" << req->expiration() << " require_compaction_stats=" << req->require_compaction_stats(); + } else if constexpr (std::is_same_v || + std::is_same_v) { + std::string debug_string = encryt_sk(req->ShortDebugString()); + debug_string = hide_ak(debug_string); + TEST_SYNC_POINT_CALLBACK("sk_begin_rpc", &debug_string); + LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() + << " original_client_ip=" << req->request_ip() << " request=" << debug_string; } else { LOG(INFO) << "begin " << func_name << " remote_caller=" << ctrl->remote_side() << " original_client_ip=" << req->request_ip() @@ -179,8 +215,10 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, const Reques << " original_client_ip=" << req->request_ip() << " status=" << res->status().ShortDebugString(); } else if constexpr (std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v) { std::string debug_string = encryt_sk(res->DebugString()); + debug_string = hide_ak(debug_string); TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string); LOG(INFO) << "finish " << func_name << " remote_caller=" << ctrl->remote_side() << " original_client_ip=" << req->request_ip() << " response=" << debug_string; diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp index 8d7f6bba6e4a31..be4c643f7e58ee 100644 --- a/cloud/src/meta-service/meta_service_http.cpp +++ b/cloud/src/meta-service/meta_service_http.cpp @@ -48,6 +48,7 @@ #include "common/configbase.h" #include "common/logging.h" #include "common/string_util.h" +#include "meta-service/meta_service_helper.h" #include "meta-store/keys.h" #include "meta-store/txn_kv.h" #include "meta-store/txn_kv_error.h" @@ -63,7 +64,7 @@ namespace doris::cloud { auto st = parse_json_message(unresolved_path, body, &req); \ if (!st.ok()) { \ std::string msg = "parse http request '" + unresolved_path + "': " + st.ToString(); \ - LOG_WARNING(msg).tag("body", body); \ + LOG_WARNING(msg).tag("body", encryt_sk(body)); \ return http_json_reply(MetaServiceCode::PROTOBUF_PARSE_ERR, msg); \ } \ } while (0) @@ -86,7 +87,7 @@ static google::protobuf::util::Status parse_json_message(const std::string& unre if (!st.ok()) { std::string msg = "failed to strictly parse http request for '" + unresolved_path + "' error: " + st.ToString(); - LOG_WARNING(msg).tag("body", body); + LOG_WARNING(msg).tag("body", encryt_sk(hide_access_key(body))); // ignore unknown fields google::protobuf::util::JsonParseOptions json_parse_options; @@ -776,6 +777,8 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller, LOG(INFO) << "rpc from " << cntl->remote_side() << " request: " << cntl->http_request().uri().path(); std::string http_request = format_http_request(cntl); + std::string http_request_for_log = encryt_sk(http_request); + http_request_for_log = hide_ak(http_request_for_log); // Auth auto token = http_query(cntl->http_request().uri(), "token"); @@ -786,7 +789,7 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller, cntl->response_attachment().append(body); cntl->response_attachment().append("\n"); LOG(WARNING) << "failed to handle http from " << cntl->remote_side() - << " request: " << http_request << " msg: " << body; + << " request: " << http_request_for_log << " msg: " << body; return; } @@ -806,7 +809,7 @@ void MetaServiceImpl::http(::google::protobuf::RpcController* controller, int ret = cntl->http_response().status_code(); LOG(INFO) << (ret == 200 ? "succ to " : "failed to ") << __PRETTY_FUNCTION__ << " " << cntl->remote_side() << " request=\n" - << http_request << "\n ret=" << ret << " msg=" << msg; + << http_request_for_log << "\n ret=" << ret << " msg=" << msg; } } // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index f8ebccc28af134..a94ceb67edb80c 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -74,7 +74,7 @@ static int encrypt_ak_sk_helper(const std::string plain_ak, const std::string pl MetaServiceCode& code, std::string& msg) { std::string key; int64_t key_id; - LOG_INFO("enter encrypt_ak_sk_helper, plain_ak {}", plain_ak); + LOG_INFO("enter encrypt_ak_sk_helper, plain_ak {}", hide_access_key(plain_ak)); int ret = get_newest_encryption_key_for_ak_sk(&key_id, &key); TEST_SYNC_POINT_CALLBACK("encrypt_ak_sk:get_encryption_key", &ret, &key, &key_id); if (ret != 0) { @@ -1659,7 +1659,10 @@ void MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle const CreateInstanceRequest* request, CreateInstanceResponse* response, ::google::protobuf::Closure* done) { + TEST_SYNC_POINT_CALLBACK("create_instance_sk_request", + const_cast(&request)); RPC_PREPROCESS(create_instance, get, put); + TEST_SYNC_POINT_RETURN_WITH_VOID("create_instance_sk_request_return"); if (request->has_ram_user()) { auto& ram_user = request->ram_user(); std::string ram_user_id = ram_user.has_user_id() ? ram_user.user_id() : ""; @@ -1737,6 +1740,10 @@ void MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle return; } + for (auto& obj_info : *instance.mutable_obj_info()) { + obj_info.set_ak(hide_access_key(obj_info.ak())); + } + LOG(INFO) << "xxx instance json=" << proto_to_json(instance); // Check existence before proceeding @@ -1976,6 +1983,8 @@ void MetaServiceImpl::get_instance(google::protobuf::RpcController* controller, const GetInstanceRequest* request, GetInstanceResponse* response, ::google::protobuf::Closure* done) { RPC_PREPROCESS(get_instance, get); + TEST_SYNC_POINT_CALLBACK("get_instance_sk_response", &response); + TEST_SYNC_POINT_RETURN_WITH_VOID("get_instance_sk_response_return"); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -2761,7 +2770,9 @@ void MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller, void MetaServiceImpl::create_stage(::google::protobuf::RpcController* controller, const CreateStageRequest* request, CreateStageResponse* response, ::google::protobuf::Closure* done) { + TEST_SYNC_POINT_CALLBACK("create_stage_sk_request", const_cast(&request)); RPC_PREPROCESS(create_stage, get, put); + TEST_SYNC_POINT_RETURN_WITH_VOID("create_stage_sk_request_return"); std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : ""; if (cloud_unique_id.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/cloud/src/resource-manager/resource_manager.cpp b/cloud/src/resource-manager/resource_manager.cpp index 7c5a95b4267898..3aa7fcb99b35f4 100644 --- a/cloud/src/resource-manager/resource_manager.cpp +++ b/cloud/src/resource-manager/resource_manager.cpp @@ -519,8 +519,12 @@ std::pair ResourceManager::add_cluster(const std:: } auto& req_cluster = cluster.cluster; + InstanceInfoPB instance_for_log {instance}; + for (auto& obj_info : *instance_for_log.mutable_obj_info()) { + obj_info.set_ak(hide_access_key(obj_info.ak())); + } LOG(INFO) << "cluster to add json=" << proto_to_json(req_cluster); - LOG(INFO) << "json=" << proto_to_json(instance); + LOG(INFO) << "json=" << proto_to_json(instance_for_log); // Check id and name, they need to be unique // One cluster id per name, name is alias of cluster id diff --git a/cloud/test/meta_service_http_test.cpp b/cloud/test/meta_service_http_test.cpp index 7ade292966354a..7fcc0f37441fc8 100644 --- a/cloud/test/meta_service_http_test.cpp +++ b/cloud/test/meta_service_http_test.cpp @@ -1603,6 +1603,124 @@ TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) { ms->get_obj_store_info(&cntl, &req1, &res1, nullptr); } +TEST(MetaServiceHttpTest, get_instance_response_sk) { + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + sp->disable_processing(); + }; + + GetInstanceResponse res; + auto* obj_info = res.mutable_instance()->add_obj_info(); + obj_info->set_ak("instance-ak"); + obj_info->set_sk("instance-sk"); + auto foo = [res](auto args) { + (*(try_any_cast(args[0])))->CopyFrom(res); + }; + sp->set_call_back("get_instance_sk_response", foo); + sp->set_call_back("get_instance_sk_response_return", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + + auto rate_limiter = std::make_shared(); + + auto ms = std::make_unique(nullptr, nullptr, rate_limiter); + + auto bar = [](auto args) { + std::cout << *try_any_cast(args[0]); + + EXPECT_TRUE((*try_any_cast(args[0])).find("instance-sk") == + std::string::npos); + EXPECT_TRUE((*try_any_cast(args[0])) + .find("md5: 79d1924d669b7412019edc42db31bd92") != std::string::npos); + }; + sp->set_call_back("sk_finish_rpc", bar); + + GetInstanceResponse res1; + GetInstanceRequest req1; + brpc::Controller cntl; + ms->get_instance(&cntl, &req1, &res1, nullptr); +} + +TEST(MetaServiceHttpTest, create_instance_request_sk) { + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + sp->disable_processing(); + }; + + CreateInstanceRequest req; + req.set_instance_id("get_value_instance_id"); + req.set_user_id("test_user"); + req.set_name("test_name"); + ObjectStoreInfoPB obj; + obj.set_ak("instance-ak"); + obj.set_sk("instance-sk"); + req.mutable_obj_info()->CopyFrom(obj); + + auto foo = [req](auto args) { + (*(try_any_cast(args[0])))->CopyFrom(req); + }; + sp->set_call_back("create_instance_sk_request", foo); + sp->set_call_back("create_instance_sk_request_return", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + + auto rate_limiter = std::make_shared(); + + auto ms = std::make_unique(nullptr, nullptr, rate_limiter); + + auto bar = [](auto args) { + std::cout << *try_any_cast(args[0]) << '\n'; + + EXPECT_TRUE((*try_any_cast(args[0])).find("instance-sk") == + std::string::npos); + EXPECT_TRUE((*try_any_cast(args[0])) + .find("md5: 79d1924d669b7412019edc42db31bd92") != std::string::npos); + }; + sp->set_call_back("sk_begin_rpc", bar); + + CreateInstanceResponse res1; + CreateInstanceRequest req1; + brpc::Controller cntl; + ms->create_instance(&cntl, &req1, &res1, nullptr); +} + +TEST(MetaServiceHttpTest, create_stage_request_sk) { + auto sp = SyncPoint::get_instance(); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + sp->disable_processing(); + }; + + CreateStageRequest req; + req.mutable_stage()->mutable_obj_info()->set_ak("stage-ak"); + req.mutable_stage()->mutable_obj_info()->set_sk("stage-sk"); + + auto foo = [req](auto args) { + (*(try_any_cast(args[0])))->CopyFrom(req); + }; + sp->set_call_back("create_stage_sk_request", foo); + sp->set_call_back("create_stage_sk_request_return", + [](auto&& args) { *try_any_cast(args.back()) = true; }); + + auto rate_limiter = std::make_shared(); + + auto ms = std::make_unique(nullptr, nullptr, rate_limiter); + + auto bar = [](auto args) { + std::cout << *try_any_cast(args[0]) << '\n'; + + EXPECT_TRUE((*try_any_cast(args[0])).find("stage-sk") == std::string::npos); + EXPECT_TRUE((*try_any_cast(args[0])) + .find("md5: f497d053066fa4b7d3b1f6564597d233") != std::string::npos); + }; + sp->set_call_back("sk_begin_rpc", bar); + + CreateStageResponse res1; + CreateStageRequest req1; + brpc::Controller cntl; + ms->create_stage(&cntl, &req1, &res1, nullptr); +} + TEST(MetaServiceHttpTest, AdjustRateLimit) { HttpContext ctx; { diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java index 61dbcb4d053399..d49c54d1367fc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java @@ -26,6 +26,7 @@ import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.cloud.proto.Cloud.ClusterPB; import org.apache.doris.cloud.proto.Cloud.InstanceInfoPB; +import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB; import org.apache.doris.cloud.qe.ComputeGroupException; import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.AnalysisException; @@ -1469,6 +1470,7 @@ public String getInstanceId(String cloudUniqueId) throws IOException { try { Cloud.GetInstanceRequest request = builder.build(); response = MetaServiceProxy.getInstance().getInstance(request); + response = hideAkSkForStorageVault(response.toBuilder()); LOG.info("get instance info, request: {}, response: {}", request, response); if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) { LOG.warn("Failed to get instance info, response: {}", response); @@ -1481,6 +1483,61 @@ public String getInstanceId(String cloudUniqueId) throws IOException { } } + private String hideKey(String key) { + if (key == null || key.isEmpty()) { + return ""; + } + + int keyLen = key.length(); + int reservedCount = (keyLen > 6) ? 6 : (keyLen > 2 ? keyLen - 2 : 0); + int xCount = keyLen - reservedCount; + + int leftXCount = xCount / 2; + StringBuilder result = new StringBuilder(); + + for (int i = 0; i < leftXCount; i++) { + result.append('x'); + } + + int startIndex = leftXCount; + int endIndex = startIndex + reservedCount; + result.append(key.substring(startIndex, endIndex)); + + int rightXCount = xCount - leftXCount; + for (int i = 0; i < rightXCount; i++) { + result.append('x'); + } + return result.toString(); + } + + public Cloud.GetInstanceResponse hideAkSkForStorageVault(Cloud.GetInstanceResponse.Builder resp) { + if (resp == null) { + return null; + } + + if (resp.getInstance().getObjInfoCount() == 0) { + return resp.build(); + } + + for (int i = 0; i < resp.getInstance().getObjInfoCount(); i++) { + ObjectStoreInfoPB objInfo = resp.getInstance().getObjInfo(i); + if (objInfo == null) { + continue; + } + if (objInfo.hasAk()) { + String ak = objInfo.getAk(); + String hiddenAk = hideKey(ak); + resp.getInstanceBuilder().getObjInfoBuilder(i).setAk(hiddenAk); + } + if (objInfo.hasSk()) { + String sk = objInfo.getSk(); + String hiddenSk = hideKey(sk); + resp.getInstanceBuilder().getObjInfoBuilder(i).setSk(hiddenSk); + } + } + return resp.build(); + } + public void renameComputeGroup(String originalName, String newGroupName) throws UserException { String cloudInstanceId = ((CloudEnv) Env.getCurrentEnv()).getCloudInstanceId(); if (Strings.isNullOrEmpty(cloudInstanceId)) { From 5d370901fe04ff2bb8c81383228a4eeb1ee6db2c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:37:22 +0800 Subject: [PATCH 547/572] branch-3.0: [Bug](routine load) Fix routine load task failed with MEM_LIMIT_EXCEED never be scheduled again #55481 (#55614) Cherry-picked from #55481 Co-authored-by: xy720 <22125576+xy720@users.noreply.github.com> --- .../routineload/RoutineLoadTaskScheduler.java | 13 +++++ .../test_routine_load_job_schedule.groovy | 57 ++++++++++++++++++- 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java index 28e03567765ba8..d1b5a6f73e8b47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java @@ -247,6 +247,13 @@ private void handleSubmitTaskFailure(RoutineLoadTaskInfo routineLoadTaskInfo, St // Check if this is a resource pressure error that should not be immediately rescheduled if (errorMsg.contains("TOO_MANY_TASKS") || errorMsg.contains("MEM_LIMIT_EXCEEDED")) { + // submit task failed (such as TOO_MANY_TASKS/MEM_LIMIT_EXCEEDED error), + // but txn has already begun. Here we will still set the ExecuteStartTime of + // this task, which means we "assume" that this task has been successfully submitted. + // And this task will then be aborted because of a timeout. + // In this way, we can prevent the entire job from being paused due to submit errors, + // and we can also relieve the pressure on BE by waiting for the timeout period. + routineLoadTaskInfo.setExecuteStartTimeMs(System.currentTimeMillis()); return; } @@ -302,6 +309,12 @@ private void submitTask(long beId, TRoutineLoadTask tTask) throws LoadException throw new LoadException("debug point FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED"); } + if (DebugPointUtil.isEnable("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED.MEM_LIMIT_EXCEEDED")) { + LOG.warn("debug point FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED.MEM_LIMIT_EXCEEDED," + + "routine load task submit failed"); + throw new LoadException("MEM_LIMIT_EXCEEDED"); + } + if (tStatus.getStatusCode() != TStatusCode.OK) { throw new LoadException("failed to submit task. error code: " + tStatus.getStatusCode() + ", msg: " + (tStatus.getErrorMsgsSize() > 0 ? tStatus.getErrorMsgs().get(0) : "NaN")); diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy index ed52e60a13ff53..5cd433283e0c01 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load_job_schedule.groovy @@ -111,5 +111,60 @@ suite("test_routine_load_job_schedule","nonConcurrent") { logger.warn("Failed to stop routine load job: ${e.message}") } } + + sql "truncate table ${tableName}" + def memJob = "test_routine_load_job_schedule_mem_limit" + try { + GetDebugPoint().enableDebugPointForAllBEs("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED.MEM_LIMIT_EXCEEDED") + testData.each { line-> + logger.info("Sending data to kafka: ${line}") + def record = new ProducerRecord<>(kafkaCsvTpoics[0], null, line) + producer.send(record) + } + + sql """ + CREATE ROUTINE LOAD ${memJob} ON ${tableName} + COLUMNS TERMINATED BY "," + FROM KAFKA + ( + "kafka_broker_list" = "${kafka_broker}", + "kafka_topic" = "${newTopic.name()}", + "property.kafka_default_offsets" = "OFFSET_BEGINNING", + "max_batch_interval" = "6" + ); + """ + + sleep(5000) + + GetDebugPoint().disableDebugPointForAllBEs("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED.MEM_LIMIT_EXCEEDED") + + def count = 0 + def maxWaitCount = 120 // > 60 = maxBatchIntervalS * Config.routine_load_task_timeout_multiplier + while (true) { + def state = sql "show routine load for ${memJob}" + def routineLoadState = state[0][8].toString() + def statistic = state[0][14].toString() + logger.info("Routine load state: ${routineLoadState}") + logger.info("Routine load statistic: ${statistic}") + def rowCount = sql "select count(*) from ${memTableName}" + if (routineLoadState == "RUNNING" && rowCount[0][0] == 5) { + break + } + if (count > maxWaitCount) { + assertEquals(1, 2) + } + sleep(1000) + count++ + } + } catch (Exception e) { + logger.error("MEM_LIMIT_EXCEEDED test failed with exception: ${e.message}") + } finally { + GetDebugPoint().disableDebugPointForAllBEs("FE.ROUTINE_LOAD_TASK_SUBMIT_FAILED.MEM_LIMIT_EXCEEDED") + try { + sql "stop routine load for test_routine_load_job_schedule_mem_limit" + } catch (Exception e) { + logger.warn("Failed to stop routine load job: ${e.message}") + } + } } -} \ No newline at end of file +} From e417ef0eceda10306f986136711cc0c98c6bc5ac Mon Sep 17 00:00:00 2001 From: walter Date: Fri, 5 Sep 2025 09:38:00 +0800 Subject: [PATCH 548/572] [fix](backup) Support backup meta/job info exceeds 2GB (#55607) pick #55599 --- .../org/apache/doris/common/GZIPUtils.java | 21 +++++++ .../apache/doris/analysis/RestoreStmt.java | 12 ++-- .../apache/doris/backup/BackupHandler.java | 12 +--- .../org/apache/doris/backup/BackupJob.java | 16 ++--- .../apache/doris/backup/BackupJobInfo.java | 8 +++ .../org/apache/doris/backup/BackupMeta.java | 2 +- .../org/apache/doris/backup/Snapshot.java | 62 ++++++++++++++----- .../doris/service/FrontendServiceImpl.java | 52 ++++++++++++---- 8 files changed, 129 insertions(+), 56 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java b/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java index 7408e2888cc3a5..4500c76f638792 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/GZIPUtils.java @@ -21,7 +21,10 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -39,10 +42,28 @@ public static byte[] compress(byte[] data) throws IOException { return bytesStream.toByteArray(); } + public static byte[] compress(File file) throws IOException { + ByteArrayOutputStream bytesStream = new ByteArrayOutputStream(); + try (FileInputStream fileInputStream = new FileInputStream(file); + GZIPOutputStream gzipStream = new GZIPOutputStream(bytesStream)) { + + byte[] buffer = new byte[8192]; // 8KB buffer + int bytesRead; + while ((bytesRead = fileInputStream.read(buffer)) != -1) { + gzipStream.write(buffer, 0, bytesRead); + } + } + return bytesStream.toByteArray(); + } + public static byte[] decompress(byte[] data) throws IOException { ByteArrayInputStream bytesStream = new ByteArrayInputStream(data); try (GZIPInputStream gzipStream = new GZIPInputStream(bytesStream)) { return IOUtils.toByteArray(gzipStream); } } + + public static InputStream lazyDecompress(byte[] data) throws IOException { + return new GZIPInputStream(new ByteArrayInputStream(data)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java index 5f141837565ad0..1deb2b48db900e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java @@ -17,6 +17,8 @@ package org.apache.doris.analysis; +import org.apache.doris.backup.BackupJobInfo; +import org.apache.doris.backup.BackupMeta; import org.apache.doris.backup.Repository; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.common.AnalysisException; @@ -58,8 +60,8 @@ public class RestoreStmt extends AbstractBackupStmt implements NotFallbackInPars private boolean isCleanPartitions = false; private boolean isAtomicRestore = false; private boolean isForceReplace = false; - private byte[] meta = null; - private byte[] jobInfo = null; + private BackupMeta meta = null; + private BackupJobInfo jobInfo = null; public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause restoreTableRefClause, Map properties) { @@ -67,7 +69,7 @@ public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefC } public RestoreStmt(LabelName labelName, String repoName, AbstractBackupTableRefClause restoreTableRefClause, - Map properties, byte[] meta, byte[] jobInfo) { + Map properties, BackupMeta meta, BackupJobInfo jobInfo) { super(labelName, repoName, restoreTableRefClause, properties); this.meta = meta; this.jobInfo = jobInfo; @@ -101,11 +103,11 @@ public boolean isLocal() { return isLocal; } - public byte[] getMeta() { + public BackupMeta getMeta() { return meta; } - public byte[] getJobInfo() { + public BackupJobInfo getJobInfo() { return jobInfo; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 188e07af5a2826..040a18ed796441 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -513,9 +513,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws private void restore(Repository repository, Database db, RestoreStmt stmt) throws DdlException { BackupJobInfo jobInfo; if (stmt.isLocal()) { - String jobInfoString = new String(stmt.getJobInfo()); - jobInfo = BackupJobInfo.genFromJson(jobInfoString); - + jobInfo = stmt.getJobInfo(); if (jobInfo.extraInfo == null) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Invalid job extra info empty"); } @@ -551,13 +549,7 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw metaVersion = jobInfo.metaVersion; } - BackupMeta backupMeta; - try { - backupMeta = BackupMeta.fromBytes(stmt.getMeta(), metaVersion); - } catch (IOException e) { - LOG.warn("read backup meta failed, current meta version {}", Env.getCurrentEnvJournalVersion(), e); - throw new DdlException("read backup meta failed", e); - } + BackupMeta backupMeta = stmt.getMeta(); String backupTimestamp = TimeUtils.longToTimeString( jobInfo.getBackupTime(), TimeUtils.getDatetimeFormatWithHyphenWithTimeZone()); restoreJob = new RestoreJob(stmt.getLabel(), backupTimestamp, diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 1b42dbbaa9daca..cba11b3dba350b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -1176,20 +1176,12 @@ public synchronized Snapshot getSnapshot() { // Avoid loading expired meta. long expiredAt = createTime + timeoutMs; if (System.currentTimeMillis() >= expiredAt) { - return new Snapshot(label, new byte[0], new byte[0], expiredAt, commitSeq); + return new Snapshot(label, null, null, expiredAt, commitSeq); } - try { - File metaInfoFile = new File(localMetaInfoFilePath); - File jobInfoFile = new File(localJobInfoFilePath); - byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath()); - byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath()); - return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt, commitSeq); - } catch (IOException e) { - LOG.warn("failed to load meta info and job info file, meta info file {}, job info file {}: ", - localMetaInfoFilePath, localJobInfoFilePath, e); - return null; - } + File metaInfoFile = new File(localMetaInfoFilePath); + File jobInfoFile = new File(localJobInfoFilePath); + return new Snapshot(label, metaInfoFile, jobInfoFile, expiredAt, commitSeq); } public synchronized List getInfo() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index 554a21c44080f7..7e77426a82a6d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -57,6 +57,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -763,6 +765,12 @@ public static BackupJobInfo genFromJson(String json) { return jobInfo; } + public static BackupJobInfo fromInputStream(InputStream inputStream) throws IOException { + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + return GsonUtils.GSON.fromJson(reader, BackupJobInfo.class); + } + } + public void writeToFile(File jobInfoFile) throws FileNotFoundException { PrintWriter printWriter = new PrintWriter(jobInfoFile); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java index 850d6f92e83c0f..124fa802ef90a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java @@ -103,7 +103,7 @@ public static BackupMeta fromBytes(byte[] bytes, int metaVersion) throws IOExcep return fromInputStream(new ByteArrayInputStream(bytes), metaVersion); } - protected static BackupMeta fromInputStream(InputStream stream, int metaVersion) throws IOException { + public static BackupMeta fromInputStream(InputStream stream, int metaVersion) throws IOException { MetaContext metaContext = new MetaContext(); metaContext.setMetaVersion(metaVersion); metaContext.setThreadLocalInfo(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java index a9f734dbc99220..2fc3ca6d146ee6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java @@ -17,19 +17,22 @@ package org.apache.doris.backup; -import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.common.GZIPUtils; +import org.apache.doris.common.Pair; import com.google.gson.annotations.SerializedName; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + public class Snapshot { @SerializedName(value = "label") private String label = null; - @SerializedName(value = "meta") - private byte[] meta = null; + private File meta = null; - @SerializedName(value = "jobInfo") - private byte[] jobInfo = null; + private File jobInfo = null; @SerializedName(value = "expired_at") private long expiredAt = 0; @@ -40,7 +43,7 @@ public class Snapshot { public Snapshot() { } - public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt, long commitSeq) { + public Snapshot(String label, File meta, File jobInfo, long expiredAt, long commitSeq) { this.label = label; this.meta = meta; this.jobInfo = jobInfo; @@ -48,12 +51,45 @@ public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt, long this.commitSeq = commitSeq; } - public byte[] getMeta() { - return meta; + public static Pair readFromBytes(byte[] meta, byte[] jobInfo) throws IOException { + BackupJobInfo backupJobInfo = BackupJobInfo.genFromJson(new String(jobInfo)); + BackupMeta backupMeta = BackupMeta.fromBytes(meta, backupJobInfo.metaVersion); + return Pair.of(backupMeta, backupJobInfo); + } + + public static Pair readFromCompressedBytes(byte[] meta, byte[] jobInfo) + throws IOException { + BackupJobInfo backupJobInfo = BackupJobInfo.fromInputStream(GZIPUtils.lazyDecompress(jobInfo)); + BackupMeta backupMeta = BackupMeta.fromInputStream(GZIPUtils.lazyDecompress(meta), backupJobInfo.metaVersion); + return Pair.of(backupMeta, backupJobInfo); + } + + public static boolean isCompressed(byte[] meta, byte[] jobInfo) { + return GZIPUtils.isGZIPCompressed(jobInfo) || GZIPUtils.isGZIPCompressed(meta); + } + + public long getMetaSize() { + return meta != null ? meta.length() : 0; + } + + public long getJobInfoSize() { + return jobInfo != null ? jobInfo.length() : 0; } - public byte[] getJobInfo() { - return jobInfo; + public byte[] getCompressedMeta() throws IOException { + return GZIPUtils.compress(meta); + } + + public byte[] getCompressedJobInfo() throws IOException { + return GZIPUtils.compress(jobInfo); + } + + public byte[] getMeta() throws IOException { + return Files.readAllBytes(meta.toPath()); + } + + public byte[] getJobInfo() throws IOException { + return Files.readAllBytes(jobInfo.toPath()); } public long getExpiredAt() { @@ -68,16 +104,10 @@ public long getCommitSeq() { return commitSeq; } - public String toJson() { - return GsonUtils.GSON.toJson(this); - } - @Override public String toString() { return "Snapshot{" + "label='" + label + '\'' - + ", meta=" + meta - + ", jobInfo=" + jobInfo + ", expiredAt=" + expiredAt + ", commitSeq=" + commitSeq + '}'; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index f3140d490c1d83..af9abb078fbcb0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -28,6 +28,8 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.backup.BackupJobInfo; +import org.apache.doris.backup.BackupMeta; import org.apache.doris.backup.Snapshot; import org.apache.doris.binlog.BinlogLagInfo; import org.apache.doris.catalog.AutoIncrementGenerator; @@ -61,7 +63,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.common.DuplicatedRequestException; -import org.apache.doris.common.GZIPUtils; import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.LabelAlreadyUsedException; import org.apache.doris.common.LoadException; @@ -3056,24 +3057,38 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_EXPIRED); result.getStatus().addToErrorMsgs(String.format("snapshot %s is expired", label)); } else { - byte[] meta = snapshot.getMeta(); - byte[] jobInfo = snapshot.getJobInfo(); + long metaSize = snapshot.getMetaSize(); + long jobInfoSize = snapshot.getJobInfoSize(); + long snapshotSize = snapshot.getMetaSize() + snapshot.getJobInfoSize(); + if (metaSize + jobInfoSize >= Integer.MAX_VALUE && !request.isEnableCompress()) { + String msg = String.format( + "Snapshot %s is too large (%d bytes > 2GB). Please enable compression to continue.", + label, snapshotSize); + LOG.warn("get snapshot failed: {}", msg); + result.getStatus().setStatusCode(TStatusCode.INTERNAL_ERROR); + result.getStatus().addToErrorMsgs(msg); + return result; + } + long expiredAt = snapshot.getExpiredAt(); long commitSeq = snapshot.getCommitSeq(); LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}, " - + "expired at: {}, commit seq: {}", label, meta.length, jobInfo.length, expiredAt, commitSeq); + + "expired at: {}, commit seq: {}", label, metaSize, jobInfoSize, expiredAt, commitSeq); if (request.isEnableCompress()) { - meta = GZIPUtils.compress(meta); - jobInfo = GZIPUtils.compress(jobInfo); + byte[] meta = snapshot.getCompressedMeta(); + byte[] jobInfo = snapshot.getCompressedJobInfo(); + result.setMeta(meta); + result.setJobInfo(jobInfo); result.setCompressed(true); if (LOG.isDebugEnabled()) { LOG.debug("get snapshot info with compress, snapshot: {}, compressed meta " + "size {}, compressed job info size {}", label, meta.length, jobInfo.length); } + } else { + result.setMeta(snapshot.getMeta()); + result.setJobInfo(snapshot.getJobInfo()); } - result.setMeta(meta); - result.setJobInfo(jobInfo); result.setExpiredAt(expiredAt); result.setCommitSeq(commitSeq); } @@ -3186,6 +3201,8 @@ private TRestoreSnapshotResult restoreSnapshotImpl(TRestoreSnapshotRequest reque } } + BackupMeta backupMeta; + BackupJobInfo backupJobInfo; byte[] meta = request.getMeta(); byte[] jobInfo = request.getJobInfo(); if (Config.enable_restore_snapshot_rpc_compression && request.isCompressed()) { @@ -3194,18 +3211,29 @@ private TRestoreSnapshotResult restoreSnapshotImpl(TRestoreSnapshotRequest reque meta.length, jobInfo.length); } try { - meta = GZIPUtils.decompress(meta); - jobInfo = GZIPUtils.decompress(jobInfo); + Pair pair = Snapshot.readFromCompressedBytes(meta, jobInfo); + backupMeta = pair.first; + backupJobInfo = pair.second; } catch (Exception e) { LOG.warn("decompress meta and job info failed", e); throw new UserException("decompress meta and job info failed", e); } - } else if (GZIPUtils.isGZIPCompressed(jobInfo) || GZIPUtils.isGZIPCompressed(meta)) { + } else if (Snapshot.isCompressed(meta, jobInfo)) { throw new UserException("The request is compressed, but the config " + "`enable_restore_snapshot_rpc_compressed` is not enabled."); + } else { + try { + Pair pair = Snapshot.readFromBytes(meta, jobInfo); + backupMeta = pair.first; + backupJobInfo = pair.second; + } catch (Exception e) { + LOG.warn("deserialize meta and job info failed", e); + throw new UserException("deserialize meta and job info failed", e); + } } - RestoreStmt restoreStmt = new RestoreStmt(label, repoName, restoreTableRefClause, properties, meta, jobInfo); + RestoreStmt restoreStmt = new RestoreStmt( + label, repoName, restoreTableRefClause, properties, backupMeta, backupJobInfo); restoreStmt.setIsBeingSynced(); LOG.debug("restore snapshot info, restoreStmt: {}", restoreStmt); try { From 10ffdbb655f7a1ed79716f8227cf1794aa64caa0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:43:21 +0800 Subject: [PATCH 549/572] branch-3.0: [fix](mow) Fix mow agg cache version check #55330 (#55474) Cherry-picked from #55330 Co-authored-by: meiyi --- be/src/olap/tablet_meta.cpp | 30 ++++++++++++------------------ be/src/olap/tablet_meta.h | 2 +- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 19250c292027bc..d9b6f76217708b 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -1012,25 +1012,19 @@ void TabletMeta::_check_mow_rowset_cache_version_size(size_t rowset_cache_versio rowset_cache_version_size > _rs_metas.size() + _stale_rs_metas.size()) { std::stringstream ss; auto rowset_ids = _delete_bitmap->get_rowset_cache_version(); - for (const auto& rowset_id : rowset_ids) { - bool found = false; + std::set tablet_rowset_ids; + { + std::shared_lock rlock(_meta_lock); for (auto& rs_meta : _rs_metas) { - if (rs_meta->rowset_id() == rowset_id) { - found = true; - break; - } - } - if (found) { - continue; + tablet_rowset_ids.emplace(rs_meta->rowset_id().to_string()); } for (auto& rs_meta : _stale_rs_metas) { - if (rs_meta->rowset_id() == rowset_id) { - found = true; - break; - } + tablet_rowset_ids.emplace(rs_meta->rowset_id().to_string()); } - if (!found) { - ss << rowset_id.to_string() << ", "; + } + for (const auto& rowset_id : rowset_ids) { + if (tablet_rowset_ids.find(rowset_id) == tablet_rowset_ids.end()) { + ss << rowset_id << ", "; } } // size(rowset_cache_version) <= size(_rs_metas) + size(_stale_rs_metas) + size(_unused_rs) @@ -1412,11 +1406,11 @@ void DeleteBitmap::clear_rowset_cache_version() { VLOG_DEBUG << "clear agg cache version for tablet=" << _tablet_id; } -std::set DeleteBitmap::get_rowset_cache_version() { - std::set set; +std::set DeleteBitmap::get_rowset_cache_version() { + std::set set; std::shared_lock l(_rowset_cache_version_lock); for (auto& [k, _] : _rowset_cache_version) { - set.insert(k); + set.insert(k.to_string()); } return set; } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index 39dba82a17e856..166bc606486cbd 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -582,7 +582,7 @@ class DeleteBitmap { void clear_rowset_cache_version(); - std::set get_rowset_cache_version(); + std::set get_rowset_cache_version(); /** * Calculate diffset with given `key_set`. All entries with keys contained in this delete bitmap but not From 6f02bc9fff766d2df69b9f126cd4da8f5a3e6c57 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:43:51 +0800 Subject: [PATCH 550/572] branch-3.0: [fix](move-memtable) fix segment number mismatch for erroneously skipped segments #55092 (#55470) Cherry-picked from #55092 Co-authored-by: hui lai --- be/src/vec/sink/writer/vtablet_writer_v2.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp index c9e63349270aa4..c534fb44334945 100644 --- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp @@ -99,9 +99,7 @@ Status VTabletWriterV2::_incremental_open_streams( tablet.set_partition_id(partition->id); tablet.set_index_id(index.index_id); tablet.set_tablet_id(tablet_id); - if (!_load_stream_map->contains(node)) { - new_backends.insert(node); - } + new_backends.insert(node); _tablets_for_node[node].emplace(tablet_id, tablet); if (known_indexes.contains(index.index_id)) [[likely]] { continue; From a50a51e6f9330b9f36456137cc2c4061ad79a3f3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:47:16 +0800 Subject: [PATCH 551/572] branch-3.0: fix ldap group filter for OpenDirectory #51379 (#55399) Cherry-picked from #51379 Co-authored-by: nsivarajan <117266407+nsivarajan@users.noreply.github.com> --- .../mysql/authenticate/ldap/LdapClient.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java index 3ae96945296942..1186469a1ec822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/LdapClient.java @@ -112,7 +112,7 @@ public boolean checkUpdate(String ldapPassword) { private void init() { LdapInfo ldapInfo = Env.getCurrentEnv().getAuth().getLdapInfo(); if (ldapInfo == null || !ldapInfo.isValid()) { - LOG.error("info is null, maybe no ldap admin password is set."); + LOG.error("LDAP info is null or invalid, LDAP admin password may not be set"); ErrorReport.report(ErrorCode.ERROR_LDAP_CONFIGURATION_ERR); throw new RuntimeException("ldapTemplate is not initialized"); } @@ -163,18 +163,18 @@ List getGroups(String userName) { return groups; } List groupDns; - - // Support Open Directory implementations - // If no group filter is configured, it defaults to querying groups based on the attribute 'member' - // for standard LDAP implementations if (!LdapConfig.ldap_group_filter.isEmpty()) { + // Support Open Directory implementations + String filter = LdapConfig.ldap_group_filter.replace("{login}", userName); groupDns = getDn(org.springframework.ldap.query.LdapQueryBuilder.query() - .base(LdapConfig.ldap_group_basedn) - .filter(getGroupFilter(LdapConfig.ldap_group_filter, userName))); + .attributes("dn") + .base(LdapConfig.ldap_group_basedn) + .filter(filter)); } else { + // Standard LDAP using member attribute groupDns = getDn(org.springframework.ldap.query.LdapQueryBuilder.query() - .base(LdapConfig.ldap_group_basedn) - .where("member").is(userDn)); + .base(LdapConfig.ldap_group_basedn) + .where("member").is(userDn)); } if (groupDns == null) { @@ -209,11 +209,12 @@ private String getUserDn(String userName) { private List getDn(LdapQuery query) { init(); try { - return clientInfo.getLdapTemplatePool().search(query, new AbstractContextMapper() { - protected String doMapFromContext(DirContextOperations ctx) { - return ctx.getNameInNamespace(); - } - }); + return clientInfo.getLdapTemplatePool().search(query, + new AbstractContextMapper() { + protected String doMapFromContext(DirContextOperations ctx) { + return ctx.getNameInNamespace(); + } + }); } catch (Exception e) { LOG.error("Get user dn fail.", e); ErrorReport.report(ErrorCode.ERROR_LDAP_CONFIGURATION_ERR); @@ -224,8 +225,4 @@ protected String doMapFromContext(DirContextOperations ctx) { private String getUserFilter(String userFilter, String userName) { return userFilter.replaceAll("\\{login}", userName); } - - private String getGroupFilter(String groupFilter, String userName) { - return groupFilter.replaceAll("\\{login}", userName); - } } From 746b8c9180ab41125a116bc4efb9f87293ca929b Mon Sep 17 00:00:00 2001 From: csding <65964830+codeDing18@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:47:49 +0800 Subject: [PATCH 552/572] =?UTF-8?q?branch-3.0:=20[Enhancement]=20reject=20?= =?UTF-8?q?schema=20change=20on=20hidden=20columns=20(#53=E2=80=A6=20(#553?= =?UTF-8?q?86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue Number: close https://github.com/apache/doris/issues/53019 picked from #53376 --- .../org/apache/doris/analysis/ColumnDef.java | 46 +++++++++-- .../java/org/apache/doris/catalog/Column.java | 1 + .../org/apache/doris/common/FeNameFormat.java | 24 ++++-- .../plans/commands/info/ColumnDefinition.java | 82 +++++++++++++++---- 4 files changed, 125 insertions(+), 28 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java index 966bc2dc18a9a5..189f54ea89d4f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java @@ -199,7 +199,12 @@ public String getValue() { private int clusterKeyId = -1; private Optional generatedColumnInfo = Optional.empty(); private Set generatedColumnsThatReferToThis = new HashSet<>(); + // if add hidden column, must set enableAddHiddenColumn true + private boolean enableAddHiddenColumn = false; + public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) { + this.enableAddHiddenColumn = enableAddHiddenColumn; + } public ColumnDef(String name, TypeDef typeDef) { this(name, typeDef, false, null, ColumnNullableType.NOT_NULLABLE, DefaultValue.NOT_SET, ""); @@ -269,44 +274,63 @@ public ColumnDef(String name, TypeDef typeDef, boolean isKey, ColumnNullableType } public static ColumnDef newDeleteSignColumnDef() { - return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, + ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), "doris delete flag hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newDeleteSignColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), false, aggregateType, + ColumnDef columnDef = new ColumnDef(Column.DELETE_SIGN, TypeDef.create(PrimitiveType.TINYINT), + false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), "doris delete flag hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newSequenceColumnDef(Type type) { - return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, null, ColumnNullableType.NULLABLE, -1, + ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, + null, ColumnNullableType.NULLABLE, -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newSequenceColumnDef(Type type, AggregateType aggregateType) { - return new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), false, aggregateType, ColumnNullableType.NULLABLE, + ColumnDef columnDef = new ColumnDef(Column.SEQUENCE_COL, new TypeDef(type), + false, aggregateType, ColumnNullableType.NULLABLE, -1, DefaultValue.NULL_DEFAULT_VALUE, "sequence column hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newRowStoreColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), false, aggregateType, + ColumnDef columnDef = new ColumnDef(Column.ROW_STORE_COL, TypeDef.create(PrimitiveType.STRING), + false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, ""), "doris row store hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newVersionColumnDef() { - return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, + ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, null, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), "doris version hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public static ColumnDef newVersionColumnDef(AggregateType aggregateType) { - return new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), false, aggregateType, + ColumnDef columnDef = new ColumnDef(Column.VERSION_COL, TypeDef.create(PrimitiveType.BIGINT), + false, aggregateType, ColumnNullableType.NOT_NULLABLE, -1, new ColumnDef.DefaultValue(true, "0"), "doris version hidden column", false, Optional.empty()); + columnDef.setEnableAddHiddenColumn(true); + return columnDef; } public boolean isAllowNull() { @@ -369,7 +393,13 @@ public void analyze(boolean isOlap) throws AnalysisException { if (name == null || typeDef == null) { throw new AnalysisException("No column name or column type in column definition."); } - FeNameFormat.checkColumnName(name); + // if enableAddHiddenColumn is true, can add hidden column. + // So does not check if the column name starts with __DORIS_ + if (enableAddHiddenColumn) { + FeNameFormat.checkColumnNameBypassHiddenColumn(name); + } else { + FeNameFormat.checkColumnName(name); + } FeNameFormat.checkColumnCommentLength(comment); typeDef.analyze(null); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 57c80a96696e9c..ecef449c67154e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -60,6 +60,7 @@ */ public class Column implements GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(Column.class); + public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_"; // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__"; public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java index c290647111151e..be1859e3af3ed1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeNameFormat.java @@ -20,6 +20,7 @@ import org.apache.doris.alter.SchemaChangeHandler; import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.ResourceTypeEnum; +import org.apache.doris.catalog.Column; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.Role; import org.apache.doris.mysql.privilege.RoleManager; @@ -86,18 +87,29 @@ public static void checkPartitionName(String partitionName) throws AnalysisExcep } public static void checkColumnName(String columnName) throws AnalysisException { + // if need check another column name prefix, add in `checkColumnNameBypassHiddenColumn` + checkColumnNameBypassHiddenColumn(columnName); + checkColumnNamePrefix(columnName, Column.HIDDEN_COLUMN_PREFIX); + } + + public static void checkColumnNameBypassHiddenColumn(String columnName) throws AnalysisException { if (Strings.isNullOrEmpty(columnName) || !columnName.matches(getColumnNameRegex())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, columnName, getColumnNameRegex()); } - if (columnName.startsWith(SchemaChangeHandler.SHADOW_NAME_PREFIX)) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_COLUMN_NAME, - columnName, getColumnNameRegex()); + checkColumnNamePrefix(columnName, SchemaChangeHandler.SHADOW_NAME_PREFIX); + checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX); + checkColumnNamePrefix(columnName, CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX); + } + + private static void checkColumnNamePrefix(String columnName, String prefix) throws AnalysisException { + int prefixLength = prefix.length(); + if (columnName.length() < prefixLength) { + return; } - if (columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX) - || columnName.startsWith(CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) { + if (columnName.substring(0, prefixLength).equalsIgnoreCase(prefix)) { throw new AnalysisException( - "Incorrect column name " + columnName + ", column name can't start with 'mv_'/'mva_'"); + "Incorrect column name " + columnName + ", column name can't start with '" + prefix + "'"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java index 239e3c0f32a355..edfe3151688e35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java @@ -68,6 +68,8 @@ public class ColumnDefinition { private int clusterKeyId = -1; private Optional generatedColumnDesc = Optional.empty(); private Set generatedColumnsThatReferToThis = new HashSet<>(); + // if add hidden column, must set enableAddHiddenColumn true + private boolean enableAddHiddenColumn = false; public ColumnDefinition(String name, DataType type, boolean isKey, AggregateType aggType, boolean isNullable, Optional defaultValue, String comment) { @@ -231,7 +233,14 @@ private void checkKeyColumnType(boolean isOlap) { public void validate(boolean isOlap, Set keysSet, Set clusterKeySet, boolean isEnableMergeOnWrite, KeysType keysType) { try { - FeNameFormat.checkColumnName(name); + // if enableAddHiddenColumn is true, can add hidden column. + // So does not check if the column name starts with __DORIS_ + if (enableAddHiddenColumn) { + FeNameFormat.checkColumnNameBypassHiddenColumn(name); + } else { + FeNameFormat.checkColumnName(name); + } + FeNameFormat.checkColumnCommentLength(comment); } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); @@ -674,35 +683,76 @@ public Column translateToCatalogStyle() { return column; } - // hidden column + /** + * add hidden column + */ public static ColumnDefinition newDeleteSignColumnDefinition() { - return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, null, + false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris delete flag hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newDeleteSignColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris delete flag hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.DELETE_SIGN, TinyIntType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris delete flag hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newSequenceColumnDefinition(DataType type) { - return new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, true, - Optional.empty(), "sequence column hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, null, + true, Optional.empty(), + "sequence column hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newSequenceColumnDefinition(DataType type, AggregateType aggregateType) { - return new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, true, - Optional.empty(), "sequence column hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.SEQUENCE_COL, type, false, aggregateType, + true, Optional.empty(), + "sequence column hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newRowStoreColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue("")), "doris row store hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue("")), + "doris row store hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } + /** + * add hidden column + */ public static ColumnDefinition newVersionColumnDefinition(AggregateType aggregateType) { - return new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, aggregateType, false, - Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), "doris version hidden column", false); + ColumnDefinition columnDefinition = new ColumnDefinition(Column.VERSION_COL, BigIntType.INSTANCE, false, + aggregateType, false, Optional.of(new DefaultValue(DefaultValue.ZERO_NUMBER)), + "doris version hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + + return columnDefinition; } public Optional getGeneratedColumnDesc() { @@ -717,6 +767,10 @@ public void addGeneratedColumnsThatReferToThis(List list) { generatedColumnsThatReferToThis.addAll(list); } + public void setEnableAddHiddenColumn(boolean enableAddHiddenColumn) { + this.enableAddHiddenColumn = enableAddHiddenColumn; + } + private void validateGeneratedColumnInfo() { // for generated column if (generatedColumnDesc.isPresent()) { From aa9bb39ae476f8002a04aa5a23f4a8c3d3dd8ec0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:48:20 +0800 Subject: [PATCH 553/572] branch-3.0: [fix](database) Fix `insert into` race with `drop table` #55264 (#55347) Cherry-picked from #55264 Co-authored-by: deardeng --- .../main/java/org/apache/doris/catalog/Database.java | 12 +++--------- .../org/apache/doris/common/proc/DbsProcDir.java | 2 +- .../org/apache/doris/datasource/InternalCatalog.java | 2 +- .../doris/transaction/DatabaseTransactionMgr.java | 2 +- .../transaction/DbUsedDataQuotaInfoCollector.java | 2 +- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index 10e20da5efbe59..2d784175c36eb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -297,20 +297,14 @@ public void setDbProperties(DatabaseProperty dbProperties) { } } - public long getUsedDataQuotaWithLock() { + public long getUsedDataQuota() { return getUsedDataSize().first; } public Pair getUsedDataSize() { long usedDataSize = 0; long usedRemoteDataSize = 0; - List
tables = new ArrayList<>(); - readLock(); - try { - tables.addAll(this.idToTable.values()); - } finally { - readUnlock(); - } + List
tables = new ArrayList<>(this.idToTable.values()); for (Table table : tables) { if (!table.isManagedTable()) { @@ -352,7 +346,7 @@ public void checkDataSizeQuota() throws DdlException { Pair quotaUnitPair = DebugUtil.getByteUint(dataQuotaBytes); String readableQuota = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(quotaUnitPair.first) + " " + quotaUnitPair.second; - long usedDataQuota = getUsedDataQuotaWithLock(); + long usedDataQuota = getUsedDataQuota(); long leftDataQuota = Math.max(dataQuotaBytes - usedDataQuota, 0); Pair leftQuotaUnitPair = DebugUtil.getByteUint(leftDataQuota); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java index 5a4fb3460ae8a1..bd9e3e9c73643a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java @@ -107,7 +107,7 @@ public ProcResult fetchResult() throws AnalysisException { dbInfo.add(dbName); dbInfo.add(tableNum); - long usedDataQuota = (db instanceof Database) ? ((Database) db).getUsedDataQuotaWithLock() : 0; + long usedDataQuota = (db instanceof Database) ? ((Database) db).getUsedDataQuota() : 0; long dataQuota = (db instanceof Database) ? ((Database) db).getDataQuota() : 0; String readableUsedQuota = DebugUtil.printByteWithUnit(usedDataQuota); String readableQuota = DebugUtil.printByteWithUnit(dataQuota); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 83ef30b0d0e808..da1b141bc8c713 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -3931,7 +3931,7 @@ public boolean enableAutoAnalyze() { public Map getUsedDataQuota() { Map dbToDataSize = new TreeMap<>(); for (Database db : this.idToDb.values()) { - dbToDataSize.put(db.getFullName(), db.getUsedDataQuotaWithLock()); + dbToDataSize.put(db.getFullName(), db.getUsedDataQuota()); } return dbToDataSize; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index c1d8018e6169b6..6263b48fabe73b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -384,7 +384,7 @@ private void checkDatabaseDataQuota() throws MetaNotFoundException, QuotaExceedE Database db = env.getInternalCatalog().getDbOrMetaException(dbId); if (usedQuotaDataBytes == -1) { - usedQuotaDataBytes = db.getUsedDataQuotaWithLock(); + usedQuotaDataBytes = db.getUsedDataQuota(); } long dataQuotaBytes = db.getDataQuota(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java index f03da231b9b114..3a85c75d284988 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DbUsedDataQuotaInfoCollector.java @@ -55,7 +55,7 @@ private void updateAllDatabaseUsedDataQuota() { continue; } try { - long usedDataQuotaBytes = db.getUsedDataQuotaWithLock(); + long usedDataQuotaBytes = db.getUsedDataQuota(); globalTransactionMgr.updateDatabaseUsedQuotaData(dbId, usedDataQuotaBytes); if (LOG.isDebugEnabled()) { LOG.debug("Update database[{}] used data quota bytes : {}.", db.getFullName(), usedDataQuotaBytes); From 0f5efd8dc090b81dda0f1683c19b887cce20d3f1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:49:01 +0800 Subject: [PATCH 554/572] branch-3.0: [bugfix](es catalog) Fix the parsing error of es catalog for special time format #54659 (#55327) Cherry-picked from #54659 Co-authored-by: lw112 <131352377+felixwluo@users.noreply.github.com> --- be/src/exec/es/es_scroll_parser.cpp | 2 +- be/test/exec/es_scroll_parser_test.cpp | 184 +++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 be/test/exec/es_scroll_parser_test.cpp diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 6067203f2ba7aa..6e72596bd4d5f0 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -199,7 +199,7 @@ Status get_date_value_int(const rapidjson::Value& col, PrimitiveType type, bool std::chrono::system_clock::time_point tp; // time_zone suffix pattern // Z/+08:00/-04:30 - RE2 time_zone_pattern(R"([+-]\d{2}:\d{2}|Z)"); + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); bool ok = false; std::string fmt; re2::StringPiece value; diff --git a/be/test/exec/es_scroll_parser_test.cpp b/be/test/exec/es_scroll_parser_test.cpp new file mode 100644 index 00000000000000..4f4f53618a0f41 --- /dev/null +++ b/be/test/exec/es_scroll_parser_test.cpp @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include + +namespace doris { + +class EsScrollParserTest : public testing::Test { +public: + void SetUp() override {} + void TearDown() override {} +}; + +// Test timezone pattern matching for ES datetime parsing fix +TEST_F(EsScrollParserTest, TestTimezonePatternMatching) { + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); + + std::vector valid_timezone_formats = { + "2025-05-23T20:56:52.052+0900", "2025-05-23T20:56:52.052-0500", + "2025-05-23T20:56:52.052+08:00", "2025-05-23T20:56:52.052-04:30", + "2025-05-23T20:56:52.052Z", "2022-08-08T12:10:10.151Z", + "2022-08-08T12:10:10+0900", "2022-08-08T12:10:10-0500"}; + + for (const auto& datetime_str : valid_timezone_formats) { + re2::StringPiece timezone_value; + bool matched = time_zone_pattern.Match(datetime_str, 0, datetime_str.size(), + RE2::UNANCHORED, &timezone_value, 1); + EXPECT_TRUE(matched) << "Failed to match timezone in: " << datetime_str; + + std::string timezone = timezone_value.as_string(); + EXPECT_FALSE(timezone.empty()) << "Empty timezone captured from: " << datetime_str; + + if (timezone == "Z") { + EXPECT_EQ(timezone, "Z"); + } else { + EXPECT_TRUE(timezone[0] == '+' || timezone[0] == '-') + << "Invalid timezone sign in: " << timezone; + // Valid timezone lengths: 5 for +0900, 6 for +08:00 + EXPECT_TRUE(timezone.length() == 5 || timezone.length() == 6) + << "Invalid timezone length in: " << timezone + << " (length: " << timezone.length() << ")"; + } + } +} + +TEST_F(EsScrollParserTest, TestInvalidTimezonePatterns) { + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); + + std::vector invalid_formats = { + "2025-05-23T20:56:52.052", "2025-05-23T20:56:52.052+9", "2025-05-23T20:56:52.052+090", + "2025-05-23T20:56:52.052+9:00"}; + + for (const auto& datetime_str : invalid_formats) { + re2::StringPiece timezone_value; + bool matched = time_zone_pattern.Match(datetime_str, 0, datetime_str.size(), + RE2::UNANCHORED, &timezone_value, 1); + if (matched) { + std::string timezone = timezone_value.as_string(); + EXPECT_TRUE(timezone.empty()) << "Should not capture timezone from: " << datetime_str; + } + } +} + +TEST_F(EsScrollParserTest, TestBugScenarioTimezoneFormat) { + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); + + std::string problematic_format = "2025-05-23T20:56:52.052+0900"; + + re2::StringPiece timezone_value; + bool matched = time_zone_pattern.Match(problematic_format, 0, problematic_format.size(), + RE2::UNANCHORED, &timezone_value, 1); + + EXPECT_TRUE(matched) << "Failed to match the bug scenario format: " << problematic_format; + + std::string timezone = timezone_value.as_string(); + EXPECT_EQ(timezone, "+0900") << "Incorrect timezone captured: " << timezone; +} + +TEST_F(EsScrollParserTest, TestEdgeCaseTimezoneFormats) { + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); + + std::vector edge_cases = {"+00:00", "-00:00", "+23:59", "-23:59", + "+99:99", "Z", "+0800", ""}; + + // Test each edge case + std::vector test_datetime_strings = { + "2025-05-23T20:56:52.052+00:00", // +00:00 (UTC with colon) + "2025-05-23T20:56:52.052-00:00", // -00:00 (UTC with colon) + "2025-05-23T20:56:52.052+23:59", // +23:59 (max valid timezone) + "2025-05-23T20:56:52.052-23:59", // -23:59 (max valid timezone) + "2025-05-23T20:56:52.052+99:99", // +99:99 (invalid but should match pattern) + "2025-05-23T20:56:52.052Z", // Z (UTC) + "2025-05-23T20:56:52.052+0800", // +0800 (no colon) + "2025-05-23T20:56:52.052" // empty timezone (no timezone) + }; + + std::vector expected_matches = {"+00:00", "-00:00", "+23:59", "-23:59", + "+99:99", "Z", "+0800", ""}; + + std::vector should_match = {true, true, true, true, true, true, true, false}; + + for (size_t i = 0; i < test_datetime_strings.size(); ++i) { + const std::string& datetime_str = test_datetime_strings[i]; + const std::string& expected_match = expected_matches[i]; + bool should_match_expected = should_match[i]; + + re2::StringPiece timezone_value; + bool matched = time_zone_pattern.Match(datetime_str, 0, datetime_str.size(), + RE2::UNANCHORED, &timezone_value, 1); + + EXPECT_EQ(matched, should_match_expected) + << "Edge case test failed for: " << datetime_str + << " (expected match: " << should_match_expected << ")"; + + if (matched && should_match_expected) { + std::string timezone = timezone_value.as_string(); + EXPECT_EQ(timezone, expected_match) + << "Incorrect timezone captured from: " << datetime_str + << " (expected: " << expected_match << ", got: " << timezone << ")"; + } + } +} + +TEST_F(EsScrollParserTest, TestSpecialTimezoneEdgeCases) { + RE2 time_zone_pattern(R"([+-]\d{2}:?\d{2}|Z)"); + + // Additional edge cases for comprehensive testing + std::vector>> special_cases = { + // {datetime_string, {expected_timezone, should_match}} + {"2025-05-23T20:56:52+0000", {"+0000", true}}, // +0000 without colon + {"2025-05-23T20:56:52-0000", {"-0000", true}}, // -0000 without colon + {"2025-05-23T20:56:52+12:30", {"+12:30", true}}, // +12:30 with colon + {"2025-05-23T20:56:52-12:30", {"-12:30", true}}, // -12:30 with colon + {"2025-05-23T20:56:52+1200", {"+1200", true}}, // +1200 without colon + {"2025-05-23T20:56:52-1200", {"-1200", true}}, // -1200 without colon + {"2025-05-23T20:56:52.000Z", {"Z", true}}, // Z with milliseconds + {"2025-05-23T20:56:52.123456+05:30", {"+05:30", true}}, // microseconds with timezone + {"2025-05-23T20:56:52.123456-05:30", {"-05:30", true}}, // microseconds with timezone + {"2025-05-23T20:56:52.123456+0530", {"+0530", true}}, // microseconds without colon + {"2025-05-23T20:56:52.123456-0530", {"-0530", true}}, // microseconds without colon + {"2025-05-23T20:56:52+14:00", {"+14:00", true}}, // +14:00 (valid max timezone) + {"2025-05-23T20:56:52-12:00", {"-12:00", true}}, // -12:00 (valid min timezone) + }; + + for (const auto& test_case : special_cases) { + const std::string& datetime_str = test_case.first; + const std::string& expected_timezone = test_case.second.first; + bool should_match = test_case.second.second; + + re2::StringPiece timezone_value; + bool matched = time_zone_pattern.Match(datetime_str, 0, datetime_str.size(), + RE2::UNANCHORED, &timezone_value, 1); + + EXPECT_EQ(matched, should_match) << "Special case test failed for: " << datetime_str + << " (expected match: " << should_match << ")"; + + if (matched && should_match) { + std::string timezone = timezone_value.as_string(); + EXPECT_EQ(timezone, expected_timezone) + << "Incorrect timezone captured from: " << datetime_str + << " (expected: " << expected_timezone << ", got: " << timezone << ")"; + } + } +} + +} // namespace doris From 0e4a785375079a3229eb33df9ab45447bd3badc5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:49:31 +0800 Subject: [PATCH 555/572] branch-3.0: [Fix] (catalog) Hive catalog query fails due to partition ID conflicts #54135 (#55288) Cherry-picked from #54135 Co-authored-by: John Zhang --- .../src/main/java/org/apache/doris/common/util/Util.java | 3 ++- .../test/java/org/apache/doris/common/util/UtilTest.java | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java index c394d9abc28709..0612b3794c637b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java @@ -46,6 +46,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.security.SecureRandom; @@ -697,7 +698,7 @@ public static String getRootCauseStack(Throwable t) { public static long sha256long(String str) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); - byte[] hash = digest.digest(str.getBytes()); + byte[] hash = digest.digest(str.getBytes(StandardCharsets.UTF_8)); ByteBuffer buffer = ByteBuffer.wrap(hash); return buffer.getLong(); } catch (NoSuchAlgorithmException e) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/UtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/UtilTest.java index 1f88cf5a662017..dc8419ddd1f56d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/UtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/UtilTest.java @@ -83,4 +83,11 @@ public void getRootCauseWithSuppressedMessageChainedExceptionWithCauseSuppressed "java.lang.Exception: Root cause message With suppressed[0]:Suppressed message", Util.getRootCauseWithSuppressedMessage(chainedException)); } + + @Test + public void sha256longEcoding() { + String str = "东南卫视"; + String str1 = "东方卫视"; + Assertions.assertNotEquals(Util.sha256long(str), Util.sha256long(str1)); + } } From bef03c9f9f9fe0ae02397b7f4b84695a23b465d1 Mon Sep 17 00:00:00 2001 From: Jack Date: Fri, 5 Sep 2025 09:49:51 +0800 Subject: [PATCH 556/572] branch-3.0: [feature](inverted index) introduce config enable_new_partition_inverted_index_v2_format to set new partiton inverted index format v2 #55069 (#55277) cherry pick from #55069 --- .../java/org/apache/doris/common/Config.java | 7 + .../datasource/CloudInternalCatalog.java | 8 +- .../doris/datasource/InternalCatalog.java | 8 +- .../datasource/CloudInternalCatalogTest.java | 775 ++++++++++++++++++ .../doris/datasource/InternalCatalogTest.java | 584 +++++++++++++ ..._inverted_index_storage_format_upgrade.out | 55 ++ ...verted_index_storage_format_upgrade.groovy | 409 +++++++++ 7 files changed, 1844 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/InternalCatalogTest.java create mode 100644 regression-test/data/inverted_index_p0/test_inverted_index_storage_format_upgrade.out create mode 100644 regression-test/suites/inverted_index_p0/test_inverted_index_storage_format_upgrade.groovy diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c09f3f315103a3..2476d927fe04a9 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2905,6 +2905,13 @@ public class Config extends ConfigBase { }) public static String inverted_index_storage_format = "V2"; + @ConfField(mutable = true, masterOnly = true, description = { + "是否为新分区启用倒排索引 V2 存储格式。启用后，新创建的分区将使用 V2 格式，而不管表的原始格式如何。", + "Enable V2 storage format for inverted indexes in new partitions. When enabled, newly created partitions " + + "will use V2 format regardless of the table's original format." + }) + public static boolean enable_new_partition_inverted_index_v2_format = false; + @ConfField(mutable = true, masterOnly = true, description = { "是否在unique表mow上开启delete语句写delete predicate。若开启，会提升delete语句的性能，" + "但delete后进行部分列更新可能会出现部分数据错误的情况。若关闭，会降低delete语句的性能来保证正确性。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java index ad7596c18fa251..98c6bed2ecbc26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java @@ -160,7 +160,13 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa Cloud.CreateTabletsRequest.Builder requestBuilder = Cloud.CreateTabletsRequest.newBuilder(); List rowStoreColumns = tbl.getTableProperty().getCopiedRowStoreColumns(); + TInvertedIndexFileStorageFormat effectiveIndexStorageFormat = + (Config.enable_new_partition_inverted_index_v2_format + && tbl.getInvertedIndexFileStorageFormat() == TInvertedIndexFileStorageFormat.V1) + ? TInvertedIndexFileStorageFormat.V2 + : tbl.getInvertedIndexFileStorageFormat(); for (Tablet tablet : index.getTablets()) { + // Use resolved format that considers global override for new partitions OlapFile.TabletMetaCloudPB.Builder builder = createTabletMetaBuilder(tbl.getId(), indexId, partitionId, tablet, tabletType, schemaHash, keysType, shortKeyColumnCount, bfColumns, tbl.getBfFpp(), indexes, columns, tbl.getDataSortInfo(), @@ -174,7 +180,7 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.disableAutoCompaction(), tbl.getRowStoreColumnsUniqueIds(rowStoreColumns), tbl.getEnableMowLightDelete(), - tbl.getInvertedIndexFileStorageFormat(), + effectiveIndexStorageFormat, tbl.rowStorePageSize(), tbl.variantEnableFlattenNested(), tbl.storagePageSize()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index da1b141bc8c713..8cfcac11202211 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2199,7 +2199,13 @@ protected Partition createPartitionWithIndices(long dbId, OlapTable tbl, long pa tbl.storagePageSize()); task.setStorageFormat(tbl.getStorageFormat()); - task.setInvertedIndexFileStorageFormat(tbl.getInvertedIndexFileStorageFormat()); + // Use resolved format that considers global override for new partitions + TInvertedIndexFileStorageFormat effectiveFormat = + (Config.enable_new_partition_inverted_index_v2_format + && tbl.getInvertedIndexFileStorageFormat() == TInvertedIndexFileStorageFormat.V1) + ? TInvertedIndexFileStorageFormat.V2 + : tbl.getInvertedIndexFileStorageFormat(); + task.setInvertedIndexFileStorageFormat(effectiveFormat); if (!CollectionUtils.isEmpty(clusterKeyIndexes)) { task.setClusterKeyIndexes(clusterKeyIndexes); LOG.info("table: {}, partition: {}, index: {}, tablet: {}, cluster key indexes: {}", diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java new file mode 100644 index 00000000000000..a7f3ddd6c1550c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java @@ -0,0 +1,775 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.datasource; + +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DataProperty; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndexMeta; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.ReplicaAllocation; +import org.apache.doris.catalog.TableProperty; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.cloud.catalog.CloudEnv; +import org.apache.doris.cloud.catalog.CloudEnvFactory; +import org.apache.doris.cloud.catalog.CloudPartition; +import org.apache.doris.cloud.catalog.ComputeGroup; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.cloud.rpc.VersionHelper; +import org.apache.doris.cloud.system.CloudSystemInfoService; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.persist.EditLog; +import org.apache.doris.proto.OlapFile; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.resource.Tag; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TCompressionType; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TStorageType; +import org.apache.doris.thrift.TTabletType; +import org.apache.doris.utframe.MockedMetaServerFactory; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import mockit.Mock; +import mockit.MockUp; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +public class CloudInternalCatalogTest { + + private static final String TEST_DB_NAME = "test_cloud_db"; + private static final String TEST_TABLE_NAME = "test_cloud_table"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private static Database db; + private ConnectContext ctx; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException, UserException { + FeConstants.runningUnitTest = true; + Config.enable_new_partition_inverted_index_v2_format = false; + + // Mock VersionHelper globally to avoid all meta service calls + new MockUp() { + @Mock + public long getVersionFromMeta(long dbId, long tableId, long partitionId) { + return 1L; + } + + @Mock + public long getVisibleVersion(long dbId, long tableId, long partitionId) { + return 1L; + } + }; + + // Mock CloudPartition globally to avoid meta service calls + new MockUp() { + @Mock + public long getVisibleVersion() { + return 1L; + } + }; + + // Setup for MetaServiceProxy mock + new MockUp(MetaServiceProxy.class) { + @Mock + public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest request) { + Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder = Cloud.BeginTxnResponse.newBuilder(); + beginTxnResponseBuilder.setTxnId(1000) + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")); + return beginTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest request) { + Cloud.TxnInfoPB.Builder txnInfoBuilder = Cloud.TxnInfoPB.newBuilder(); + txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1); + txnInfoBuilder.addAllTableIds(Lists.newArrayList(CatalogTestUtil.testTableId1)); + txnInfoBuilder.setLabel("test_label"); + txnInfoBuilder.setListenerId(-1); + Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder = Cloud.CommitTxnResponse.newBuilder(); + commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")) + .setTxnInfo(txnInfoBuilder.build()); + return commitTxnResponseBuilder.build(); + } + + @Mock + public Cloud.CheckTxnConflictResponse checkTxnConflict(Cloud.CheckTxnConflictRequest request) { + Cloud.CheckTxnConflictResponse.Builder checkTxnConflictResponseBuilder = + Cloud.CheckTxnConflictResponse.newBuilder(); + checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")) + .setFinished(true); + return checkTxnConflictResponseBuilder.build(); + } + + @Mock + public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest request) { + Cloud.GetClusterResponse.Builder getClusterResponseBuilder = Cloud.GetClusterResponse.newBuilder(); + Cloud.ClusterPB.Builder clusterBuilder = Cloud.ClusterPB.newBuilder(); + clusterBuilder.setClusterId("test_id").setClusterName("test_group"); + + Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder(); + node1.setCloudUniqueId("test_cloud") + .setName("host1") + .setIp("host1") + .setHost("host1") + .setHeartbeatPort(123) + .setEditLogPort(125) + .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING); + clusterBuilder.addNodes(node1.build()); + getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")) + .addCluster(clusterBuilder.build()); + return getClusterResponseBuilder.build(); + } + + @Mock + public Cloud.CreateTabletsResponse createTablets(Cloud.CreateTabletsRequest request) { + Cloud.CreateTabletsResponse.Builder responseBuilder = Cloud.CreateTabletsResponse.newBuilder(); + responseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.FinishTabletJobResponse finishTabletJob(Cloud.FinishTabletJobRequest request) { + Cloud.FinishTabletJobResponse.Builder responseBuilder = Cloud.FinishTabletJobResponse.newBuilder(); + responseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")); + return responseBuilder.build(); + } + + @Mock + public Cloud.GetCurrentMaxTxnResponse getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) { + Cloud.GetCurrentMaxTxnResponse.Builder builder = Cloud.GetCurrentMaxTxnResponse.newBuilder(); + builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK).setMsg("OK")) + .setCurrentMaxTxnId(1000); + return builder.build(); + } + }; + + Config.cloud_unique_id = "test_cloud"; + Config.meta_service_endpoint = MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo(); + fakeEnv = new FakeEnv(); + FakeEnv.setSystemInfo(cloudSystemInfo); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; // Will be set by MockUp + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + ctx.setCloudCluster("test_group"); + + Assert.assertTrue(envFactory instanceof CloudEnvFactory); + Assert.assertTrue(masterEnv instanceof CloudEnv); + + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + testEditLog = new EditLog("test") { + // Override to avoid initialization issues + }; + } + return testEditLog; + } + + @Mock + public AccessControllerManager getAccessManager() { + return new AccessControllerManager(masterEnv.getAuth()) { + @Override + public boolean checkTblPriv(ConnectContext ctx, String ctl, String db, String tbl, + PrivPredicate wanted) { + return true; // Allow all access for test + } + }; + } + }; + + new MockUp() { + @Mock + public String getDefaultCloudCluster(String user) { + return "test_group"; // Return default cluster for test + } + + @Mock + public ComputeGroup getComputeGroup(String user) { + // Return a test compute group for the mock + return new ComputeGroup("test_id", "test_group", ComputeGroup.ComputeTypeEnum.SQL); + } + }; + + // Mock cloud environment permissions + new MockUp() { + @Mock + public void checkCloudClusterPriv(String cluster) throws Exception { + // Always allow for tests + } + }; + + // Mock ConnectContext to avoid compute group permission check + new MockUp() { + @Mock + public String getCloudCluster() { + return "test_group"; + } + + @Mock + public UserIdentity getCurrentUserIdentity() { + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + return rootUser; + } + }; + + // Setup CloudSystemInfoService directly like CloudIndexTest + Assert.assertTrue(Env.getCurrentSystemInfo() instanceof CloudSystemInfoService); + CloudSystemInfoService systemInfo = (CloudSystemInfoService) Env.getCurrentSystemInfo(); + Backend backend = new Backend(10001L, "host1", 123); + backend.setAlive(true); + backend.setBePort(456); + backend.setHttpPort(789); + backend.setBrpcPort(321); + Map newTagMap = Tag.DEFAULT_BACKEND_TAG.toMap(); + newTagMap.put(Tag.CLOUD_CLUSTER_STATUS, "NORMAL"); + newTagMap.put(Tag.CLOUD_CLUSTER_NAME, "test_group"); + newTagMap.put(Tag.CLOUD_CLUSTER_ID, "test_id"); + newTagMap.put(Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, ""); + newTagMap.put(Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, ""); + newTagMap.put(Tag.CLOUD_UNIQUE_ID, "test_cloud"); + backend.setTagMap(newTagMap); + List backends = Lists.newArrayList(backend); + systemInfo.updateCloudClusterMapNoLock(backends, new ArrayList<>()); + + db = new Database(CatalogTestUtil.testDbId1, TEST_DB_NAME); + masterEnv.unprotectCreateDb(db); + } + + @Test + public void testCloudMixedFormatPartitions() throws Exception { + // Test: Old partitions keep V1, new partitions use V2 when config is enabled + + // Step 1: Create initial partition with V1 format (config disabled) + Config.enable_new_partition_inverted_index_v2_format = false; + Map partitionFormats = Maps.newHashMap(); + + // Mock sendCreateTabletsRpc to avoid actual meta service calls + new MockUp() { + @Mock + public Cloud.CreateTabletsResponse sendCreateTabletsRpc(Cloud.CreateTabletsRequest.Builder requestBuilder) + throws DdlException { + return Cloud.CreateTabletsResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK) + .setMsg("OK")) + .build(); + } + }; + + // Mock createTabletMetaBuilder to capture formats for each partition + new MockUp() { + @Mock + public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId, + long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, + short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, + List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, + String storagePolicy, boolean isInMemory, boolean isShadow, + String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, + boolean storeRowColumn, int schemaVersion, String compactionPolicy, + Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, + Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, + Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, + List rowStoreColumnUniqueIds, boolean enableMowLightDelete, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize, + boolean variantEnableFlattenNested, long storagePageSize) throws DdlException { + + // Track format for each partition + partitionFormats.put(partitionId, invertedIndexFileStorageFormat); + return OlapFile.TabletMetaCloudPB.newBuilder(); + } + }; + + CloudInternalCatalog cloudCatalog = (CloudInternalCatalog) masterEnv.getInternalCatalog(); + + // Create MaterializedIndexMeta for base index + long baseIndexId = 2000L; + MaterializedIndexMeta indexMeta = + new MaterializedIndexMeta( + baseIndexId, + Lists.newArrayList(new Column("col1", + PrimitiveType.INT)), + 0, // schema version + 100, // schema hash + (short) 1, // short key column count + TStorageType.COLUMN, + KeysType.DUP_KEYS, + new OriginStatement("CREATE TABLE test", 0) // origin stmt + ); + Map indexIdToMeta = Maps.newHashMap(); + indexIdToMeta.put(baseIndexId, indexMeta); + + // Mock OlapTable with V1 format + new MockUp() { + @Mock + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return TInvertedIndexFileStorageFormat.V1; // Table has V1 format + } + + @Mock + public long getId() { + return 1000L; + } + + @Mock + public long getBaseIndexId() { + return baseIndexId; + } + + @Mock + public String getStorageVaultId() { + return "vault_id"; + } + + @Mock + public String getStorageVaultName() { + return "vault_name"; + } + + @Mock + public String getName() { + return "test_table"; + } + + @Mock + public java.util.List getIndexes() { + return Lists.newArrayList(); + } + + @Mock + public TableProperty getTableProperty() { + return new TableProperty(Maps.newHashMap()); + } + + @Mock + public double getBfFpp() { + return 0.05; + } + + @Mock + public DataSortInfo getDataSortInfo() { + return null; + } + }; + + // Create initial partition + long partition1Id = 3000L; + try { + OlapTable table = new OlapTable(); + cloudCatalog.createPartitionWithIndices( + db.getId(), table, partition1Id, "p1", + indexIdToMeta, // Pass proper index metadata + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + // Expected in test environment + } + + // Verify partition1 uses V1 format (config was disabled) + Assert.assertEquals("First partition should use V1 format when config is disabled", + TInvertedIndexFileStorageFormat.V1, partitionFormats.get(partition1Id)); + + // Step 2: Enable config and create new partition + Config.enable_new_partition_inverted_index_v2_format = true; + + long partition2Id = 3001L; + try { + OlapTable table = new OlapTable(); + cloudCatalog.createPartitionWithIndices( + db.getId(), table, partition2Id, "p2", + indexIdToMeta, // Pass proper index metadata + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + // Expected in test environment + } + + // Step 3: Verify mixed formats + Assert.assertEquals("First partition should still be V1", + TInvertedIndexFileStorageFormat.V1, partitionFormats.get(partition1Id)); + Assert.assertEquals("Second partition should be upgraded to V2", + TInvertedIndexFileStorageFormat.V2, partitionFormats.get(partition2Id)); + } + + @Test + public void testCloudV1FormatRemainsWhenConfigDisabled() throws Exception { + // Test: V1 table format should remain V1 when config is disabled + Config.enable_new_partition_inverted_index_v2_format = false; + + AtomicReference capturedFormat = new AtomicReference<>(); + + // Mock sendCreateTabletsRpc to avoid actual meta service calls + new MockUp() { + @Mock + public Cloud.CreateTabletsResponse sendCreateTabletsRpc(Cloud.CreateTabletsRequest.Builder requestBuilder) + throws DdlException { + return Cloud.CreateTabletsResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK) + .setMsg("OK")) + .build(); + } + }; + + // Mock createTabletMetaBuilder to capture the actual format used during partition creation + new MockUp() { + @Mock + public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId, + long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, + short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, + List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, + String storagePolicy, boolean isInMemory, boolean isShadow, + String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, + boolean storeRowColumn, int schemaVersion, String compactionPolicy, + Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, + Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, + Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, + List rowStoreColumnUniqueIds, boolean enableMowLightDelete, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize, + boolean variantEnableFlattenNested, long storagePageSize) throws DdlException { + + // Capture the actual format passed to createTabletMetaBuilder + capturedFormat.set(invertedIndexFileStorageFormat); + return OlapFile.TabletMetaCloudPB.newBuilder(); + } + }; + + CloudInternalCatalog cloudCatalog = (CloudInternalCatalog) masterEnv.getInternalCatalog(); + + // Create MaterializedIndexMeta for base index + long baseIndexId = 2000L; + MaterializedIndexMeta indexMeta = + new MaterializedIndexMeta( + baseIndexId, + Lists.newArrayList(new Column("col1", + PrimitiveType.INT)), + 0, // schema version + 100, // schema hash + (short) 1, // short key column count + TStorageType.COLUMN, + KeysType.DUP_KEYS, + new OriginStatement("CREATE TABLE test", 0) // origin stmt + ); + Map indexIdToMeta = Maps.newHashMap(); + indexIdToMeta.put(baseIndexId, indexMeta); + + // Create a mock OlapTable with V1 format + new MockUp() { + @Mock + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return TInvertedIndexFileStorageFormat.V1; // Table originally has V1 format + } + + @Mock + public long getId() { + return 1000L; + } + + @Mock + public long getBaseIndexId() { + return baseIndexId; + } + + @Mock + public String getStorageVaultId() { + return "vault_id"; + } + + @Mock + public String getStorageVaultName() { + return "vault_name"; + } + + @Mock + public String getName() { + return "test_table"; + } + + @Mock + public java.util.List getIndexes() { + return Lists.newArrayList(); + } + + @Mock + public TableProperty getTableProperty() { + return new TableProperty(Maps.newHashMap()); + } + + @Mock + public double getBfFpp() { + return 0.05; + } + + @Mock + public DataSortInfo getDataSortInfo() { + return null; + } + }; + + try { + OlapTable table = new OlapTable(); + + // Call the actual createPartitionWithIndices method to test no upgrade when config disabled + cloudCatalog.createPartitionWithIndices( + db.getId(), table, 3000L, "test_partition", + indexIdToMeta, // Pass proper index metadata + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + // It's expected to fail in test environment, we only care about the format capture + } + + // Verify that V1 table format remains V1 when config is disabled + Assert.assertEquals("V1 table format should remain V1 when config is disabled", + TInvertedIndexFileStorageFormat.V1, capturedFormat.get()); + } + + @Test + public void testCloudV2TableFormatBehavior() throws Exception { + // Test V2 table format behavior in cloud mode - should remain V2 regardless of config + Config.enable_new_partition_inverted_index_v2_format = true; + AtomicReference capturedFormat = new AtomicReference<>(); + // Mock sendCreateTabletsRpc to avoid actual meta service calls + new MockUp() { + @Mock + public Cloud.CreateTabletsResponse sendCreateTabletsRpc(Cloud.CreateTabletsRequest.Builder requestBuilder) + throws DdlException { + return Cloud.CreateTabletsResponse.newBuilder() + .setStatus(Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.OK) + .setMsg("OK")) + .build(); + } + }; + + // Mock createTabletMetaBuilder to capture the actual format used during partition creation + new MockUp() { + @Mock + public OlapFile.TabletMetaCloudPB.Builder createTabletMetaBuilder(long tableId, long indexId, + long partitionId, Tablet tablet, TTabletType tabletType, int schemaHash, KeysType keysType, + short shortKeyColumnCount, Set bfColumns, double bfFpp, List indexes, + List schemaColumns, DataSortInfo dataSortInfo, TCompressionType compressionType, + String storagePolicy, boolean isInMemory, boolean isShadow, + String tableName, long ttlSeconds, boolean enableUniqueKeyMergeOnWrite, + boolean storeRowColumn, int schemaVersion, String compactionPolicy, + Long timeSeriesCompactionGoalSizeMbytes, Long timeSeriesCompactionFileCountThreshold, + Long timeSeriesCompactionTimeThresholdSeconds, Long timeSeriesCompactionEmptyRowsetsThreshold, + Long timeSeriesCompactionLevelThreshold, boolean disableAutoCompaction, + List rowStoreColumnUniqueIds, boolean enableMowLightDelete, + TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat, long pageSize, + boolean variantEnableFlattenNested, long storagePageSize) throws DdlException { + // Capture the actual format passed to createTabletMetaBuilder + capturedFormat.set(invertedIndexFileStorageFormat); + return OlapFile.TabletMetaCloudPB.newBuilder(); + } + }; + CloudInternalCatalog cloudCatalog = (CloudInternalCatalog) masterEnv.getInternalCatalog(); + // Create MaterializedIndexMeta for base index + long baseIndexId = 2000L; + MaterializedIndexMeta indexMeta = + new MaterializedIndexMeta( + baseIndexId, + Lists.newArrayList(new Column("col1", + PrimitiveType.INT)), + 0, // schema version + 100, // schema hash + (short) 1, // short key column count + TStorageType.COLUMN, + KeysType.DUP_KEYS, + new OriginStatement("CREATE TABLE test", 0) // origin stmt + ); + Map indexIdToMeta = Maps.newHashMap(); + indexIdToMeta.put(baseIndexId, indexMeta); + + // Create a mock OlapTable with V2 format + new MockUp() { + @Mock + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return TInvertedIndexFileStorageFormat.V2; // Table originally has V2 format + } + + @Mock + public long getId() { + return 1000L; + } + + @Mock + public long getBaseIndexId() { + return baseIndexId; + } + + @Mock + public String getStorageVaultId() { + return "vault_id"; + } + + @Mock + public String getStorageVaultName() { + return "vault_name"; + } + + @Mock + public String getName() { + return "test_table"; + } + + @Mock + public java.util.List getIndexes() { + return Lists.newArrayList(); + } + + @Mock + public TableProperty getTableProperty() { + return new TableProperty(Maps.newHashMap()); + } + + @Mock + public double getBfFpp() { + return 0.05; + } + + @Mock + public DataSortInfo getDataSortInfo() { + return null; + } + }; + + try { + OlapTable table = new OlapTable(); + + // Call the actual createPartitionWithIndices method to test V2 format behavior + cloudCatalog.createPartitionWithIndices( + db.getId(), table, 3000L, "test_partition", + indexIdToMeta, // Pass proper index metadata + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + // It's expected to fail in test environment, we only care about the format capture + } + + // Verify that V2 table format remains V2 when config is enabled + Assert.assertEquals("V2 table format should remain V2 when config is enabled", + TInvertedIndexFileStorageFormat.V2, capturedFormat.get()); + + // Test with config disabled - V2 should still remain V2 + capturedFormat.set(null); // Reset + Config.enable_new_partition_inverted_index_v2_format = false; + + try { + OlapTable table = new OlapTable(); + cloudCatalog.createPartitionWithIndices( + db.getId(), table, 3001L, "test_partition2", + indexIdToMeta, // Pass proper index metadata + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + // It's expected to fail in test environment, we only care about the format capture + } + + // Verify that V2 table format remains V2 even when config is disabled + Assert.assertEquals("V2 table format should remain V2 when config is disabled", + TInvertedIndexFileStorageFormat.V2, capturedFormat.get()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/InternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/InternalCatalogTest.java new file mode 100644 index 00000000000000..4b310b1dcf1383 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/InternalCatalogTest.java @@ -0,0 +1,584 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.analysis.DataSortInfo; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.CatalogTestUtil; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DataProperty; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.FakeEditLog; +import org.apache.doris.catalog.FakeEnv; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.MaterializedIndexMeta; +import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.Replica; +import org.apache.doris.catalog.Replica.ReplicaState; +import org.apache.doris.catalog.ReplicaAllocation; +import org.apache.doris.catalog.TableProperty; +import org.apache.doris.catalog.Tablet; +import org.apache.doris.catalog.TabletMeta; +import org.apache.doris.common.Config; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.MarkedCountDownLatch; +import org.apache.doris.common.Status; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.task.CreateReplicaTask; +import org.apache.doris.thrift.TInvertedIndexFileStorageFormat; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.thrift.TStorageType; +import org.apache.doris.thrift.TTabletType; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import mockit.Invocation; +import mockit.Mock; +import mockit.MockUp; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.InvocationTargetException; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +public class InternalCatalogTest { + + private static final String TEST_DB_NAME = "test_db"; + private static final String TEST_TABLE_NAME = "test_table"; + + private static FakeEditLog fakeEditLog; + private static FakeEnv fakeEnv; + private static Env masterEnv; + private static EditLog testEditLog; + private static Database db; + private ConnectContext ctx; + + @Before + public void setUp() throws InstantiationException, IllegalAccessException, IllegalArgumentException, + InvocationTargetException, NoSuchMethodException, SecurityException { + FeConstants.runningUnitTest = true; + Config.enable_new_partition_inverted_index_v2_format = false; + + EnvFactory envFactory = EnvFactory.getInstance(); + masterEnv = envFactory.createEnv(false); + fakeEnv = new FakeEnv(); + + // Create SystemInfoService with a live backend + SystemInfoService systemInfoService = new SystemInfoService(); + Backend backend = new Backend(0, "127.0.0.1", 9050); + backend.updateOnce(9060, 8040, 9070); // bePort, httpPort, beRpcPort + systemInfoService.addBackend(backend); + + FakeEnv.setSystemInfo(systemInfoService); + + fakeEditLog = new FakeEditLog(); + testEditLog = null; + FakeEnv.setEnv(masterEnv); + + ctx = new ConnectContext(); + ctx.setEnv(masterEnv); + UserIdentity rootUser = new UserIdentity("root", "%"); + rootUser.setIsAnalyzed(); + ctx.setCurrentUserIdentity(rootUser); + ctx.setThreadLocalInfo(); + + new MockUp() { + @Mock + public Env getCurrentEnv() { + return masterEnv; + } + + @Mock + public EditLog getEditLog() { + if (testEditLog == null) { + testEditLog = new EditLog("test") { + }; + } + return testEditLog; + } + }; + + db = new Database(CatalogTestUtil.testDbId1, TEST_DB_NAME); + masterEnv.unprotectCreateDb(db); + } + + @Test + public void testMixedFormatPartitions() throws Exception { + // Test: Old partitions keep V1, new partitions use V2 when config is enabled + + // Step 1: Create initial partition with V1 format (config disabled) + Config.enable_new_partition_inverted_index_v2_format = false; + Map partitionFormats = Maps.newHashMap(); + + // Mock MarkedCountDownLatch to immediately return success + new MockUp() { + @Mock + public boolean await(long time, java.util.concurrent.TimeUnit unit) { + return true; // Immediately return success + } + + @Mock + public Status getStatus() { + return Status.OK; + } + }; + + new MockUp() { + @Mock + public TStorageMedium createTablets(MaterializedIndex index, ReplicaState replicaState, + DistributionInfo distributionInfo, long version, ReplicaAllocation replicaAlloc, + TabletMeta tabletMeta, Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer, + boolean isStorageMediumSpecified) throws DdlException { + Tablet tablet = new org.apache.doris.catalog.Tablet(10001); + Replica replica = new Replica(10031, 0, 0, replicaState); + tablet.addReplica(replica, true); + index.addTablet(tablet, tabletMeta); + tabletIdSet.add(tablet.getId()); + return TStorageMedium.HDD; + } + }; + // Mock CreateReplicaTask to capture the format set for each partition + new MockUp() { + @Mock + public void setInvertedIndexFileStorageFormat(Invocation inv, TInvertedIndexFileStorageFormat format) { + // Capture the format for this partition + // We'll use a simple approach to capture the format without calling the real method + // since we're in a mock context + CreateReplicaTask self = inv.getInvokedInstance(); + long pid = self.getPartitionId(); + partitionFormats.put(pid, format); // Use a default key for now + } + }; + + InternalCatalog internalCatalog = (InternalCatalog) masterEnv.getInternalCatalog(); + + // Create MaterializedIndexMeta for base index + long baseIndexId = 2000L; + MaterializedIndexMeta indexMeta = + new MaterializedIndexMeta( + baseIndexId, + Lists.newArrayList(new Column("col1", + PrimitiveType.INT)), + 0, + 100, + (short) 1, + TStorageType.COLUMN, + KeysType.DUP_KEYS, + new OriginStatement("CREATE TABLE test", 0) + ); + Map indexIdToMeta = Maps.newHashMap(); + indexIdToMeta.put(baseIndexId, indexMeta); + + // Mock OlapTable with V1 format + new MockUp() { + @Mock + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return TInvertedIndexFileStorageFormat.V1; + } + + @Mock + public long getId() { + return 1000L; + } + + @Mock + public long getBaseIndexId() { + return baseIndexId; + } + + @Mock + public String getName() { + return "test_table"; + } + + @Mock + public java.util.List getIndexes() { + return Lists.newArrayList(); + } + + @Mock + public TableProperty getTableProperty() { + return new TableProperty(Maps.newHashMap()); + } + + @Mock + public double getBfFpp() { + return 0.05; + } + + @Mock + public DataSortInfo getDataSortInfo() { + return null; + } + }; + + // Create initial partition + long partition1Id = 3000L; + try { + OlapTable table = new OlapTable(); + internalCatalog.createPartitionWithIndices( + db.getId(), table, partition1Id, "p1", + indexIdToMeta, + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + e.printStackTrace(); + // Expected in test environment + } + + // Verify partition1 uses V1 format (config was disabled) + Assert.assertEquals("First partition should use V1 format when config is disabled", + TInvertedIndexFileStorageFormat.V1, partitionFormats.get(partition1Id)); + + // Step 2: Enable config and create new partition + Config.enable_new_partition_inverted_index_v2_format = true; + + long partition2Id = 3001L; + try { + OlapTable table = new OlapTable(); + internalCatalog.createPartitionWithIndices( + db.getId(), table, partition2Id, "p2", + indexIdToMeta, + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + e.printStackTrace(); + // Expected in test environment + } + + // Step 3: Verify mixed formats + Assert.assertEquals("First partition should still be V1", + TInvertedIndexFileStorageFormat.V1, partitionFormats.get(partition1Id)); + Assert.assertEquals("Second partition should be upgraded to V2", + TInvertedIndexFileStorageFormat.V2, partitionFormats.get(partition2Id)); + } + + @Test + public void testV1FormatRemainsWhenConfigDisabled() throws Exception { + // Test: V1 table format should remain V1 when config is disabled + Config.enable_new_partition_inverted_index_v2_format = false; + + AtomicReference capturedFormat = new AtomicReference<>(); + + // Mock MarkedCountDownLatch to immediately return success + new MockUp() { + @Mock + public boolean await(long time, java.util.concurrent.TimeUnit unit) { + return true; // Immediately return success + } + + @Mock + public Status getStatus() { + return Status.OK; + } + }; + + new MockUp() { + @Mock + public TStorageMedium createTablets(MaterializedIndex index, ReplicaState replicaState, + DistributionInfo distributionInfo, long version, ReplicaAllocation replicaAlloc, + TabletMeta tabletMeta, Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer, + boolean isStorageMediumSpecified) throws DdlException { + Tablet tablet = new org.apache.doris.catalog.Tablet(10001); + Replica replica = new Replica(10031, 0, 0, replicaState); + tablet.addReplica(replica, true); + index.addTablet(tablet, tabletMeta); + tabletIdSet.add(tablet.getId()); + return TStorageMedium.HDD; + } + }; + // Mock CreateReplicaTask to capture the format set for each partition + new MockUp() { + @Mock + public void setInvertedIndexFileStorageFormat(Invocation inv, TInvertedIndexFileStorageFormat format) { + // Capture the format for this partition + // We'll use a simple approach to capture the format without calling the real method + // since we're in a mock context + capturedFormat.set(format); // Use a default key for now + } + }; + + InternalCatalog internalCatalog = (InternalCatalog) masterEnv.getInternalCatalog(); + + // Create MaterializedIndexMeta for base index + long baseIndexId = 2000L; + MaterializedIndexMeta indexMeta = + new MaterializedIndexMeta( + baseIndexId, + Lists.newArrayList(new Column("col1", + PrimitiveType.INT)), + 0, + 100, + (short) 1, + TStorageType.COLUMN, + KeysType.DUP_KEYS, + new OriginStatement("CREATE TABLE test", 0) + ); + Map indexIdToMeta = Maps.newHashMap(); + indexIdToMeta.put(baseIndexId, indexMeta); + + // Create a mock OlapTable with V1 format + new MockUp() { + @Mock + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return TInvertedIndexFileStorageFormat.V1; + } + + @Mock + public long getId() { + return 1000L; + } + + @Mock + public long getBaseIndexId() { + return baseIndexId; + } + + @Mock + public String getName() { + return "test_table"; + } + + @Mock + public java.util.List getIndexes() { + return Lists.newArrayList(); + } + + @Mock + public TableProperty getTableProperty() { + return new TableProperty(Maps.newHashMap()); + } + + @Mock + public double getBfFpp() { + return 0.05; + } + + @Mock + public DataSortInfo getDataSortInfo() { + return null; + } + }; + + try { + OlapTable table = new OlapTable(); + + // Call the actual createPartitionWithIndices method to test no upgrade when config disabled + internalCatalog.createPartitionWithIndices( + db.getId(), table, 3000L, "test_partition", + indexIdToMeta, + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + e.printStackTrace(); + // It's expected to fail in test environment, we only care about the format capture + } + + // Verify that V1 table format remains V1 when config is disabled + Assert.assertEquals("V1 table format should remain V1 when config is disabled", + TInvertedIndexFileStorageFormat.V1, capturedFormat.get()); + } + + @Test + public void testV2TableFormatBehavior() throws Exception { + // Test V2 table format behavior - should remain V2 regardless of config + Config.enable_new_partition_inverted_index_v2_format = true; + AtomicReference capturedFormat = new AtomicReference<>(); + + // Mock MarkedCountDownLatch to immediately return success + new MockUp() { + @Mock + public boolean await(long time, java.util.concurrent.TimeUnit unit) { + return true; // Immediately return success + } + + @Mock + public Status getStatus() { + return Status.OK; + } + }; + + new MockUp() { + @Mock + public TStorageMedium createTablets(MaterializedIndex index, ReplicaState replicaState, + DistributionInfo distributionInfo, long version, ReplicaAllocation replicaAlloc, + TabletMeta tabletMeta, Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer, + boolean isStorageMediumSpecified) throws DdlException { + Tablet tablet = new org.apache.doris.catalog.Tablet(10001); + Replica replica = new Replica(10031, 0, 0, replicaState); + tablet.addReplica(replica, true); + index.addTablet(tablet, tabletMeta); + tabletIdSet.add(tablet.getId()); + return TStorageMedium.HDD; + } + }; + // Mock CreateReplicaTask to capture the format set for each partition + new MockUp() { + @Mock + public void setInvertedIndexFileStorageFormat(Invocation inv, TInvertedIndexFileStorageFormat format) { + // Capture the format for this partition + // We'll use a simple approach to capture the format without calling the real method + // since we're in a mock context + capturedFormat.set(format); // Use a default key for now + } + }; + + InternalCatalog internalCatalog = (InternalCatalog) masterEnv.getInternalCatalog(); + + // Create MaterializedIndexMeta for base index + long baseIndexId = 2000L; + MaterializedIndexMeta indexMeta = + new MaterializedIndexMeta( + baseIndexId, + Lists.newArrayList(new Column("col1", + PrimitiveType.INT)), + 0, + 100, + (short) 1, + TStorageType.COLUMN, + KeysType.DUP_KEYS, + new OriginStatement("CREATE TABLE test", 0) + ); + Map indexIdToMeta = Maps.newHashMap(); + indexIdToMeta.put(baseIndexId, indexMeta); + + // Create a mock OlapTable with V2 format + new MockUp() { + @Mock + public TInvertedIndexFileStorageFormat getInvertedIndexFileStorageFormat() { + return TInvertedIndexFileStorageFormat.V2; + } + + @Mock + public long getId() { + return 1000L; + } + + @Mock + public long getBaseIndexId() { + return baseIndexId; + } + + @Mock + public String getName() { + return "test_table"; + } + + @Mock + public java.util.List getIndexes() { + return Lists.newArrayList(); + } + + @Mock + public TableProperty getTableProperty() { + return new TableProperty(Maps.newHashMap()); + } + + @Mock + public double getBfFpp() { + return 0.05; + } + + @Mock + public DataSortInfo getDataSortInfo() { + return null; + } + }; + + try { + OlapTable table = new OlapTable(); + + // Call the actual createPartitionWithIndices method to test V2 format behavior + internalCatalog.createPartitionWithIndices( + db.getId(), table, 3000L, "test_partition", + indexIdToMeta, + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + e.printStackTrace(); + // It's expected to fail in test environment, we only care about the format capture + } + + // Verify that V2 table format remains V2 when config is enabled + Assert.assertEquals("V2 table format should remain V2 when config is enabled", + TInvertedIndexFileStorageFormat.V2, capturedFormat.get()); + + // Test with config disabled - V2 should still remain V2 + capturedFormat.set(null); // Reset + Config.enable_new_partition_inverted_index_v2_format = false; + + try { + OlapTable table = new OlapTable(); + internalCatalog.createPartitionWithIndices( + db.getId(), table, 3001L, "test_partition2", + indexIdToMeta, + new HashDistributionInfo(1, Lists.newArrayList()), + new DataProperty(TStorageMedium.HDD), + new ReplicaAllocation((short) 1), + 1L, Sets.newHashSet(), Sets.newHashSet(), + false, + TTabletType.TABLET_TYPE_DISK, + "", null, null, false); + } catch (Exception e) { + e.printStackTrace(); + // It's expected to fail in test environment, we only care about the format capture + } + + // Verify that V2 table format remains V2 even when config is disabled + Assert.assertEquals("V2 table format should remain V2 when config is disabled", + TInvertedIndexFileStorageFormat.V2, capturedFormat.get()); + } + +} diff --git a/regression-test/data/inverted_index_p0/test_inverted_index_storage_format_upgrade.out b/regression-test/data/inverted_index_p0/test_inverted_index_storage_format_upgrade.out new file mode 100644 index 00000000000000..b812262f0e1b5f --- /dev/null +++ b/regression-test/data/inverted_index_p0/test_inverted_index_storage_format_upgrade.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_v1_original -- +1 alice alice loves programming 95 + +-- !sql_v1_original_text -- +1 alice alice loves programming 95 + +-- !sql_v1_original_numeric -- +1 alice alice loves programming 95 + +-- !sql_mixed_all -- +1 alice alice loves programming 95 +2 bob bob enjoys coding 88 +50 charlie charlie studies algorithms 92 +150 david david develops applications 89 +180 eve eve explores databases 94 + +-- !sql_mixed_p1 -- +1 alice alice loves programming 95 + +-- !sql_mixed_p2 -- +150 david david develops applications 89 + +-- !sql_mixed_text -- +1 alice alice loves programming 95 +180 eve eve explores databases 94 + +-- !sql_v1_preserve -- +1 test content for V1 format + +-- !sql_v1_preserve_new -- +1 test content for V1 format +150 new content in second partition + +-- !sql_v2_enabled -- +1 test content for V2 format + +-- !sql_v2_disabled_config -- +1 test content for V2 format +150 new content should still use V2 + +-- !sql_v1_func -- +1 apple red apple is sweet + +-- !sql_v2_func -- +1 apple red apple is sweet + +-- !sql_v1_text_func -- +1 apple red apple is sweet +4 date brown date is sweet + +-- !sql_v2_text_func -- +1 apple red apple is sweet +4 date brown date is sweet + diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index_storage_format_upgrade.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index_storage_format_upgrade.groovy new file mode 100644 index 00000000000000..230ef1e078d2f3 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_inverted_index_storage_format_upgrade.groovy @@ -0,0 +1,409 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_inverted_index_storage_format_upgrade", "nonConcurrent") { + def tableName = "test_inverted_index_format_upgrade" + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort) + + def show_nested_index_file_on_tablet = { ip, port, tablet -> + return http_client("GET", String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s", ip, port, tablet)) + } + + def get_tablet_inverted_index_format = { tblName, partitionName = null -> + def targetTablet = null + if (partitionName != null) { + def tablets = sql_return_maparray """ show tablets from ${tblName} partition ${partitionName}; """ + logger.info("tablets: ${tablets}") + if (tablets.size() >= 0) { + targetTablet = tablets[0] + } + } else { + def tablets = sql_return_maparray """ show tablets from ${tblName}; """ + logger.info("tablets: ${tablets}") + if (tablets.size() >= 0) { + targetTablet = tablets[0] + } + } + + if (targetTablet == null) { + logger.error("No tablet found for table: ${tblName}, partition: ${partitionName}") + return null + } + + String tablet_id = targetTablet.TabletId + String backend_id = targetTablet.BackendId + String ip = backendId_to_backendIP.get(backend_id) + String port = backendId_to_backendHttpPort.get(backend_id) + + def (code, out, err) = show_nested_index_file_on_tablet(ip, port, tablet_id) + logger.info("Get tablet inverted index format: tablet_id=${tablet_id}, partition=${partitionName}, code=" + code) + + if (code == 0 && out != null) { + def jsonResponse = parseJson(out.trim()) + if (jsonResponse.rowsets != null && jsonResponse.rowsets.size() > 0) { + // Return the format from the first rowset + def format = jsonResponse.rowsets[0].index_storage_format + logger.info("Tablet ${tablet_id} in partition ${partitionName} has format: ${format}") + return format + } + } + + logger.warn("Could not determine format for tablet ${tablet_id}") + return null + } + + def get_fe_config = { key -> + def result = sql "SHOW FRONTEND CONFIG LIKE '${key}'" + if (result.size() > 0) { + return result[0][1] + } + return null + } + + def set_fe_config = { key, value -> + sql "ADMIN SET FRONTEND CONFIG ('${key}' = '${value}')" + // Wait a bit for config to take effect + sleep(2000) + } + + def getJobState = { tblName -> + def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE TableName='${tblName}' ORDER BY createtime DESC LIMIT 1 """ + return jobStateResult.size() > 0 ? jobStateResult[0][9] : "FINISHED" + } + + def waitForJob = { tblName -> + int max_try_secs = 60 + while (max_try_secs--) { + String res = getJobState(tblName) + if (res == "FINISHED" || res == "CANCELLED") { + assertEquals("FINISHED", res) + sleep(3000) + break + } else { + Thread.sleep(1000) + if (max_try_secs < 1) { + println "test timeout," + "state:" + res + assertEquals("FINISHED", res) + } + } + } + } + + def originalConfigValue + try { + // Get original config value to restore later + originalConfigValue = get_fe_config("enable_new_partition_inverted_index_v2_format") + logger.info("Original enable_new_partition_inverted_index_v2_format value: ${originalConfigValue}") + + // Test 1: Mixed format partitions - V1 table with config disabled/enabled + logger.info("=== Test 1: Mixed Format Partitions ===") + + // Step 1: Disable config, create table with V1 format + set_fe_config("enable_new_partition_inverted_index_v2_format", "false") + sql "DROP TABLE IF EXISTS ${tableName}" + + sql """ + CREATE TABLE ${tableName} ( + id int(11) NOT NULL, + name varchar(255) NOT NULL, + description text, + score int(11), + INDEX idx_name (name) USING INVERTED, + INDEX idx_description (description) USING INVERTED PROPERTIES("parser"="english"), + INDEX idx_score (score) USING INVERTED + ) + DUPLICATE KEY(id) + PARTITION BY RANGE(id) ( + PARTITION p1 VALUES [("1"), ("100")) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V1" + ); + """ + + // Verify table was created with V1 format + def tableInfo = sql "SHOW CREATE TABLE ${tableName}" + assertTrue(tableInfo[0][1].contains("inverted_index_storage_format") && tableInfo[0][1].contains("V1")) + logger.info("Table created with V1 format successfully") + + // Insert some test data + sql "INSERT INTO ${tableName} VALUES (1, 'alice', 'alice loves programming', 95)" + sql "INSERT INTO ${tableName} VALUES (2, 'bob', 'bob enjoys coding', 88)" + sql "INSERT INTO ${tableName} VALUES (50, 'charlie', 'charlie studies algorithms', 92)" + + // Sync data to ensure tablets are created + sql "SELECT * FROM ${tableName};" + + // Verify inverted index works with V1 format + qt_sql_v1_original "SELECT * FROM ${tableName} WHERE name MATCH 'alice' ORDER BY id" + qt_sql_v1_original_text "SELECT * FROM ${tableName} WHERE description MATCH 'programming' ORDER BY id" + qt_sql_v1_original_numeric "SELECT * FROM ${tableName} WHERE score = 95 ORDER BY id" + + // Verify p1 partition uses V1 format through API + def p1_format = get_tablet_inverted_index_format(tableName, "p1") + assertEquals("V1", p1_format) + + // Step 2: Enable config for V2 format upgrade + set_fe_config("enable_new_partition_inverted_index_v2_format", "true") + + // Add new partition - should use V2 format due to config upgrade + sql "ALTER TABLE ${tableName} ADD PARTITION p2 VALUES [('100'), ('200'))" + waitForJob(tableName) + + // Insert data into new partition + sql "INSERT INTO ${tableName} VALUES (150, 'david', 'david develops applications', 89)" + sql "INSERT INTO ${tableName} VALUES (180, 'eve', 'eve explores databases', 94)" + + // Sync data + sql "SELECT * FROM ${tableName};" + + // Verify both partitions work correctly + qt_sql_mixed_all "SELECT * FROM ${tableName} ORDER BY id" + qt_sql_mixed_p1 "SELECT * FROM ${tableName} WHERE id < 100 AND name MATCH 'alice' ORDER BY id" + qt_sql_mixed_p2 "SELECT * FROM ${tableName} WHERE id >= 100 AND name MATCH 'david' ORDER BY id" + qt_sql_mixed_text "SELECT * FROM ${tableName} WHERE description MATCH 'programming databases' ORDER BY id" + + // Verify formats through API - this is the key validation + def p1_format_after = get_tablet_inverted_index_format(tableName, "p1") + def p2_format = get_tablet_inverted_index_format(tableName, "p2") + + assertEquals("V1", p1_format_after) // p1 should still be V1 + assertEquals("V2", p2_format) // p2 should be upgraded to V2 + + logger.info("Mixed partition format test completed successfully - p1: ${p1_format_after}, p2: ${p2_format}") + + // Test 2: V1 format preservation when config is disabled + logger.info("=== Test 2: V1 Format Preservation ===") + + def tableName2 = "${tableName}_v1_preserve" + set_fe_config("enable_new_partition_inverted_index_v2_format", "false") + + sql "DROP TABLE IF EXISTS ${tableName2}" + sql """ + CREATE TABLE ${tableName2} ( + id int(11) NOT NULL, + content varchar(255), + INDEX idx_content (content) USING INVERTED PROPERTIES("parser"="unicode") + ) + DUPLICATE KEY(id) + PARTITION BY RANGE(id) ( + PARTITION p1 VALUES [("1"), ("100")) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V1" + ); + """ + + sql "INSERT INTO ${tableName2} VALUES (1, 'test content for V1 format')" + + // Sync data + sql "SELECT * FROM ${tableName2};" + + qt_sql_v1_preserve "SELECT * FROM ${tableName2} WHERE content MATCH 'test' ORDER BY id" + + // Verify initial partition format + def initial_format = get_tablet_inverted_index_format(tableName2, "p1") + assertEquals("V1", initial_format) + + // Verify V1 format is preserved when adding partitions with config disabled + sql "ALTER TABLE ${tableName2} ADD PARTITION p2 VALUES [('100'), ('200'))" + waitForJob(tableName2) + + sql "INSERT INTO ${tableName2} VALUES (150, 'new content in second partition')" + + // Sync data + sql "SELECT * FROM ${tableName2};" + + qt_sql_v1_preserve_new "SELECT * FROM ${tableName2} WHERE content MATCH 'content' ORDER BY id" + + // Verify both partitions use V1 format when config is disabled + def p1_v1_format = get_tablet_inverted_index_format(tableName2, "p1") // initial partition + def p2_v1_format = get_tablet_inverted_index_format(tableName2, "p2") + + // Both should be V1 since config is disabled + assertEquals("V1", p1_v1_format) + assertEquals("V1", p2_v1_format) + + logger.info("V1 format preservation test completed successfully") + + // Test 3: V2 format behavior - remains V2 regardless of config + logger.info("=== Test 3: V2 Format Behavior ===") + + def tableName3 = "${tableName}_v2_behavior" + set_fe_config("enable_new_partition_inverted_index_v2_format", "true") + + sql "DROP TABLE IF EXISTS ${tableName3}" + sql """ + CREATE TABLE ${tableName3} ( + id int(11) NOT NULL, + content varchar(255), + INDEX idx_content (content) USING INVERTED PROPERTIES("parser"="unicode") + ) + DUPLICATE KEY(id) + PARTITION BY RANGE(id) ( + PARTITION p1 VALUES [("1"), ("100")) + ) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ); + """ + + sql "INSERT INTO ${tableName3} VALUES (1, 'test content for V2 format')" + + // Sync data + sql "SELECT * FROM ${tableName3};" + + qt_sql_v2_enabled "SELECT * FROM ${tableName3} WHERE content MATCH 'test' ORDER BY id" + + // Verify initial partition uses V2 format + def p1_v2_format = get_tablet_inverted_index_format(tableName3, "p1") + assertEquals("V2", p1_v2_format) + + // Disable config and verify V2 format is still preserved + set_fe_config("enable_new_partition_inverted_index_v2_format", "false") + + sql "ALTER TABLE ${tableName3} ADD PARTITION p2 VALUES [('100'), ('200'))" + waitForJob(tableName3) + + sql "INSERT INTO ${tableName3} VALUES (150, 'new content should still use V2')" + + // Sync data + sql "SELECT * FROM ${tableName3};" + + qt_sql_v2_disabled_config "SELECT * FROM ${tableName3} WHERE content MATCH 'content' ORDER BY id" + + // Verify both partitions still use V2 format even when config is disabled + def p1_v2_after = get_tablet_inverted_index_format(tableName3, "p1") + def p2_v2_format = get_tablet_inverted_index_format(tableName3, "p2") + + assertEquals("V2", p1_v2_after) // p1 should remain V2 + assertEquals("V2", p2_v2_format) // p2 should also be V2 (V2 table format preserved) + + // Verify table info still shows V2 format + def tableInfo3 = sql "SHOW CREATE TABLE ${tableName3}" + assertTrue(tableInfo3[0][1].contains("inverted_index_storage_format") && tableInfo3[0][1].contains("V2")) + + logger.info("V2 format behavior test completed successfully") + + // Test 4: Performance comparison between formats (basic functionality test) + logger.info("=== Test 4: Format Functionality Verification ===") + + def testData = [ + [1, "apple", "red apple is sweet"], + [2, "banana", "yellow banana is nutritious"], + [3, "cherry", "red cherry is sour"], + [4, "date", "brown date is sweet"], + [5, "elderberry", "purple elderberry is rare"] + ] + + // Test with V1 format + set_fe_config("enable_new_partition_inverted_index_v2_format", "false") + def tableName4V1 = "${tableName}_func_v1" + sql "DROP TABLE IF EXISTS ${tableName4V1}" + sql """ + CREATE TABLE ${tableName4V1} ( + id int(11) NOT NULL, + name varchar(255), + description text, + INDEX idx_name (name) USING INVERTED, + INDEX idx_description (description) USING INVERTED PROPERTIES("parser"="english") + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V1" + ); + """ + + // Insert test data for V1 + for (data in testData) { + sql "INSERT INTO ${tableName4V1} VALUES (${data[0]}, '${data[1]}', '${data[2]}')" + } + + // Sync data + sql "SELECT * FROM ${tableName4V1};" + + // Test with V2 format + set_fe_config("enable_new_partition_inverted_index_v2_format", "true") + def tableName4V2 = "${tableName}_func_v2" + sql "DROP TABLE IF EXISTS ${tableName4V2}" + sql """ + CREATE TABLE ${tableName4V2} ( + id int(11) NOT NULL, + name varchar(255), + description text, + INDEX idx_name (name) USING INVERTED, + INDEX idx_description (description) USING INVERTED PROPERTIES("parser"="english") + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "inverted_index_storage_format" = "V2" + ); + """ + + // Insert test data for V2 + for (data in testData) { + sql "INSERT INTO ${tableName4V2} VALUES (${data[0]}, '${data[1]}', '${data[2]}')" + } + + // Sync data + sql "SELECT * FROM ${tableName4V2};" + + // Verify formats through API + def v1_format = get_tablet_inverted_index_format(tableName4V1) + def v2_format = get_tablet_inverted_index_format(tableName4V2) + + assertEquals("V1", v1_format) + assertEquals("V2", v2_format) + + // Compare functionality between V1 and V2 + qt_sql_v1_func "SELECT * FROM ${tableName4V1} WHERE name MATCH 'apple' ORDER BY id" + qt_sql_v2_func "SELECT * FROM ${tableName4V2} WHERE name MATCH 'apple' ORDER BY id" + + qt_sql_v1_text_func "SELECT * FROM ${tableName4V1} WHERE description MATCH 'sweet' ORDER BY id" + qt_sql_v2_text_func "SELECT * FROM ${tableName4V2} WHERE description MATCH 'sweet' ORDER BY id" + + logger.info("Format functionality verification test completed successfully") + + // Clean up test tables + sql "DROP TABLE IF EXISTS ${tableName}" + sql "DROP TABLE IF EXISTS ${tableName}_v1_preserve" + sql "DROP TABLE IF EXISTS ${tableName}_v2_behavior" + sql "DROP TABLE IF EXISTS ${tableName4V1}" + sql "DROP TABLE IF EXISTS ${tableName4V2}" + + logger.info("All inverted index storage format upgrade tests completed successfully") + + } finally { + // Restore original config + if (originalConfigValue != null) { + set_fe_config("enable_new_partition_inverted_index_v2_format", originalConfigValue) + logger.info("Restored enable_new_partition_inverted_index_v2_format to: ${originalConfigValue}") + } + } +} \ No newline at end of file From 3c00017c0b70830d16388c9186f4441f6ed39d56 Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 5 Sep 2025 09:54:32 +0800 Subject: [PATCH 557/572] [Bug](function) fix wrong result about array_agg_foreach #55075 (#55419) pick from #55075 --- .../aggregate_function_collect.h | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/be/src/vec/aggregate_functions/aggregate_function_collect.h b/be/src/vec/aggregate_functions/aggregate_function_collect.h index 305fe3276c5649..08a1f751206753 100644 --- a/be/src/vec/aggregate_functions/aggregate_function_collect.h +++ b/be/src/vec/aggregate_functions/aggregate_function_collect.h @@ -471,15 +471,7 @@ struct AggregateFunctionArrayAggData { } void merge(const Self& rhs) { - const auto size = rhs.null_map->size(); - null_map->resize(size); - nested_column->reserve(size); - for (size_t i = 0; i < size; i++) { - const auto null_value = rhs.null_map->data()[i]; - const auto data_value = rhs.nested_column->get_data()[i]; - null_map->data()[i] = null_value; - nested_column->get_data().push_back(data_value); - } + column_data->insert_range_from(*rhs.column_data, 0, rhs.column_data->size()); } }; @@ -568,15 +560,7 @@ struct AggregateFunctionArrayAggData { } void merge(const Self& rhs) { - const auto size = rhs.null_map->size(); - null_map->resize(size); - nested_column->reserve(size); - for (size_t i = 0; i < size; i++) { - const auto null_value = rhs.null_map->data()[i]; - auto s = rhs.nested_column->get_data_at(i); - null_map->data()[i] = null_value; - nested_column->insert_data(s.data, s.size); - } + column_data->insert_range_from(*rhs.column_data, 0, rhs.column_data->size()); } }; From e6899fe1c722dac1782fc40e7815b4309e2ef8da Mon Sep 17 00:00:00 2001 From: zhangdong Date: Fri, 5 Sep 2025 09:56:18 +0800 Subject: [PATCH 558/572] branch-3.0:[enhance](auth)support ldap user show grants (#54087) (#54765) pick: https://github.com/apache/doris/pull/54087 --- .../java/org/apache/doris/analysis/ShowGrantsStmt.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java index 0439544ae0e3a1..b6e473c4f0e466 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowGrantsStmt.java @@ -80,16 +80,17 @@ public void analyze(Analyzer analyzer) throws AnalysisException { userIdent = ConnectContext.get().getCurrentUserIdentity(); } } + boolean isSelf = userIdent != null && ConnectContext.get().getCurrentUserIdentity().equals(userIdent); Preconditions.checkState(isAll || userIdent != null); - UserIdentity self = ConnectContext.get().getCurrentUserIdentity(); - // if show all grants, or show other user's grants, need global GRANT priv. - if (isAll || !self.equals(userIdent)) { + if (isAll || !isSelf) { if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.GRANT)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "GRANT"); } } - if (userIdent != null && !Env.getCurrentEnv().getAccessManager().getAuth().doesUserExist(userIdent)) { + // ldap user not exist in userManager, so should not check + if (userIdent != null && !isSelf && !Env.getCurrentEnv().getAccessManager().getAuth() + .doesUserExist(userIdent)) { throw new AnalysisException(String.format("User: %s does not exist", userIdent)); } } From 07ef370b48fe9fa842ecaafb59954f67a1e4468f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 09:58:08 +0800 Subject: [PATCH 559/572] branch-3.0: [opt](nereids) optimize parallel of insert command when olap table have auto partitions #54983 (#55028) Cherry-picked from #54983 Co-authored-by: 924060929 --- .../plans/physical/PhysicalOlapTableSink.java | 2 +- .../apache/doris/load/loadv2/InsertTest.java | 40 +++++++++++++++++++ .../doris/nereids/util/PlanChecker.java | 16 ++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/load/loadv2/InsertTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java index a04912f5119879..99969cdfb373e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java @@ -198,7 +198,7 @@ isPartialUpdate, dmlCommandType, groupExpression, getLogicalProperties(), * get output physical properties */ public PhysicalProperties getRequirePhysicalProperties() { - if (targetTable.isPartitionDistributed()) { + if (targetTable.isPartitionDistributed() || targetTable.getPartitionInfo().enableAutomaticPartition()) { DistributionInfo distributionInfo = targetTable.getDefaultDistributionInfo(); if (distributionInfo instanceof HashDistributionInfo) { // Do not enable shuffle for duplicate key tables when its tablet num is less than threshold. diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/InsertTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/InsertTest.java new file mode 100644 index 00000000000000..41140e58047932 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/InsertTest.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.load.loadv2; + +import org.apache.doris.nereids.trees.plans.commands.insert.AbstractInsertExecutor; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class InsertTest extends TestWithFeService { + @Test + public void testParallelOfInsertAutoPartition() throws Exception { + createDatabase("test"); + useDatabase("test"); + + createTable("create table test.tbl(id int, name varchar(255)) auto partition by list(name)()properties('replication_num'='1')"); + + AbstractInsertExecutor insertExecutor = PlanChecker.from(connectContext) + .getInsertExecutor( + "insert into test.tbl select * from (select 1, 'test' union all select 2, 'doris')a"); + Assertions.assertEquals(1, insertExecutor.getCoordinator().getFragments().size()); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 6962572d07a483..d56092b625e136 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -58,6 +58,8 @@ import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.commands.insert.AbstractInsertExecutor; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -66,6 +68,8 @@ import org.apache.doris.planner.PlanFragment; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.thrift.TUniqueId; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -75,6 +79,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.UUID; import java.util.function.Consumer; import java.util.function.Supplier; @@ -116,6 +121,17 @@ public PlanChecker checkParse(String sql, Consumer consumer) { return this; } + public AbstractInsertExecutor getInsertExecutor(String sql) throws Exception { + StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); + LogicalPlan parsedPlan = new NereidsParser().parseSingle(sql); + UUID uuid = UUID.randomUUID(); + connectContext.setQueryId(new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits())); + InsertIntoTableCommand insertIntoTableCommand = (InsertIntoTableCommand) parsedPlan; + LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(parsedPlan, statementContext); + return insertIntoTableCommand.initPlan(connectContext, + new StmtExecutor(connectContext, logicalPlanAdapter)); + } + public PlanChecker parse(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); this.cascadesContext.toMemo(); From 224641e12eb651d8da7a56aca532b7eac75cfb5d Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 5 Sep 2025 10:00:24 +0800 Subject: [PATCH 560/572] branch-3.0: [feat](nereids) Add session variable enable_insert_value_auto_cast for insert value truncate long string #55325 (#55423) cherry pick from #55325 --- .../nereids/rules/analysis/BindSink.java | 4 +- .../org/apache/doris/qe/SessionVariable.java | 12 +++++ .../insert_into_table/insert_values.out | 10 +++- .../insert_into_table/insert_values.groovy | 54 +++++++++++++++++-- 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 5bf20d7069586b..a2d7decee123f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -239,6 +239,8 @@ private LogicalProject getOutputProjectByCoercion(List tableSchema, L // add cast project List castExprs = Lists.newArrayList(); + ConnectContext connCtx = ConnectContext.get(); + final boolean truncateString = connCtx == null || connCtx.getSessionVariable().enableInsertValueAutoCast; for (int i = 0; i < tableSchema.size(); ++i) { Column col = tableSchema.get(i); NamedExpression expr = columnToOutput.get(col.getName()); @@ -258,7 +260,7 @@ private LogicalProject getOutputProjectByCoercion(List tableSchema, L int targetLength = ((CharacterType) targetType).getLen(); if (sourceLength == targetLength) { castExpr = TypeCoercionUtils.castIfNotSameType(castExpr, targetType); - } else if (sourceLength > targetLength && targetLength >= 0) { + } else if (truncateString && sourceLength > targetLength && targetLength >= 0) { castExpr = new Substring(castExpr, Literal.of(1), Literal.of(targetLength)); } else if (targetType.isStringType()) { castExpr = new Cast(castExpr, StringType.INSTANCE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 3bc7c92ca8bbf6..2c26d066b2bb8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -139,6 +139,7 @@ public class SessionVariable implements Serializable, Writable { public static final String MAX_INSTANCE_NUM = "max_instance_num"; public static final String DML_PLAN_RETRY_TIMES = "DML_PLAN_RETRY_TIMES"; public static final String ENABLE_INSERT_STRICT = "enable_insert_strict"; + public static final String ENABLE_INSERT_VALUE_AUTO_CAST = "enable_insert_value_auto_cast"; public static final String INSERT_MAX_FILTER_RATIO = "insert_max_filter_ratio"; public static final String ENABLE_SERVER_SIDE_PREPARED_STATEMENT = "enable_server_side_prepared_statement"; @@ -1047,6 +1048,13 @@ public enum IgnoreSplitType { @VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true) public boolean enableInsertStrict = true; + @VariableMgr.VarAttr(name = ENABLE_INSERT_VALUE_AUTO_CAST, needForward = true, description = { + "INSERT VALUE 语句是否自动类型转换。当前只针对长字符串自动截短。默认开。", + "INSERT VALUE statement whether to automatically type cast. Only use for truncate long string. " + + "ON by default." + }) + public boolean enableInsertValueAutoCast = true; + @VariableMgr.VarAttr(name = INSERT_MAX_FILTER_RATIO, needForward = true) public double insertMaxFilterRatio = 1.0; @@ -3205,6 +3213,10 @@ public void setEnableInsertStrict(boolean enableInsertStrict) { this.enableInsertStrict = enableInsertStrict; } + public boolean getEnableInsertValueAutoCast() { + return enableInsertValueAutoCast; + } + public double getInsertMaxFilterRatio() { return insertMaxFilterRatio; } diff --git a/regression-test/data/nereids_p0/insert_into_table/insert_values.out b/regression-test/data/nereids_p0/insert_into_table/insert_values.out index 36d51ffcb2a6fc..59ba6ff19ac53e 100644 --- a/regression-test/data/nereids_p0/insert_into_table/insert_values.out +++ b/regression-test/data/nereids_p0/insert_into_table/insert_values.out @@ -47,8 +47,16 @@ 3 30 3 3 3.0 2000-01-03 5 5 5 5.0 2000-01-05 5 -- !select_all_default -- -true 10 10000 10000000 92233720368547758 19223372036854775807 3.142 hello world, today is 15/06/2023 2023-06-15 2023-06-15T16:10:15 +true 10 10000 10000000 92233720368547758 19223372036854775807 3.14159 hello world, today is 15/06/2023 2023-06-15 2023-06-15T16:10:15 -- !mv -- -4 -4 -4 d +-- !select_test_insert_cast_interval -- +1 2020-02-02 + +-- !select_test_insert_more_string -- +1 ab +2 abcd +5 o + diff --git a/regression-test/suites/nereids_p0/insert_into_table/insert_values.groovy b/regression-test/suites/nereids_p0/insert_into_table/insert_values.groovy index 84461ca2ba8476..42aa9bd711118b 100644 --- a/regression-test/suites/nereids_p0/insert_into_table/insert_values.groovy +++ b/regression-test/suites/nereids_p0/insert_into_table/insert_values.groovy @@ -21,8 +21,6 @@ suite('nereids_insert_into_values') { sql 'set enable_nereids_dml=true' sql 'set enable_strict_consistency_dml=true' - sql 'use nereids_insert_into_table_test' - def t1 = 'value_t1' def t2 = 'value_t2' def t3 = 'value_t3' @@ -143,4 +141,54 @@ suite('nereids_insert_into_values') { sql "insert into agg_have_dup_base_value values (-4, -4, -4, 'd')" sql "sync" qt_mv "select * from agg_have_dup_base_value" -} \ No newline at end of file + + multi_sql """ + DROP TABLE IF EXISTS test_insert_cast_interval; + CREATE TABLE test_insert_cast_interval ( + `id` int NULL, + `dt` date NULL + ) ENGINE=OLAP + DUPLICATE KEY(`id`, `dt`) + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + + INSERT INTO test_insert_cast_interval values(1, date_floor('2020-02-02', interval 1 second)); + """ + + qt_select_test_insert_cast_interval "select * from test_insert_cast_interval" + + multi_sql """ + drop table if exists test_insert_more_string; + CREATE TABLE test_insert_more_string ( + `r_regionkey` int NULL, + `r_name` varchar(4) NULL + ) + DUPLICATE KEY(`r_regionkey`) + DISTRIBUTED BY HASH(`r_regionkey`) + BUCKETS 1 PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + // shorter varchar is ok + sql "insert into test_insert_more_string values (1, 'ab')" + + // set enable_insert_value_auto_cast = true + // longer varchar will truncate + sql "insert into test_insert_more_string values (2, 'abcdefg')" + + // when disable string auto cast and in insert strict mode, insert will failed + sql 'set enable_insert_value_auto_cast = false' + test { + sql "insert into test_insert_more_string values (3, 'hi'), (4, 'jklmn')" + exception 'Insert has filtered data in strict mode' + } + + // when disable insert strict, the longer varchar row will be filtered, other rows will succ + sql 'set enable_insert_strict = false' + sql "insert into test_insert_more_string values (5, 'o'), (6, 'pqrst')" + + order_qt_select_test_insert_more_string "select * from test_insert_more_string" +} From 636db03892ad1f95e735bca987f398f843ad71a7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 10:01:37 +0800 Subject: [PATCH 561/572] branch-3.0: [Fix](case) Fix some cases #55606 (#55655) Cherry-picked from #55606 Co-authored-by: bobhan1 --- .../partial_update/test_delete_publish_skip_read.groovy | 2 ++ .../test_partial_update_conflict_skip_compaction.groovy | 2 ++ .../test_partial_update_with_delete_col_in_publish.groovy | 3 ++- .../partial_update/test_partial_update_merge_type.groovy | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_delete_publish_skip_read.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_delete_publish_skip_read.groovy index 6fce2245a9a205..7f2ddb94112537 100644 --- a/regression-test/suites/fault_injection_p0/partial_update/test_delete_publish_skip_read.groovy +++ b/regression-test/suites/fault_injection_p0/partial_update/test_delete_publish_skip_read.groovy @@ -98,6 +98,8 @@ suite("test_delete_publish_skip_read", "nonConcurrent") { t1.join() t2.join() + sleep(2000) + sql "sync;" order_qt_sql "select * from ${table1};" sql "set skip_delete_sign=true;" diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_conflict_skip_compaction.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_conflict_skip_compaction.groovy index 08eba337af3327..007c76ec24af03 100644 --- a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_conflict_skip_compaction.groovy +++ b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_conflict_skip_compaction.groovy @@ -180,6 +180,8 @@ suite("test_partial_update_conflict_skip_compaction", "nonConcurrent") { t1.join() t2.join() + sleep(2000) + sql "sync;" order_qt_sql "select * from ${table1};" check_rs_metas(3, {int startVersion, int endVersion, int numSegments, int numRows, String overlapPb -> diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy index a1d4409b0e6267..1438b5b6df5dc1 100644 --- a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy +++ b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy @@ -92,7 +92,8 @@ suite("test_partial_update_with_delete_col_in_publish", "nonConcurrent") { disable_block_in_publish() threads.each { t -> t.join() } - + sleep(2000) + sql "sync;" qt_sql "select * from ${tableName} order by k;" } catch(Exception e) { logger.info(e.getMessage()) diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy index dfaa9aca3068f0..663d7c2a0b5425 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_merge_type.groovy @@ -47,6 +47,7 @@ suite("test_partial_update_merge_type", "p0") { UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1 PROPERTIES( "replication_num" = "1", + "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName} select number,number,number,number from numbers("number"="9");""" @@ -118,6 +119,7 @@ suite("test_partial_update_merge_type", "p0") { PROPERTIES( "replication_num" = "1", "enable_unique_key_merge_on_write" = "true", + "disable_auto_compaction" = "true", "function_column.sequence_type" = "BIGINT", "store_row_column" = "${use_row_store}"); """ sql """insert into ${tableName}(k,c1,c2,c3,__DORIS_SEQUENCE_COL__) select number,number,number,number,1 from numbers("number"="9");""" From 1ca5eb185cb46ae4678bbaf1cbbb116b915b049a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 10:06:05 +0800 Subject: [PATCH 562/572] branch-3.0: [test](warmup) fix flaky periodic warmup cases #55365 (#55452) Cherry-picked from #55365 Co-authored-by: Kaijie Chen --- .../test_warm_up_cluster_periodic.groovy | 31 ++++++++++++++- ...warm_up_cluster_periodic_add_new_be.groovy | 39 +++++++++++++++++-- ...est_warm_up_cluster_periodic_rename.groovy | 19 ++++++++- 3 files changed, 83 insertions(+), 6 deletions(-) diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy index 3ad8c820f7d254..cc6609081b7e1f 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic.groovy @@ -84,6 +84,23 @@ suite('test_warm_up_cluster_periodic', 'docker') { return getBrpcMetrics(ip, port, "ttl_cache_size") } + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> def backends = sql """SHOW BACKENDS""" @@ -113,6 +130,16 @@ suite('test_warm_up_cluster_periodic', 'docker') { assertEquals(srcSum, tgtSum) } + def waitUntil = { condition, timeoutMs -> + long start = System.currentTimeMillis() + while (System.currentTimeMillis() - start < timeoutMs) { + if (condition()) { + return + } + sleep(1000) + } + } + docker(options) { def clusterName1 = "warmup_source" def clusterName2 = "warmup_target" @@ -164,12 +191,14 @@ suite('test_warm_up_cluster_periodic', 'docker') { sql """SELECT * FROM customer""" } - sleep(10000) + waitUntil({ getClusterTTLCacheSizeSum(clusterName1) > 0 }, 30000) + waitUntil({ getClusterTTLCacheSizeSum(clusterName1) == getClusterTTLCacheSizeSum(clusterName2) }, 60000) def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" logger.info("hotspot: {}", hotspot) logFileCacheDownloadMetrics(clusterName2) + assertTrue(getClusterTTLCacheSizeSum(clusterName1) > 0) checkTTLCacheSizeSumEqual(clusterName1, clusterName2) def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy index 5c58da8a564410..1ab3a12ded19d1 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_add_new_be.groovy @@ -84,6 +84,23 @@ suite('test_warm_up_cluster_periodic_add_new_be', 'docker') { return getBrpcMetrics(ip, port, "ttl_cache_size") } + def getClusterTTLCacheSizeSum = { cluster -> + def backends = sql """SHOW BACKENDS""" + + def cluster_bes = backends.findAll { it[19].contains("""\"compute_group_name\" : \"${cluster}\"""") } + + long sum = 0 + for (be in cluster_bes) { + def ip = be[1] + def port = be[5] + def size = getTTLCacheSize(ip, port) + sum += size + logger.info("be be ${ip}:${port} ttl cache size ${size}") + } + + return sum + } + def checkTTLCacheSizeSumEqual = { cluster1, cluster2 -> def backends = sql """SHOW BACKENDS""" @@ -94,14 +111,18 @@ suite('test_warm_up_cluster_periodic_add_new_be', 'docker') { for (src in srcBes) { def ip = src[1] def port = src[5] - srcSum += getTTLCacheSize(ip, port) + def size = getTTLCacheSize(ip, port) + srcSum += size + logger.info("src be ${ip}:${port} ttl cache size ${size}") } long tgtSum = 0 for (tgt in tgtBes) { def ip = tgt[1] def port = tgt[5] - tgtSum += getTTLCacheSize(ip, port) + def size = getTTLCacheSize(ip, port) + tgtSum += size + logger.info("dst be ${ip}:${port} ttl cache size ${size}") } logger.info("ttl_cache_size: src=${srcSum} dst=${tgtSum}") @@ -109,6 +130,16 @@ suite('test_warm_up_cluster_periodic_add_new_be', 'docker') { assertEquals(srcSum, tgtSum) } + def waitUntil = { condition, timeoutMs -> + long start = System.currentTimeMillis() + while (System.currentTimeMillis() - start < timeoutMs) { + if (condition()) { + return + } + sleep(1000) + } + } + docker(options) { def clusterName1 = "warmup_source" def clusterName2 = "warmup_target" @@ -162,12 +193,14 @@ suite('test_warm_up_cluster_periodic_add_new_be', 'docker') { sql """SELECT * FROM customer""" } - sleep(15000) + waitUntil({ getClusterTTLCacheSizeSum(clusterName1) > 0 }, 60000) + waitUntil({ getClusterTTLCacheSizeSum(clusterName1) == getClusterTTLCacheSizeSum(clusterName2) }, 60000) def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" logger.info("hotspot: {}", hotspot) logFileCacheDownloadMetrics(clusterName2) + assertTrue(getClusterTTLCacheSizeSum(clusterName1) > 0) checkTTLCacheSizeSumEqual(clusterName1, clusterName2) def jobInfo = sql """SHOW WARM UP JOB WHERE ID = ${jobId}""" diff --git a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy index 271fec8bbfc9b0..53f65605a439f9 100644 --- a/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy +++ b/regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_rename.groovy @@ -77,6 +77,11 @@ suite('test_warm_up_cluster_periodic_rename', 'docker') { def failed = getBrpcMetrics(ip, port, "file_cache_download_failed_num") logger.info("${cluster} be ${ip}:${port}, downloader submitted=${submitted}" + ", finished=${finished}, failed=${failed}") + + def submitted_segment = getBrpcMetrics(ip, port, "file_cache_once_or_periodic_warm_up_submitted_segment_num") + def finished_segment = getBrpcMetrics(ip, port, "file_cache_once_or_periodic_warm_up_finished_segment_num") + logger.info("${cluster} be ${ip}:${port}, warmup submitted=${submitted_segment}" + + ", finished=${finished_segment}") } } @@ -130,6 +135,16 @@ suite('test_warm_up_cluster_periodic_rename', 'docker') { assertEquals(srcSum, tgtSum) } + def waitUntil = { condition, timeoutMs -> + long start = System.currentTimeMillis() + while (System.currentTimeMillis() - start < timeoutMs) { + if (condition()) { + return + } + sleep(1000) + } + } + docker(options) { def clusterName1 = "warmup_source" def clusterName2 = "warmup_target" @@ -184,7 +199,7 @@ suite('test_warm_up_cluster_periodic_rename', 'docker') { sql """SELECT * FROM customer""" } - sleep(5000) + waitUntil({ getClusterTTLCacheSizeSum(clusterName1) > 0 }, 60000) def hotspot = sql """select * from __internal_schema.cloud_cache_hotspot;""" logger.info("hotspot: {}", hotspot) @@ -194,7 +209,7 @@ suite('test_warm_up_cluster_periodic_rename', 'docker') { assertEquals(0, getClusterTTLCacheSizeSum(clusterName3)) sql """ALTER SYSTEM RENAME COMPUTE GROUP ${clusterName3} ${clusterName2}""" - sleep(5000) + waitUntil({ getClusterTTLCacheSizeSum(clusterName1) == getClusterTTLCacheSizeSum(clusterName2) }, 60000) logFileCacheDownloadMetrics(clusterName2) checkTTLCacheSizeSumEqual(clusterName1, clusterName2) From 448d4537689dee5f38a02ef7db2158f546e6f7a4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 10:06:40 +0800 Subject: [PATCH 563/572] branch-3.0: [chore](tablet) ignore_load_tablet_failure default to be true #55109 (#55439) Cherry-picked from #55109 Co-authored-by: Yongqiang YANG --- be/src/common/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 34b68cfe0215a4..b555435b43f9b1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -807,7 +807,7 @@ DEFINE_Int32(txn_map_shard_size, "1024"); DEFINE_Int32(txn_shard_size, "1024"); // Whether to continue to start be when load tablet from header failed. -DEFINE_Bool(ignore_load_tablet_failure, "false"); +DEFINE_Bool(ignore_load_tablet_failure, "true"); // Whether to continue to start be when load tablet from header failed. DEFINE_mBool(ignore_rowset_stale_unconsistent_delete, "false"); From 568a8d09e80babbd87a7900c99a86403e0d90b25 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Fri, 5 Sep 2025 12:09:10 +0800 Subject: [PATCH 564/572] branch-30: [Bug](agg) fix agg with topn limit coredump with ctor same key twice (#54137) (#54271) ### What problem does this PR solve? Problem Summary: cherry-pick from master (#54137) ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/pipeline/exec/aggregation_sink_operator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/pipeline/exec/aggregation_sink_operator.cpp b/be/src/pipeline/exec/aggregation_sink_operator.cpp index 39dfb4e7f3c84a..63d43e075115c7 100644 --- a/be/src/pipeline/exec/aggregation_sink_operator.cpp +++ b/be/src/pipeline/exec/aggregation_sink_operator.cpp @@ -601,6 +601,7 @@ bool AggSinkLocalState::_emplace_into_hash_table_limit(vectorized::AggregateData try { HashMethodType::try_presis_key_and_origin(key, origin, *_agg_arena_pool); + _shared_state->refresh_top_limit(i, key_columns); auto mapped = _shared_state->aggregate_data_container->append_data( origin); @@ -609,7 +610,6 @@ bool AggSinkLocalState::_emplace_into_hash_table_limit(vectorized::AggregateData throw Exception(st.code(), st.to_string()); } ctor(key, mapped); - _shared_state->refresh_top_limit(i, key_columns); } catch (...) { // Exception-safety - if it can not allocate memory or create status, // the destructors will not be called. From 22470f03b3a053033bd1fe8f0425572d6935a4f4 Mon Sep 17 00:00:00 2001 From: Yongqiang YANG Date: Fri, 5 Sep 2025 14:37:52 +0800 Subject: [PATCH 565/572] [fix](case) fix some failure cases (#55687) --- ...on_remove_old_version_delete_bitmap.groovy | 24 +++++++++---------- ...ud_mow_stream_load_with_commit_fail.groovy | 2 +- .../test_cloud_version_already_merged.groovy | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy index 7b072666c1dacd..25c6470a675468 100644 --- a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy +++ b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy @@ -92,9 +92,9 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { String command = sb.toString() logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() + def process = command.execute() + def code = process.waitFor() + def out = process.getText() logger.info("Get tablet status: =" + code + ", out=" + out) assertEquals(code, 0) def tabletStatus = parseJson(out.trim()) @@ -112,9 +112,9 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { String command = sb.toString() logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() + def process = command.execute() + def code = process.waitFor() + def out = process.getText() logger.info("Get compaction status: code=" + code + ", out=" + out) assertEquals(code, 0) def compactionStatus = parseJson(out.trim()) @@ -132,9 +132,9 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { String command = sb.toString() logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() + def process = command.execute() + def code = process.waitFor() + def out = process.getText() logger.info("Get local delete bitmap count status: =" + code + ", out=" + out) assertEquals(code, 0) def deleteBitmapStatus = parseJson(out.trim()) @@ -150,9 +150,9 @@ suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") { String command = sb.toString() logger.info(command) - process = command.execute() - code = process.waitFor() - out = process.getText() + def process = command.execute() + def code = process.waitFor() + def out = process.getText() logger.info("Get ms delete bitmap count status: =" + code + ", out=" + out) assertEquals(code, 0) def deleteBitmapStatus = parseJson(out.trim()) diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy index aaeb596fa07f3d..77555980dbac69 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_mow_stream_load_with_commit_fail.groovy @@ -146,7 +146,7 @@ suite("test_cloud_mow_stream_load_with_commit_fail", "nonConcurrent") { file "test_stream_load0.csv" time 10000 // limit inflight 10s - directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId) + directToBe backendIdToBackendIP.get(backendId), backendId_to_backendHttpPort.get(backendId).toInteger() check { result, exception, startTime, endTime -> log.info("Stream load result: ${result}") diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy index f7453c93300fcc..111fe556dfc7c0 100644 --- a/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_version_already_merged.groovy @@ -116,7 +116,7 @@ suite("test_cloud_version_already_merged", "nonConcurrent") { test { sql """ SELECT * from ${tblName} ORDER BY k1 """ - exception "version already merged, meet error during remote capturing rowsets" + exception "versions are already compacted" } } finally { From a98e95d8c18da3d0f60b8df62e6c9ee6c0e50a81 Mon Sep 17 00:00:00 2001 From: Lei Zhang Date: Fri, 5 Sep 2025 17:44:49 +0800 Subject: [PATCH 566/572] [fix](txn lazy commit) fix txn lazy commit conflict with schema change (#55697) * step1: MS txn lazy commit convert tmp rowsets and make txn visible commit failed due to unexpected fdb error * step2: BE schema change job convert historical data and prepare/commit rowsets with the same txn_id and tablet_id for new tablet rowset meta, tmp rowset key is decided by (txn_id, tablet_id) * step3: MS retry lazy commit will convert tmp rowset which write by schema change job ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- cloud/src/meta-service/txn_lazy_committer.cpp | 14 ++ cloud/test/txn_lazy_commit_test.cpp | 177 +++++++++++++++++- 2 files changed, 186 insertions(+), 5 deletions(-) diff --git a/cloud/src/meta-service/txn_lazy_committer.cpp b/cloud/src/meta-service/txn_lazy_committer.cpp index aaff8e948566b8..5b9ae571bb3adc 100644 --- a/cloud/src/meta-service/txn_lazy_committer.cpp +++ b/cloud/src/meta-service/txn_lazy_committer.cpp @@ -121,6 +121,7 @@ void convert_tmp_rowsets( LOG(WARNING) << msg; return; } + VersionPB version_pb; if (!version_pb.ParseFromString(ver_val)) { code = MetaServiceCode::PROTOBUF_PARSE_ERR; @@ -130,6 +131,17 @@ void convert_tmp_rowsets( } LOG(INFO) << "txn_id=" << txn_id << " key=" << hex(ver_key) << " version_pb:" << version_pb.ShortDebugString(); + + if (version_pb.pending_txn_ids_size() == 0 || version_pb.pending_txn_ids(0) != txn_id) { + LOG(INFO) << "txn_id=" << txn_id << " partition_id=" << tmp_rowset_pb.partition_id() + << " tmp_rowset_key=" << hex(tmp_rowset_key) + << " version has already been converted." + << " version_pb:" << version_pb.ShortDebugString(); + TEST_SYNC_POINT_CALLBACK("convert_tmp_rowsets::already_been_converted", + &version_pb); + return; + } + partition_versions.emplace(tmp_rowset_pb.partition_id(), version_pb); DCHECK_EQ(partition_versions.size(), 1) << partition_versions.size(); } @@ -279,6 +291,8 @@ void make_committed_txn_visible(const std::string& instance_id, int64_t db_id, i txn->put(recycle_key, recycle_val); LOG(INFO) << "put recycle_key=" << hex(recycle_key) << " txn_id=" << txn_id; + TEST_SYNC_POINT_RETURN_WITH_VOID("TxnLazyCommitTask::make_committed_txn_visible::commit", + &code); err = txn->commit(); if (err != TxnErrorCode::TXN_OK) { code = cast_as(err); diff --git a/cloud/test/txn_lazy_commit_test.cpp b/cloud/test/txn_lazy_commit_test.cpp index 5b1686eed28745..d75ebca81846cd 100644 --- a/cloud/test/txn_lazy_commit_test.cpp +++ b/cloud/test/txn_lazy_commit_test.cpp @@ -156,8 +156,8 @@ static void create_tablet_with_db_id(MetaServiceProxy* meta_service, int64_t db_ ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << tablet_id; } -static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, int index_id, - int partition_id, int64_t version = -1, +static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, int64_t index_id, + int64_t partition_id, int64_t version = -1, int num_rows = 100) { doris::RowsetMetaCloudPB rowset; rowset.set_rowset_id(0); // required @@ -178,9 +178,9 @@ static doris::RowsetMetaCloudPB create_rowset(int64_t txn_id, int64_t tablet_id, return rowset; } -static doris::RowsetMetaCloudPB create_huge_rowset(int64_t txn_id, int64_t tablet_id, int index_id, - int partition_id, int64_t version = -1, - int num_rows = 100) { +static doris::RowsetMetaCloudPB create_huge_rowset(int64_t txn_id, int64_t tablet_id, + int64_t index_id, int64_t partition_id, + int64_t version = -1, int num_rows = 100) { doris::RowsetMetaCloudPB rowset; rowset.set_rowset_id(0); // required rowset.set_rowset_id_v2(next_rowset_id()); @@ -3088,4 +3088,171 @@ TEST(TxnLazyCommitTest, CommitTxnEventuallyWithHugeRowsetMetaTest) { config::txn_lazy_max_rowsets_per_batch = 2; } +TEST(TxnLazyCommitTest, CommitTxnEventuallyWithSchemaChangeTest) { + // =========================================================================== + // threads concurrent execution flow: + // + // thread1 lazy thread1 thread3 + // | | | + // commit_txn_eventually begin | | + // | | | + // lazy commit wait | | + // | | | + // | make_committed_txn_visible | + // | | | + // | inject TXN_TOO_OLD fdb error | + // | | sc create new tablet tmp rowset + // | | | + // | | | + // retry commit_txn | | + // v v + config::txn_lazy_max_rowsets_per_batch = 1000; + auto txn_kv = get_mem_txn_kv(); + int64_t db_id = 4534445675; + int64_t table_id = 4365676543; + int64_t index_id = 665453237; + int64_t partition_id = 2136776543678; + + bool go = false; + std::mutex go_mutex; + std::condition_variable go_cv; + std::atomic make_committed_txn_visible_count = {0}; + std::atomic sc_create_tmp_rowset_count = {0}; + std::atomic sc_create_tmp_rowset_finish_count = {0}; + std::atomic tmp_rowsets_been_already_converted = {0}; + + auto sp = SyncPoint::get_instance(); + sp->set_call_back("TxnLazyCommitTask::make_committed_txn_visible::commit", [&](auto&& args) { + LOG(INFO) << "zhangleiyyy"; + { + std::unique_lock _lock(go_mutex); + if (make_committed_txn_visible_count == 0) { + make_committed_txn_visible_count++; + if (sc_create_tmp_rowset_count == 0) { + go_cv.wait(_lock, [&] { return sc_create_tmp_rowset_count == 1; }); + } + MetaServiceCode* code = try_any_cast(args[0]); + *code = MetaServiceCode::KV_TXN_CONFLICT; + bool* pred = try_any_cast(args.back()); + *pred = true; + LOG(INFO) << "inject kv error KV_TXN_CONFLICT"; + go_cv.notify_all(); + } + } + }); + + sp->set_call_back("convert_tmp_rowsets::already_been_converted", [&](auto&& args) { + auto version_pb = *try_any_cast(args[0]); + LOG(INFO) << "version_pb:" << version_pb.ShortDebugString(); + std::unique_lock _lock(go_mutex); + tmp_rowsets_been_already_converted++; + go_cv.notify_all(); + }); + + sp->enable_processing(); + + auto meta_service = get_meta_service(txn_kv, true); + brpc::Controller cntl; + BeginTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + TxnInfoPB txn_info_pb; + txn_info_pb.set_db_id(db_id); + txn_info_pb.set_label("test_sc_with_commit_txn_label"); + txn_info_pb.add_table_ids(table_id); + txn_info_pb.set_timeout_ms(36000); + req.mutable_txn_info()->CopyFrom(txn_info_pb); + BeginTxnResponse res; + meta_service->begin_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + int64_t txn_id = res.txn_id(); + + // mock rowset and tablet + int64_t tablet_id_base = 3131124; + for (int i = 0; i < 1000; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, partition_id, + tablet_id_base + i); + auto tmp_rowset = create_huge_rowset(txn_id, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + + std::thread thread1([&] { + { + { + std::unique_lock _lock(go_mutex); + go_cv.wait(_lock, [&] { return go; }); + } + + brpc::Controller cntl; + CommitTxnRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_db_id(db_id); + req.set_txn_id(txn_id); + req.set_is_2pc(false); + req.set_enable_txn_lazy_commit(true); + CommitTxnResponse res; + meta_service->commit_txn(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), + &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + LOG(INFO) << "thread1 finish"; + }); + + std::thread thread2([&] { + { + { + std::unique_lock _lock(go_mutex); + go_cv.wait(_lock, [&] { return go; }); + } + + { + std::unique_lock _lock(go_mutex); + sc_create_tmp_rowset_count++; + if (make_committed_txn_visible_count == 0) { + go_cv.wait(_lock, [&] { return make_committed_txn_visible_count > 0; }); + } + for (int i = 0; i < 1000; ++i) { + create_tablet_with_db_id(meta_service.get(), db_id, table_id, index_id, + partition_id, tablet_id_base + i); + auto tmp_rowset = + create_huge_rowset(txn_id, tablet_id_base + i, index_id, partition_id); + CreateRowsetResponse res; + commit_rowset(meta_service.get(), tmp_rowset, res); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK); + } + LOG(INFO) << "sc_create_tmp_rowset_finish_count finish"; + sc_create_tmp_rowset_finish_count++; + go_cv.notify_all(); + } + LOG(INFO) << "thread2 finish"; + } + }); + + std::unique_lock go_lock(go_mutex); + go = true; + go_lock.unlock(); + go_cv.notify_all(); + + thread1.join(); + thread2.join(); + + ASSERT_GT(tmp_rowsets_been_already_converted, 1); + { + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string mock_instance = "test_instance"; + for (int i = 0; i < 1000; ++i) { + int64_t tablet_id = tablet_id_base + i; + check_tablet_idx_db_id(txn, db_id, tablet_id); + check_tmp_rowset_exist(txn, tablet_id, txn_id); + check_rowset_meta_exist(txn, tablet_id, 2); + } + } + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); +} + } // namespace doris::cloud From e0d08ca6f67d9eca67eaa29bb08e4ba41deea54a Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Fri, 5 Sep 2025 17:47:38 +0800 Subject: [PATCH 567/572] branch-3.0: [Fix](cloud) `calc_sync_versions` should consider full compaction (#55630) (#55699) pick https://github.com/apache/doris/pull/55630 --- .../cloud_cumulative_compaction_policy.cpp | 11 ++ be/src/cloud/cloud_full_compaction.cpp | 11 ++ be/src/cloud/cloud_meta_mgr.cpp | 2 + be/src/cloud/cloud_tablet.h | 3 + cloud/src/meta-service/meta_service.cpp | 22 ++- cloud/src/meta-service/meta_service_job.cpp | 1 + cloud/test/meta_service_test.cpp | 55 ++++-- gensrc/proto/cloud.proto | 3 +- .../cloud/test_cloud_calc_sync_version.out | 55 ++++++ .../cloud/test_cloud_calc_sync_version.groovy | 179 ++++++++++++++++++ 10 files changed, 324 insertions(+), 18 deletions(-) create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_calc_sync_version.out create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_calc_sync_version.groovy diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp b/be/src/cloud/cloud_cumulative_compaction_policy.cpp index 24bd61db8fafe6..f4178320044ea6 100644 --- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp +++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp @@ -219,6 +219,17 @@ int64_t CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point( int64_t last_cumulative_point) { TEST_INJECTION_POINT_RETURN_WITH_VALUE("new_cumulative_point", int64_t(0), output_rowset.get(), last_cumulative_point); + DBUG_EXECUTE_IF("CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point", { + auto target_tablet_id = dp->param("tablet_id", -1); + auto cumu_point = dp->param("cumu_point", -1); + if (target_tablet_id == tablet->tablet_id() && cumu_point != -1) { + LOG_INFO( + "[CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point] " + "tablet_id={}, cumu_point={}", + target_tablet_id, cumu_point); + return cumu_point; + } + }); // for MoW table, if there's too many versions, the delete bitmap will grow to // a very big size, which may cause the tablet meta too big and the `save_meta` // operation too slow. diff --git a/be/src/cloud/cloud_full_compaction.cpp b/be/src/cloud/cloud_full_compaction.cpp index 7358f6d19156a1..08e43ab2142ae9 100644 --- a/be/src/cloud/cloud_full_compaction.cpp +++ b/be/src/cloud/cloud_full_compaction.cpp @@ -149,6 +149,16 @@ Status CloudFullCompaction::pick_rowsets_to_compact() { } Status CloudFullCompaction::execute_compact() { + DBUG_EXECUTE_IF("CloudFullCompaction::execute_compact.block", { + auto target_tablet_id = dp->param("tablet_id", -1); + LOG_INFO( + "[verbose] CloudFullCompaction::execute_compact.block, target_tablet_id={}, " + "tablet_id={}", + target_tablet_id, cloud_tablet()->tablet_id()); + if (target_tablet_id == cloud_tablet()->tablet_id()) { + DBUG_BLOCK; + } + }); TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudFullCompaction::execute_compact_impl", Status::OK(), this); #ifndef __APPLE__ @@ -272,6 +282,7 @@ Status CloudFullCompaction::modify_rowsets() { cloud_tablet()->delete_rowsets(_input_rowsets, wrlock); cloud_tablet()->add_rowsets({_output_rowset}, false, wrlock); cloud_tablet()->set_base_compaction_cnt(stats.base_compaction_cnt()); + cloud_tablet()->set_full_compaction_cnt(stats.full_compaction_cnt()); cloud_tablet()->set_cumulative_layer_point(stats.cumulative_point()); if (output_rowset_delete_bitmap) { _tablet->tablet_meta()->delete_bitmap()->merge(*output_rowset_delete_bitmap); diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index 904e8e2e099405..a7a560f67193c6 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -521,6 +521,7 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, } req.set_base_compaction_cnt(tablet->base_compaction_cnt()); req.set_cumulative_compaction_cnt(tablet->cumulative_compaction_cnt()); + req.set_full_compaction_cnt(tablet->full_compaction_cnt()); req.set_cumulative_point(tablet->cumulative_layer_point()); } req.set_end_version(-1); @@ -772,6 +773,7 @@ Status CloudMetaMgr::sync_tablet_rowsets_unlocked(CloudTablet* tablet, tablet->last_cumu_compaction_success_time_ms = stats.last_cumu_compaction_time_ms(); tablet->set_base_compaction_cnt(stats.base_compaction_cnt()); tablet->set_cumulative_compaction_cnt(stats.cumulative_compaction_cnt()); + tablet->set_full_compaction_cnt(stats.full_compaction_cnt()); tablet->set_cumulative_layer_point(stats.cumulative_point()); tablet->reset_approximate_stats(stats.num_rowsets(), stats.num_segments(), stats.num_rows(), stats.data_size()); diff --git a/be/src/cloud/cloud_tablet.h b/be/src/cloud/cloud_tablet.h index d1ea7dec379684..1f3c202d06a6ef 100644 --- a/be/src/cloud/cloud_tablet.h +++ b/be/src/cloud/cloud_tablet.h @@ -123,12 +123,14 @@ class CloudTablet final : public BaseTablet { int64_t max_version_unlocked() const override { return _max_version; } int64_t base_compaction_cnt() const { return _base_compaction_cnt; } int64_t cumulative_compaction_cnt() const { return _cumulative_compaction_cnt; } + int64_t full_compaction_cnt() const { return _full_compaction_cnt; } int64_t cumulative_layer_point() const { return _cumulative_point.load(std::memory_order_relaxed); } void set_base_compaction_cnt(int64_t cnt) { _base_compaction_cnt = cnt; } void set_cumulative_compaction_cnt(int64_t cnt) { _cumulative_compaction_cnt = cnt; } + void set_full_compaction_cnt(int64_t cnt) { _full_compaction_cnt = cnt; } void set_cumulative_layer_point(int64_t new_point); int64_t last_cumu_compaction_failure_time() { return _last_cumu_compaction_failure_millis; } @@ -322,6 +324,7 @@ class CloudTablet final : public BaseTablet { int64_t _base_compaction_cnt = 0; int64_t _cumulative_compaction_cnt = 0; + int64_t _full_compaction_cnt = 0; int64_t _max_version = -1; int64_t _base_size = 0; int64_t _alter_version = -1; diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp index 2fa746e5f69f18..68ee07cd2680c5 100644 --- a/cloud/src/meta-service/meta_service.cpp +++ b/cloud/src/meta-service/meta_service.cpp @@ -1603,6 +1603,7 @@ void internal_get_rowset(Transaction* txn, int64_t start, int64_t end, std::vector> calc_sync_versions(int64_t req_bc_cnt, int64_t bc_cnt, int64_t req_cc_cnt, int64_t cc_cnt, int64_t req_cp, int64_t cp, + int64_t req_fc_cnt, int64_t fc_cnt, int64_t req_start, int64_t req_end) { using Version = std::pair; // combine `v1` `v2` to `v1`, return true if success @@ -1628,8 +1629,8 @@ std::vector> calc_sync_versions(int64_t req_bc_cnt, if (req_cc_cnt < cc_cnt) { Version cc_version; - if (req_cp < cp && req_cc_cnt + 1 == cc_cnt) { - // * only one CC happened and CP changed + if (req_cp < cp && req_cc_cnt + 1 == cc_cnt && req_fc_cnt == fc_cnt) { + // * only one CC happened and CP changed, and no full compaction happened // BE [=][=][=][=][=====][=][=] // ^~~~~ req_cp // MS [=][=][=][=][xxxxxxxxxxxxxx][=======][=][=] @@ -1653,6 +1654,13 @@ std::vector> calc_sync_versions(int64_t req_bc_cnt, // ^_____________________^ related_versions: [req_cp, max] // there may be holes if we don't return all version // after ms_cp, however it can be optimized. + // * one CC happened and CP changed, and full compaction happened + // BE [=][=][=][=][=][=][=][=][=][=] + // ^~~~~ req_cp + // MS [xxxxxxxxxxxxxx][xxxxxxxxxxxxxx][=======][=][=] + // ^~~~~~~ ms_cp + // ^___________________________^ related_versions: [req_cp, max] + // cc_version = {req_cp, std::numeric_limits::max() - 1}; } if (versions.empty() || !combine_if_overlapping(versions.front(), cc_version)) { @@ -1723,6 +1731,7 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, } int64_t req_bc_cnt = request->base_compaction_cnt(); int64_t req_cc_cnt = request->cumulative_compaction_cnt(); + int64_t req_fc_cnt = request->has_full_compaction_cnt() ? request->full_compaction_cnt() : 0; int64_t req_cp = request->cumulative_point(); do { @@ -1807,6 +1816,8 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, int64_t bc_cnt = tablet_stat.base_compaction_cnt(); int64_t cc_cnt = tablet_stat.cumulative_compaction_cnt(); + int64_t fc_cnt = + tablet_stat.has_full_compaction_cnt() ? tablet_stat.full_compaction_cnt() : 0; int64_t cp = tablet_stat.cumulative_point(); response->mutable_stats()->CopyFrom(tablet_stat); @@ -1818,17 +1829,18 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller, //========================================================================== // Find version ranges to be synchronized due to compaction //========================================================================== - if (req_bc_cnt > bc_cnt || req_cc_cnt > cc_cnt || req_cp > cp) { + if (req_bc_cnt > bc_cnt || req_cc_cnt > cc_cnt || req_cp > cp || req_fc_cnt > fc_cnt) { code = MetaServiceCode::INVALID_ARGUMENT; ss << "no valid compaction_cnt or cumulative_point given. req_bc_cnt=" << req_bc_cnt << ", bc_cnt=" << bc_cnt << ", req_cc_cnt=" << req_cc_cnt << ", cc_cnt=" << cc_cnt - << ", req_cp=" << req_cp << ", cp=" << cp << " tablet_id=" << tablet_id; + << " req_fc_cnt=" << req_fc_cnt << ", fc_cnt=" << fc_cnt << ", req_cp=" << req_cp + << ", cp=" << cp << " tablet_id=" << tablet_id; msg = ss.str(); LOG(WARNING) << msg; return; } auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, - req_start, req_end); + req_fc_cnt, fc_cnt, req_start, req_end); for (auto [start, end] : versions) { internal_get_rowset(txn.get(), start, end, instance_id, tablet_id, code, msg, response); if (code != MetaServiceCode::OK) { diff --git a/cloud/src/meta-service/meta_service_job.cpp b/cloud/src/meta-service/meta_service_job.cpp index 7b1f13462037f1..76b6a8d0c432fc 100644 --- a/cloud/src/meta-service/meta_service_job.cpp +++ b/cloud/src/meta-service/meta_service_job.cpp @@ -755,6 +755,7 @@ void process_compaction_job(MetaServiceCode& code, std::string& msg, std::string } else if (compaction.type() == TabletCompactionJobPB::FULL) { // clang-format off stats->set_base_compaction_cnt(stats->base_compaction_cnt() + 1); + stats->set_full_compaction_cnt(stats->has_full_compaction_cnt() ? stats->full_compaction_cnt() + 1 : 1); if (compaction.output_cumulative_point() > stats->cumulative_point()) { // After supporting parallel cumu compaction, compaction with older cumu point may be committed after // new cumu point has been set, MUST NOT set cumu point back to old value diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index 8150dced5bd9d1..6f94ec3b46ae75 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -3781,7 +3781,7 @@ TEST(MetaServiceTest, FilterCopyFilesTest) { extern std::vector> calc_sync_versions( int64_t req_bc_cnt, int64_t bc_cnt, int64_t req_cc_cnt, int64_t cc_cnt, int64_t req_cp, - int64_t cp, int64_t req_start, int64_t req_end); + int64_t cp, int64_t req_fc_cnt, int64_t fc_cnt, int64_t req_start, int64_t req_end); TEST(MetaServiceTest, CalcSyncVersionsTest) { using Versions = std::vector>; @@ -3797,7 +3797,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 1}; auto [req_cp, cp] = std::tuple {5, 5}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{8, 12}})); } @@ -3813,7 +3813,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; auto [req_cp, cp] = std::tuple {5, 10}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{5, 12}})); // [5, 9] v [8, 12] } @@ -3822,7 +3822,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; auto [req_cp, cp] = std::tuple {5, 15}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{5, 14}})); // [5, 14] v [8, 12] } @@ -3839,7 +3839,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; auto [req_cp, cp] = std::tuple {5, 5}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{5, INT64_MAX - 1}})); // [5, max] v [8, 12] } @@ -3855,7 +3855,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 3}; auto [req_cp, cp] = std::tuple {5, 5}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{5, INT64_MAX - 1}})); // [5, max] v [8, 12] } @@ -3870,7 +3870,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 3}; auto [req_cp, cp] = std::tuple {5, 15}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{5, INT64_MAX - 1}})); // [5, max] v [8, 12] } @@ -3886,7 +3886,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 1}; auto [req_cp, cp] = std::tuple {5, 5}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{0, 4}, {8, 12}})); } @@ -3895,7 +3895,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 1}; auto [req_cp, cp] = std::tuple {8, 8}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{0, 12}})); // [0, 7] v [8, 12] } @@ -3904,7 +3904,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; auto [req_cp, cp] = std::tuple {5, 10}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{0, 12}})); // [0, 4] v [5, 9] v [8, 12] } @@ -3913,7 +3913,7 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; auto [req_cp, cp] = std::tuple {5, 15}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); ASSERT_EQ(versions, (Versions {{0, 14}})); // [0, 4] v [5, 14] v [8, 12] } @@ -3922,11 +3922,42 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; auto [req_cp, cp] = std::tuple {5, 5}; auto [req_start, req_end] = std::tuple {8, 12}; - auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, req_start, req_end); // [0, 4] v [5, max] v [8, 12] ASSERT_EQ(versions, (Versions {{0, INT64_MAX - 1}})); } + + { + // when there exists full compaction, we can't optimize by "* only one CC happened and CP changed" + + // * one CC happened and CP changed, and full compaction happened + // BE [=][=][=][=][=][=][=][=][=][=] + // ^~~~~ req_cp + // MS [xxxxxxxxxxxxxx][xxxxxxxxxxxxxx][=======][=][=] + // ^~~~~~~ ms_cp + // ^___________________________^ related_versions: [req_cp, max] + // + auto [req_bc_cnt, bc_cnt] = std::tuple {0, 1}; + auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; + auto [req_cp, cp] = std::tuple {4, 7}; + auto [req_start, req_end] = std::tuple {9, 12}; + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 1, + req_start, req_end); + ASSERT_EQ(versions, (Versions {{0, INT64_MAX - 1}})); + } + + { + // abnormal case: + auto [req_bc_cnt, bc_cnt] = std::tuple {0, 1}; + auto [req_cc_cnt, cc_cnt] = std::tuple {1, 2}; + auto [req_cp, cp] = std::tuple {4, 7}; + auto [req_start, req_end] = std::tuple {9, 12}; + auto versions = calc_sync_versions(req_bc_cnt, bc_cnt, req_cc_cnt, cc_cnt, req_cp, cp, 0, 0, + req_start, req_end); + // when not considering full compaction, the returned versions is wrong becasue rowsets in [7-8] are missed + ASSERT_EQ(versions, (Versions {{0, 6}, {9, 12}})); + } } TEST(MetaServiceTest, StageTest) { diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index bc4a219d02691f..345e03e8574264 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -624,7 +624,7 @@ message TabletStatsPB { optional int64 cumulative_point = 9; optional int64 last_base_compaction_time_ms = 10; optional int64 last_cumu_compaction_time_ms = 11; - optional int64 full_compaction_cnt = 12; + optional int64 full_compaction_cnt = 12; // used by calc_sync_versions() only optional int64 last_full_compaction_time_ms = 13; optional int64 index_size = 14; optional int64 segment_size = 15; @@ -1043,6 +1043,7 @@ message GetRowsetRequest { // for compability reason we use FILL_WITH_DICT as default optional SchemaOp schema_op = 8 [default = FILL_WITH_DICT]; optional string request_ip = 9; + optional int64 full_compaction_cnt = 10; } message GetRowsetResponse { diff --git a/regression-test/data/fault_injection_p0/cloud/test_cloud_calc_sync_version.out b/regression-test/data/fault_injection_p0/cloud/test_cloud_calc_sync_version.out new file mode 100644 index 00000000000000..ed930452a5d142 --- /dev/null +++ b/regression-test/data/fault_injection_p0/cloud/test_cloud_calc_sync_version.out @@ -0,0 +1,55 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 10 +2 20 +3 30 + +-- !sql -- +1 10 +2 20 +3 30 + +-- !write_cluster_new_write -- +1 60 +2 70 +3 30 +4 40 +5 50 + +-- !read_cluster_query -- +1 60 +2 70 +3 30 +4 40 +5 50 + +-- !write_cluster_full_compaction -- +1 60 +2 70 +3 30 +4 40 +5 50 + +-- !write_cluster_cumu_compaction -- +1 60 +2 70 +3 30 +4 40 +5 50 + +-- !write_cluster_new_write -- +1 80 +2 70 +3 30 +4 40 +5 50 + +-- !read_cluster_check_dup_key -- + +-- !read_cluster_res -- +1 80 +2 70 +3 30 +4 40 +5 50 + diff --git a/regression-test/suites/fault_injection_p0/cloud/test_cloud_calc_sync_version.groovy b/regression-test/suites/fault_injection_p0/cloud/test_cloud_calc_sync_version.groovy new file mode 100644 index 00000000000000..12bdd2dd007bf9 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/cloud/test_cloud_calc_sync_version.groovy @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.suite.ClusterOptions + +suite("test_cloud_calc_sync_version","docker") { + def options = new ClusterOptions() + options.feConfigs += [ + 'cloud_cluster_check_interval_second=1', + 'cloud_tablet_rebalancer_interval_second=1', + ] + options.enableDebugPoints() + options.cloudMode = true + + docker(options) { + def write_cluster = "write_cluster" + def read_cluster = "read_cluster" + + // Add two clusters + cluster.addBackend(1, write_cluster) + cluster.addBackend(1, read_cluster) + + sql "use @${write_cluster}" + logger.info("==== switch to write cluster") + def tableName = "test_cloud_calc_sync_version" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k` int , + `v` int , + ) engine=olap + UNIQUE KEY(k) + DISTRIBUTED BY HASH(`k`) BUCKETS 1 + properties( + "replication_num" = "1", + "disable_auto_compaction" = "true") + """ + + sql """ INSERT INTO ${tableName} VALUES (1,10)""" + sql """ INSERT INTO ${tableName} VALUES (2,20)""" + sql """ INSERT INTO ${tableName} VALUES (3,30)""" + qt_sql "select * from ${tableName} order by k;" + + def check_rs_metas = { tbl, check_func -> + def compactionUrl = sql_return_maparray("show tablets from ${tbl};").get(0).CompactionStatus + def (code, out, err) = curl("GET", compactionUrl) + assert code == 0 + def jsonMeta = parseJson(out.trim()) + logger.info("==== rowsets: ${jsonMeta.rowsets}, cumu point: ${jsonMeta["cumulative point"]}") + check_func(jsonMeta.rowsets, jsonMeta["cumulative point"]) + } + + def tabletStats = sql_return_maparray("show tablets from ${tableName};") + def tabletId = tabletStats[0].TabletId + def tabletBackendId = tabletStats[0].BackendId + def tabletBackend + def backends = sql_return_maparray('show backends') + for (def be : backends) { + if (be.BackendId == tabletBackendId) { + tabletBackend = be + break; + } + } + logger.info("==== tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); + + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + + def do_cumu_compaction = { def tbl, def tablet_id, int start, int end, int cp -> + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point", [tablet_id: "${tablet_id}", cumu_point: "${cp}"]) + GetDebugPoint().enableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets", [tablet_id: "${tablet_id}", start_version: "${start}", end_version: "${end}"]) + + trigger_and_wait_compaction(tbl, "cumulative") + + GetDebugPoint().disableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::new_cumulative_point") + GetDebugPoint().disableDebugPointForAllBEs("CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets.set_input_rowsets") + } + + try { + // [2-2],[3-3],[4-4] -> [2,4] + do_cumu_compaction(tableName, tabletId, 2, 4, 5) + qt_sql "select * from ${tableName} order by k;" + check_rs_metas(tableName, {def rowsets, def cumu_point -> + assert rowsets.size() == 2 + assert cumu_point as int == 5 + assert rowsets[1].contains("[2-4]") + }) + + sql """ INSERT INTO ${tableName} VALUES (4,40)""" // ver=5 + sql """ INSERT INTO ${tableName} VALUES (5,50)""" // ver=6 + sql "sync;" + + GetDebugPoint().enableDebugPointForAllBEs("CloudFullCompaction::execute_compact.block", [tablet_id: "${tabletId}"]) + def t1 = thread("full compaction") { + // [2,4],[5-5],[6-6] -> [2,6] + sql "use @${write_cluster}" + trigger_and_wait_compaction(tableName, "full") + } + + sleep(1500) + sql """ INSERT INTO ${tableName} VALUES (1,60)""" // ver=7 + sql """ INSERT INTO ${tableName} VALUES (2,70)""" // ver=8 + sql "sync;" + qt_write_cluster_new_write "select * from ${tableName} order by k;" + + + // read cluster sync rowsets [2-4],[5-5],[6-6],[7-7],[8-8], bc_cnt=0, cc_cnt=1, cp=4 + sql "use @${read_cluster}" + logger.info("==== switch to read cluster") + qt_read_cluster_query "select * from ${tableName} order by k;" + check_rs_metas(tableName, {def rowsets, def cumu_point -> + assert rowsets.size() == 6 + assert cumu_point as int == 5 + assert rowsets[1].contains("[2-4]") + assert rowsets[2].contains("[5-5]") + assert rowsets[3].contains("[6-6]") + assert rowsets[4].contains("[7-7]") + assert rowsets[5].contains("[8-8]") + }) + + + sql "use @${write_cluster}" + logger.info("==== switch to write cluster") + GetDebugPoint().disableDebugPointForAllBEs("CloudFullCompaction::execute_compact.block") + t1.get() + qt_write_cluster_full_compaction "select * from ${tableName} order by k;" + check_rs_metas(tableName, {def rowsets, def cumu_point -> + assert rowsets.size() == 4 + assert cumu_point as int == 7 // updated by full compaction + assert rowsets[1].contains("[2-6]") + assert rowsets[2].contains("[7-7]") + assert rowsets[3].contains("[8-8]") + }) + + + do_cumu_compaction(tableName, tabletId, 7, 8, 7) + qt_write_cluster_cumu_compaction "select * from ${tableName} order by k;" + check_rs_metas(tableName, {def rowsets, def cumu_point -> + assert rowsets.size() == 3 + assert cumu_point as int == 7 + assert rowsets[1].contains("[2-6]") + assert rowsets[2].contains("[7-8]") + }) + + sql """ INSERT INTO ${tableName} VALUES (1,80)""" // ver=9 + sql "sync;" + qt_write_cluster_new_write "select * from ${tableName} order by k;" + + + // read cluster will read dup keys of ver=9 to ver=7 because it will not sync rowset [7-8] + sql "use @${read_cluster}" + logger.info("==== switch to read cluster") + sql "set disable_nereids_rules=ELIMINATE_GROUP_BY;" + qt_read_cluster_check_dup_key "select k,count() from ${tableName} group by k having count()>1;" + qt_read_cluster_res "select * from ${tableName} order by k;" + + } catch (Exception e) { + logger.info(e.getMessage()) + throw e + } finally { + GetDebugPoint().clearDebugPointsForAllBEs() + GetDebugPoint().clearDebugPointsForAllFEs() + } + } +} From 32710fd03a2460873473e566605ecd5bb842ec62 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 5 Sep 2025 17:49:34 +0800 Subject: [PATCH 568/572] branch-3.0: [Fix](oss-hdfs)Fix insert failure on Hive partitioned table with oss-hdfs (#55565) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … ### What problem does this PR solve? When inserting into a Hive partitioned table stored on oss-hdfs, the following issue occurs: First insert succeeds: Since the partition does not exist yet, HiveTableSink#setPartitionValues does not set storage-related information for the partition. Subsequent inserts fail: Once the partition exists, the system tries to resolve the partition’s storage information. At this stage, oss-hdfs is incorrectly treated as s3 instead of being recognized as hdfs, leading to insert failure. This PR fixes the storage type handling logic so that oss-hdfs partitions are correctly recognized as hdfs. #### How to Reproduce ``` Step 1: Create a Hive catalog whose storage is configured to use oss-hdfs. CREATE TABLE hive_partition_table ( `ts` DATETIME COMMENT 'ts', `col1` BOOLEAN COMMENT 'col1', `col2` INT COMMENT 'col2', `col3` BIGINT COMMENT 'col3', `col4` FLOAT COMMENT 'col4', `col5` DOUBLE COMMENT 'col5', `col6` DECIMAL(9,4) COMMENT 'col6', `col7` STRING COMMENT 'col7', `col8` DATE COMMENT 'col8', `col9` DATETIME COMMENT 'col9', `pt1` STRING COMMENT 'pt1', `pt2` STRING COMMENT 'pt2' ) PARTITION BY LIST (day(ts), pt1, pt2) () PROPERTIES ( 'write-format'='orc', 'compression-codec'='zlib' ); -- First insert (works fine) INSERT INTO hive_partition_table VALUES ('2023-01-01 00:00:00', true, 1, 1, 1.0, 1.0, 1.0000, '1', '2023-01-01', '2023-01-01 00:00:00', 'a', '1'), ('2023-01-02 00:00:00', false, 2, 2, 2.0, 2.0, 2.0000, '2', '2023-01-02', '2023-01-02 00:00:00', 'b', '2'), ('2023-01-03 00:00:00', true, 3, 3, 3.0, 3.0, 3.0000, '3', '2023-01-03', '2023-01-03 00:00:00', 'c', '3'); -- Second insert (fails) INSERT INTO hive_partition_table VALUES ('2023-01-01 00:00:00', true, 1, 1, 1.0, 1.0, 1.0000, '1', '2023-01-01', '2023-01-01 00:00:00', 'a', '1'), ('2023-01-02 00:00:00', false, 2, 2, 2.0, 2.0, 2.0000, '2', '2023-01-02', '2023-01-02 00:00:00', 'b', '2'), ('2023-01-03 00:00:00', true, 3, 3, 3.0, 3.0, 3.0000, '3', '2023-01-03', '2023-01-03 00:00:00', 'c', '3'); Error message on the second insert: [INVALID_ARGUMENT] Invalid S3 URI: oss://emr-ssss-oss.cn-beijing.oss-dls.aliyuncs.com/tmp/.sss/root/4118a835d5d948f8adc34107230c9b9b/pt1=a/pt2=1/727bd17a7b9541db-8f4bb2fbfda35b86_6ec0a4b4-cacc-4dd3-b3fc-b130cadcd508-0.zlib.orc ``` ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../org/apache/doris/common/util/LocationPath.java | 14 +------------- .../doris/fs/remote/RemoteFileSystemTest.java | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java index 798e13d321fe88..ad16ce1c74c079 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/LocationPath.java @@ -137,19 +137,7 @@ public LocationPath(String originLocation, Map props, boolean co tmpLocation = convertPath ? convertToS3(tmpLocation) : tmpLocation; break; case FeConstants.FS_PREFIX_OSS: - String endpoint = ""; - if (props.containsKey(OssProperties.ENDPOINT)) { - endpoint = props.get(OssProperties.ENDPOINT); - if (endpoint.startsWith(OssProperties.OSS_PREFIX)) { - // may use oss.oss-cn-beijing.aliyuncs.com - endpoint = endpoint.replace(OssProperties.OSS_PREFIX, ""); - } - } else if (props.containsKey(S3Properties.ENDPOINT)) { - endpoint = props.get(S3Properties.ENDPOINT); - } else if (props.containsKey(S3Properties.Env.ENDPOINT)) { - endpoint = props.get(S3Properties.Env.ENDPOINT); - } - if (isHdfsOnOssEndpoint(endpoint)) { + if (isHdfsOnOssEndpoint(originLocation)) { this.scheme = Scheme.OSS_HDFS; } else { if (useS3EndPoint(props)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java index 3fc15ab8e374fa..2b94dbf8bad92e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/remote/RemoteFileSystemTest.java @@ -68,7 +68,7 @@ public void testFilesystemAndAuthType() throws UserException { dfsPaths.add("ofs://a/b/c"); dfsPaths.add("gfs://a/b/c"); dfsPaths.add("hdfs://a/b/c"); - dfsPaths.add("oss://a/b/c"); // if endpoint contains 'oss-dls.aliyuncs', will use dfs filesystem + dfsPaths.add("oss://emr-ssss-oss.cn-beijing.oss-dls.aliyuncs.com/tmp/"); // if endpoint contains 'oss-dls.aliyuncs', will use dfs filesystem new MockUp(UserGroupInformation.class) { @Mock From 1b68a0f2e6919906ffd9929ec20aec97026b4395 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Sep 2025 17:49:47 +0800 Subject: [PATCH 569/572] branch-3.0: [fix](maxcompute)fix maxcompute catalog international user can not access. #55256 (#55559) Cherry-picked from #55256 Co-authored-by: daidai --- .../maxcompute/MaxComputeExternalCatalog.java | 19 +++++++++++++++++++ .../property/constants/MCProperties.java | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java index 1c8979ee1db5fc..ff46f3ad65a994 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java @@ -31,6 +31,7 @@ import com.aliyun.odps.Partition; import com.aliyun.odps.Project; import com.aliyun.odps.account.Account; +import com.aliyun.odps.account.AccountFormat; import com.aliyun.odps.account.AliyunAccount; import com.aliyun.odps.security.SecurityManager; import com.aliyun.odps.table.configuration.RestOptions; @@ -78,6 +79,8 @@ public class MaxComputeExternalCatalog extends ExternalCatalog { public boolean dateTimePredicatePushDown; + AccountFormat accountFormat = AccountFormat.DISPLAYNAME; + private static final Map REGION_ZONE_MAP; private static final List REQUIRED_PROPERTIES = ImmutableList.of( MCProperties.PROJECT, @@ -211,6 +214,14 @@ protected void initLocalObjectsImpl() { this.odps = new Odps(account); odps.setDefaultProject(defaultProject); odps.setEndpoint(endpoint); + + String accountFormatProp = props.getOrDefault(MCProperties.ACCOUNT_FORMAT, MCProperties.DEFAULT_ACCOUNT_FORMAT); + if (accountFormatProp.equals(MCProperties.ACCOUNT_FORMAT_NAME)) { + accountFormat = AccountFormat.DISPLAYNAME; + } else if (accountFormatProp.equals(MCProperties.ACCOUNT_FORMAT_ID)) { + accountFormat = AccountFormat.ID; + } + odps.setAccountFormat(accountFormat); Credentials credentials = Credentials.newBuilder().withAccount(odps.getAccount()) .withAppAccount(odps.getAppAccount()).build(); @@ -427,6 +438,14 @@ public void checkProperties() throws DdlException { + MCProperties.SPLIT_ROW_COUNT + "must be an integer"); } + String accountFormatProp = props.getOrDefault(MCProperties.ACCOUNT_FORMAT, MCProperties.DEFAULT_ACCOUNT_FORMAT); + if (accountFormatProp.equals(MCProperties.ACCOUNT_FORMAT_NAME)) { + accountFormat = AccountFormat.DISPLAYNAME; + } else if (accountFormatProp.equals(MCProperties.ACCOUNT_FORMAT_ID)) { + accountFormat = AccountFormat.ID; + } else { + throw new DdlException("property " + MCProperties.ACCOUNT_FORMAT + "only support name and id"); + } try { connectTimeout = Integer.parseInt( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java index 8282134c5a19b9..28375b4f371de2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java @@ -75,6 +75,13 @@ public class MCProperties extends BaseProperties { "mc.datetime_predicate_push_down"; public static final String DEFAULT_DATETIME_PREDICATE_PUSH_DOWN = "true"; + // The account systems for Alibaba Cloud China and International are different. If the primary account is an + // International user, specify ACCOUNT_FORMAT as ACCOUNT_FORMAT_ID. Otherwise, specify ACCOUNT_FORMAT_NAME. + public static final String ACCOUNT_FORMAT = "mc.account_format"; + public static final String ACCOUNT_FORMAT_NAME = "name"; + public static final String ACCOUNT_FORMAT_ID = "id"; + public static final String DEFAULT_ACCOUNT_FORMAT = ACCOUNT_FORMAT_NAME; + public static CloudCredential getCredential(Map props) { return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN); } From 701dc404589116fa98c7589d3f7f130a68b23b61 Mon Sep 17 00:00:00 2001 From: walter Date: Sat, 6 Sep 2025 16:34:28 +0800 Subject: [PATCH 570/572] branch-3.0: [chore](cloud) Supports to report the conflicting ranges during commit txn #55340 (#55715) cherry pick from #55340 --- cloud/src/common/config.h | 2 + cloud/src/meta-store/txn_kv.cpp | 94 ++++++++++++++++++++++++++++++++- cloud/src/meta-store/txn_kv.h | 10 ++++ cloud/test/txn_kv_test.cpp | 39 ++++++++++++++ 4 files changed, 143 insertions(+), 2 deletions(-) diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h index ffdf3837ee61d2..28b3a440df9b6d 100644 --- a/cloud/src/common/config.h +++ b/cloud/src/common/config.h @@ -325,4 +325,6 @@ CONF_mString(ca_cert_file_paths, "/etc/ssl/ca-bundle.pem"); CONF_Bool(enable_check_fe_drop_in_safe_time, "true"); +CONF_mBool(enable_logging_conflict_keys, "false"); + } // namespace doris::cloud::config diff --git a/cloud/src/meta-store/txn_kv.cpp b/cloud/src/meta-store/txn_kv.cpp index 2b07870a321e72..23df51d5ac88f8 100644 --- a/cloud/src/meta-store/txn_kv.cpp +++ b/cloud/src/meta-store/txn_kv.cpp @@ -317,6 +317,18 @@ TxnErrorCode Transaction::init() { return cast_as_txn_code(err); } + if (config::enable_logging_conflict_keys) { + err = fdb_transaction_set_option( + txn_, FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS, nullptr, 0); + if (err) { + LOG_WARNING("fdb_transaction_set_option error: ") + .tag("option", "FDB_TR_OPTION_REPORT_CONFLICTING_KEYS") + .tag("code", err) + .tag("msg", fdb_get_error(err)); + return cast_as_txn_code(err); + } + } + return TxnErrorCode::TXN_OK; } @@ -551,10 +563,15 @@ TxnErrorCode Transaction::commit() { if (err) { LOG(WARNING) << "fdb commit error, code=" << err << " msg=" << fdb_get_error(err); - fdb_error_is_txn_conflict(err) ? g_bvar_txn_kv_commit_conflict_counter << 1 - : g_bvar_txn_kv_commit_error_counter << 1; + if (fdb_error_is_txn_conflict(err)) { + g_bvar_txn_kv_commit_conflict_counter << 1; + static_cast(report_conflicting_range()); // don't overwrite the original error. + } else { + g_bvar_txn_kv_commit_error_counter << 1; + } return cast_as_txn_code(err); } + return TxnErrorCode::TXN_OK; } @@ -596,6 +613,79 @@ TxnErrorCode Transaction::abort() { return TxnErrorCode::TXN_OK; } +TxnErrorCode Transaction::get_conflicting_range( + std::vector>* values) { + constexpr std::string_view start = "\xff\xff/transaction/conflicting_keys/"; + constexpr std::string_view end = "\xff\xff/transaction/conflicting_keys/\xff"; + + int limit = 0; + int target_bytes = 0; + FDBStreamingMode mode = FDB_STREAMING_MODE_WANT_ALL; + int iteration = 0; + fdb_bool_t snapshot = 0; + fdb_bool_t reverse = 0; + FDBFuture* future = fdb_transaction_get_range( + txn_, FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)start.data(), start.size()), + FDB_KEYSEL_FIRST_GREATER_OR_EQUAL((uint8_t*)end.data(), end.size()), limit, + target_bytes, mode, iteration, snapshot, reverse); + + DORIS_CLOUD_DEFER { + fdb_future_destroy(future); + }; + + RETURN_IF_ERROR(await_future(future)); + + FDBKeyValue const* out_kvs; + int out_kvs_count; + fdb_bool_t out_more; + do { + fdb_error_t err = + fdb_future_get_keyvalue_array(future, &out_kvs, &out_kvs_count, &out_more); + if (err) { + LOG(WARNING) << "get_conflicting_range get keyvalue array error: " + << fdb_get_error(err); + return cast_as_txn_code(err); + } + for (int i = 0; i < out_kvs_count; i++) { + std::string_view key((char*)out_kvs[i].key, out_kvs[i].key_length); + std::string_view value((char*)out_kvs[i].value, out_kvs[i].value_length); + key.remove_prefix(start.size()); + values->emplace_back(key, value); + } + } while (out_more); + + return TxnErrorCode::TXN_OK; +} + +TxnErrorCode Transaction::report_conflicting_range() { + if (!config::enable_logging_conflict_keys) { + return TxnErrorCode::TXN_OK; + } + + std::vector> key_values; + RETURN_IF_ERROR(get_conflicting_range(&key_values)); + + // See https://github.com/apple/foundationdb/pull/2257/files for detail. + if (key_values.size() % 2 != 0) { + LOG(WARNING) << "the conflicting range is not well-formed, size=" << key_values.size(); + return TxnErrorCode::TXN_UNIDENTIFIED_ERROR; + } + + std::string out; + for (size_t i = 0; i < key_values.size(); i += 2) { + std::string_view start = key_values[i].first; + std::string_view end = key_values[i + 1].first; + std::string_view conflict_count = key_values[i].second; + if (!out.empty()) { + out += ", "; + } + out += fmt::format("[{}, {}): {}", hex(start), hex(end), conflict_count); + } + LOG(WARNING) << "conflicting key ranges: " << out; + + return TxnErrorCode::TXN_OK; +} + TxnErrorCode RangeGetIterator::init() { if (fut_ == nullptr) return TxnErrorCode::TXN_UNIDENTIFIED_ERROR; idx_ = 0; diff --git a/cloud/src/meta-store/txn_kv.h b/cloud/src/meta-store/txn_kv.h index 45d3b1cc8100e0..c2a8a3f044a3c1 100644 --- a/cloud/src/meta-store/txn_kv.h +++ b/cloud/src/meta-store/txn_kv.h @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -503,6 +504,8 @@ class RangeGetIterator : public cloud::RangeGetIterator { }; class Transaction : public cloud::Transaction { + FRIEND_TEST(TxnKvTest, ReportConflictingRange); + public: friend class Database; friend class FullRangeGetIterator; @@ -604,6 +607,13 @@ class Transaction : public cloud::Transaction { size_t get_bytes() const override { return get_bytes_; } private: + // Return the conflicting range when the transaction commit returns TXN_CONFLICT. + // + // It only works when the report_conflicting_ranges option is enabled. + TxnErrorCode get_conflicting_range( + std::vector>* key_values); + TxnErrorCode report_conflicting_range(); + std::shared_ptr db_ {nullptr}; bool commited_ = false; bool aborted_ = false; diff --git a/cloud/test/txn_kv_test.cpp b/cloud/test/txn_kv_test.cpp index b63773f38c4cb3..47805fed980d10 100644 --- a/cloud/test/txn_kv_test.cpp +++ b/cloud/test/txn_kv_test.cpp @@ -819,3 +819,42 @@ TEST(TxnKvTest, FullRangeGetIterator) { << "ms" << std::endl; } } + +TEST(TxnKvTest, ReportConflictingRange) { + config::enable_logging_conflict_keys = true; + + constexpr std::string_view key_prefix = "txn_kv_test__report_conflicting_range"; + std::string key = std::string(key_prefix) + std::to_string(time(nullptr)); + + { + // 1. write a common key + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, "value0"); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + } + + // 2. two txns, conflicting writes + std::unique_ptr txn1, txn2; + ASSERT_EQ(txn_kv->create_txn(&txn1), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn_kv->create_txn(&txn2), TxnErrorCode::TXN_OK); + + std::string val1, val2; + ASSERT_EQ(txn1->get(key, &val1), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn2->get(key, &val2), TxnErrorCode::TXN_OK); + + txn1->put(key, "value1"); + txn2->put(key, "value2"); + + ASSERT_EQ(txn1->commit(), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn2->commit(), TxnErrorCode::TXN_CONFLICT); + + // 3. get the conflicting ranges. + std::vector> values; + ASSERT_EQ(reinterpret_cast(txn2.get())->get_conflicting_range(&values), + TxnErrorCode::TXN_OK); + ASSERT_EQ(values.size(), 2); + ASSERT_EQ(values[0].first, key); + ASSERT_EQ(values[1].second, "0"); + ASSERT_TRUE(values[1].first.starts_with(key)); +} From 1d4e210c55fab5f799ad9e517c382c30b82450d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 6 Sep 2025 16:56:27 +0800 Subject: [PATCH 571/572] branch-3.0: [regression-test](point query) remove test_point_query nonconcurrent #52004 (#55707) Cherry-picked from #52004 Co-authored-by: lihangyu --- .../point_query_p0/test_point_query.groovy | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy b/regression-test/suites/point_query_p0/test_point_query.groovy index 21c4306f02d9ea..38855044153086 100644 --- a/regression-test/suites/point_query_p0/test_point_query.groovy +++ b/regression-test/suites/point_query_p0/test_point_query.groovy @@ -28,16 +28,7 @@ import java.sql.PreparedStatement import java.sql.ResultSet import java.util.concurrent.CopyOnWriteArrayList -suite("test_point_query", "nonConcurrent") { - def backendId_to_backendIP = [:] - def backendId_to_backendHttpPort = [:] - getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); - def set_be_config = { key, value -> - for (String backend_id: backendId_to_backendIP.keySet()) { - def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) - logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) - } - } +suite("test_point_query") { def user = context.config.jdbcUser def password = context.config.jdbcPassword def realDb = "regression_test_serving_p0" @@ -56,9 +47,8 @@ suite("test_point_query", "nonConcurrent") { // set server side prepared statement url def prepare_url = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true" try { - set_be_config.call("disable_storage_row_cache", "false") - sql "set global enable_fallback_to_original_planner = false" - sql """set global enable_nereids_planner=true""" + sql """set enable_fallback_to_original_planner = false""" + sql """set enable_nereids_planner=true""" def tableName = realDb + ".tbl_point_query" sql "CREATE DATABASE IF NOT EXISTS ${realDb}" @@ -309,8 +299,6 @@ suite("test_point_query", "nonConcurrent") { WHERE aaaid = '1111111'""" } finally { - set_be_config.call("disable_storage_row_cache", "true") - sql """set global enable_nereids_planner=true""" } // test partial update/delete From 7cffc132102d9815d3dd9edc7c014da16032cea4 Mon Sep 17 00:00:00 2001 From: zhaochangle Date: Tue, 2 Sep 2025 18:01:55 +0800 Subject: [PATCH 572/572] 1 --- .../apache/doris/insertoverwrite/InsertOverwriteManager.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java index cb01ff90339c12..a24b35efafabe4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java @@ -115,6 +115,7 @@ public long registerTaskGroup() { * for iot auto detect. register task first. then put in group. */ public void registerTaskInGroup(long groupId, long taskId) { + //FIXME: we should move log of ADD here and rethink the logic when transfer to master. LOG.info("register task " + taskId + " in group " + groupId); taskGroups.get(groupId).add(taskId); } @@ -278,6 +279,10 @@ private boolean rollback(long taskId) { } catch (DdlException e) { LOG.warn("can not get table, task: {}, reason: {}", task, e.getMessage()); return true; + } catch (NullPointerException e) { + // cannot get task + LOG.warn("can not get task {}, maybe removed", task); + return true; } return InsertOverwriteUtil.dropPartitions(olapTable, task.getTempPartitionNames()); }